diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f8e046c7b..aaad5838a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -148,6 +148,7 @@ jobs: node test-live.js node test-coverage-gate.js node test-node-reach-coverage.js + node test-reach-rank.js node test-issue-1107-live-layout.js node test-issue-1532-live-fullscreen.js node test-issue-1619-feed-detail-card-draggable.js @@ -506,6 +507,7 @@ jobs: CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1487-byop-modal-layout-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1630-reach-mobile-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-node-reach-coverage-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-reach-rank-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1640-compare-discovery-e2e.js 2>&1 | tee -a e2e-output.txt # #1616: slide-over focus-restore flake-gate. Runs the slide-over diff --git a/cmd/server/node_reach.go b/cmd/server/node_reach.go index f9c9192f6..4fbcdb437 100644 --- a/cmd/server/node_reach.go +++ b/cmd/server/node_reach.go @@ -199,13 +199,20 @@ type NodeReachWindow struct { Days int `json:"days"` Since string `json:"since"` } + +// NodeReachImportance: NeighborDegree / DegreeRank / NodesWithEdges come from +// the shared degree snapshot (see reach_rank.go) and match /api/reach-rank for +// the same RankSnapshotAt. DegreeRank is 0 unless RankStatus is "ranked"; +// NodesWithEdges is the ranked (visible) population. type NodeReachImportance struct { - NeighborDegree int `json:"neighbor_degree"` - DegreeRank int `json:"degree_rank"` - NodesWithEdges int `json:"nodes_with_edges"` - RelayObservations int `json:"relay_observations"` - BidirectionalLinks int `json:"bidirectional_links"` - DirectObservers int `json:"direct_observers"` + NeighborDegree int `json:"neighbor_degree"` + DegreeRank int `json:"degree_rank"` + NodesWithEdges int `json:"nodes_with_edges"` + RankStatus string `json:"rank_status"` // "ranked" | "unranked" | "unavailable" + RankSnapshotAt string `json:"rank_snapshot_at"` // RFC3339 UTC; "" when unavailable + RelayObservations int `json:"relay_observations"` + BidirectionalLinks int `json:"bidirectional_links"` + DirectObservers int `json:"direct_observers"` } type NodeReachObserver struct { Pubkey string `json:"pubkey"` @@ -271,16 +278,23 @@ const ( reachCacheMax = 256 ) -// reachCacheEntry keeps the computed report (resp, unfiltered) and the body -// last served for it. Visibility is applied on every serve (visibleReach), not -// at compute time, so a cached report never outlives a blacklist, hidden-prefix -// or rename change; hiddenKey records which identities the cached body omits so -// an unchanged result reuses it without re-marshalling. +// reachCacheEntry keeps the computed report (resp, unfiltered, no rank +// applied) and the body last served for it. Visibility and rank are both +// applied on every serve, not at compute time, so a cached report never +// outlives a blacklist, hidden-prefix, rename or rank-view change: +// - hiddenKey records which identities the served body's links/observers +// omit (visibleReach) — see identity_visibility.go and #1181. +// - viewID records which reachRankView (reach_rank.go) the served body's +// rank fields were applied from. +// +// raw is only re-marshalled when either one moved on from what it was last +// computed under (reachBody). type reachCacheEntry struct { at time.Time resp NodeReachResponse raw []byte hiddenKey string + viewID uint64 found bool // false only for the "node not found" result, which is never cached } @@ -301,8 +315,18 @@ type reachState struct { // round-2, adversarial #5). lastSeenBlacklistGen atomic.Uint64 - degreeMu sync.Mutex - degreeSnap *degreeSnapshot + // degreeMu guards the shared degree snapshot, the last rebuild failure + // and the ranked view built from the snapshot (reach_rank.go). degreeSF + // collapses concurrent rebuilds into one set of DB queries; viewBuildMu + // serialises view rebuilds so a burst after a change builds one view. + degreeMu sync.Mutex + degreeSnap *degreeSnapshot + degreeFailAt time.Time + degreeFailErr error + degreeSF singleflight.Group + rankView *reachRankView + rankViewSeq uint64 + viewBuildMu sync.Mutex // inactiveNodesTable caches a positive inactive_nodes probe for the // visibility name lookup (identity_visibility.go). @@ -378,11 +402,11 @@ func (s *Server) reachCachePut(key string, e reachCacheEntry) { // reachCacheSetBody stores a re-marshalled body for the entry computed at at, // keeping its original TTL; a no-op if the entry was replaced or evicted. -func (s *Server) reachCacheSetBody(key string, at time.Time, raw []byte, hiddenKey string) { +func (s *Server) reachCacheSetBody(key string, at time.Time, raw []byte, hiddenKey string, viewID uint64) { s.reach.cacheMu.Lock() defer s.reach.cacheMu.Unlock() if e, ok := s.reach.cache[key]; ok && e.at.Equal(at) { - e.raw, e.hiddenKey = raw, hiddenKey + e.raw, e.hiddenKey, e.viewID = raw, hiddenKey, viewID s.reach.cache[key] = e } } @@ -395,7 +419,9 @@ func (s *Server) reachCacheSetBody(key string, at time.Time, raw []byte, hiddenK // Otherwise every hidden identity is removed from links and direct_observers // and the counts derived from those lists are recomputed; hiddenKey lists the // removed pubkeys (sorted, comma-joined; "" when none, in which case resp is -// returned as is, without copying). +// returned as is, without copying). Rank fields (NeighborDegree and friends) +// are untouched here — they are counts over every valid edge, including +// hidden neighbours (see reach_rank.go); applyReachRank sets them separately. func (s *Server) visibleReach(ctx context.Context, resp NodeReachResponse) (out NodeReachResponse, hiddenKey string, targetHidden bool, err error) { var live map[string][]string // hiding names by pubkey; nil without prefixes if len(s.cfg.EnforcedHiddenNamePrefixes()) > 0 { @@ -461,28 +487,44 @@ func (s *Server) visibleReach(ctx context.Context, resp NodeReachResponse) (out keys = append(keys, pk) } sort.Strings(keys) - out = resp // Node, Window, ReliableTokens are the target's own + out = resp // Node, Window, ReliableTokens, Importance are the target's own out.Links, out.DirectObservers = links, obs out.Importance.BidirectionalLinks, out.Importance.DirectObservers = bidir, len(obs) return out, strings.Join(keys, ","), false, nil } -// reachBody returns e's body with visibility applied now, re-marshalling (and -// refreshing the cached body) only when the set of hidden identities changed. -// targetHidden means the caller must answer 404 instead. -func (s *Server) reachBody(ctx context.Context, key string, e reachCacheEntry) (raw []byte, targetHidden bool, err error) { +// currentReachRankView is the rank view for a Reach response, or nil when no +// snapshot can be read — the report then renders with rank_status +// "unavailable" instead of failing (applyReachRank handles a nil view). +func (s *Server) currentReachRankView(ctx context.Context) *reachRankView { + v, err := s.reachRankView(ctx) + if err != nil { + return nil + } + return v +} + +// reachBody returns e's body with visibility and the current rank view v both +// applied now, re-marshalling (and refreshing the cached body) only when the +// set of hidden identities or the rank view changed since raw was last +// marshalled. targetHidden means the caller must answer 404 instead — a +// target that was visible when e was computed can still be hidden by a rename +// or a blacklist/prefix change since. +func (s *Server) reachBody(ctx context.Context, key string, e reachCacheEntry, v *reachRankView) (raw []byte, targetHidden bool, err error) { vis, hiddenKey, targetHidden, err := s.visibleReach(ctx, e.resp) if err != nil || targetHidden { return nil, targetHidden, err } - if e.raw != nil && hiddenKey == e.hiddenKey { + viewID := v.viewID() + if e.raw != nil && hiddenKey == e.hiddenKey && viewID == e.viewID { return e.raw, false, nil } + applyReachRank(&vis.Importance, vis.Node.Pubkey, v) raw, err = json.Marshal(vis) if err != nil { return nil, false, err } - s.reachCacheSetBody(key, e.at, raw, hiddenKey) + s.reachCacheSetBody(key, e.at, raw, hiddenKey, viewID) return raw, false, nil } @@ -584,20 +626,23 @@ func (s *Server) handleNodeReach(w http.ResponseWriter, r *http.Request) { if !ok { return reachCacheEntry{}, nil } - // Marshal the visible body once here so the waiters sharing this - // result reuse it (each still re-checks visibility in reachBody). + // Marshal the visible, ranked body once here so the waiters sharing + // this result reuse it (each still re-checks visibility and rank in + // reachBody on later serves). e := reachCacheEntry{at: time.Now(), resp: resp, found: true} vis, hiddenKey, targetHidden, vErr := s.visibleReach(r.Context(), resp) if vErr != nil { return nil, vErr } if !targetHidden { // else leave raw nil; writeReachEntry re-checks and 404s + view := s.currentReachRankView(r.Context()) + applyReachRank(&vis.Importance, pubkey, view) raw, mErr := json.Marshal(vis) if mErr != nil { log.Printf("[reach] marshal failed for %s: %v", cacheKey, mErr) return nil, mErr } - e.raw, e.hiddenKey = raw, hiddenKey + e.raw, e.hiddenKey, e.viewID = raw, hiddenKey, view.viewID() } s.reachCachePut(cacheKey, e) return e, nil @@ -614,11 +659,11 @@ func (s *Server) handleNodeReach(w http.ResponseWriter, r *http.Request) { s.writeReachEntry(w, r, cacheKey, e) } -// writeReachEntry writes a found entry's body with visibility applied now: -// 404 when the target itself became hidden. A failed live-name lookup fails -// closed (500), never serving unfiltered data. +// writeReachEntry writes a found entry's body with visibility and the current +// rank view both applied now: 404 when the target itself became hidden. A +// failed live-name lookup fails closed (500), never serving unfiltered data. func (s *Server) writeReachEntry(w http.ResponseWriter, r *http.Request, key string, e reachCacheEntry) { - raw, targetHidden, err := s.reachBody(r.Context(), key, e) + raw, targetHidden, err := s.reachBody(r.Context(), key, e, s.currentReachRankView(r.Context())) if err != nil { log.Printf("[reach] serving %s failed: %v", key, err) writeError(w, 500, "reach computation failed") @@ -662,10 +707,9 @@ func (s *Server) computeNodeReach(ctx context.Context, pubkey string, days int) d = dirCounts{we: map[string]int{}, they: map[string]int{}, obs: map[string]obsAgg{}} } - // importance: neighbor_edges degree + rank (all-time). Served from a - // coarse-TTL snapshot so the full UNION+GROUP-BY aggregate runs at most - // once per snapshotTTL, not on every cache miss. - degree, rank, nodesWithEdges := s.reachDegreeRank(ctx, pubkey) + // importance: the all-time neighbour degree + rank are NOT computed here — + // the handler applies them from the shared rank view (applyReachRank) at + // serve time, so a cached report always carries the leaderboard's rank. // node first_seen comes from nodeInfo (buildNodeInfoMap folds it in via a // single bulk SELECT). Missing → empty string (the node may be @@ -740,7 +784,6 @@ func (s *Server) computeNodeReach(ctx context.Context, pubkey string, days int) Window: NodeReachWindow{Days: days, Since: since.Format(time.RFC3339)}, ReliableTokens: toks, Importance: NodeReachImportance{ - NeighborDegree: degree, DegreeRank: rank, NodesWithEdges: nodesWithEdges, RelayObservations: d.relay, BidirectionalLinks: bidir, DirectObservers: len(directObs), }, DirectObservers: directObs, @@ -748,80 +791,6 @@ func (s *Server) computeNodeReach(ctx context.Context, pubkey string, days int) }, true, nil } -// --- neighbor-degree snapshot --------------------------------------------- -// The degree/rank importance is identical across all reach requests except the -// pubkey match, so the full neighbor_edges aggregate is computed once and shared -// behind a coarse TTL. Rank is a binary search over the descending degree list. -const reachDegreeTTL = 60 * time.Second - -type degreeSnapshot struct { - at time.Time - total int // nodes that have any edge - deg map[string]int // lowercase pubkey → neighbour count - sortedDesc []int // degrees sorted descending, for rank -} - -func (s *Server) reachDegreeRank(ctx context.Context, pubkey string) (degree, rank, total int) { - snap := s.getDegreeSnapshot(ctx) - if snap == nil { - return 0, 0, 0 - } - degree = snap.deg[pubkey] - if degree == 0 { - // No edges → not ranked. rank=0 is the documented "off-the-list" value; - // avoids the nonsensical "#N+1 / N" the binary search would produce. - return 0, 0, snap.total - } - // rank = 1 + (number of nodes with strictly higher degree). sortedDesc is - // descending, so the count of entries > degree is the first index whose - // value is <= degree. - rank = 1 + sort.Search(len(snap.sortedDesc), func(i int) bool { return snap.sortedDesc[i] <= degree }) - return degree, rank, snap.total -} - -func (s *Server) getDegreeSnapshot(ctx context.Context) *degreeSnapshot { - // Fast path: serve a fresh snapshot under a short lock. - s.reach.degreeMu.Lock() - if s.reach.degreeSnap != nil && time.Since(s.reach.degreeSnap.at) < reachDegreeTTL { - snap := s.reach.degreeSnap - s.reach.degreeMu.Unlock() - return snap - } - stale := s.reach.degreeSnap - s.reach.degreeMu.Unlock() - - // Rebuild WITHOUT holding the lock so concurrent reach requests aren't - // serialized behind the aggregate query. A brief cold-start herd may run a - // few redundant queries; the last writer wins. - rows, err := s.db.conn.QueryContext(ctx, ` - SELECT pk, COUNT(*) neigh FROM ( - SELECT node_a pk FROM neighbor_edges - UNION ALL SELECT node_b FROM neighbor_edges - ) GROUP BY pk`) - if err != nil { - log.Printf("[reach] degree snapshot query failed: %v (serving stale)", err) - return stale // serve stale on error rather than zeroing - } - defer rows.Close() - deg := make(map[string]int) - var sortedDesc []int - for rows.Next() { - var pk string - var neigh int - if rows.Scan(&pk, &neigh) != nil { - continue - } - deg[strings.ToLower(pk)] = neigh - sortedDesc = append(sortedDesc, neigh) - } - sort.Sort(sort.Reverse(sort.IntSlice(sortedDesc))) - snap := °reeSnapshot{at: time.Now(), total: len(deg), deg: deg, sortedDesc: sortedDesc} - s.reach.degreeMu.Lock() - s.reach.degreeSnap = snap - s.reach.degreeMu.Unlock() - return snap -} - // scanReachRows reads windowed observations whose path contains any reliable // token, with the originator + observer + snr needed for attribution. Observer // id and originator pubkey are lowercased in SQL (not per row), the path slice diff --git a/cmd/server/node_reach_endpoint_test.go b/cmd/server/node_reach_endpoint_test.go index 679602467..9fbf8690e 100644 --- a/cmd/server/node_reach_endpoint_test.go +++ b/cmd/server/node_reach_endpoint_test.go @@ -44,6 +44,8 @@ func resetReachState(t *testing.T, servers ...*Server) { s.reach.cacheMu.Unlock() s.reach.degreeMu.Lock() s.reach.degreeSnap = nil + s.reach.rankView = nil + s.reach.degreeFailAt, s.reach.degreeFailErr = time.Time{}, nil s.reach.degreeMu.Unlock() } } @@ -68,7 +70,7 @@ func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { stmts := []string{ `CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER)`, `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (id TEXT)`, + `CREATE TABLE observers (id TEXT, name TEXT)`, `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE neighbor_edges (node_a TEXT, node_b TEXT, count INTEGER)`, } diff --git a/cmd/server/node_reach_visibility_test.go b/cmd/server/node_reach_visibility_test.go index d89170f7b..41e5652a6 100644 --- a/cmd/server/node_reach_visibility_test.go +++ b/cmd/server/node_reach_visibility_test.go @@ -338,6 +338,10 @@ func TestNodeReach_NothingHiddenBodyUnchanged(t *testing.T) { if err != nil || !ok { t.Fatalf("compute: ok=%v err=%v", ok, err) } + // computeNodeReach itself leaves the rank fields zero — the handler + // applies them from the shared rank view at serve time (applyReachRank), + // same as visibleReach does for the HTTP body compared below. + applyReachRank(&resp.Importance, resp.Node.Pubkey, f.srv.currentReachRankView(context.Background())) var got NodeReachResponse if err := json.Unmarshal([]byte(body), &got); err != nil { t.Fatal(err) @@ -432,20 +436,22 @@ func TestHiddenIdentityNames_PrefixMatchIsExact(t *testing.T) { // reachCacheSetBody never attaches a body to an entry computed at another // time: a body built from an older report must not be reused for a newer one. +// Covers both freshness keys it carries: hiddenKey (visibility) and viewID +// (rank). func TestReachCacheSetBody_StaleEntryUntouched(t *testing.T) { srv := &Server{} resetReachState(t, srv) old := time.Now().Add(-time.Minute) - cur := reachCacheEntry{at: time.Now(), raw: []byte(`{"current":true}`), hiddenKey: "", found: true} + cur := reachCacheEntry{at: time.Now(), raw: []byte(`{"current":true}`), hiddenKey: "", viewID: 1, found: true} srv.reachCachePut("k", cur) - srv.reachCacheSetBody("k", old, []byte(`{"stale":true}`), "x") + srv.reachCacheSetBody("k", old, []byte(`{"stale":true}`), "x", 2) got, ok := srv.reachCacheGet("k") - if !ok || string(got.raw) != `{"current":true}` || got.hiddenKey != "" { - t.Fatalf("stale body attached to a newer entry: %s %q", got.raw, got.hiddenKey) + if !ok || string(got.raw) != `{"current":true}` || got.hiddenKey != "" || got.viewID != 1 { + t.Fatalf("stale body attached to a newer entry: %s %q view=%d", got.raw, got.hiddenKey, got.viewID) } - srv.reachCacheSetBody("k", cur.at, []byte(`{"refreshed":true}`), "y") - if got, _ := srv.reachCacheGet("k"); string(got.raw) != `{"refreshed":true}` || got.hiddenKey != "y" { - t.Fatalf("matching entry not refreshed: %s %q", got.raw, got.hiddenKey) + srv.reachCacheSetBody("k", cur.at, []byte(`{"refreshed":true}`), "y", 3) + if got, _ := srv.reachCacheGet("k"); string(got.raw) != `{"refreshed":true}` || got.hiddenKey != "y" || got.viewID != 3 { + t.Fatalf("matching entry not refreshed: %s %q view=%d", got.raw, got.hiddenKey, got.viewID) } } @@ -594,10 +600,19 @@ func TestNodeReach_CacheHitMissAndInvalidation(t *testing.T) { } } -// The filtering does not touch the rank/total contract: neighbour degree, rank -// and total are identical with and without hidden identities configured. +// The filtering does not touch the numeric neighbour count: NeighborDegree +// counts every valid edge — including ones to a hidden or blacklisted +// neighbour — the same with or without hiding configured (it is a count, not +// an identity, so it does not leak who the neighbour is). DegreeRank and +// NodesWithEdges, by contrast, ARE affected by hiding: they are computed over +// the visible (ranked) population, so a hidden or blacklisted node never +// itself occupies a placement or is counted in the total (the agreed "visible +// population" contract — see reach_rank.go and the PR description). func TestNodeReach_FilteringLeavesRankFieldsUnchanged(t *testing.T) { f := newReachVisibilityDB(t) + // n–hidName, n–visible, n–blNode, visible–hidName: n has 3 neighbours; + // hidName and blNode are each hidden from ranking one way (name prefix, + // node blacklist) with the fixture's own config. for _, pair := range [][2]string{{f.n, f.hidName}, {f.n, f.visible}, {f.n, f.blNode}, {f.visible, f.hidName}} { a, b := pair[0], pair[1] if a > b { @@ -617,7 +632,83 @@ func TestNodeReach_FilteringLeavesRankFieldsUnchanged(t *testing.T) { t.Fatal(err) } h, p := hiddenCfg.Importance, noHiding.Importance - if h.NeighborDegree != 3 || h.NeighborDegree != p.NeighborDegree || h.DegreeRank != p.DegreeRank || h.NodesWithEdges != p.NodesWithEdges { - t.Fatalf("rank fields changed by filtering: with hiding %+v, without %+v", h, p) + if h.NeighborDegree != 3 || p.NeighborDegree != 3 { + t.Fatalf("NeighborDegree changed by filtering: with hiding %d, without %d, want 3 both (hidden neighbours still counted)", h.NeighborDegree, p.NeighborDegree) + } + // With hiding: only n and visible are ranked (hidName is name-hidden, + // blNode is blacklisted) — total 2. Without hiding: all four edge + // endpoints are ranked — total 4. + if h.NodesWithEdges != 2 { + t.Fatalf("NodesWithEdges with hiding = %d, want 2 (hidden/blacklisted nodes excluded from the ranked population)", h.NodesWithEdges) + } + if p.NodesWithEdges != 4 { + t.Fatalf("NodesWithEdges without hiding = %d, want 4 (nothing excluded)", p.NodesWithEdges) + } + // n has the highest degree (3) in both populations, so removing the two + // lower-degree nodes from the ranking does not change n's own placement. + if h.DegreeRank != 1 || p.DegreeRank != 1 { + t.Fatalf("DegreeRank changed unexpectedly: with hiding %d, without %d, want 1 both (n is always the top neighbour count)", h.DegreeRank, p.DegreeRank) + } +} + +// Cross-endpoint integration: a hidden neighbour of a visible node (a) never +// appears in that node's Reach links, (b) never appears in the leaderboard's +// rows or search — by pubkey or by name — yet (c) is still counted in the +// visible node's own NeighborDegree, and the leaderboard's total reflects +// only the visible population. Exercises /api/nodes/{pk}/reach and +// /api/reach-rank together, on the same fixture config (hidName is +// name-hidden, blNode is node-blacklisted). +func TestNodeReach_HiddenNeighbourCountedNotListedOrRanked(t *testing.T) { + f := newReachVisibilityDB(t) + for _, pair := range [][2]string{{f.n, f.hidName}, {f.n, f.visible}, {f.n, f.blNode}} { + a, b := pair[0], pair[1] + if a > b { + a, b = b, a + } + if _, err := f.db.conn.Exec(`INSERT INTO neighbor_edges (node_a, node_b) VALUES (?, ?)`, a, b); err != nil { + t.Fatal(err) + } + } + + // (a) + (c): n's own Reach page. + _, resp, body := f.get(t, f.n) + for _, l := range resp.Links { + if l.Pubkey == f.hidName || l.Pubkey == f.blNode { + t.Fatalf("hidden/blacklisted neighbour listed in links: %+v", l) + } + } + for _, leak := range []string{f.hidName, f.blNode, "🚫 private A", "Bravo"} { + if strings.Contains(strings.ToLower(body), strings.ToLower(leak)) { + t.Fatalf("hidden identity %q leaked into n's Reach body: %s", leak, body) + } + } + if resp.Importance.NeighborDegree != 3 { + t.Fatalf("n's NeighborDegree = %d, want 3 (counts hidden/blacklisted neighbours numerically)", resp.Importance.NeighborDegree) + } + + // (b): the leaderboard. + lb := getRank(t, f.srv, "/api/reach-rank") + for _, row := range lb.Rows { + if row.Pubkey == f.hidName || row.Pubkey == f.blNode { + t.Fatalf("hidden/blacklisted node placed on the leaderboard: %+v", row) + } + } + // n and visible are the only ranked pubkeys from this edge set (hidName, + // blNode are excluded); n is #1 with 3 neighbours. + if lb.Total != 2 { + t.Fatalf("leaderboard total = %d, want 2", lb.Total) + } + if len(lb.Rows) == 0 || lb.Rows[0].Pubkey != f.n || lb.Rows[0].Neighbors != 3 { + t.Fatalf("leaderboard top row = %+v, want n with 3 neighbours", lb.Rows) + } + for _, q := range []string{f.hidName, f.hidName[:10], "private", "🚫", f.blNode, f.blNode[:10], "bravo"} { + if r := getRank(t, f.srv, "/api/reach-rank?q="+q); r.Matched != 0 { + t.Fatalf("search %q found %d hidden/blacklisted rows, want 0: %+v", q, r.Matched, r.Rows) + } + } + + // Cross-check: the hidden node's own Reach page is still 404. + if rr := serveRankRoutes(f.srv, "/api/nodes/"+f.hidName+"/reach?days=30"); rr.Code != http.StatusNotFound { + t.Fatalf("hidden neighbour's own reach page: %d want 404", rr.Code) } } diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 1440eef50..b454f3724 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -103,6 +103,12 @@ func routeDescriptions() map[string]routeMeta { QueryParams: []paramMeta{ {Name: "days", Description: "Time window in days, 1-365.", Type: "integer"}, }}, + "GET /api/reach-rank": {Summary: "Reach leaderboard", Description: "Nodes ranked by all-time neighbour count: distinct neighbours over valid neighbor_edges rows (both endpoints 64-hex pubkeys, not equal; within the ingestor's edge retention) — a historical count, not a measure of radio quality, range or traffic. Only nodes with a Reach page (node row, or named observer row) that are not blacklisted or hidden are ranked. Rank is competition ranking (1, 1, 3) with ties listed in pubkey order; a search or page never renumbers. Served from a shared snapshot (60s TTL, refreshed in the background) that also backs the Rank on /api/nodes/{pubkey}/reach; snapshot_at is when it was read.", Tag: "nodes", + QueryParams: []paramMeta{ + {Name: "q", Description: "Case-insensitive substring of node name or pubkey (max 64 characters)", Type: "string"}, + {Name: "offset", Description: "Rows to skip within the (filtered) list; non-negative (default 0)", Type: "integer"}, + {Name: "limit", Description: "Rows per page (default 50; above 100 → 100; zero, negative or non-numeric → 50)", Type: "integer"}, + }}, "GET /api/nodes/{pubkey}/neighbors": {Summary: "Get node neighbors", Description: "Returns the queried node's first-hop neighbors with affinity scores and observation metadata (count, SNR, distance, observers). Ambiguous edges carry candidate pubkeys.", Tag: "nodes", Response: schemaRef("NodeNeighborsResponse")}, // Analytics diff --git a/cmd/server/reach_rank.go b/cmd/server/reach_rank.go new file mode 100644 index 000000000..da0a452ef --- /dev/null +++ b/cmd/server/reach_rank.go @@ -0,0 +1,654 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log" + "net/http" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" + + "golang.org/x/sync/singleflight" +) + +// reach_rank.go — the neighbour-degree snapshot shared by the Reach page's +// Rank card (/api/nodes/{pubkey}/reach) and the Reach leaderboard +// (/api/reach-rank). One definition, one snapshot: +// +// - Valid edge: a neighbor_edges row whose endpoints are both MeshCore +// pubkeys (exactly 64 hex characters, case-insensitive) and differ from +// each other. Legacy rows that fail this (e.g. an empty endpoint) are +// filtered here, never deleted. +// - Neighbours: distinct neighbours of a pubkey over every valid edge +// (all-time, i.e. within the ingestor's edge retention). +// - Ranked population: pubkeys with at least one valid edge that have a node +// row or a named observer row — exactly the pubkeys buildNodeInfoMap knows, +// so every placement has a Reach page — and are not node-blacklisted, +// observer-blacklisted, or hidden by any current node/observer name (or +// its inactive_nodes name while it has no named nodes row). +// Hidden nodes never occupy a placement, so no rank gap or total reveals +// them. +// - Rank: 1 + the number of ranked pubkeys with strictly more neighbours +// (competition ranking: 1, 1, 3). Ties are listed in pubkey order. +// - Total: the size of the ranked population. +// +// It is NOT a measure of radio quality, range or traffic. + +const ( + reachDegreeTTL = 60 * time.Second + reachDegreeQueryTimeout = 15 * time.Second + + reachRankDefaultLimit = 50 + reachRankMaxLimit = 100 + reachRankMaxQueryLen = 64 // runes + + reachRankRanked = "ranked" + reachRankUnranked = "unranked" + reachRankUnavailable = "unavailable" +) + +// reachDegreeRetryBackoff is how long a failed snapshot rebuild suppresses the +// next attempt, so a failing DB is not re-queried on every request. var (not +// const) so tests can shorten it. +var reachDegreeRetryBackoff = 15 * time.Second + +var errReachRankNoDB = errors.New("reach rank: no database") + +// onDegreeSnapshotLoad runs at the start of every snapshot DB load. A no-op in +// production; tests swap it to count loads and to hold one in flight. +var onDegreeSnapshotLoad = func() {} + +// edgeExistChunk bounds the row-value IN list of canonicalEdgesExisting +// (2 parameters per pair) well under SQLite's variable limit. +const edgeExistChunk = 400 + +// degreeSnapshot is one complete read of the neighbour graph plus the node / +// observer records of its endpoints. Immutable once published. +type degreeSnapshot struct { + at time.Time + deg map[string]int // lowercase pubkey → distinct neighbours over valid edges + ident map[string]rankIdent // endpoints that have a Reach page (node row or named observer row) +} + +type rankIdent struct { + name string // display name, as the Reach page header shows it + names []string // every non-empty current node/observer name, for the hidden-prefix check + named bool // has a nodes row with a name (so an inactive_nodes name is stale) +} + +// reachRankRow is one leaderboard placement. The unexported nameLower is the +// precomputed search key; it is never serialised. +type reachRankRow struct { + Rank int `json:"rank"` + Pubkey string `json:"pubkey"` + Name string `json:"name"` + Neighbors int `json:"neighbors"` + nameLower string +} + +// reachRankView is the ranked, visibility-filtered projection of one snapshot +// under one blacklist / hidden-prefix generation. Immutable once published. +type reachRankView struct { + id uint64 // unique per build, lets cached Reach bodies detect a stale rank + snap *degreeSnapshot + blGen uint64 + hidGen uint64 + rows []reachRankRow // neighbours desc, pubkey asc + pos map[string]int // pubkey → index into rows +} + +func (v *reachRankView) viewID() uint64 { + if v == nil { + return 0 + } + return v.id +} + +// lookup returns a pubkey's neighbour count, placement and the ranked total. +// degree comes from the full snapshot, so an unranked-but-visible node still +// shows its neighbour count. +func (v *reachRankView) lookup(pubkey string) (degree, rank, total int, ranked bool) { + degree = v.snap.deg[pubkey] + if i, ok := v.pos[pubkey]; ok { + return degree, v.rows[i].Rank, len(v.rows), true + } + return degree, 0, len(v.rows), false +} + +// page returns the rows matching q (case-insensitive substring of name or +// pubkey; "" matches all) in [offset, offset+limit), plus the match count. +// Placements are the global ones — a search never renumbers. +func (v *reachRankView) page(q string, offset, limit int) ([]reachRankRow, int) { + if q == "" { + n := len(v.rows) + if offset >= n { + return []reachRankRow{}, n + } + return v.rows[offset:min(offset+limit, n)], n + } + q = strings.ToLower(q) + out := make([]reachRankRow, 0, min(limit, 16)) + matched := 0 + for i := range v.rows { + r := &v.rows[i] + if !strings.Contains(r.Pubkey, q) && !strings.Contains(r.nameLower, q) { + continue + } + if matched >= offset && len(out) < limit { + out = append(out, *r) + } + matched++ + } + return out, matched +} + +// reachRankVisible reports whether a pubkey may occupy a placement. It is the +// same identityHidden rule the Reach page enforces (identity_visibility.go, +// #1181) — blacklisted, observer-blacklisted, or hidden by any current +// node/observer/inactive-node name — never a separate check, so the two +// endpoints cannot drift apart. +func reachRankVisible(cfg *Config, pubkey string, id rankIdent) bool { + return !identityHidden(cfg, pubkey, id.names...) +} + +func buildReachRankView(snap *degreeSnapshot, cfg *Config, id, blGen, hidGen uint64) *reachRankView { + rows := make([]reachRankRow, 0, len(snap.ident)) + for pk, ident := range snap.ident { + n := snap.deg[pk] + if n <= 0 || !reachRankVisible(cfg, pk, ident) { + continue + } + rows = append(rows, reachRankRow{Pubkey: pk, Name: ident.name, Neighbors: n, nameLower: strings.ToLower(ident.name)}) + } + sort.Slice(rows, func(i, j int) bool { + if rows[i].Neighbors != rows[j].Neighbors { + return rows[i].Neighbors > rows[j].Neighbors + } + return rows[i].Pubkey < rows[j].Pubkey + }) + pos := make(map[string]int, len(rows)) + for i := range rows { + if i > 0 && rows[i].Neighbors == rows[i-1].Neighbors { + rows[i].Rank = rows[i-1].Rank + } else { + rows[i].Rank = i + 1 + } + pos[rows[i].Pubkey] = i + } + return &reachRankView{id: id, snap: snap, blGen: blGen, hidGen: hidGen, rows: rows, pos: pos} +} + +// currentRankView returns the published view if it matches snap and the +// generations, else nil. +func (s *Server) currentRankView(snap *degreeSnapshot, blGen, hidGen uint64) *reachRankView { + s.reach.degreeMu.Lock() + defer s.reach.degreeMu.Unlock() + if v := s.reach.rankView; v != nil && v.snap == snap && v.blGen == blGen && v.hidGen == hidGen { + return v + } + return nil +} + +// reachRankView returns the current ranked view, rebuilding it when the +// snapshot or a visibility generation moved. The generations are read before +// filtering, so a concurrent blacklist / prefix change at worst tags the view +// with the older generation and forces one more rebuild on the next request. +// Rebuilds (CPU-only, no DB) are serialised on viewBuildMu so a burst of +// requests after a change builds one view; the fast path never takes it. +func (s *Server) reachRankView(ctx context.Context) (*reachRankView, error) { + snap, err := s.getDegreeSnapshot(ctx) + if err != nil { + return nil, err + } + var blGen, hidGen uint64 + if s.cfg != nil { + blGen, hidGen = s.cfg.BlacklistGeneration(), s.cfg.HiddenNamePrefixesGeneration() + } + if v := s.currentRankView(snap, blGen, hidGen); v != nil { + return v, nil + } + s.reach.viewBuildMu.Lock() + defer s.reach.viewBuildMu.Unlock() + if v := s.currentRankView(snap, blGen, hidGen); v != nil { + return v, nil // built by the request we queued behind + } + s.reach.degreeMu.Lock() + s.reach.rankViewSeq++ + id := s.reach.rankViewSeq + s.reach.degreeMu.Unlock() + + v := buildReachRankView(snap, s.cfg, id, blGen, hidGen) + + s.reach.degreeMu.Lock() + if s.reach.degreeSnap == snap { // not superseded by a newer snapshot meanwhile + s.reach.rankView = v + } + s.reach.degreeMu.Unlock() + return v, nil +} + +// getDegreeSnapshot returns the shared snapshot. While fresh it is served +// directly. Once expired, the old snapshot is still served immediately while +// one background rebuild refreshes it (stale-while-revalidate), so cached +// Reach responses never wait on the aggregate; its timestamp tells readers its +// age. Only a cold start (no snapshot yet) waits for the rebuild, shared by all +// concurrent callers. After a failed rebuild, no new attempt starts for +// reachDegreeRetryBackoff; a cold caller then gets the last error at once. +func (s *Server) getDegreeSnapshot(ctx context.Context) (*degreeSnapshot, error) { + s.reach.degreeMu.Lock() + cur, failAt, failErr := s.reach.degreeSnap, s.reach.degreeFailAt, s.reach.degreeFailErr + s.reach.degreeMu.Unlock() + if cur != nil && time.Since(cur.at) < reachDegreeTTL { + return cur, nil + } + backingOff := !failAt.IsZero() && time.Since(failAt) < reachDegreeRetryBackoff + if cur != nil { + if !backingOff { + // Starts one background refresh, or joins the running one + // (singleflight); not awaited — the channel is buffered. + s.refreshDegreeSnapshot(ctx) + } + return cur, nil + } + if backingOff { + return nil, failErr + } + select { + case res := <-s.refreshDegreeSnapshot(ctx): + if res.Err != nil { + return nil, res.Err + } + return res.Val.(*degreeSnapshot), nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// refreshDegreeSnapshot starts (or joins) the single in-flight rebuild. It runs +// on a context detached from the triggering request and bounded by +// reachDegreeQueryTimeout, so one client disconnecting cannot fail it. A +// failed or partial read is never published; a panic is turned into an error +// (DoChan would otherwise re-panic outside any handler and crash the server). +func (s *Server) refreshDegreeSnapshot(ctx context.Context) <-chan singleflight.Result { + return s.reach.degreeSF.DoChan("degree", func() (v interface{}, err error) { + attempted := false // only a real load attempt records a failure + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("reach rank snapshot load panicked: %v", r) + } + if attempted && err != nil { + s.reach.degreeMu.Lock() + s.reach.degreeFailAt, s.reach.degreeFailErr = time.Now(), err + stale := s.reach.degreeSnap + s.reach.degreeMu.Unlock() + if stale != nil { + log.Printf("[reach] degree snapshot rebuild failed: %v (serving snapshot from %s)", err, stale.at.UTC().Format(time.RFC3339)) + } else { + log.Printf("[reach] degree snapshot rebuild failed: %v (no snapshot to serve)", err) + } + } + }() + s.reach.degreeMu.Lock() + c, failAt, failErr := s.reach.degreeSnap, s.reach.degreeFailAt, s.reach.degreeFailErr + s.reach.degreeMu.Unlock() + if c != nil && time.Since(c.at) < reachDegreeTTL { + return c, nil // refreshed while this caller queued + } + if !failAt.IsZero() && time.Since(failAt) < reachDegreeRetryBackoff { + // A rebuild failed while this caller queued: honour the backoff. + // (A nil err with a stale c is fine — callers serve it.) + if c != nil { + return c, nil + } + return nil, failErr + } + attempted = true + onDegreeSnapshotLoad() + qctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), reachDegreeQueryTimeout) + defer cancel() + snap, err := s.loadDegreeSnapshot(qctx) + if err != nil { + return nil, err + } + s.reach.degreeMu.Lock() + s.reach.degreeSnap = snap + s.reach.degreeFailAt, s.reach.degreeFailErr = time.Time{}, nil + s.reach.degreeMu.Unlock() + return snap, nil + }) +} + +// rankQueryer is the read surface of the snapshot load (a *sql.Tx). +type rankQueryer interface { + QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) +} + +// loadDegreeSnapshot reads the neighbour degrees and the node / observer +// records of every edge endpoint with bulk queries (no per-row lookups), all +// in one read transaction so they see one consistent database state. Any +// query, scan or iteration error fails the whole load. The records are read +// here rather than via getCachedNodesAndPM because that cache swallows DB +// errors — an empty node list would silently drop every node from the ranking +// (or skip their hidden-name check). Which records make a pubkey "known" +// mirrors buildNodeInfoMap exactly (any node row; an observer row only with a +// non-NULL id and name), so every placement has a Reach page. An +// inactive_nodes name (a node aged out of `nodes`) feeds the hidden-name check +// unless the pubkey has a nodes row with a name: rows there are never removed +// when a node returns, so a current name supersedes it (a nameless return +// does not). +func (s *Server) loadDegreeSnapshot(ctx context.Context) (*degreeSnapshot, error) { + if s.db == nil || s.db.conn == nil { + return nil, errReachRankNoDB + } + at := time.Now() + tx, err := s.db.conn.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("reach rank begin: %w", err) + } + defer tx.Rollback() // read-only: nothing to commit + deg, err := loadValidDegrees(ctx, tx) + if err != nil { + return nil, err + } + ident := make(map[string]rankIdent) + addName := func(id *rankIdent, name string) { + if name != "" { + id.names = append(id.names, name) + } + } + err = scanRankRows(ctx, tx, "nodes", `SELECT public_key, name FROM nodes WHERE public_key IS NOT NULL`, + func(scan func(...interface{}) error) error { + var pk string + var name sql.NullString + if err := scan(&pk, &name); err != nil { + return err + } + pk = strings.ToLower(pk) + if deg[pk] == 0 { + return nil + } + id := ident[pk] + id.name = name.String + id.named = id.named || name.String != "" + addName(&id, name.String) + ident[pk] = id + return nil + }) + if err != nil { + return nil, err + } + err = scanRankRows(ctx, tx, "observers", `SELECT id, name FROM observers WHERE id IS NOT NULL`, + func(scan func(...interface{}) error) error { + var pk string + var name sql.NullString + if err := scan(&pk, &name); err != nil { + return err + } + pk = strings.ToLower(pk) // observer ids are stored upper-case + if deg[pk] == 0 { + return nil + } + id, known := ident[pk] + if !name.Valid { + // buildNodeInfoMap skips a NULL-name observer row, so it does + // not give the pubkey a Reach page; it has no name to hide. + return nil + } + if !known { + id.name = name.String // node rows win the display name + } + addName(&id, name.String) + ident[pk] = id + return nil + }) + if err != nil { + return nil, err + } + var hasInactive int + err = scanRankRows(ctx, tx, "schema", `SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'inactive_nodes'`, + func(scan func(...interface{}) error) error { return scan(&hasInactive) }) + if err != nil { + return nil, err + } + if hasInactive > 0 { // always in production (dbschema); minimal DBs may lack it + err = scanRankRows(ctx, tx, "inactive nodes", `SELECT public_key, name FROM inactive_nodes WHERE public_key IS NOT NULL AND name IS NOT NULL`, + func(scan func(...interface{}) error) error { + var pk, name string + if err := scan(&pk, &name); err != nil { + return err + } + pk = strings.ToLower(pk) + if id, known := ident[pk]; known && !id.named { + addName(&id, name) + ident[pk] = id + } + return nil + }) + if err != nil { + return nil, err + } + } + return °reeSnapshot{at: at, deg: deg, ident: ident}, nil +} + +// degreeCounter counts neighbours per pubkey; each count lives in a slice so +// an edge costs one map lookup and no map write once the pubkey is known. +type degreeCounter struct { + idx map[string]int + n []int +} + +func (d *degreeCounter) add(pk string) { + if i, ok := d.idx[pk]; ok { + d.n[i]++ + return + } + d.idx[pk] = len(d.n) + d.n = append(d.n, 1) +} + +// pubkeyForm reports whether s is a MeshCore pubkey (exactly 64 hex +// characters, any case) and whether it is already lower-case. +func pubkeyForm(s string) (valid, lower bool) { + if len(s) != 64 { + return false, false + } + lower = true + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c >= '0' && c <= '9', c >= 'a' && c <= 'f': + case c >= 'A' && c <= 'F': + lower = false + default: + return false, false + } + } + return true, lower +} + +// loadValidDegrees counts, per pubkey, its distinct neighbours over valid +// edges: rows whose endpoints are both pubkeys (case-insensitive) and differ. +// One plain scan of neighbor_edges; the checks run in Go — much cheaper than +// per-row SQL string functions (GLOB / ltrim / lower + DISTINCT). A row already in the form +// the ingestor writes (lower-case, node_a < node_b) is unique by the primary +// key and is counted directly. Any other valid row (upper-case hex or reversed +// order) is lower-cased and canonicalised, deduplicated against the other such +// rows and against an existing canonical row (one bulk lookup in the same +// read transaction as the scan), then counted. +func loadValidDegrees(ctx context.Context, tx rankQueryer) (map[string]int, error) { + counter := degreeCounter{idx: make(map[string]int)} + odd := map[[2]string]bool{} + err := scanRankRows(ctx, tx, "degree", `SELECT node_a, node_b FROM neighbor_edges WHERE node_a IS NOT NULL AND node_b IS NOT NULL`, func(scan func(...interface{}) error) error { + var a, b string + if err := scan(&a, &b); err != nil { + return err + } + aOK, aLower := pubkeyForm(a) + bOK, bLower := pubkeyForm(b) + if !aOK || !bOK { + return nil + } + if aLower && bLower && a < b { + counter.add(a) + counter.add(b) + return nil + } + la, lb := strings.ToLower(a), strings.ToLower(b) + if la == lb { + return nil // self-edge, in any case + } + if la > lb { + la, lb = lb, la + } + odd[[2]string{la, lb}] = true + return nil + }) + if err != nil { + return nil, err + } + if len(odd) > 0 { + existing, err := canonicalEdgesExisting(ctx, tx, odd) + if err != nil { + return nil, err + } + for pair := range odd { + if !existing[pair] { + counter.add(pair[0]) + counter.add(pair[1]) + } + } + } + deg := make(map[string]int, len(counter.idx)) + for pk, i := range counter.idx { + deg[pk] = counter.n[i] + } + return deg, nil +} + +// canonicalEdgesExisting returns which of the given lower-case canonical pairs +// also exist as a canonical row (and were therefore already counted). +func canonicalEdgesExisting(ctx context.Context, tx rankQueryer, pairs map[[2]string]bool) (map[[2]string]bool, error) { + list := make([][2]string, 0, len(pairs)) + for p := range pairs { + list = append(list, p) + } + found := make(map[[2]string]bool) + for start := 0; start < len(list); start += edgeExistChunk { + chunk := list[start:min(start+edgeExistChunk, len(list))] + args := make([]interface{}, 0, 2*len(chunk)) + for _, p := range chunk { + args = append(args, p[0], p[1]) + } + q := `SELECT node_a, node_b FROM neighbor_edges WHERE (node_a, node_b) IN (VALUES ` + + strings.TrimSuffix(strings.Repeat("(?,?),", len(chunk)), ",") + `)` + rows, err := tx.QueryContext(ctx, q, args...) + if err != nil { + return nil, fmt.Errorf("reach rank edge lookup: %w", err) + } + for rows.Next() { + var a, b string + if err := rows.Scan(&a, &b); err != nil { + rows.Close() + return nil, fmt.Errorf("reach rank edge lookup scan: %w", err) + } + found[[2]string{a, b}] = true + } + err = rows.Err() + rows.Close() + if err != nil { + return nil, fmt.Errorf("reach rank edge lookup rows: %w", err) + } + } + return found, nil +} + +// scanRankRows runs one snapshot query and feeds each row to fn, turning any +// query / scan / iteration error into a load failure. +func scanRankRows(ctx context.Context, tx rankQueryer, what, q string, fn func(scan func(...interface{}) error) error) error { + rows, err := tx.QueryContext(ctx, q) + if err != nil { + return fmt.Errorf("reach rank %s query: %w", what, err) + } + defer rows.Close() + for rows.Next() { + if err := fn(rows.Scan); err != nil { + return fmt.Errorf("reach rank %s scan: %w", what, err) + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("reach rank %s rows: %w", what, err) + } + return nil +} + +// applyReachRank fills the snapshot-derived importance fields of a Reach +// response from v (nil = no snapshot could be read). +func applyReachRank(imp *NodeReachImportance, pubkey string, v *reachRankView) { + if v == nil { + imp.NeighborDegree, imp.DegreeRank, imp.NodesWithEdges = 0, 0, 0 + imp.RankStatus, imp.RankSnapshotAt = reachRankUnavailable, "" + return + } + degree, rank, total, ranked := v.lookup(pubkey) + imp.NeighborDegree, imp.DegreeRank, imp.NodesWithEdges = degree, rank, total + imp.RankStatus = reachRankUnranked + if ranked { + imp.RankStatus = reachRankRanked + } + imp.RankSnapshotAt = v.snap.at.UTC().Format(time.RFC3339) +} + +// ReachRankResponse is the /api/reach-rank body. +type ReachRankResponse struct { + SnapshotAt string `json:"snapshot_at"` // when the neighbour graph was read (RFC3339, UTC) + Total int `json:"total"` // ranked nodes in the whole leaderboard + Matched int `json:"matched"` // rows matching q (== total without q) + Offset int `json:"offset"` + Limit int `json:"limit"` + Query string `json:"q"` + Rows []reachRankRow `json:"rows"` +} + +// handleReachRank serves one page of the Reach leaderboard from the cached +// ranked view: O(1) for an unfiltered page, one linear pass for a search. +func (s *Server) handleReachRank(w http.ResponseWriter, r *http.Request) { + qv := r.URL.Query() + q := strings.TrimSpace(qv.Get("q")) + if !utf8.ValidString(q) || utf8.RuneCountInString(q) > reachRankMaxQueryLen { + writeError(w, 400, "invalid q: expected at most "+strconv.Itoa(reachRankMaxQueryLen)+" characters") + return + } + limit := clampLimit(qv.Get("limit"), reachRankDefaultLimit, reachRankMaxLimit) + offset := 0 + if raw := qv.Get("offset"); raw != "" { + n, err := strconv.Atoi(raw) + if err != nil || n < 0 { + writeError(w, 400, "invalid offset: expected a non-negative integer") + return + } + offset = n + } + view, err := s.reachRankView(r.Context()) + if err != nil { + // 500, not 503: the SPA's api() treats 503 as server warm-up and + // retries for ~a minute. A failed read is never an empty leaderboard. + writeError(w, 500, "reach rank unavailable") + return + } + rows, matched := view.page(q, offset, limit) + writeJSON(w, ReachRankResponse{ + SnapshotAt: view.snap.at.UTC().Format(time.RFC3339), + Total: len(view.rows), + Matched: matched, + Offset: offset, + Limit: limit, + Query: q, + Rows: rows, + }) +} diff --git a/cmd/server/reach_rank_bench_test.go b/cmd/server/reach_rank_bench_test.go new file mode 100644 index 000000000..62880e925 --- /dev/null +++ b/cmd/server/reach_rank_bench_test.go @@ -0,0 +1,238 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "math/rand" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/gorilla/mux" + _ "modernc.org/sqlite" +) + +// Synthetic, reproducible datasets for the Reach leaderboard benchmarks. +// Each node gets edgesPerNode random neighbours (fixed seed, canonical a c { + a, c = c, a + } + if _, err := tx.Exec(`INSERT OR IGNORE INTO neighbor_edges (node_a, node_b, last_seen) VALUES (?, ?, '2026-09-01T00:00:00Z')`, a, c); err != nil { + b.Fatal(err) + } + } + } + if err := tx.Commit(); err != nil { + b.Fatal(err) + } + return &DB{conn: conn}, pks +} + +func benchRankServer(db *DB) *Server { + cfg := &Config{HiddenNamePrefixes: []string{"🚫"}} + return &Server{store: &PacketStore{db: db}, db: db, cfg: cfg, perfStats: NewPerfStats()} +} + +func (s *Server) benchDropDegreeSnapshot() { + s.reach.degreeMu.Lock() + s.reach.degreeSnap, s.reach.rankView = nil, nil + s.reach.degreeMu.Unlock() +} + +// BenchmarkReachRankColdBuild: a snapshot rebuild (three bulk queries) plus +// the ranked view — what one request pays once per 60s TTL. +func BenchmarkReachRankColdBuild(b *testing.B) { + for _, ds := range reachRankBenchSets { + b.Run(ds.name, func(b *testing.B) { + db, _ := benchRankDB(b, ds.nodes, ds.edgesPerNode) + srv := benchRankServer(db) + ctx := context.Background() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + srv.benchDropDegreeSnapshot() + v, err := srv.reachRankView(ctx) + if err != nil || len(v.rows) == 0 { + b.Fatalf("view: %v", err) + } + } + }) + } +} + +// BenchmarkReachRankViewRebuild: the CPU-only re-rank after a blacklist / +// hidden-prefix change (no DB). +func BenchmarkReachRankViewRebuild(b *testing.B) { + for _, ds := range reachRankBenchSets { + b.Run(ds.name, func(b *testing.B) { + db, _ := benchRankDB(b, ds.nodes, ds.edgesPerNode) + srv := benchRankServer(db) + snap, err := srv.loadDegreeSnapshot(context.Background()) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if v := buildReachRankView(snap, srv.cfg, uint64(i+1), 0, 0); len(v.rows) == 0 { + b.Fatal("empty view") + } + } + }) + } +} + +// BenchmarkReachRankWarm: full HTTP handler on a warm snapshot — a plain page, +// a deep page, a broad name search, and a pubkey search. +func BenchmarkReachRankWarm(b *testing.B) { + for _, ds := range reachRankBenchSets { + db, pks := benchRankDB(b, ds.nodes, ds.edgesPerNode) + srv := benchRankServer(db) + cases := []struct{ name, path string }{ + {"page1", "/api/reach-rank"}, + {"deep_page", fmt.Sprintf("/api/reach-rank?offset=%d", ds.nodes/2)}, + {"search_name", "/api/reach-rank?q=node-01"}, + {"search_pubkey", "/api/reach-rank?q=" + pks[ds.nodes/3][:12]}, + } + router := mux.NewRouter() + router.HandleFunc("/api/reach-rank", srv.handleReachRank).Methods("GET") + get := func(path string) int { + rr := httptest.NewRecorder() + router.ServeHTTP(rr, httptest.NewRequest("GET", path, nil)) + return rr.Code + } + for _, c := range cases { + b.Run(ds.name+"/"+c.name, func(b *testing.B) { + if code := get(c.path); code != http.StatusOK { + b.Fatalf("warm-up %s: %d", c.path, code) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if code := get(c.path); code != http.StatusOK { + b.Fatalf("%s: %d", c.path, code) + } + } + }) + } + } +} + +// BenchmarkNodeReachCacheHit: /api/nodes/{pk}/reach served from the response +// cache. Uses only APIs that also exist on master so the same benchmark can +// run before/after: the branch adds the shared rank-view lookup to this path. +func BenchmarkNodeReachCacheHit(b *testing.B) { + db, pks := benchRankDB(b, 1000, 4) + srv := benchRankServer(db) + path := "/api/nodes/" + pks[0] + "/reach?days=7" + if rr := serveReach(srv, path); rr.Code != http.StatusOK { + b.Fatalf("warm-up: %d %s", rr.Code, rr.Body.String()) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if rr := serveReach(srv, path); rr.Code != http.StatusOK { + b.Fatalf("status %d", rr.Code) + } + } +} + +// BenchmarkReachAndRankParallel: concurrent GOMAXPROCS goroutines hitting a +// warm cache/view — 3 parts /api/reach-rank (page1) to 1 part +// /api/nodes/{pk}/reach, resembling the leaderboard driving traffic to +// individual Reach pages. Confirms the shared snapshot/view mutex is not a +// bottleneck under concurrent readers. +func BenchmarkReachAndRankParallel(b *testing.B) { + for _, ds := range reachRankBenchSets { + b.Run(ds.name, func(b *testing.B) { + db, pks := benchRankDB(b, ds.nodes, ds.edgesPerNode) + srv := benchRankServer(db) + router := mux.NewRouter() + router.HandleFunc("/api/nodes/{pubkey}/reach", srv.handleNodeReach).Methods("GET") + router.HandleFunc("/api/reach-rank", srv.handleReachRank).Methods("GET") + get := func(path string) int { + rr := httptest.NewRecorder() + router.ServeHTTP(rr, httptest.NewRequest("GET", path, nil)) + return rr.Code + } + reachPath := "/api/nodes/" + pks[0] + "/reach?days=7" + if code := get("/api/reach-rank"); code != http.StatusOK { + b.Fatalf("warm-up rank: %d", code) + } + if code := get(reachPath); code != http.StatusOK { + b.Fatalf("warm-up reach: %d", code) + } + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + i++ + path := "/api/reach-rank" + if i%4 == 0 { + path = reachPath + } + if code := get(path); code != http.StatusOK { + b.Fatalf("%s: %d", path, code) + } + } + }) + }) + } +} diff --git a/cmd/server/reach_rank_test.go b/cmd/server/reach_rank_test.go new file mode 100644 index 000000000..5136fd8b5 --- /dev/null +++ b/cmd/server/reach_rank_test.go @@ -0,0 +1,1126 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/mux" + _ "modernc.org/sqlite" +) + +// ---- fixtures --------------------------------------------------------------- + +type rankTestNode struct { + pk, name string +} + +type rankTestObserver struct { + id, name string // id as stored (observer ids are upper-case in production) +} + +// newReachRankDB builds an in-memory DB with the columns the Reach and +// leaderboard paths read. edges are [a, b] pubkey pairs; each is one +// neighbor_edges row (canonical order is irrelevant to the degree count). +func newReachRankDB(t testing.TB, nodes []rankTestNode, observers []rankTestObserver, edges [][2]string) *DB { + t.Helper() + conn, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + conn.SetMaxOpenConns(1) // one shared :memory: database + stmts := []string{ + `CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER DEFAULT 0, battery_mv INTEGER, temperature_c REAL, foreign_advert INTEGER DEFAULT 0)`, + `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT)`, + `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, + `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, + `CREATE TABLE neighbor_edges (node_a TEXT NOT NULL, node_b TEXT NOT NULL, count INTEGER DEFAULT 1, last_seen TEXT, PRIMARY KEY (node_a, node_b))`, + } + for _, s := range stmts { + if _, err := conn.Exec(s); err != nil { + t.Fatal(err) + } + } + tx, err := conn.Begin() + if err != nil { + t.Fatal(err) + } + for _, n := range nodes { + if _, err := tx.Exec(`INSERT INTO nodes (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) VALUES (?, ?, 'repeater', 56.1, 10.2, '2026-09-01T00:00:00Z', '2026-06-01T00:00:00Z', 1)`, n.pk, n.name); err != nil { + t.Fatal(err) + } + } + for _, o := range observers { + if _, err := tx.Exec(`INSERT INTO observers (id, name) VALUES (?, ?)`, o.id, o.name); err != nil { + t.Fatal(err) + } + } + for _, e := range edges { + if _, err := tx.Exec(`INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, 1, '2026-09-01T00:00:00Z')`, e[0], e[1]); err != nil { + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + db := &DB{conn: conn} + db.isV3Flag.forceTrue() + t.Cleanup(func() { conn.Close() }) + return db +} + +func newReachRankServer(t *testing.T, db *DB, cfg *Config) *Server { + t.Helper() + srv := &Server{store: newTestStoreWithDB(t, db, cfg), db: db, cfg: cfg, perfStats: NewPerfStats()} + resetReachState(t, srv) + return srv +} + +func serveRankRoutes(srv *Server, path string) *httptest.ResponseRecorder { + router := mux.NewRouter() + router.HandleFunc("/api/nodes/{pubkey}/reach", srv.handleNodeReach).Methods("GET") + router.HandleFunc("/api/reach-rank", srv.handleReachRank).Methods("GET") + req := httptest.NewRequest("GET", path, nil) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + return rr +} + +func getRank(t *testing.T, srv *Server, path string) ReachRankResponse { + t.Helper() + rr := serveRankRoutes(srv, path) + if rr.Code != http.StatusOK { + t.Fatalf("GET %s: status=%d body=%s", path, rr.Code, rr.Body.String()) + } + var resp ReachRankResponse + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatalf("GET %s: bad json: %v", path, err) + } + return resp +} + +func getReachImportance(t *testing.T, srv *Server, pk string) NodeReachImportance { + t.Helper() + rr := serveRankRoutes(srv, "/api/nodes/"+pk+"/reach?days=30") + if rr.Code != http.StatusOK { + t.Fatalf("reach %s: status=%d body=%s", pk, rr.Code, rr.Body.String()) + } + var resp NodeReachResponse + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatalf("reach %s: bad json: %v", pk, err) + } + return resp.Importance +} + +// star returns edges joining hub to each of spokes. +func star(hub string, spokes ...string) [][2]string { + out := make([][2]string, 0, len(spokes)) + for _, s := range spokes { + out = append(out, [2]string{hub, s}) + } + return out +} + +func snapOf(deg map[string]int, ident map[string]rankIdent) *degreeSnapshot { + return °reeSnapshot{at: time.Date(2026, 9, 18, 12, 0, 0, 0, time.UTC), deg: deg, ident: ident} +} + +func node(name string) rankIdent { + id := rankIdent{name: name} + if name != "" { + id.names = []string{name} + } + return id +} + +// expireDegreeSnapshot replaces the published snapshot with a copy backdated +// past its TTL (a new value, so readers of the old pointer never race) and +// returns its timestamp. +func expireDegreeSnapshot(srv *Server) time.Time { + srv.reach.degreeMu.Lock() + defer srv.reach.degreeMu.Unlock() + old := *srv.reach.degreeSnap + old.at = old.at.Add(-reachDegreeTTL - time.Minute) + srv.reach.degreeSnap = &old + return old.at +} + +// clearDegreeBackoff forgets the last rebuild failure so the next request +// retries at once. +func clearDegreeBackoff(srv *Server) { + srv.reach.degreeMu.Lock() + srv.reach.degreeFailAt, srv.reach.degreeFailErr = time.Time{}, nil + srv.reach.degreeMu.Unlock() +} + +// publishedSnap returns the currently published snapshot pointer. +func publishedSnap(srv *Server) *degreeSnapshot { + srv.reach.degreeMu.Lock() + defer srv.reach.degreeMu.Unlock() + return srv.reach.degreeSnap +} + +// waitForSnapshotChange polls until the published snapshot is no longer prev. +func waitForSnapshotChange(t *testing.T, srv *Server, prev *degreeSnapshot) *degreeSnapshot { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if cur := publishedSnap(srv); cur != prev { + return cur + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("background snapshot refresh never published") + return nil +} + +func rowPKs(rows []reachRankRow) []string { + out := make([]string, len(rows)) + for i, r := range rows { + out[i] = r.Pubkey + } + return out +} + +func rowRanks(rows []reachRankRow) []int { + out := make([]int, len(rows)) + for i, r := range rows { + out[i] = r.Rank + } + return out +} + +// ---- ranking contract (pure) ----------------------------------------------- + +func TestReachRankView_CompetitionRankingDeterministicOrder(t *testing.T) { + a, b, c, d, e := pk64("a1"), pk64("b2"), pk64("c3"), pk64("d4"), pk64("e5") + snap := snapOf( + map[string]int{e: 5, a: 5, d: 3, c: 3, b: 1}, + map[string]rankIdent{a: node("A"), b: node("B"), c: node("C"), d: node("D"), e: node("E")}, + ) + want := []string{a, e, c, d, b} + for i := 0; i < 25; i++ { // map iteration order is randomised per build + v := buildReachRankView(snap, &Config{}, 1, 0, 0) + if got := rowPKs(v.rows); fmt.Sprint(got) != fmt.Sprint(want) { + t.Fatalf("build %d: order=%v want %v (neighbours desc, pubkey asc)", i, got, want) + } + if got := rowRanks(v.rows); fmt.Sprint(got) != "[1 1 3 3 5]" { + t.Fatalf("build %d: ranks=%v want [1 1 3 3 5] (competition ranking)", i, got) + } + } + v := buildReachRankView(snap, &Config{}, 1, 0, 0) + if deg, rank, total, ok := v.lookup(d); !ok || deg != 3 || rank != 3 || total != 5 { + t.Fatalf("lookup(d)=(%d,%d,%d,%v) want (3,3,5,true)", deg, rank, total, ok) + } +} + +func TestReachRankView_EmptyAndUnranked(t *testing.T) { + v := buildReachRankView(snapOf(map[string]int{}, map[string]rankIdent{}), &Config{}, 1, 0, 0) + if len(v.rows) != 0 { + t.Fatalf("empty snapshot: %d rows", len(v.rows)) + } + rows, matched := v.page("", 0, 50) + if rows == nil || len(rows) != 0 || matched != 0 { + t.Fatalf("empty page: rows=%v matched=%d (want non-nil empty, 0)", rows, matched) + } + if deg, rank, total, ok := v.lookup(pk64("aa")); ok || deg != 0 || rank != 0 || total != 0 { + t.Fatalf("lookup on empty view = (%d,%d,%d,%v)", deg, rank, total, ok) + } + + // Known node without edges, and edge endpoints with no node/observer + // record (incl. the empty pubkey legacy rows carry) are not ranked. + lonely, a, ghost := pk64("10"), pk64("a1"), pk64("99") + snap := snapOf( + map[string]int{a: 2, ghost: 9, "": 55}, + map[string]rankIdent{lonely: node("Lonely"), a: node("A")}, + ) + v = buildReachRankView(snap, &Config{}, 1, 0, 0) + if got := rowPKs(v.rows); fmt.Sprint(got) != fmt.Sprint([]string{a}) { + t.Fatalf("ranked=%v want only %s", got, a) + } + if _, rank, total, ok := v.lookup(a); !ok || rank != 1 || total != 1 { + t.Fatalf("a must be #1/1 once unknown endpoints are excluded, got #%d/%d", rank, total) + } + if deg, _, _, ok := v.lookup(lonely); ok || deg != 0 { + t.Fatalf("node without edges must be unranked with degree 0, got degree=%d ranked=%v", deg, ok) + } +} + +func TestReachRankView_SearchAndPagingKeepGlobalRanks(t *testing.T) { + deg := map[string]int{} + ident := map[string]rankIdent{} + for i := 0; i < 120; i++ { + pk := pk64(fmt.Sprintf("%04x", i)) + deg[pk] = 200 - i/2 // pairs tie: ranks 1,1,3,3,... + name := fmt.Sprintf("Relay-%03d", i) + if i%10 == 0 { + name = fmt.Sprintf("Aarhus Hub %03d", i) + } + ident[pk] = node(name) + } + v := buildReachRankView(snapOf(deg, ident), &Config{}, 1, 0, 0) + + p2, matched := v.page("", 50, 50) + if matched != 120 || len(p2) != 50 || p2[0].Rank != 51 || p2[0].Pubkey != pk64("0032") { + t.Fatalf("page 2: matched=%d len=%d first=%+v", matched, len(p2), p2[0]) + } + if last, m := v.page("", 100, 50); m != 120 || len(last) != 20 { + t.Fatalf("last page: len=%d matched=%d want 20/120", len(last), m) + } + if past, m := v.page("", 500, 50); len(past) != 0 || m != 120 { + t.Fatalf("offset past end: len=%d matched=%d", len(past), m) + } + + // Case-insensitive name search keeps the global placements. + hubs, m := v.page("aarhus HUB", 0, 5) + if m != 12 || len(hubs) != 5 { + t.Fatalf("hub search: matched=%d len=%d want 12/5", m, len(hubs)) + } + for _, r := range hubs { + if want := v.rows[v.pos[r.Pubkey]].Rank; r.Rank != want { + t.Fatalf("search renumbered %s: rank %d want global %d", r.Name, r.Rank, want) + } + } + if hubs[1].Rank != 11 { // Aarhus Hub 010: index 10 ties index 11 → rank 11 + t.Fatalf("second hub rank=%d want 11", hubs[1].Rank) + } + more, m2 := v.page("aarhus hub", 5, 5) + if m2 != 12 || len(more) != 5 || more[0].Name != "Aarhus Hub 050" { + t.Fatalf("hub search page 2: matched=%d first=%+v", m2, more) + } + + // Pubkey search (upper-case input, prefix) resolves to the global rank. + byPK, m := v.page("0045", 0, 50) + if m != 1 || byPK[0].Pubkey != pk64("0045") || byPK[0].Rank != 69 { + t.Fatalf("pubkey search: matched=%d rows=%+v (want rank 69)", m, byPK) + } + if none, m := v.page("no such node", 0, 50); len(none) != 0 || m != 0 { + t.Fatalf("no-match search: len=%d matched=%d", len(none), m) + } +} + +func TestReachRankView_HiddenAndBlacklistedNeverPlaced(t *testing.T) { + top, bl, obsBl, hidNode, hidObs, a, b := pk64("f0"), pk64("f1"), pk64("f2"), pk64("f3"), pk64("f4"), pk64("a1"), pk64("b2") + snap := snapOf( + map[string]int{top: 50, bl: 40, obsBl: 30, hidNode: 20, hidObs: 10, a: 5, b: 5}, + map[string]rankIdent{ + top: node("🚫 private top"), + bl: node("Blacklisted"), + obsBl: node("Blocked observer"), + hidNode: {name: "", names: []string{"🚫 hidden obs name"}}, // unnamed node, hidden observer name + hidObs: {name: "Visible node", names: []string{"Visible node", "🚫 observer"}}, // any hidden name hides + a: node("A"), + b: node("B"), + }, + ) + cfg := &Config{ + NodeBlacklist: []string{strings.ToUpper(bl)}, + ObserverBlacklist: []string{obsBl}, + HiddenNamePrefixes: []string{"🚫"}, + } + v := buildReachRankView(snap, cfg, 1, 0, 0) + if got := rowPKs(v.rows); fmt.Sprint(got) != fmt.Sprint([]string{a, b}) { + t.Fatalf("ranked=%v want only the visible nodes %v", got, []string{a, b}) + } + if got := rowRanks(v.rows); fmt.Sprint(got) != "[1 1]" { + t.Fatalf("ranks=%v: hidden nodes above must not leave a gap", got) + } + for _, pk := range []string{top, bl, obsBl, hidNode, hidObs} { + if _, rank, total, ok := v.lookup(pk); ok || rank != 0 || total != 2 { + t.Fatalf("lookup(%s)=rank %d total %d ranked %v; want unranked, total 2", pk[:4], rank, total, ok) + } + if rows, m := v.page(pk[:8], 0, 50); len(rows) != 0 || m != 0 { + t.Fatalf("hidden pubkey %s leaked via search", pk[:4]) + } + } + for _, q := range []string{"private", "blacklisted", "blocked", "hidden", "visible", "🚫"} { + if rows, m := v.page(q, 0, 50); len(rows) != 0 || m != 0 { + t.Fatalf("hidden name leaked via search %q: %+v", q, rows) + } + } +} + +// ---- endpoint ----------------------------------------------------------------- + +func TestHandleReachRank_ShapeAndInputValidation(t *testing.T) { + a, b, c, d := pk64("a1"), pk64("b2"), pk64("c3"), pk64("d4") + db := newReachRankDB(t, + []rankTestNode{{a, "Alpha "}, {b, "Bravo"}, {c, ""}, {d, "Delta"}}, + []rankTestObserver{{strings.ToUpper(d), "Delta observer"}}, + append(star(a, b, c, d), [2]string{b, c})) + srv := newReachRankServer(t, db, &Config{}) + + resp := getRank(t, srv, "/api/reach-rank") + if resp.Total != 4 || resp.Matched != 4 || resp.Offset != 0 || resp.Limit != 50 || resp.Query != "" { + t.Fatalf("meta=%+v", resp) + } + if _, err := time.Parse(time.RFC3339, resp.SnapshotAt); err != nil { + t.Fatalf("snapshot_at %q: %v", resp.SnapshotAt, err) + } + if got := fmt.Sprint(rowRanks(resp.Rows)); got != "[1 2 2 4]" { + t.Fatalf("ranks=%s want [1 2 2 4]", got) + } + if resp.Rows[0].Name != "Alpha " || resp.Rows[0].Neighbors != 3 { + t.Fatalf("row 0=%+v (name returned verbatim as data)", resp.Rows[0]) + } + // The raw body must not carry a literal tag (encoding/json escapes <, >). + if rr := serveRankRoutes(srv, "/api/reach-rank"); strings.Contains(rr.Body.String(), " + diff --git a/public/node-reach.css b/public/node-reach.css index 7c830fcbd..e7466b487 100644 --- a/public/node-reach.css +++ b/public/node-reach.css @@ -43,6 +43,26 @@ .nq-tier { font-weight:700; } .nq-tier-glyph { font-size:10px; margin-left:4px; letter-spacing:1px; } +/* Rank card link to the Reach leaderboard. New links use --link-color (the + AA-contrast link token, #1668 M5) rather than --accent. */ +.nq-rank-link { display:inline-block; margin-top:4px; font-size:12px; color:var(--link-color); } + +/* Reach leaderboard (#/reach-rank, reach-rank.js) — reuses the report styles + above; these are the leaderboard-specific bits. */ +.rr-meta { margin:8px 0 0; color:var(--text-muted); font-size:12px; font-weight:500; } +.rr-search { display:flex; flex-direction:column; gap:4px; max-width:360px; margin:10px 0 4px; } +.rr-search label { font-size:12px; font-weight:600; color:var(--text-muted); } +.rr-search .nodes-search { flex:none; width:100%; box-sizing:border-box; font-size:14px; } +.rr-table { font-size:14px; margin:8px 0 12px; } +.rr-table td { vertical-align:top; } +.rr-rank { width:56px; } +.rr-node { overflow-wrap:anywhere; } +.rr-node .nq-link { color:var(--link-color); } +.rr-pk { font-family:var(--mono); font-size:12px; font-weight:500; color:var(--text-muted); } +.rr-msg { color:var(--text-muted); padding:16px 8px; } +.rr-error { padding:12px 0; text-align:left; } +.rr-pager { display:flex; flex-wrap:wrap; align-items:center; gap:12px; margin:0 0 24px; font-size:13px; } + @media print { /* Scope the print to the report only — the SPA shell (nav/header/sidebar) must not bleed onto the page. */ diff --git a/public/node-reach.js b/public/node-reach.js index 2d4f2b1f0..514c7add9 100644 --- a/public/node-reach.js +++ b/public/node-reach.js @@ -43,6 +43,36 @@ '
' + escapeHtml(descShort) + '
'; } + // Rank card: one snapshot with the leaderboard (#/reach-rank), so the same + // placement and total show on both. rank_status distinguishes a placed node + // from one outside the ranked population and from a failed snapshot read. + function rankValue(imp) { + if (imp.rank_status === 'ranked') return '#' + imp.degree_rank + ' / ' + imp.nodes_with_edges; + if (imp.rank_status === 'unavailable') return '—'; + return 'Not ranked'; + } + + function rankCard(imp) { + var desc = imp.rank_status === 'unavailable' ? 'Rank unavailable' : 'Rank by neighbour count'; + return '
' + + '
Rank
' + + '
' + escapeHtml(rankValue(imp)) + '
' + + '
' + escapeHtml(desc) + '
' + + 'View leaderboard
'; + } + + // positionHtml renders the all-time "Network position" cards (neighbours + + // rank). They come from neighbor_edges, not from path tokens, so they show + // for every node — including one without a reliable path-hash token. + function positionHtml(imp) { + return '
Network position (all-time)
' + + '
' + + statCard('Neighbours', imp.rank_status === 'unavailable' ? '—' : imp.neighbor_degree, 'All-time distinct neighbours', + 'Distinct neighbours in the all-time neighbour graph (advert first-hop + observer last-hop, geo-filtered).') + + rankCard(imp) + + '
'; + } + function linkRow(i, l) { var dist = l.distance_km != null ? Number(l.distance_km).toFixed(1) : '—'; var dir = l.bidir ? '' : (l.we_hear > 0 ? 'incoming' : 'outgoing'); @@ -146,6 +176,7 @@ // nodeName is already escaped; build then assign (keeps it off the // innerHTML line for the XSS-sink gate, like statsHtml below). var emptyHtml = '
' + headerHtml(n, nodeName, days) + + '
' + positionHtml(imp) + '
' + '
This node has no unique 1–3 byte prefix, so it cannot be reliably identified in paths — no link data available.
'; container.innerHTML = emptyHtml; wireTimeRange(container, pubkey); @@ -154,13 +185,7 @@ var statsHtml = headerHtml(n, nodeName, days) + '
' + - '
Network position (all-time)
' + - '
' + - statCard('Neighbours', imp.neighbor_degree, 'All-time distinct neighbours', - 'Distinct neighbours in the all-time neighbour graph (advert first-hop + observer last-hop, geo-filtered).') + - statCard('Rank', '#' + imp.degree_rank + ' / ' + imp.nodes_with_edges, 'Rank by neighbour count', - 'Rank by neighbour count among all nodes with edges. #1 = most-connected node in the network.') + - '
' + + positionHtml(imp) + '
Last ' + d.window.days + ' days
' + '
' + statCard('Links', d.links.length, 'Neighbours seen this window', @@ -278,4 +303,7 @@ } registerPage('node-reach', { init: init, destroy: destroy }); + + // Pure helpers, exposed for test-reach-rank.js (vm sandbox). + window.NodeReach = { rankValue: rankValue, positionHtml: positionHtml }; })(); diff --git a/public/reach-rank.js b/public/reach-rank.js new file mode 100644 index 000000000..6e79ba12f --- /dev/null +++ b/public/reach-rank.js @@ -0,0 +1,208 @@ +/* === CoreScope — reach-rank.js === + Reach leaderboard (#/reach-rank): every visible node ranked by all-time + neighbour count — the same Rank shown on each node's Reach page + (#/nodes//reach). One page of rows per request from + /api/reach-rank; search and paging run server-side over one cached + snapshot, and placements are global (a search never renumbers). + Reuses the .nq-* report styles from node-reach.css. */ +'use strict'; +(function () { + var PAGE_SIZE = 50; // mirrors the server default (reachRankDefaultLimit) + var MAX_QUERY = 64; // mirrors reachRankMaxQueryLen (code points, like the server's runes) + var MAX_PAGE = 100000; // keeps offset a safe integer; past-the-end pages snap back + var SEARCH_DEBOUNCE_MS = 250; + var loadGen = 0; // bumped per fetch + on destroy; drops stale responses + var state = null; // { q, page } while the page is mounted + var searchTimer = null; + + // clipQuery trims and caps a query at MAX_QUERY code points. Array.from + // splits by code point, so an emoji is never cut in half (a lone surrogate + // would make encodeURIComponent throw). + function clipQuery(q) { + return Array.from(String(q || '').trim()).slice(0, MAX_QUERY).join(''); + } + + // parseState reads ?q=&page= (1-based) from the hash query string. + function parseState(params) { + var page = parseInt(params.get('page'), 10); + return { q: clipQuery(params.get('q')), page: isFinite(page) && page > 1 ? Math.min(page, MAX_PAGE) : 1 }; + } + + // Keep search + page in the URL (replaceState: no router re-run, no history + // entry) so Back from a node's Reach page returns to the same view. + function syncHash() { + var parts = []; + if (state.q) parts.push('q=' + encodeURIComponent(state.q)); + if (state.page > 1) parts.push('page=' + state.page); + try { history.replaceState(null, '', '#/reach-rank' + (parts.length ? '?' + parts.join('&') : '')); } catch (e) {} + } + + // rowHtml renders one placement. Every node-controlled value is escaped; + // a node without a name falls back to its pubkey (as on the Reach page). + function rowHtml(r) { + var pk = String(r.pubkey || ''); + var label = r.name ? escapeHtml(r.name) : escapeHtml(pk.slice(0, 12)); + var href = '#/nodes/' + encodeURIComponent(pk) + '/reach'; + return '' + + '#' + escapeHtml(String(r.rank)) + '' + + '' + + 'Reach page for ' + label + '' + + (r.name ? ' ' + escapeHtml(pk.slice(0, 8)) + '' : '') + + '' + + '' + escapeHtml(String(r.neighbors)) + ''; + } + + function messageRow(text) { + return '' + escapeHtml(text) + ''; + } + + // statusText summarises the visible slice, e.g. "Showing 51–100 of 212 + // ranked nodes" or "Showing 1–3 of 3 matches for “ab” (212 ranked nodes)". + function statusText(d) { + var n = d.rows.length; + if (d.q) { + if (!d.matched) return 'No ranked node matches “' + d.q + '” (' + d.total + ' ranked nodes).'; + return 'Showing ' + (d.offset + 1) + '–' + (d.offset + n) + ' of ' + d.matched + + ' match' + (d.matched === 1 ? '' : 'es') + ' for “' + d.q + '” (' + d.total + ' ranked nodes).'; + } + if (!d.total) return 'No ranked nodes yet.'; + if (!n) return 'No rows on this page (' + d.total + ' ranked nodes).'; + return 'Showing ' + (d.offset + 1) + '–' + (d.offset + n) + ' of ' + d.total + ' ranked nodes.'; + } + + // snapshotText states the age of the data behind every placement. + function snapshotText(iso) { + var t = new Date(iso); + if (!iso || !isFinite(t.getTime())) return 'Snapshot time unknown.'; + return 'Snapshot ' + t.toLocaleString() + ' (' + timeAgo(iso) + ').'; + } + + function shellHtml(q) { + return '
' + + '

Reach leaderboard

' + + '
Nodes ranked by all-time neighbour count — the Rank shown on each node’s Reach page.
' + + '

Historical neighbour count — not a measure of radio quality or range. ' + + 'Counts distinct neighbours in the all-time neighbour graph. Equal counts share a placement (1, 1, 3). Hidden nodes are not ranked.

' + + '

' + + '
' + + '
' + + '' + + '' + + '' + + '' + + '
Nodes ranked by all-time neighbour count
RankNodeNeighbours
Loading leaderboard…
' + + '
'; + } + + function el(id) { return document.getElementById(id); } + + function setError(msg) { + var box = el('rrError'); + if (!box) return; + box.textContent = msg || ''; + box.hidden = !msg; + } + + // Disabling the focused pager button would drop focus to ; hand it + // to the other button instead so keyboard users keep their place. + function setPager(hasPrev, hasNext) { + var prev = el('rrPrev'), next = el('rrNext'); + var active = document.activeElement; + prev.disabled = !hasPrev; + next.disabled = !hasNext; + if (active === prev && prev.disabled && !next.disabled) next.focus(); + else if (active === next && next.disabled && !prev.disabled) prev.focus(); + } + + async function fetchPage() { + var myGen = ++loadGen; + var offset = (state.page - 1) * PAGE_SIZE; + var qs = 'offset=' + offset + '&limit=' + PAGE_SIZE + (state.q ? '&q=' + encodeURIComponent(state.q) : ''); + var table = el('rrTable'); + if (table) table.setAttribute('aria-busy', 'true'); + var d; + try { + d = await api('/reach-rank?' + qs, { ttl: 30000 }); + } catch (e) { + if (myGen !== loadGen) return; + if (table) table.removeAttribute('aria-busy'); + setError('Failed to load the leaderboard: ' + e.message); + el('rrRows').innerHTML = messageRow('Leaderboard unavailable.'); + el('rrStatus').textContent = ''; + setPager(state.page > 1, false); + return; + } + if (myGen !== loadGen) return; // a newer search/page (or destroy) won + table.removeAttribute('aria-busy'); + setError(''); + // A page past the end (e.g. a stale ?page= link) snaps back to the last one. + var count = d.q ? d.matched : d.total; + var lastPage = Math.max(1, Math.ceil(count / PAGE_SIZE)); + if (state.page > lastPage) { + state.page = lastPage; + syncHash(); + fetchPage(); + return; + } + var rows = d.rows || []; + var rowsHtml = rows.length ? rows.map(rowHtml).join('') : messageRow(statusText(d)); // escaped per field + el('rrRows').innerHTML = rowsHtml; + el('rrStatus').textContent = statusText(d); + el('rrSnapshot').textContent = snapshotText(d.snapshot_at); + setPager(state.page > 1, offset + rows.length < count); + } + + function setQuery(q) { + q = clipQuery(q); + if (q === state.q) return; + state.q = q; + state.page = 1; + syncHash(); + fetchPage(); + } + + function wire() { + var input = el('rrSearch'); + input.addEventListener('input', function () { + clearTimeout(searchTimer); + searchTimer = setTimeout(function () { setQuery(input.value); }, SEARCH_DEBOUNCE_MS); + }); + el('rrSearchForm').addEventListener('submit', function (e) { + e.preventDefault(); // Enter searches immediately + clearTimeout(searchTimer); + setQuery(input.value); + }); + el('rrPrev').addEventListener('click', function () { + if (state.page > 1) { state.page--; syncHash(); fetchPage(); } + }); + el('rrNext').addEventListener('click', function () { + if (state.page < MAX_PAGE) { state.page++; syncHash(); fetchPage(); } + }); + } + + function init(container) { + state = parseState(getHashParams()); + container.innerHTML = shellHtml(state.q); + wire(); + fetchPage(); + } + + function destroy() { + loadGen++; + clearTimeout(searchTimer); + searchTimer = null; + state = null; + } + + registerPage('reach-rank', { init: init, destroy: destroy }); + + // Pure helpers, exposed for test-reach-rank.js (vm sandbox). + window.ReachRank = { rowHtml: rowHtml, statusText: statusText, snapshotText: snapshotText, parseState: parseState, clipQuery: clipQuery }; +})(); diff --git a/test-a11y-axe-1668.js b/test-a11y-axe-1668.js index a30ab88d4..708235e8d 100644 --- a/test-a11y-axe-1668.js +++ b/test-a11y-axe-1668.js @@ -84,6 +84,7 @@ const ROUTES = [ '/analytics?tab=wardriving', '/analytics?tab=areas', '/audio-lab', + '/reach-rank', // Reach leaderboard (reach-rank.js) ]; // Source-of-truth for ROUTES reciprocity. Keep these in sync with the @@ -93,6 +94,7 @@ const REGISTERED_PAGES = [ 'home', 'packets', 'packet-detail', 'nodes', 'node-analytics', 'node-reach', 'channels', 'live', 'map', 'observers', 'observer-detail', 'compare', 'analytics', 'audio-lab', 'perf', 'traces', 'path-inspector', 'tools-landing', + 'reach-rank', ]; const REGISTERED_ANALYTICS_TABS = [ 'overview', 'rf', 'topology', 'channels', 'hashsizes', 'collisions', diff --git a/test-reach-rank-e2e.js b/test-reach-rank-e2e.js new file mode 100644 index 000000000..a7031593f --- /dev/null +++ b/test-reach-rank-e2e.js @@ -0,0 +1,333 @@ +/** + * E2E: Reach leaderboard (#/reach-rank) + the Reach page's Rank card link. + * + * 1. /api/reach-rank shape; the page renders the API's first page (ranks, + * names, neighbour counts, Reach links) with the "historical neighbour + * count" disclaimer and the snapshot time. + * 2. Paging keeps global ranks; search shows global ranks (no renumbering); + * no-match and past-the-end states. + * 3. Keyboard: search → Enter → Tab to a row link → Enter opens that node's + * Reach page, whose Rank card matches the leaderboard (same snapshot) and + * links back; Back restores the search from the hash. + * 4. Mobile 375×812: no horizontal scroll, all three columns visible. + * 5. Hostile node names render as text (API response intercepted); a failed + * API call shows an error, never an empty leaderboard. + * + * Defaults to localhost:13581 — NEVER point at prod (AGENTS.md). CI sets BASE_URL. + * Run: BASE_URL=http://localhost:13581 node test-reach-rank-e2e.js + */ +'use strict'; +const { chromium } = require('playwright'); + +const BASE = process.env.BASE_URL || 'http://localhost:13581'; +const DISCLAIMER = 'Historical neighbour count — not a measure of radio quality or range.'; + +let passed = 0, failed = 0; +async function step(name, fn) { + try { await fn(); passed++; console.log(' ✓ ' + name); } + catch (e) { failed++; console.error(' ✗ ' + name + ': ' + e.message); } +} +function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } + +async function getJson(page, path) { + const r = await page.request.get(BASE + path); + if (!r.ok()) throw new Error('GET ' + path + ' → HTTP ' + r.status()); + return r.json(); +} + +async function openBoard(page, hash) { + await page.goto(BASE + '/#/reach-rank' + (hash || '')); + await page.waitForSelector('#rrRows tr'); + await page.waitForFunction(() => document.getElementById('rrTable') && + !document.getElementById('rrTable').hasAttribute('aria-busy')); +} + +async function tableRows(page) { + return page.$$eval('#rrRows tr', trs => trs.map(tr => { + const tds = tr.querySelectorAll('td'); + const a = tr.querySelector('a.nq-link'); + return { + cells: tds.length, + rank: tds[0] ? tds[0].textContent.trim() : '', + node: a ? a.textContent.replace('Reach page for ', '').trim() : '', + href: a ? a.getAttribute('href') : '', + neighbors: tds[2] ? tds[2].textContent.trim() : '', + }; + })); +} + +function nodeLabel(r) { return r.name || r.pubkey.slice(0, 12); } + +// Reach + leaderboard read one shared snapshot. Fetch both until they report +// the same snapshot time (a 60s refresh can land between two requests). +async function sameSnapshot(page, pubkey) { + for (let i = 0; i < 3; i++) { + const reach = await getJson(page, '/api/nodes/' + pubkey + '/reach'); + const board = await getJson(page, '/api/reach-rank?limit=100&q=' + pubkey); + if (reach.importance.rank_snapshot_at === board.snapshot_at) return { reach, board }; + } + throw new Error('Reach and leaderboard never reported the same snapshot'); +} + +// Top node of the committed CI fixture: 10 neighbours before legacy +// empty-endpoint edges were excluded, 9 real ones after. +const FIXTURE_TOP = '1000009e310d729534b70faa33a1abe5cd5e45d594f72f786febccbb770b7e74'; + +async function main() { + let browser; + try { + browser = await chromium.launch({ + headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'], + }); + } catch (err) { + if (process.env.CHROMIUM_REQUIRE === '1') { + console.error('test-reach-rank-e2e.js: FAIL — Chromium required but unavailable: ' + err.message); + process.exit(1); + } + console.log('test-reach-rank-e2e.js: SKIP (Chromium unavailable: ' + err.message.split('\n')[0] + ')'); + process.exit(0); + } + const ctx = await browser.newContext({ viewport: { width: 1200, height: 900 } }); + const page = await ctx.newPage(); + page.setDefaultTimeout(20000); + // Leaving a node's Reach page while its Leaflet map is still animating + // throws "reading '_leaflet_pos'" — a pre-existing map-teardown race that + // reproduces on master's frontend and is unrelated to the leaderboard. This + // test leaves the Reach page quickly, so ignore that one message only. + const LEAFLET_TEARDOWN = /reading '_leaflet_pos'/; + const pageErrors = []; + page.on('pageerror', e => { if (!LEAFLET_TEARDOWN.test(e.message)) pageErrors.push(e.message); }); + + const api = await getJson(page, '/api/reach-rank'); + console.log('reach-rank E2E: total=' + api.total + ' snapshot_at=' + api.snapshot_at); + + await step('API shape', async () => { + for (const k of ['snapshot_at', 'total', 'matched', 'offset', 'limit', 'q', 'rows']) { + assert(k in api, 'missing key ' + k); + } + assert(api.limit === 50 && api.offset === 0 && api.matched === api.total, 'defaults: ' + JSON.stringify(api)); + assert(api.total > 0, 'fixture must rank at least one node'); + assert(api.rows[0].rank === 1, 'first placement must be #1'); + for (let i = 1; i < api.rows.length; i++) { + const a = api.rows[i - 1], b = api.rows[i]; + if (a.neighbors === b.neighbors) { + assert(a.rank === b.rank && a.pubkey < b.pubkey, 'tie at ' + i + ' must share a rank, pubkey-ordered'); + } else { + assert(a.neighbors > b.neighbors && b.rank === i + 1, 'competition rank broken at ' + i); + } + } + }); + + await step('desktop: first page matches the API', async () => { + await openBoard(page); + const rows = await tableRows(page); + assert(rows.length === api.rows.length, 'rows ' + rows.length + ' != ' + api.rows.length); + for (let i = 0; i < rows.length; i++) { + const r = api.rows[i]; + assert(rows[i].rank === '#' + r.rank, 'row ' + i + ' rank ' + rows[i].rank + ' != #' + r.rank); + assert(rows[i].node === nodeLabel(r), 'row ' + i + ' name ' + rows[i].node + ' != ' + nodeLabel(r)); + assert(rows[i].neighbors === String(r.neighbors), 'row ' + i + ' neighbours'); + assert(rows[i].href === '#/nodes/' + r.pubkey + '/reach', 'row ' + i + ' href ' + rows[i].href); + } + const note = await page.textContent('.rr-note'); + assert(note.includes(DISCLAIMER), 'disclaimer missing: ' + note); + const snap = await page.textContent('#rrSnapshot'); + assert(/^Snapshot .+ ago\)\.$/.test(snap.trim()), 'snapshot text: ' + snap); + assert(await page.isDisabled('#rrPrev'), 'Previous disabled on page 1'); + assert((await page.isDisabled('#rrNext')) === !(api.total > 50), 'Next state vs total'); + }); + + if (api.total > 50) { + await step('paging keeps global ranks and updates the hash', async () => { + const p2 = await getJson(page, '/api/reach-rank?offset=50'); + await page.click('#rrNext'); + await page.waitForFunction(r => document.querySelector('#rrRows tr td') && + document.querySelector('#rrRows tr td').textContent.trim() === '#' + r, p2.rows[0].rank); + assert(page.url().endsWith('#/reach-rank?page=2'), 'hash: ' + page.url()); + const status = await page.textContent('#rrStatus'); + assert(status.startsWith('Showing 51–'), 'status: ' + status); + await page.click('#rrPrev'); + await page.waitForFunction(() => document.querySelector('#rrRows tr td').textContent.trim() === '#1'); + }); + } + + // A ranked node below #1 whose Reach page renders the stats (it needs a + // reliable path-hash token; without one the page shows only a notice). + let target = null; + for (const r of api.rows.slice(1)) { + const rep = await getJson(page, '/api/nodes/' + r.pubkey + '/reach'); + if (rep.reliable_tokens && rep.reliable_tokens.length) { target = r; break; } + } + if (!target) throw new Error('fixture has no ranked node with a Reach stats card'); + console.log('reach-rank E2E: target #' + target.rank + ' ' + nodeLabel(target)); + await step('search shows global ranks (no renumbering)', async () => { + const q = target.name ? target.name : target.pubkey.slice(0, 10); + const expect = await getJson(page, '/api/reach-rank?q=' + encodeURIComponent(q)); + await openBoard(page); + await page.fill('#rrSearch', q); + await page.press('#rrSearch', 'Enter'); + await page.waitForFunction(n => document.querySelectorAll('#rrRows a.nq-link').length === n, expect.rows.length); + const rows = await tableRows(page); + for (let i = 0; i < rows.length; i++) { + assert(rows[i].rank === '#' + expect.rows[i].rank, 'search row ' + i + ' rank ' + rows[i].rank); + } + assert(rows.some(r => r.href === '#/nodes/' + target.pubkey + '/reach' && r.rank === '#' + target.rank), + 'target keeps its global rank #' + target.rank); + assert(page.url().includes('q=' + encodeURIComponent(q)), 'hash carries q: ' + page.url()); + }); + + await step('search by pubkey prefix and no-match message', async () => { + await page.fill('#rrSearch', target.pubkey.slice(0, 16).toUpperCase()); + await page.press('#rrSearch', 'Enter'); + await page.waitForFunction(h => { + const links = document.querySelectorAll('#rrRows a.nq-link'); + return links.length === 1 && links[0].getAttribute('href') === h; + }, '#/nodes/' + target.pubkey + '/reach'); + await page.fill('#rrSearch', 'zz-no-such-node-zz'); + await page.press('#rrSearch', 'Enter'); + await page.waitForFunction(() => /No ranked node matches/.test(document.getElementById('rrRows').textContent)); + assert(await page.isDisabled('#rrNext') && await page.isDisabled('#rrPrev'), 'pager disabled on no match'); + }); + + await step('past-the-end ?page snaps back to the last page', async () => { + await openBoard(page, '?page=999'); + await page.waitForFunction(() => !/page=999/.test(location.hash)); + assert((await tableRows(page)).length > 0, 'rows rendered after snapping back'); + }); + + await step('keyboard: search → Tab → Enter opens Reach; Rank card matches; Back restores search', async () => { + const q = target.name ? target.name : target.pubkey.slice(0, 10); + await openBoard(page); + await page.focus('#rrSearch'); + await page.keyboard.type(q); + await page.keyboard.press('Enter'); + await page.waitForFunction(h => [...document.querySelectorAll('#rrRows a.nq-link')].some(a => a.getAttribute('href') === h), + '#/nodes/' + target.pubkey + '/reach'); + // Tab from the search box walks the rows' links in order. + let found = false; + for (let i = 0; i < 60 && !found; i++) { + await page.keyboard.press('Tab'); + found = await page.evaluate(h => document.activeElement && document.activeElement.getAttribute('href') === h, + '#/nodes/' + target.pubkey + '/reach'); + } + assert(found, 'target row link not reachable by Tab'); + await page.keyboard.press('Enter'); + await page.waitForSelector('.nq-rank-link'); + assert(page.url().endsWith('#/nodes/' + target.pubkey + '/reach'), 'navigated to ' + page.url()); + const { reach, board } = await sameSnapshot(page, target.pubkey); + const row = board.rows.find(r => r.pubkey === target.pubkey); + assert(row, 'target missing from leaderboard search'); + assert(reach.importance.rank_status === 'ranked', 'rank_status ' + reach.importance.rank_status); + assert(reach.importance.degree_rank === row.rank && reach.importance.nodes_with_edges === board.total && + reach.importance.neighbor_degree === row.neighbors, + 'Reach #' + reach.importance.degree_rank + '/' + reach.importance.nodes_with_edges + ' (' + reach.importance.neighbor_degree + + ') != board #' + row.rank + '/' + board.total + ' (' + row.neighbors + ')'); + const cardText = await page.$$eval('.analytics-stat-card', cs => cs.map(c => c.textContent).join('|')); + assert(cardText.includes('#' + target.rank + ' / ' + board.total), 'Rank card text: ' + cardText); + assert(await page.getAttribute('.nq-rank-link', 'href') === '#/reach-rank', 'View leaderboard link'); + await page.goBack(); + await page.waitForSelector('#rrSearch'); + assert(await page.inputValue('#rrSearch') === q, 'search restored after Back'); + }); + + await step('legacy empty-endpoint edges are not neighbours (fixture top node shows 9, not 10)', async () => { + const board = await getJson(page, '/api/reach-rank?q=' + FIXTURE_TOP); + if (!board.rows.length) { console.log(' (fixture top node not present — skipped)'); return; } + const row = board.rows[0]; + assert(row.neighbors === 9 && row.rank === 1, 'fixture top node: #' + row.rank + ' with ' + row.neighbors + ' neighbours'); + const reach = await getJson(page, '/api/nodes/' + FIXTURE_TOP + '/reach'); + assert(reach.importance.neighbor_degree === 9, 'Reach Neighbours ' + reach.importance.neighbor_degree); + await page.goto(BASE + '/#/nodes/' + FIXTURE_TOP + '/reach'); + await page.waitForSelector('.nq-rank-link'); + const cards = await page.$$eval('.analytics-stat-card', cs => cs.map(c => c.innerText.replace(/\s+/g, ' '))); + assert(cards.some(t => /NEIGHBOURS 9 /i.test(t + ' ')), 'Neighbours card: ' + cards.join(' | ')); + }); + + await step('ranked node without a reliable path token still shows its Rank card', async () => { + const all = (await getJson(page, '/api/reach-rank?limit=100')).rows + .concat((await getJson(page, '/api/reach-rank?limit=100&offset=100')).rows); + let noToken = null; + for (const r of all) { + const rep = await getJson(page, '/api/nodes/' + r.pubkey + '/reach'); + if (!rep.reliable_tokens || !rep.reliable_tokens.length) { noToken = r; break; } + } + if (!noToken) { console.log(' (no ranked node without tokens in dataset — skipped)'); return; } + await page.goto(BASE + '/#/nodes/' + noToken.pubkey + '/reach'); + await page.waitForSelector('.nq-msg'); + await page.waitForSelector('.nq-rank-link'); + const text = await page.$$eval('.analytics-stat-card', cs => cs.map(c => c.innerText).join(' | ')); + assert(text.includes('#' + noToken.rank + ' / '), 'no-token Rank card: ' + text); + }); + + await step('Reach card: "View leaderboard" opens the leaderboard', async () => { + await page.goto(BASE + '/#/nodes/' + target.pubkey + '/reach'); + await page.waitForSelector('.nq-rank-link'); + await page.click('.nq-rank-link'); + await page.waitForSelector('#rrRows a.nq-link'); + assert(page.url().endsWith('#/reach-rank'), 'url ' + page.url()); + }); + + await step('mobile 375×812: no horizontal scroll, three columns visible', async () => { + const m = await browser.newContext({ viewport: { width: 375, height: 812 }, isMobile: true, hasTouch: true }); + const mp = await m.newPage(); + await openBoard(mp); + const dims = await mp.evaluate(() => { + const se = document.scrollingElement; + const t = document.getElementById('rrTable'); + const ths = [...t.querySelectorAll('thead th')].filter(th => th.offsetWidth > 0).length; + return { sw: se.scrollWidth, cw: se.clientWidth, tw: t.scrollWidth, tcw: t.parentElement.clientWidth, ths }; + }); + assert(dims.sw <= dims.cw + 1, 'page scrolls horizontally: ' + JSON.stringify(dims)); + assert(dims.tw <= dims.tcw + 1, 'table overflows: ' + JSON.stringify(dims)); + assert(dims.ths === 3, 'visible columns ' + dims.ths); + assert(await mp.isVisible('#rrSearch') && await mp.isVisible('#rrStatus'), 'search + status visible'); + await m.close(); + }); + + await step('hostile node names render as text (intercepted API)', async () => { + const x = await browser.newContext({ viewport: { width: 1200, height: 900 } }); + const xp = await x.newPage(); + const evilPk = 'ab'.repeat(32); + await xp.route('**/api/reach-rank*', route => route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + snapshot_at: new Date().toISOString(), total: 1, matched: 1, offset: 0, limit: 50, q: '', + rows: [{ rank: 1, pubkey: evilPk, name: '', neighbors: 3 }], + }), + })); + await openBoard(xp); + const cell = await xp.textContent('#rrRows td.rr-node a'); + assert(cell.includes(''), 'name not shown literally: ' + cell); + assert(await xp.$('#rrRows img') === null, 'markup injected'); + assert(!(await xp.evaluate(() => window.__rrXss)), 'script executed'); + await x.close(); + }); + + await step('API failure shows an error, not an empty leaderboard', async () => { + const f = await browser.newContext({ viewport: { width: 1200, height: 900 } }); + const fp = await f.newPage(); + await fp.route('**/api/reach-rank*', route => route.fulfill({ + status: 500, contentType: 'application/json', body: '{"error":"reach rank unavailable"}', + })); + await fp.goto(BASE + '/#/reach-rank'); + await fp.waitForSelector('#rrError:not([hidden])'); + const err = await fp.textContent('#rrError'); + assert(/Failed to load the leaderboard/.test(err), 'error text: ' + err); + const body = await fp.textContent('#rrRows'); + assert(/unavailable/i.test(body) && !/No ranked nodes/.test(body), 'rows: ' + body); + await f.close(); + }); + + await step('no page errors', async () => { + assert(pageErrors.length === 0, pageErrors.join(' | ')); + }); + + await browser.close(); + console.log('\ntest-reach-rank-e2e.js: ' + passed + ' passed, ' + failed + ' failed'); + if (failed) process.exit(1); +} + +main().catch(e => { console.error(e); process.exit(1); }); diff --git a/test-reach-rank.js b/test-reach-rank.js new file mode 100644 index 000000000..67b4bcaaa --- /dev/null +++ b/test-reach-rank.js @@ -0,0 +1,134 @@ +'use strict'; +// Unit test for reach-rank.js (Reach leaderboard page). Loads the browser IIFE +// in a vm sandbox (pattern from test-node-reach-coverage.js) with the real +// escapeHtml from app.js, and exercises the pure render helpers: escaping of +// node-controlled fields, the pubkey fallback, global ranks in the status +// line, the snapshot-age text and hash-state parsing. +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const appSrc = fs.readFileSync(path.join(__dirname, 'public', 'app.js'), 'utf8'); +const escSrc = appSrc.match(/function escapeHtml\(s\) \{[\s\S]*?\n\}/); +assert.ok(escSrc, 'escapeHtml not found in public/app.js'); + +const registered = {}; +const sandbox = { + window: {}, + document: {}, + registerPage: function (name, mod) { registered[name] = mod; }, + timeAgo: function () { return '2m ago'; }, + URLSearchParams: URLSearchParams, +}; +vm.createContext(sandbox); +vm.runInContext(escSrc[0], sandbox); +vm.runInContext(fs.readFileSync(path.join(__dirname, 'public', 'reach-rank.js'), 'utf8'), sandbox); + +assert.ok(registered['reach-rank'] && typeof registered['reach-rank'].init === 'function', 'page reach-rank registered'); +const { rowHtml, statusText, snapshotText, parseState } = sandbox.window.ReachRank; + +const PK = 'a1b2c3d4e5f6' + '0'.repeat(52); + +// --- rowHtml: link, rank, neighbours -------------------------------------- +{ + const html = rowHtml({ rank: 3, pubkey: PK, name: 'Aarhus Hub', neighbors: 42 }); + assert.ok(html.includes('href="#/nodes/' + PK + '/reach"'), 'links to the node Reach page: ' + html); + assert.ok(html.includes('>#3<'), 'global rank rendered: ' + html); + assert.ok(html.includes('>42<'), 'neighbour count rendered'); + assert.ok(html.includes('>Aarhus Hub'), 'name is the link text'); + assert.ok(html.includes('>a1b2c3d4<'), 'short pubkey shown next to a named node'); + assert.ok(html.includes('title="' + PK + '"'), 'full pubkey in the title'); +} + +// --- rowHtml: pubkey fallback for a node without a name ------------------------ +{ + const html = rowHtml({ rank: 1, pubkey: PK, name: '', neighbors: 7 }); + assert.ok(html.includes('>a1b2c3d4e5f6'), 'unnamed node falls back to its pubkey: ' + html); + assert.ok(!html.includes('rr-pk'), 'no duplicate pubkey span for the fallback'); +} + +// --- rowHtml: XSS — every node-controlled field is escaped ---------------------- +{ + const evil = '\'"&'; + const html = rowHtml({ rank: '1', pubkey: '">', name: evil, neighbors: '9' }); + assert.ok(!/|/i.test(html), 'no raw markup survives: ' + html); + assert.ok(html.includes('<img src=x onerror="alert(1)">'"&'), 'name escaped verbatim'); + assert.ok(html.includes('href="#/nodes/%22%3E%3Cscript%3Ex()%3C%2Fscript%3E/reach"'), 'pubkey URL-encoded in href: ' + html); + assert.ok(!html.includes('title=""'), 'title attribute not broken out of'); +} + +// --- statusText: global placements, no renumbering in search ----------------------- +assert.strictEqual(statusText({ q: '', total: 212, matched: 212, offset: 50, rows: new Array(50) }), + 'Showing 51–100 of 212 ranked nodes.'); +assert.strictEqual(statusText({ q: 'hub', total: 212, matched: 3, offset: 0, rows: new Array(3) }), + 'Showing 1–3 of 3 matches for “hub” (212 ranked nodes).'); +assert.strictEqual(statusText({ q: 'x', total: 212, matched: 1, offset: 0, rows: new Array(1) }), + 'Showing 1–1 of 1 match for “x” (212 ranked nodes).'); +assert.strictEqual(statusText({ q: 'zzz', total: 212, matched: 0, offset: 0, rows: [] }), + 'No ranked node matches “zzz” (212 ranked nodes).'); +assert.strictEqual(statusText({ q: '', total: 0, matched: 0, offset: 0, rows: [] }), 'No ranked nodes yet.'); + +// --- snapshotText: states the data age; tolerates bad input ------------------------- +assert.ok(/^Snapshot .+ \(2m ago\)\.$/.test(snapshotText('2026-09-18T12:00:00Z')), snapshotText('2026-09-18T12:00:00Z')); +assert.strictEqual(snapshotText(''), 'Snapshot time unknown.'); +assert.strictEqual(snapshotText('not a date'), 'Snapshot time unknown.'); + +// --- parseState: hash ?q=&page= ------------------------------------------------------ +{ + let s = parseState(new URLSearchParams('q=%20Hub%20&page=3')); + assert.strictEqual(s.q, 'Hub'); + assert.strictEqual(s.page, 3); + s = parseState(new URLSearchParams('page=-2')); + assert.strictEqual(s.page, 1, 'negative page clamps to 1'); + s = parseState(new URLSearchParams('page=abc&q=' + 'a'.repeat(100))); + assert.strictEqual(s.page, 1, 'garbage page → 1'); + assert.strictEqual(s.q.length, 64, 'query capped at the server limit'); + s = parseState(new URLSearchParams('')); + assert.deepStrictEqual({ q: s.q, page: s.page }, { q: '', page: 1 }); + s = parseState(new URLSearchParams('page=200000000000000000')); + assert.strictEqual(s.page, 100000, 'absurd page clamps so the offset stays a safe integer'); +} + +// --- clipQuery: caps by code point, never splits an emoji ------------------------- +{ + const { clipQuery } = sandbox.window.ReachRank; + const q = clipQuery('a'.repeat(63) + '💥💥'); + assert.strictEqual(Array.from(q).length, 64, 'capped at 64 code points'); + assert.ok(q.endsWith('💥'), 'last emoji kept whole'); + assert.doesNotThrow(() => encodeURIComponent(q), 'no lone surrogate left behind'); + assert.doesNotThrow(() => encodeURIComponent(parseState(new URLSearchParams('q=' + encodeURIComponent('a'.repeat(63) + '💥x'))).q)); + assert.strictEqual(clipQuery(' hub '), 'hub'); + assert.strictEqual(clipQuery(null), ''); +} + +// --- node-reach.js: Rank card states and the leaderboard link -------------------------- +{ + const reachBox = { window: {}, document: {}, registerPage: function () {} }; + vm.createContext(reachBox); + vm.runInContext(escSrc[0], reachBox); + vm.runInContext(fs.readFileSync(path.join(__dirname, 'public', 'node-reach.js'), 'utf8'), reachBox); + const { rankValue, positionHtml } = reachBox.window.NodeReach; + + assert.strictEqual(rankValue({ rank_status: 'ranked', degree_rank: 3, nodes_with_edges: 106 }), '#3 / 106'); + assert.strictEqual(rankValue({ rank_status: 'unranked', degree_rank: 0, nodes_with_edges: 106 }), 'Not ranked', + 'unranked never renders "#0 / N"'); + assert.strictEqual(rankValue({ rank_status: 'unavailable' }), '—'); + + const ranked = positionHtml({ rank_status: 'ranked', degree_rank: 2, nodes_with_edges: 106, neighbor_degree: 5 }); + assert.ok(ranked.includes('href="#/reach-rank"') && ranked.includes('View leaderboard'), 'Rank card links to the leaderboard'); + assert.ok(ranked.includes('>#2 / 106<') && ranked.includes('>5<'), 'rank and neighbours rendered: ' + ranked); + const down = positionHtml({ rank_status: 'unavailable', degree_rank: 0, nodes_with_edges: 0, neighbor_degree: 0 }); + assert.ok(down.includes('Rank unavailable') && !down.includes('>0<'), 'unavailable shows no fake zero: ' + down); + + // The no-token branch (node cannot be identified in paths) still renders the + // position cards — the rank does not depend on path tokens. + const reachSrc = fs.readFileSync(path.join(__dirname, 'public', 'node-reach.js'), 'utf8'); + const emptyBranch = reachSrc.slice(reachSrc.indexOf('var emptyHtml'), reachSrc.indexOf('container.innerHTML = emptyHtml')); + assert.ok(emptyBranch.includes('positionHtml(imp)'), 'no-token Reach page renders the Rank card'); + + const html = fs.readFileSync(path.join(__dirname, 'public', 'index.html'), 'utf8'); + assert.ok(html.includes('