From cc5882c82ab156e1fd1387dd085e7bb8bd49494e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 22:12:36 +0000 Subject: [PATCH 01/13] Report API: the numbers behind a period report GET /api/report?from&to&if answers what a printable report needs in one request: totals for the period, its average, peak and 95th percentile, a row per interface, the top hosts, applications and countries, and the findings by severity. With if=exporter/ifindex the whole report is about that one interface. Two measurements of an interface exist and they do not agree, so each row says which one it is rather than mixing them: the device's own counters where they were read (what a carrier bills against, with the share of the period that was measured), the sampled flows where there are none. Utilisation is given only where the interface speed is known. The peak and the percentile leave out the bucket that is still filling, so a report reaching up to now no longer takes its peak from a half-counted minute. --- CHANGELOG.md | 7 + docs/ROADMAP.md | 4 +- internal/api/report.go | 301 ++++++++++++++++++++++++++++++++++++ internal/api/report_test.go | 195 +++++++++++++++++++++++ internal/api/server.go | 1 + 5 files changed, 506 insertions(+), 2 deletions(-) create mode 100644 internal/api/report.go create mode 100644 internal/api/report_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index d36e8cf..fb880a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,13 @@ The release notes on GitHub are taken from this file: the section whose heading is the version number. +## 2.4.0 + +- `GET /api/report?from&to&if` answers the numbers behind a period report: traffic and flow totals, average, peak and 95th percentile of the whole period, a row per interface, the top hosts, applications and countries, and how many findings the detection rules raised, by severity. `if=exporter/ifindex` narrows the whole report to one interface (its totals, its top lists, its row). +- Each interface row says which measurement it rests on: `counters` where the device's own interface counters were read (what a carrier bills against — average, peak and 95th percentile per direction, with the share of the period that was measured), `flows` where there are none and the sampled flows are the only measurement there is. Utilisation against the interface speed is given only where that speed is known, rather than as a percentage of a guess. +- The peak and the percentile leave out the bucket that is still filling, so a report that reaches up to now no longer takes its peak from a half-counted minute. +- `TestAttackFound` no longer fails now and then: the test builds its traffic from a fixed hour of the day, so the 1:4096 sampling of the demo attack draws the same records on every run and on every platform (the port scan reaches detection as 18 distinct ports against a threshold of 5, instead of coming within one port of it at 15:00). + ## 2.3.9 - NetFlow / IPFIX loss compensation starts much sooner: a session is compensated once 100 records have arrived and the factor is recomputed every 5 seconds (was 30 seconds), so totals in the first minute after start are no longer low when the exporter or the kernel drops packets. With 37% loss the compensated total at 30 s rose from 0.62 to 0.98 of the real total. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 6217830..5e100f9 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -6,8 +6,8 @@ separate: merged work reaches users with the next release. ## Reports (target 7 Nov) -- [ ] Make `TestAttackFound` deterministic (it fails now and then with sampled scans) -- [ ] `GET /api/report?from&to&if`: totals, average, peak and 95th percentile per interface, utilisation against interface speed, top hosts, applications and countries, findings by severity; with tests +- [x] Make `TestAttackFound` deterministic (it fails now and then with sampled scans) +- [x] `GET /api/report?from&to&if`: totals, average, peak and 95th percentile per interface, utilisation against interface speed, top hosts, applications and countries, findings by severity; with tests - [ ] Report page: printable layout (print CSS), period and interface picker, in all 15 languages - [ ] Report: CSV export of the same tables - [ ] Report: comparison with the previous period (change in %, new top hosts) diff --git a/internal/api/report.go b/internal/api/report.go new file mode 100644 index 0000000..48e675e --- /dev/null +++ b/internal/api/report.go @@ -0,0 +1,301 @@ +package api + +import ( + "fmt" + "math" + "net/http" + "net/netip" + "sort" + "strconv" + "strings" + "time" + + "github.com/githubflyideas/traffic66/internal/store" +) + +// The report is the one page somebody prints and hands to a customer or a +// manager, so every number on it has to say where it came from. Two +// measurements of the same interface exist here and they do not agree: +// +// - the device's own interface counters, read over SNMP into five-minute +// rate buckets. They are what a carrier bills against and they count +// every byte, sampled or not. +// - the flows, which are sampled and therefore an estimate, but are the +// only measurement for an interface no device reports counters for. +// +// Each interface row says which of the two it is (Source), rather than +// mixing them into one number nobody can argue with. Utilisation is only +// given where the interface speed is known: a percentage of an unknown +// capacity is not a figure, it is a guess with a unit. +const ( + srcCounters = "counters" // the device's interface counters + srcFlows = "flows" // sampled flow records +) + +// reportIface is one interface's figures over the period. +type reportIface struct { + Exporter string `json:"exporter"` + IfIndex uint32 `json:"ifindex"` + Name string `json:"name,omitempty"` + Device string `json:"device,omitempty"` + Speed uint64 `json:"speed,omitempty"` // bits/s, 0 when unknown + Source string `json:"source"` + Billed bool `json:"billed,omitempty"` + + InWire uint64 `json:"in_wire"` // bytes on the wire, from the flows + OutWire uint64 `json:"out_wire"` + + InAvg float64 `json:"in_avg"` // bits/s + OutAvg float64 `json:"out_avg"` + InPeak float64 `json:"in_peak"` + OutPeak float64 `json:"out_peak"` + In95 float64 `json:"in95"` + Out95 float64 `json:"out95"` + + // Util* are per cent of Speed, omitted when the speed is unknown. + UtilAvg float64 `json:"util_avg,omitempty"` + UtilPeak float64 `json:"util_peak,omitempty"` + Util95 float64 `json:"util95,omitempty"` + + // Coverage is the share of the period that was measured, for counter + // rows: a 95th percentile taken over half the period is not wrong, but + // it is not the period either. + Coverage float64 `json:"coverage,omitempty"` + Samples int64 `json:"samples,omitempty"` +} + +// reportOut is the whole report. It is a struct rather than a map so that +// the printable page and the CSV export read the same field names. +type reportOut struct { + From int64 `json:"from"` + To int64 `json:"to"` + SpanS int64 `json:"span_s"` + Generated int64 `json:"generated"` + Iface string `json:"if,omitempty"` + + Totals store.Totals `json:"totals"` + AvgBps float64 `json:"avg_bps"` + PeakBps float64 `json:"peak_bps"` + PeakAt int64 `json:"peak_at,omitempty"` + P95Bps float64 `json:"p95_bps"` + // Step is the width of the buckets the peak and the percentile above + // were taken over, in seconds. A peak over a one-minute bucket and a + // peak over an hour are different claims. + Step int64 `json:"step"` + + Ifaces []reportIface `json:"ifaces"` + + TopHosts []store.TopRow `json:"top_hosts"` + TopApps []store.TopRow `json:"top_apps"` + TopCountries []store.TopRow `json:"top_countries"` + + Findings map[string]int `json:"findings"` + Top []store.Finding `json:"top_findings,omitempty"` + + Notice string `json:"notice,omitempty"` +} + +// percentile returns the 95th percentile of v by nearest rank, which is +// the sample a device's own counters could be checked against rather than +// an interpolation between two of them (see store.Bills). +func percentile(v []float64, p float64) float64 { + if len(v) == 0 { + return 0 + } + s := append([]float64(nil), v...) + sort.Float64s(s) + i := int(math.Ceil(p*float64(len(s)))) - 1 + if i < 0 { + i = 0 + } + if i >= len(s) { + i = len(s) - 1 + } + return s[i] +} + +// totalSeries sums every application's series into one line of bits/s and +// drops the bucket that is still filling: the last minute of a live range +// is half counted, and a report that takes its peak or its percentile from +// that minute understates both. +func (s *Server) totalSeries(q store.Query) (step int64, times []int64, bps []float64, err error) { + se, err := s.Store.SeriesByApp(q, 1<<30) + if err != nil { + return 0, nil, nil, err + } + for i, t := range se.Times { + if se.SettledTo > 0 && t >= se.SettledTo { + break + } + sum := 0.0 + for _, v := range se.Values { + sum += v[i] + } + times, bps = append(times, t), append(bps, sum) + } + return se.Step, times, bps, nil +} + +// report answers GET /api/report?from&to&if — the numbers behind the +// printable report: traffic over the period, what each interface carried +// and how full it was, what the top of the period was made of, and what +// the detection rules found while it ran. +func (s *Server) report(w http.ResponseWriter, r *http.Request) { + q, err := parseQuery(r) + if err != nil { + fail(w, err) + return + } + only := r.URL.Query().Get("if") + if only != "" { + // One interface: every flow figure below is then this interface's, + // including the top hosts and applications, which is what somebody + // reporting on one circuit asked for. + if err := checkIface(only); err != nil { + fail(w, err) + return + } + q.Filters = append(q.Filters, store.Filter{Field: "iface", Value: only}) + } + span := q.To.Sub(q.From) + out := reportOut{ + From: q.From.UnixMilli(), To: q.To.UnixMilli(), SpanS: int64(span.Seconds()), + Generated: time.Now().UTC().UnixMilli(), Iface: only, Notice: s.notice(), + Findings: map[string]int{"high": 0, "medium": 0, "low": 0}, + } + if out.Totals, err = s.Store.Totals(q); err != nil { + fail(w, err) + return + } + if span > 0 { + out.AvgBps = float64(out.Totals.Wire) * 8 / span.Seconds() + } + step, times, bps, err := s.totalSeries(q) + if err != nil { + fail(w, err) + return + } + out.Step = step + for i, v := range bps { + if v > out.PeakBps { + out.PeakBps, out.PeakAt = v, times[i] + } + } + out.P95Bps = percentile(bps, 0.95) + + // Interfaces. The device's counters are read once for the whole period + // and indexed, rather than queried per interface. + bills := map[string]store.Bill{} + if bl, err := s.Store.Bills(q.From, q.To, ""); err == nil { + for _, b := range bl { + bills[ifKey(b.Exporter, b.IfIndex)] = b + } + } + var billed map[string]bool + if s.Inv != nil { + billed = s.Inv.Billed() + } + list, err := s.Store.Interfaces(q) + if err != nil { + fail(w, err) + return + } + for _, f := range list { + k := ifKey(f.Exporter, f.IfIndex) + if only != "" && k != only { + continue + } + it := reportIface{Exporter: f.Exporter, IfIndex: f.IfIndex, Speed: f.Speed, + InWire: f.InWire, OutWire: f.OutWire, Source: srcFlows, Billed: billed[k]} + if a, err := netip.ParseAddr(f.Exporter); err == nil && s.Inv != nil { + ifc := s.Inv.Iface(a, f.IfIndex) + it.Name, it.Device = ifc.Name, s.Inv.Device(a) + if it.Speed == 0 { + it.Speed = ifc.Speed + } + } + if b, ok := bills[k]; ok && b.Buckets > 0 { + it.Source = srcCounters + it.InAvg, it.OutAvg = b.InAvg, b.OutAvg + it.InPeak, it.OutPeak = b.InMax, b.OutMax + it.In95, it.Out95 = b.In95, b.Out95 + it.Coverage, it.Samples = b.Coverage(), b.Samples + } else if span > 0 { + // No counters: the flows are the only measurement there is. + // They are sampled, so the row says so through Source. + it.InAvg = float64(f.InWire) * 8 / span.Seconds() + it.OutAvg = float64(f.OutWire) * 8 / span.Seconds() + qi := q + if only == "" { + qi.Filters = append(append([]store.Filter{}, q.Filters...), store.Filter{Field: "iface", Value: k}) + } + if _, _, b, err := s.totalSeries(qi); err == nil { + for _, v := range b { + it.InPeak = math.Max(it.InPeak, v) + } + it.In95 = percentile(b, 0.95) + // One series per interface covers both directions; it is + // reported as the inbound figure with the outbound left + // empty rather than counted twice. + } + } + if it.Speed > 0 { + pc := func(v float64) float64 { return v / float64(it.Speed) * 100 } + it.UtilAvg = pc(math.Max(it.InAvg, it.OutAvg)) + it.UtilPeak = pc(math.Max(it.InPeak, it.OutPeak)) + it.Util95 = pc(math.Max(it.In95, it.Out95)) + } + out.Ifaces = append(out.Ifaces, it) + if len(out.Ifaces) >= 50 { + break + } + } + + top := func(dim string) []store.TopRow { + rows, err := s.Store.TopN(q, dim, 10) + if err != nil { + return nil + } + return rows + } + out.TopHosts, out.TopApps, out.TopCountries = top("client"), top("app"), top("country") + + fs, err := s.Store.Findings(store.FindingQuery{From: q.From, To: q.To, Status: "all", Limit: 1000}) + if err != nil { + fail(w, err) + return + } + name := map[int]string{3: "high", 2: "medium", 1: "low"} + for _, f := range fs { + if n, ok := name[f.Sev]; ok { + out.Findings[n]++ + } + } + if len(fs) > 20 { + fs = fs[:20] + } + out.Top = fs + writeJSON(w, http.StatusOK, out) +} + +// checkIface refuses an interface that is not written as the store's +// filter expects it (exporter/ifindex), here rather than inside a query, +// so the caller is told what is wrong with the request instead of being +// handed a database error. +func checkIface(v string) error { + e, idx, ok := strings.Cut(v, "/") + if !ok || e == "" { + return fmt.Errorf("if must be written exporter/ifindex, not %q", v) + } + if _, err := netip.ParseAddr(e); err != nil { + return fmt.Errorf("if: %q is not an exporter address", e) + } + if _, err := strconv.ParseUint(idx, 10, 32); err != nil { + return fmt.Errorf("if: %q is not an interface number", idx) + } + return nil +} + +func ifKey(exporter string, ifindex uint32) string { + return exporter + "/" + strconv.FormatUint(uint64(ifindex), 10) +} diff --git a/internal/api/report_test.go b/internal/api/report_test.go new file mode 100644 index 0000000..4d096a1 --- /dev/null +++ b/internal/api/report_test.go @@ -0,0 +1,195 @@ +package api + +import ( + "encoding/json" + "fmt" + "math" + "net/http/httptest" + "testing" + "time" + + "github.com/githubflyideas/traffic66/internal/store" +) + +// An hour of traffic over two interfaces: one the device reports counters +// for, one it does not. The report has to speak for both, and say which +// measurement each row rests on. +func TestReport(t *testing.T) { + st, err := store.Open(store.Options{Dir: t.TempDir(), Threads: 1}) + if err != nil { + t.Fatal(err) + } + defer st.Close() + to := time.Now().UTC().Truncate(5 * time.Minute) + from := to.Add(-time.Hour) + + // flows: the counted interface carries twice what the other one does, + // and one minute in the middle carries ten times its usual minute so + // that the peak and the percentile cannot be the same number. + const perMin = 60_000_000 // wire bytes a minute on iface 1 + var rows []store.Row + for i := 0; i < 60; i++ { + ts := from.Add(time.Duration(i)*time.Minute + 30*time.Second) + w := uint64(perMin) + if i == 30 { + w *= 10 + } + rows = append(rows, + store.Row{TS: ts, Exporter: "10.0.0.1", InIf: 1, OutIf: 9, Wire: w, Bytes: w, Pkts: w / 1000, Flows: 1, + Client: "10.1.1.5", Server: "203.0.113.9", Cli4: ip4(10, 1, 1, 5), Srv4: ip4(203, 0, 113, 9), + App: "https", SvcPort: 443, Proto: 6, Dir: store.DirOutbound, PeerCC: "US", CliInt: true}, + store.Row{TS: ts, Exporter: "10.0.0.1", InIf: 2, OutIf: 9, Wire: w / 2, Bytes: w / 2, Pkts: w / 2000, Flows: 1, + Client: "10.1.1.6", Server: "198.51.100.7", Cli4: ip4(10, 1, 1, 6), Srv4: ip4(198, 51, 100, 7), + App: "dns", SvcPort: 53, Proto: 17, Dir: store.DirOutbound, PeerCC: "NL", CliInt: true}) + } + if err := st.AppendRows(rows); err != nil { + t.Fatal(err) + } + // counters for iface 1 only: a steady megabyte a second, so 8 Mbit/s + // against a gigabit interface — 0.8% full. + for i := 0; i <= 120; i++ { // every 30 s across the hour + ts := from.Add(time.Duration(i) * 30 * time.Second) + oct := uint64(i) * 30 * 1_000_000 + if err := st.AppendCounter(ts, "10.0.0.1", 1, 1e9, oct, oct/2, 0, 0); err != nil { + t.Fatal(err) + } + } + if err := st.RollRates(to); err != nil { + t.Fatal(err) + } + st.SaveFinding(store.Finding{Kind: "scan", Sev: 3, Src: "10.1.1.5", First: from.Add(10 * time.Minute), Last: from.Add(20 * time.Minute)}, 30*time.Minute) + st.SaveFinding(store.Finding{Kind: "threat", Sev: 1, Src: "10.1.1.6", Dst: "203.0.113.9", First: from.Add(time.Minute), Last: to}, 30*time.Minute) + + s := &Server{Store: st} + get := func(extra string) (int, reportOut) { + w := httptest.NewRecorder() + url := fmt.Sprintf("/api/report?from=%d&to=%d%s", from.UnixMilli(), to.UnixMilli(), extra) + s.report(w, httptest.NewRequest("GET", url, nil)) + var out reportOut + json.Unmarshal(w.Body.Bytes(), &out) + return w.Code, out + } + code, out := get("") + if code != 200 { + t.Fatalf("%d", code) + } + if out.From != from.UnixMilli() || out.To != to.UnixMilli() || out.SpanS != 3600 { + t.Errorf("period %d..%d span %d", out.From, out.To, out.SpanS) + } + // 60 minutes on iface 1 plus half as much on iface 2, with one minute + // at ten times the rate on each + wantWire := uint64(perMin*69) + uint64(perMin*69)/2 + if out.Totals.Wire != wantWire { + t.Errorf("wire %d, want %d", out.Totals.Wire, wantWire) + } + if want := float64(wantWire) * 8 / 3600; !near(out.AvgBps, want, 0.001) { + t.Errorf("avg %.0f bits/s, want %.0f", out.AvgBps, want) + } + if out.Step != 60 { + t.Errorf("step %d s, want per-minute buckets", out.Step) + } + // the busy minute is the peak; the percentile is not dragged up to it + if !near(out.PeakBps, float64(perMin)*10*1.5*8/60, 0.02) { + t.Errorf("peak %.0f bits/s", out.PeakBps) + } + if out.P95Bps >= out.PeakBps || out.P95Bps <= out.AvgBps/2 { + t.Errorf("p95 %.0f against peak %.0f and average %.0f", out.P95Bps, out.PeakBps, out.AvgBps) + } + if out.PeakAt < out.From || out.PeakAt >= out.To { + t.Errorf("peak at %d, outside the period", out.PeakAt) + } + + byKey := map[string]reportIface{} + for _, f := range out.Ifaces { + byKey[ifKey(f.Exporter, f.IfIndex)] = f + } + one, ok := byKey["10.0.0.1/1"] + if !ok { + t.Fatalf("no row for the counted interface: %+v", out.Ifaces) + } + if one.Source != srcCounters { + t.Errorf("counted interface reported from %s", one.Source) + } + if !near(one.InAvg, 8e6, 0.02) || !near(one.In95, 8e6, 0.02) { + t.Errorf("counters: avg %.0f, p95 %.0f bits/s, want 8M", one.InAvg, one.In95) + } + if one.Speed != 1e9 || !near(one.Util95, 0.8, 0.05) { + t.Errorf("speed %d, 95th percentile utilisation %.3f%%, want 0.8", one.Speed, one.Util95) + } + if one.Coverage < 0.9 || one.Samples == 0 { + t.Errorf("coverage %.2f over %d samples", one.Coverage, one.Samples) + } + two, ok := byKey["10.0.0.1/2"] + if !ok { + t.Fatalf("no row for the interface without counters: %+v", out.Ifaces) + } + if two.Source != srcFlows { + t.Errorf("interface without counters reported from %s", two.Source) + } + if want := float64(perMin*69) / 2 * 8 / 3600; !near(two.InAvg, want, 0.001) { + t.Errorf("flows: avg %.0f bits/s, want %.0f", two.InAvg, want) + } + if two.In95 == 0 || two.InPeak <= two.In95 { + t.Errorf("flows: p95 %.0f bits/s is not below the peak %.0f", two.In95, two.InPeak) + } + // nothing says how fast that interface is, so no utilisation is given + // rather than a percentage of a guess + if two.Speed != 0 || two.Util95 != 0 || two.UtilPeak != 0 { + t.Errorf("utilisation without a known speed: %+v", two) + } + + if len(out.TopHosts) == 0 || out.TopHosts[0].Key != "10.1.1.5" { + t.Errorf("top hosts %+v", out.TopHosts) + } + if len(out.TopApps) == 0 || out.TopApps[0].Key != "https" { + t.Errorf("top applications %+v", out.TopApps) + } + if len(out.TopCountries) == 0 || out.TopCountries[0].Key != "US" { + t.Errorf("top countries %+v", out.TopCountries) + } + if out.Findings["high"] != 1 || out.Findings["low"] != 1 || out.Findings["medium"] != 0 { + t.Errorf("findings %v", out.Findings) + } + if len(out.Top) != 2 || out.Top[0].Kind != "scan" { + t.Errorf("top findings %+v", out.Top) + } + // an unregistered installation marks its own report + if out.Notice == "" { + t.Error("no evaluation notice on an unregistered installation") + } + + // one interface: the whole report is then about that interface + code, only := get("&if=10.0.0.1/2") + if code != 200 { + t.Fatalf("%d", code) + } + if only.Iface != "10.0.0.1/2" || len(only.Ifaces) != 1 || only.Ifaces[0].IfIndex != 2 { + t.Fatalf("if=: %+v", only.Ifaces) + } + if only.Totals.Wire != uint64(perMin*69)/2 { + t.Errorf("one interface's wire %d", only.Totals.Wire) + } + if len(only.TopApps) != 1 || only.TopApps[0].Key != "dns" { + t.Errorf("one interface's applications %+v", only.TopApps) + } + + for _, bad := range []string{"&if=eth0", "&if=10.0.0.1/eth0", "&if=hostname/1"} { + if code, _ := get(bad); code != 400 { + t.Errorf("%s: %d, want 400", bad, code) + } + } + w := httptest.NewRecorder() + s.report(w, httptest.NewRequest("GET", "/api/report?from=zzz&to=1", nil)) + if w.Code != 400 { + t.Errorf("bad period accepted: %d", w.Code) + } +} + +func near(got, want, tol float64) bool { + if want == 0 { + return got == 0 + } + return math.Abs(got-want)/math.Abs(want) <= tol +} + +func ip4(a, b, c, d uint32) uint32 { return a<<24 | b<<16 | c<<8 | d } diff --git a/internal/api/server.go b/internal/api/server.go index 37ccc3f..2d69d47 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -154,6 +154,7 @@ func (s *Server) Handler() http.Handler { api("GET /api/threats", s.data((*Server).threats)) api("GET /api/rings", s.data((*Server).rings)) api("GET /api/geolines", s.data((*Server).geoLines)) + api("GET /api/report", s.data((*Server).report)) api("GET /api/ifaces", s.ifaces) api("GET /api/recon", s.recon) api("GET /api/billing", s.billing) From fc2969f054fb45c1a3c6147fcf6bf60faafc585b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 23:18:45 +0000 Subject: [PATCH 02/13] Top 66 opens on the two ends; narrower top end for the map's lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Top 66 opened on the grouped table, which answers "what kind of traffic is this" — but the question somebody opens the page with is "who is talking", and that is the other tab. It now opens on the two ends of each conversation, and only the table mode is written into the address so a shared link still lands where the sender was. The map's three line widths were 1.5 / 4 / 8 px. The widest was so much heavier than the rest that it drowned the five shades that carry the same figure. They are now 1.5 / 3 / 4.5 px: the widest is about what the middle one was, and the colour is read again. --- internal/web/static/app.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/web/static/app.js b/internal/web/static/app.js index 5ee270b..9546919 100644 --- a/internal/web/static/app.js +++ b/internal/web/static/app.js @@ -322,7 +322,7 @@ function readHash() { if (RANGES.includes(p.get('r'))) state.r = p.get('r'); if (p.get('r') === 'custom' && +p.get('from') > 0 && +p.get('to') > +p.get('from')) { state.r = 'custom'; state.cf = +p.get('from'); state.ct = +p.get('to'); } if (p.get('dim')) state.dim = p.get('dim'); - state.tm = p.get('tm') === 'talkers' ? 'talkers' : 'table'; // Top 66 opens on the table + state.tm = p.get('tm') === 'table' ? 'table' : 'talkers'; // Top 66 opens on the two ends state.sk = ['segment', 'conv'].includes(p.get('by')) ? p.get('by') : 'host'; state.ra = p.get('ra') === 'client' ? 'client' : 'server'; state.ds = p.get('ds') === 'sb' ? 'sb' : ''; @@ -339,7 +339,7 @@ function writeHash(push) { const p = new URLSearchParams({v: state.v, r: state.r}); if (state.r === 'custom') { p.set('from', state.cf); p.set('to', state.ct); } if (state.ds) p.set('ds', state.ds); - if (state.v === 'topn') { if (state.tm === 'talkers') p.set('tm', 'talkers'); else p.set('dim', state.dim); } + if (state.v === 'topn') { if (state.tm === 'table') { p.set('tm', 'table'); p.set('dim', state.dim); } } if (state.v === 'sankey' && state.sk !== 'host') p.set('by', state.sk); if (state.v === 'records' && state.rl) p.set('live', '1'); if (state.ifv && state.ifv !== 'all' && IFV_VIEWS.has(state.v)) p.set('if', state.ifv); @@ -1482,8 +1482,11 @@ function drawMap(el, w, rows, gl) { // over the lines' own range, cut into 15 equal steps), not by quantiles: // a few huge flows then do not flatten the rest, and equal traffic always // gets an equal level. Level l -> width LINE_W[floor(l/5)], shade l%5. +// The three widths are close together on purpose: a widest line several +// times the narrowest one dominated the map and the shades stopped being +// read at all, so the step carries less of the difference than the colour. // The palette follows the theme unless one is picked in the legend. -const LINE_W = [1.5, 4, 8]; +const LINE_W = [1.5, 3, 4.5]; const LINE_PAL = { gb: ['#7fd6a0', '#3fb8a0', '#1f97b5', '#1f6fb8', '#163f8f'], or: ['#ffc266', '#ff9a3c', '#f26a2e', '#d63a2a', '#9e1622'], From 0f97932e3339e6e50c34986430ae10b83cc812ef Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 23:19:06 +0000 Subject: [PATCH 03/13] The evaluation mark on every page, a little denser Four pages were left out of the mark: the overview, the sources, the account and the help page. The overview is the one somebody screenshots into a report, and keeping a list of exceptions right as pages are added is work nobody will do, so there are no exceptions now. The login screen is not one of these views and stays unmarked. The tile is 360x235 instead of 460x300, which puts about half again as many of the words on a screen. The glyph, its size and its faint ink are unchanged: the mark is easier to see as a mark without making anything under it harder to read. --- internal/web/static/app.js | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/internal/web/static/app.js b/internal/web/static/app.js index 9546919..83b5ec5 100644 --- a/internal/web/static/app.js +++ b/internal/web/static/app.js @@ -2,17 +2,14 @@ 'use strict'; // ------------------------------------------------------------ the evaluation mark -// Every page an unregistered installation might hand to somebody else -// carries the mark; these three do not. The overview is the first thing a -// new installation sees and the mark would be the first thing it says, and -// the settings and the account are a person working on their own machine -// rather than a page anybody reads twice. +// Every page of an unregistered installation carries the mark, with no +// exceptions. The overview, the sources, the account and the help page +// used to be left out as pages nobody hands on; but a screenshot of the +// overview is exactly what gets pasted into a report, and a list of +// exceptions is a list somebody has to keep right as pages are added. // -// It is a list of what is NOT marked on purpose: a page added later is -// marked unless somebody decides otherwise, which is the right way round — -// the cost of forgetting is then an extra mark, not a report that leaves -// the building looking finished when it is not. -const NOMARK = new Set(['overview', 'sources', 'account', 'help']); +// The login screen is not one of these views, so it is never marked: +// nobody has been shown anything yet. let marked = false; // from the licence: true until the installation is registered let markNotice = ''; // the server's wording, so a copy says what a file says @@ -26,10 +23,14 @@ function markTile() { // read here and written into the tile instead, which is also why the // tile is rebuilt whenever the theme changes. const ink = getComputedStyle(document.body).getPropertyValue('--ink-3').trim() || '#8a8a8a'; - // One large glyph per tile rather than many small ones: the tile is - // repeated across the whole view. - const svg = `` + - `` + + `${esc(text)}`; return `url("data:image/svg+xml;charset=utf-8,${encodeURIComponent(svg)}")`; @@ -37,7 +38,7 @@ function markTile() { // applyMark decides whether the view on screen carries the mark. function applyMark() { - const on = marked && !NOMARK.has(state.v); + const on = marked; document.body.classList.toggle('marked', on); if (on) document.body.style.setProperty('--mark', markTile()); markChip(on); From 2cf7605439d7070eb4be8021a18453d0a9369710 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 23:20:24 +0000 Subject: [PATCH 04/13] Every live page refreshes itself, on a beat that follows the range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rule named four pages and two ranges, so the traffic, Top 66, flow paths, geo and interface pages stood still on a 6-hour or 24-hour range until somebody clicked something. The records were arriving and the numbers were right; the page was simply old, which is what "the chart lags" looks like from the outside. Every page that shows incoming data now redraws on its own, once what is on screen is older than its range's beat: 20 s on 15 minutes, 30 s on an hour, a minute on 6 and 24 hours, five minutes on 7 and 30 days — the width of the buckets the range is drawn in, so no redraw is asked for figures that cannot have changed. A range typed by hand ends in the past and has no beat. The check runs every 5 s, so changing the range takes effect at once rather than at the end of a cycle nobody can see. --- internal/web/static/app.js | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/internal/web/static/app.js b/internal/web/static/app.js index 83b5ec5..8be8292 100644 --- a/internal/web/static/app.js +++ b/internal/web/static/app.js @@ -2936,6 +2936,8 @@ async function loadSB() { if (offlineMode && sbInfo.ready) state.ds = 'sb'; // nothing else to show } +// When the page on screen was last drawn, for the refresh beat below. +let drawnAt = 0; async function render(push) { // the default password of a new installation is changed before anything else if (me?.must_change) state.v = 'account'; @@ -2964,7 +2966,7 @@ async function render(push) { } catch (e) { if (e.message !== 'login' && seq === loadSeq) el.innerHTML = errorBox(e); } - if (seq === loadSeq) resolveNames(); + if (seq === loadSeq) { drawnAt = Date.now(); resolveNames(); } } function go(v) { state.v = v; render(true); window.scrollTo(0, 0); } @@ -3053,6 +3055,19 @@ $('#loginForm').addEventListener('submit', async e => { render(); startTimers(); }); +// How often a page redraws itself, by range. It follows the width of the +// buckets the range is drawn in: a 15-minute or 1-hour range is drawn per +// minute, so half a minute is as often as it can say anything new, while a +// 30-day range is drawn in three-hour buckets and a redraw every five +// minutes is already generous. A range the user typed ends in the past and +// never changes, so it has no beat at all. +const REFRESH_MS = {'15m': 20000, '1h': 30000, '6h': 60000, '24h': 60000, '7d': 300000, '30d': 300000}; + +// The pages that show incoming data. The others (help, the account, the +// asset and name pages, data cleanup, the sandbox, the bill) change when +// somebody changes them, not on their own. +const LIVE_VIEWS = new Set(['overview', 'findings', 'topn', 'traffic', 'detail', 'sankey', 'geo', 'threats', 'records', 'ifaces', 'sources']); + // The refresh timers start once, whether the page opened signed in or after // signing in (before, signing in left the page without automatic refresh). let timers = false; @@ -3066,13 +3081,24 @@ function startTimers() { if (document.activeElement?.closest?.('.cols, #recPer')) return; render(); }, 5000); + // Every live page, on a beat that follows the range. The old rule named + // four pages and two ranges, which left the traffic, Top 66, flow paths, + // geo and interface pages standing still on a 6-hour or a 24-hour range + // until somebody clicked something: the records were arriving and the + // numbers were right, and the page was simply old. + // + // The beat is checked every few seconds and a page is redrawn once what + // is on it is older than its range's beat, so changing the range takes + // effect at once instead of at the end of a cycle somebody cannot see. setInterval(() => { // not while something is being typed or picked (the names form, filters) if (document.hidden || pop.style.display === 'block' || ['TEXTAREA', 'INPUT', 'SELECT'].includes(document.activeElement?.tagName)) return; - if (state.v === 'records' && state.rl) return; - if (state.v === 'help') return; - if (state.v === 'sources' || state.v === 'records' || state.v === 'overview' || state.v === 'findings' || state.r === '15m' || state.r === '1h') render(); - }, 30000); + if (state.v === 'records' && state.rl) return; // its own, faster beat + if (!LIVE_VIEWS.has(state.v)) return; + const beat = REFRESH_MS[state.r]; + if (!beat || Date.now() - drawnAt < beat) return; // a range in the past never moves + render(); + }, 5000); } $('#logout').onclick = async () => { await fetch('/api/logout', {method: 'POST'}); me = null; showLogin(); }; $('#acct').onclick = () => go('account'); From e63bdd8fea5bf7cc63015cd3c2bebf063c88797a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 23:23:09 +0000 Subject: [PATCH 05/13] The network charts count traffic whose AS is unknown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The source and destination network charts asked for peer_asn <> 0, so traffic the AS database could not place was left out of them entirely. The AS table on the same page counts it, as "unknown", so the two disagreed; and where few flows carry an AS number, whole buckets came back empty and the chart was drawn as blocks with nothing between them — the shorter the range, the worse, because a one-minute bucket needs a placed AS in that minute while an hour-long one only needs it somewhere in the hour. Unplaced traffic now counts as AS 0 and is drawn as "unknown", like the table. A bucket is empty only when nothing went through it. --- internal/store/asnseries_test.go | 73 ++++++++++++++++++++++++++++++++ internal/store/series.go | 14 ++++-- internal/web/static/app.js | 2 +- 3 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 internal/store/asnseries_test.go diff --git a/internal/store/asnseries_test.go b/internal/store/asnseries_test.go new file mode 100644 index 0000000..2a2d2ab --- /dev/null +++ b/internal/store/asnseries_test.go @@ -0,0 +1,73 @@ +package store + +import ( + "testing" + "time" +) + +// The two network charts are drawn from asn_src and asn_dst. Traffic whose +// peer has no AS number has to count, as AS 0: an installation without a +// good AS database has minutes that carry nothing else, and leaving those +// minutes out drew the chart as a run of blocks with gaps between them +// while the AS table under it showed the same traffic as "unknown". +func TestASSeriesCountsUnknownAS(t *testing.T) { + s, err := Open(Options{Dir: t.TempDir(), Threads: 1}) + if err != nil { + t.Fatal(err) + } + defer s.Close() + to := time.Now().UTC().Truncate(time.Minute) + from := to.Add(-30 * time.Minute) + // Every minute carries traffic. Only every third minute has a peer with + // an AS number; the rest is traffic the AS database could not place. + var rows []Row + for i := 0; i < 30; i++ { + ts := from.Add(time.Duration(i)*time.Minute + 10*time.Second) + r := Row{TS: ts, Src: "203.0.113.9", Dst: "10.1.1.5", Client: "10.1.1.5", Server: "203.0.113.9", + Cli4: 0x0A010105, Srv4: 0xCB007109, Dir: DirOutbound, App: "HTTPS", SvcPort: 443, Proto: 6, + Wire: 1_000_000, Bytes: 1_000_000, Pkts: 800, Flows: 1, Exporter: "10.0.0.1", InIf: 1, CliInt: true} + if i%3 == 0 { + r.PeerASN, r.PeerOrg = 64501, "Example Net" + } + // the other half of the same conversation, so both charts have data + b := r + b.Src, b.Dst = r.Dst, r.Src + b.Wire, b.Bytes, b.Pkts = r.Wire/4, r.Bytes/4, r.Pkts/4 + rows = append(rows, r, b) + } + if err := s.AppendRows(rows); err != nil { + t.Fatal(err) + } + q := Query{From: from, To: to} + for _, by := range []string{"asn_src", "asn_dst"} { + se, err := s.SeriesBy(q, by, "wire", 8) + if err != nil { + t.Fatal(err) + } + if len(se.Names) == 0 { + t.Fatalf("%s: no series", by) + } + var unknown, known bool + for _, n := range se.Names { + switch { + case n == "0\t": + unknown = true + case n == "64501\tExample Net": + known = true + } + } + if !unknown || !known { + t.Fatalf("%s: series %q, want both the placed AS and AS 0", by, se.Names) + } + // no bucket of a minute that carried traffic may be empty + for i := range se.Times { + sum := 0.0 + for _, v := range se.Values { + sum += v[i] + } + if sum == 0 { + t.Errorf("%s: bucket %d of %d is empty although that minute carried traffic", by, i, len(se.Times)) + } + } + } +} diff --git a/internal/store/series.go b/internal/store/series.go index 4084306..63045ae 100644 --- a/internal/store/series.go +++ b/internal/store/series.go @@ -38,9 +38,17 @@ var seriesDims = map[string]seriesDim{ return `SELECT ` + b + `, val, sum(wire), sum(pkts) FROM r_dim WHERE ts >= ? AND ts < ? AND dim = 'port' AND NOT starts_with(val, '0/') GROUP BY 1,2` }}, // AS of the sender and of the receiver of each packet, when that side - // is the external one - "asn_src": {key: "CAST(peer_asn AS VARCHAR) || chr(9) || peer_org", cond: "peer_asn <> 0 AND src = " + peerExpr}, - "asn_dst": {key: "CAST(peer_asn AS VARCHAR) || chr(9) || peer_org", cond: "peer_asn <> 0 AND dst = " + peerExpr}, + // is the external one. + // + // Traffic whose peer has no AS number counts as AS 0, the way the AS + // table beside these charts counts it. Leaving it out instead made the + // chart disagree with the table under it, and worse: on an installation + // without a good AS database most minutes carried no AS at all, so + // whole buckets came back empty and the chart was drawn as a run of + // blocks with nothing between them. A bucket is empty here only when + // nothing went through it. + "asn_src": {key: "CAST(peer_asn AS VARCHAR) || chr(9) || peer_org", cond: "src = " + peerExpr}, + "asn_dst": {key: "CAST(peer_asn AS VARCHAR) || chr(9) || peer_org", cond: "dst = " + peerExpr}, // interfaces of the flow exporters, as "exporter/ifindex" "if_in": {key: "exporter || '/' || in_if", cond: "in_if <> 0"}, "if_out": {key: "exporter || '/' || out_if", cond: "out_if <> 0"}, diff --git a/internal/web/static/app.js b/internal/web/static/app.js index 8be8292..3bcd80e 100644 --- a/internal/web/static/app.js +++ b/internal/web/static/app.js @@ -1117,7 +1117,7 @@ function seriesLabel(by, k) { if (k === '__other__') return t('other'); if (by === 'client' || by === 'server') return names.get(k) || k; if (by === 'service') { const [p, a] = k.split('\t'); return a ? p + ' ' + a : p; } - if (by.startsWith('asn')) { const [n, o] = k.split('\t'); return 'AS' + n + (o ? ' ' + o : ''); } + if (by.startsWith('asn')) { const [n, o] = k.split('\t'); return n === '0' ? t('unknown') : 'AS' + n + (o ? ' ' + o : ''); } if (by.startsWith('if_')) return ifaceNames.get(k) || k; return appLabel(k); } From f3c4e4bf3f1034a58f88f5012cee03affd24554c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 23:25:12 +0000 Subject: [PATCH 06/13] The sponsor banner is a strip across the top of the page It sat at the foot of the sidebar, a 728x90 banner squeezed into 208 px and shown one at a time. It is now the full width of the browser above the sidebar and the content, and because the banners in the manifest are 728x90 the strip puts as many of them side by side as the window has room for (one per 420 px, up to six fetched) rather than stretching one. The rest rotate in together, with dots only when more exist than fit. It scrolls away with the page instead of holding a band of every screen, it is left out of print, and the sidebar's height is measured against it so the language picker and the sign-out button stay on screen. --- internal/web/static/app.css | 28 ++++++--- internal/web/static/index.html | 2 +- internal/web/static/sponsor.js | 103 ++++++++++++++++++++++++--------- 3 files changed, 96 insertions(+), 37 deletions(-) diff --git a/internal/web/static/app.css b/internal/web/static/app.css index e0378f3..7991c60 100644 --- a/internal/web/static/app.css +++ b/internal/web/static/app.css @@ -54,7 +54,7 @@ button,input,select,textarea{font:inherit;color:inherit} [hidden]{display:none!important} .app{display:grid;grid-template-columns:236px 1fr;min-height:100vh} -aside{background:var(--side);border-inline-end:1px solid var(--side-line);display:flex;flex-direction:column;position:sticky;top:0;height:100vh} +aside{background:var(--side);border-inline-end:1px solid var(--side-line);display:flex;flex-direction:column;position:sticky;top:0;height:calc(100vh - var(--sp-h, 0px))} .brand{padding:12px 14px 10px;flex-wrap:wrap;position:relative;font-weight:700;font-size:18px;letter-spacing:-.01em;border-bottom:1px solid var(--side-line);display:flex;align-items:center;gap:8px} /* the logo fills the sidebar's width (236px less the brand padding = 208px) and keeps its aspect ratio; 72px tall at most. Recommended upload: @@ -95,18 +95,28 @@ nav.dash button[aria-current="page"] .ni{opacity:1} .self .live i{width:8px;height:8px;border-radius:50%;background:currentColor} .self .live.ok{color:var(--good)}.self .live.idle{color:var(--ink-3)} .foot{display:flex;flex-direction:column;gap:8px;padding:0 16px 14px} -.sponsor{display:flex;flex-direction:column;gap:4px} +/* The sponsor strip of an unregistered installation: the full width of the + browser at the top of the page, above the sidebar and the content, and it + scrolls away with the page rather than taking a band of every screen. + + The banners in the manifest are 728x90, so a wide window is filled with + several of them side by side instead of one stretched one; the strip + decides how many fit and rotates through the rest. Its height is the + cell's aspect ratio, which is why the sidebar's own height is measured + against it (--sp-h, set by sponsor.js) instead of a fixed 100vh. */ +.sponsor{grid-column:1/-1;background:var(--side);border-bottom:1px solid var(--side-line)} .sponsor[hidden]{display:none} -.sponsor .sp-h{font-size:10.5px;letter-spacing:.06em;text-transform:uppercase;color:var(--ink-3)} -.sponsor .sp-text{border:1px dashed var(--line-2);border-radius:6px;padding:6px 8px;font-size:11.5px;line-height:1.35;color:var(--ink-3);overflow-wrap:anywhere} +.sponsor .sp-h{font-size:10.5px;letter-spacing:.06em;text-transform:uppercase;color:var(--ink-3);padding:3px 12px 2px} +.sponsor .sp-text{margin:0 12px 8px;border:1px dashed var(--line-2);border-radius:6px;padding:6px 8px;font-size:11.5px;line-height:1.35;color:var(--ink-3);overflow-wrap:anywhere} .sponsor .sp-text[hidden],.spc[hidden]{display:none} -.spc-track{position:relative;aspect-ratio:728/90;width:100%;border-radius:6px;overflow:hidden;background:var(--surface);border:1px solid var(--line-2)} -.spc-slide{position:absolute;inset:0;opacity:0;transition:opacity .45s ease;pointer-events:none;display:block} -.spc-slide.on{opacity:1;pointer-events:auto} -.spc-slide img{width:100%;height:100%;object-fit:cover;display:block} -.spc-dots{display:flex;gap:6px;justify-content:center} +.spc-strip{display:grid;grid-auto-flow:column;grid-auto-columns:minmax(0,1fr);gap:1px;background:var(--side-line)} +.spc-cell{position:relative;aspect-ratio:728/90;overflow:hidden;background:var(--surface)} +.spc-cell>a,.spc-cell>div{display:block;width:100%;height:100%} +.spc-cell img{width:100%;height:100%;object-fit:cover;display:block} +.spc-dots{display:flex;gap:6px;justify-content:center;padding:4px 0 5px} .spc-dot{width:6px;height:6px;padding:0;border-radius:50%;border:0;cursor:pointer;background:var(--line-2)} .spc-dot.on{background:var(--accent)} +@media print{.sponsor{display:none}} .foot .pick{display:grid;grid-template-columns:1fr auto;gap:6px} #theme{display:flex;align-items:center;gap:6px;border:1px solid var(--line-2);border-radius:6px;padding:4px 10px;background:var(--surface);color:var(--ink);cursor:pointer;font-size:13px;white-space:nowrap} #theme i{width:12px;height:12px;border-radius:50%;background:var(--accent);box-shadow:inset 0 0 0 2px var(--surface),0 0 0 1px var(--line-2)} diff --git a/internal/web/static/index.html b/internal/web/static/index.html index ffae465..845523e 100644 --- a/internal/web/static/index.html +++ b/internal/web/static/index.html @@ -11,6 +11,7 @@