diff --git a/changelog.d/added-inbox-upload-links.md b/changelog.d/added-inbox-upload-links.md new file mode 100644 index 0000000000..73c7666ef5 --- /dev/null +++ b/changelog.d/added-inbox-upload-links.md @@ -0,0 +1 @@ +- Add links to the Inbox: paste a URL onto the Inbox or type it into + Add, and the hive fetches it once and reads it like a dropped file (a web page becomes readable markdown titled by its ``; a linked JSON, CSV or Markdown file becomes that kind; `.html` files can be dropped too). The fetch is SSRF-guarded: http/https only, no credentials in the URL, the hive's own DNS lookup with the connection pinned to the checked address (private, loopback, link-local, CGNAT, multicast, IPv6 ULA and cloud metadata addresses refused), every redirect re-checked (at most 3), size and time caps, and `User-Agent: hive-inbox/<version>`; see `docs/inbox-uploads.md`. diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 9866891443..d8b9bdc365 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -38693,7 +38693,8 @@ "jsonl", "yaml", "csv", - "tsv" + "tsv", + "html" ] }, "code": { @@ -38758,7 +38759,8 @@ "jsonl", "yaml", "csv", - "tsv" + "tsv", + "html" ] }, "code": { @@ -38823,7 +38825,8 @@ "jsonl", "yaml", "csv", - "tsv" + "tsv", + "html" ] }, "code": { @@ -38894,7 +38897,8 @@ "jsonl", "yaml", "csv", - "tsv" + "tsv", + "html" ] }, "code": { @@ -38959,7 +38963,8 @@ "jsonl", "yaml", "csv", - "tsv" + "tsv", + "html" ] }, "code": { @@ -39024,7 +39029,8 @@ "jsonl", "yaml", "csv", - "tsv" + "tsv", + "html" ] }, "code": { @@ -39089,7 +39095,8 @@ "jsonl", "yaml", "csv", - "tsv" + "tsv", + "html" ] }, "code": { @@ -39792,6 +39799,125 @@ } } }, + "/api/inbox/uploads/url": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Add a link to the Inbox", + "description": "Owner or read-write. The hive fetches the URL itself under an SSRF guard: http/https only, no user name or password, its own DNS lookup with the connection pinned to the checked address (any private, loopback, link-local, CGNAT, multicast, IPv6 ULA or cloud metadata answer refuses the link), every redirect re-checked (at most 3), the body capped at `inbox.uploads.max_file_bytes` and the whole fetch at 60 s; `HTTP(S)_PROXY` is not used, a matching `network.upstream_proxy` is asked to CONNECT to the checked IP. The body goes through the same type pipeline as an uploaded file (an HTML page becomes readable markdown titled by its `<title>`). Audited as `inbox_upload` with `origin=link` and the host.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "url": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "201": { + "description": "One file added", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "filename": { + "type": "string" + }, + "ok": { + "type": "boolean" + }, + "source": { + "type": "string", + "enum": [ + "upload" + ] + }, + "item_id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "markdown", + "text", + "json", + "jsonl", + "yaml", + "csv", + "tsv", + "html" + ] + }, + "code": { + "type": "string", + "enum": [ + "empty", + "too_large", + "unsupported_type", + "parse_failed", + "quota_exceeded", + "too_many_files", + "internal" + ] + }, + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "400": { + "description": "No url, or the link breaks the fetch rules (`link_refused`)" + }, + "403": { + "description": "Not owner or read-write, or a cross-site request" + }, + "404": { + "description": "The hive has no upload store" + }, + "413": { + "description": "The linked file is too large" + }, + "415": { + "description": "Unsupported type" + }, + "422": { + "description": "The linked file does not parse" + }, + "502": { + "description": "The site could not be reached or answered an error (`fetch_failed`)" + }, + "507": { + "description": "The per-hive upload quota is full" + } + } + } + }, "/api/inbox/transcriptions": { "get": { "tags": [ diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go index 73e1bf39f9..9592467a26 100644 --- a/src/cmd/hive/boot_inbox.go +++ b/src/cmd/hive/boot_inbox.go @@ -57,6 +57,9 @@ func (b *boot) bootInbox(baseDir string) { ConnectorsDir: filepath.Join(baseDir, "connectors"), FactWritten: b.inboxUploadFactWritten, Logger: b.logger, + // Links are fetched by the hive with the SSRF guard (own DNS + // lookup, blocked ranges, every redirect re-checked). + Fetcher: &inbox.LinkFetcher{UserAgent: "hive-inbox/" + version}, } svc.EnableUploads(uploads) b.startInboxUploadRetention(uploads) diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index 7c1782c20a..8d9e6dfb22 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -76,7 +76,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/inbox/transcriptions` | Dashboard auth | Live progress of queued and running transcriptions in queue order (`{active: [{source, id, state}]}` with `phase`, `progress`, `estimated`, `started_at`, `audio_seconds`, `processed_seconds`, `eta_seconds`, `queue_position`); reads no source, so the Inbox polls it every 3 s while a job runs; `Cache-Control: no-store`; see [inbox-transcription.md](../inbox-transcription.md) | `pkg/dashboard/api_inbox.go:34` | | `GET` | `/api/inbox/{source}/{id}/audio` | Owner or read-write | Mint a fresh short-lived audio/video link for one item (`{url, expires_at}`; Plaud: presigned, 24h; Zoom: audio-only or video `download_url` with a one-hour download token; Webex: the recording's temporary direct audio (else MP4) download link, valid 3 hours; Teams and Google Meet: 404, no browser-playable link; Slack huddles: 404, Slack keeps no huddle audio; Otter: the conversation's MP3 download link (`GET /conversations/{id}/audio`), minted per click; Granola: 404, Granola keeps no meeting audio; Limitless: 404, Pendant audio is served only behind the API key; Fathom: the signed url of a recording download (`POST /recordings/{id}/download`, audio-only recordings complete at once, video is polled), valid about 24 hours; Fireflies: the transcript's hashed `audio_url` (else `video_url`), valid 24 hours, Pro plan or higher); never stored by the hive, replied with `Cache-Control: no-store`, audited as `inbox_audio` (403 role, 404 unknown item or no audio, 502 source unreachable) | `pkg/dashboard/api_inbox.go:30` | | `GET` | `/api/inbox/{source}/{id}/audio/stream` | Owner or read-write | Stream one item's audio from the dashboard origin, for the Inbox player (the CSP refuses cross-origin media). The hive mints the link as above, fetches it server-side and copies the bytes with the media `Content-Type` (from storage, else the link's extension); a single `Range: bytes=…` is passed through (206 + `Content-Range`; 200 whole file when storage ignores it; 416 passed through). The link stays in memory for one playback's Range requests (10 min, never past expiry), is never logged, returned or stored, and is re-minted once when storage refuses it; nothing is written to disk. `Cache-Control: no-store`; audited as `inbox_audio_stream` once per minted link (403 role, 404 unknown item or no audio, 413 over the 2 GiB cap, 502 source or storage unreachable, or link refused/expired) | `pkg/dashboard/api_inbox.go:31` | -| `POST` | `/api/inbox/uploads` | Owner or read-write | Add dropped files to the Inbox: `multipart/form-data` with one or more `file` fields (at most `inbox.uploads.max_files_per_drop`, default 20, each up to `inbox.uploads.max_file_bytes`, default 25 MiB). Each file is typed by content (magic bytes, UTF-8, a parse check), never by name or client MIME alone; md, txt, json, jsonl, yaml, csv and tsv become `upload` items with extracted markdown and a Knowledge fact. One file: 201 `{results: [{filename, ok, source, item_id, title, kind}]}` or its refusal (400 `empty`/`too_many_files`, 413 `too_large`, 415 `unsupported_type`, 422 `parse_failed`, 507 `quota_exceeded`, each with `code` and `error`); several files: 200 with per-file results unless all failed. Cross-site requests are refused; every file is audited as `inbox_upload` (403 role, 404 no upload store); see [inbox-uploads.md](../inbox-uploads.md) | `pkg/dashboard/api_inbox_uploads.go:26` | +| `POST` | `/api/inbox/uploads` | Owner or read-write | Add dropped files to the Inbox: `multipart/form-data` with one or more `file` fields (at most `inbox.uploads.max_files_per_drop`, default 20, each up to `inbox.uploads.max_file_bytes`, default 25 MiB). Each file is typed by content (magic bytes, UTF-8, a parse check), never by name or client MIME alone; md, txt, json, jsonl, yaml, csv, tsv and html become `upload` items with extracted markdown and a Knowledge fact. One file: 201 `{results: [{filename, ok, source, item_id, title, kind}]}` or its refusal (400 `empty`/`too_many_files`, 413 `too_large`, 415 `unsupported_type`, 422 `parse_failed`, 507 `quota_exceeded`, each with `code` and `error`); several files: 200 with per-file results unless all failed. Cross-site requests are refused; every file is audited as `inbox_upload` (403 role, 404 no upload store); see [inbox-uploads.md](../inbox-uploads.md) | `pkg/dashboard/api_inbox_uploads.go:27` | +| `POST` | `/api/inbox/uploads/url` | Owner or read-write | Add a link: `{url}`. The hive fetches it under the SSRF guard (http/https only, no credentials, its own DNS lookup with the connection pinned to the checked address, private/loopback/link-local/CGNAT/multicast/ULA/metadata ranges refused, every redirect re-checked, at most 3, body capped at `inbox.uploads.max_file_bytes`, 60 s) and runs the body through the upload type pipeline (an HTML page becomes readable markdown titled by its `<title>`). 201 `{results: [{filename: host, ok, source, item_id, title, kind}]}`; 400 `link_refused`, 413, 415, 422, 502 `fetch_failed`, 507; audited as `inbox_upload` with `origin=link` and the host (403 role, 404 no upload store) | `pkg/dashboard/api_inbox_uploads.go:28` | | `POST` | `/api/inbox/recordings` | Owner or read-write | Start a dashboard recording (● Record): `{title, mime: audio/webm\|audio/mp4, consent_version, includes_tab_audio}`. `consent_version` must name the current consent notice; it is stored with the user in the recording's metadata. One recording per user at a time. 201 `{id, max_chunk_bytes, max_seconds, retention_days, transcribes}`; audited as `inbox_recording_start` (400 consent missing or stale, 403 role, 409 `{active_id}` already recording, 412 `inbox.recording.enabled` off, 415 other media types, 507 `inbox.recording.quota_bytes` used up); see [inbox-recording.md](../inbox-recording.md) | `pkg/dashboard/api_inbox_recordings.go:20` | | `POST` | `/api/inbox/recordings/{id}/chunks` | The recorder (owner or read-write) | Store one MediaRecorder chunk `?seq=N`, the raw bytes as the body (at most `max_chunk_bytes`, 512 KiB). In order only; each part is fsynced before the 200 `{id, seq, next_seq, bytes, duplicate?}`. The first chunk must start with a WebM or MP4 signature; the hive never decodes audio. A re-send with the same bytes is acknowledged again (`duplicate: true`) (400 bad seq, 403 someone else's, 409 `{expected_seq}` gap, other bytes for a stored seq, or already ended, 413 chunk over 512 KiB or past `max_bytes` / `max_seconds`, 415 not WebM/MP4, 507 quota) | `pkg/dashboard/api_inbox_recordings.go:21` | | `POST` | `/api/inbox/recordings/{id}/finalize` | The recorder | End a recording: `{duration_ms?}` (the recorder's count, pauses excluded). The chunks are concatenated in order into `audio.webm` or `audio.mp4`, a pending Knowledge fact (personal layer) is written and the item appears in the Inbox. 200 `{id, state, title, duration_ms, bytes, chunks, transcribes}`; audited as `inbox_recording_finalize` (400 no chunk received, 403, 404, 409 already ended) | `pkg/dashboard/api_inbox_recordings.go:22` | diff --git a/src/docs/inbox-uploads.md b/src/docs/inbox-uploads.md index 061898d0d1..2f6e1f6832 100644 --- a/src/docs/inbox-uploads.md +++ b/src/docs/inbox-uploads.md @@ -2,8 +2,9 @@ Besides the recordings that [voice and meeting connectors](inbox-voice-sources.md) bring in, the owner can add material to the Inbox directly: drop a file onto -the Inbox, choose it with **+ Add**, or paste it with Ctrl/Cmd+V while the -Inbox has focus. Each file becomes an Inbox item like any other: it gets a +the Inbox, choose it with **+ Add**, paste it with Ctrl/Cmd+V while the +Inbox has focus, or add a link (type it into + Add, or paste a URL onto the +Inbox). Each file becomes an Inbox item like any other: it gets a Knowledge fact, the 🔗 Share ▾ menu (RFC, Knowledge, DIBS) and the route history. @@ -28,6 +29,7 @@ as plain text otherwise. | JSON Lines | `.jsonl`, `.ndjson` | the record count and the first 20 records in a fenced block; every line is validated | | YAML | `.yaml`, `.yml` | a one-line summary and the document in a fenced `yaml` block (parsed with `gopkg.in/yaml.v3`) | | CSV / TSV | `.csv`, `.tsv`, `.tab` | row and column counts and a markdown table of the first 20 rows (20 columns at most) | +| HTML | `.html`, `.htm`, and web pages behind a link | readable markdown: the `<title>` names the item; the `<article>` or `<main>` (else `<body>`) is kept with headings, paragraphs, lists, quotes, code, tables, link targets and image alt text; scripts, styles, forms, navigation, footers, asides and hidden elements are dropped. Parsed with `golang.org/x/net/html` (BSD-3-Clause); nothing is executed and nothing the page references is loaded | Text is validated as UTF-8 (a UTF-8 BOM is dropped; UTF-16 is refused with "save the file as UTF-8") and newlines are normalised to `\n`. The fenced @@ -68,6 +70,41 @@ Every file, accepted or refused, is audited as `inbox_upload` with the user, the item id, the detected kind, the size and the outcome or refusal code; the file's content and name are not written to the audit log. +## Links + +A link is fetched by the hive itself, once, when it is added, and its body +goes through the same type pipeline as a dropped file: an HTML page becomes +text, a link to a `.json`, `.csv` or `.md` file becomes that kind (the file +name in the URL or `Content-Disposition`, else the response `Content-Type`, +picks the format, and the content must still pass the sniff and the parse). +YouTube and other streaming pages are read as HTML pages, never as video; +links to audio or video files are refused for now. + +The item is titled by the page `<title>`, else the file name, else the URL, +and shows **link · host**. The URL is kept as the item's provenance (and the +fact's `source_url`) only. + +Fetch rules (SSRF protection): + +- `http` and `https` only; a link with a user name or password is refused. +- The hive resolves the host itself and connects to the address it checked; + if **any** answer is blocked the link is refused, so a second + (rebinding) answer is never used. Blocked: loopback, RFC 1918 private, + link-local (including the cloud metadata address `169.254.169.254`), + CGNAT `100.64.0.0/10`, multicast, reserved and documentation ranges, + `0.0.0.0/8`, IPv6 unspecified, loopback, unique-local `fc00::/7` + (including `fd00:ec2::254`), link-local and site-local, Teredo, and IPv4 + addresses embedded in IPv4-mapped, NAT64 and 6to4 addresses; host names + `localhost`, `*.localhost`, `*.local`, `*.internal` and `metadata`. +- Every redirect hop is checked the same way, at most 3 redirects, and a + redirect never carries `Authorization` or `Cookie`. +- The body is streamed through a limit (`inbox.uploads.max_file_bytes`); + connect 10 s, TLS 10 s, response headers 15 s, whole fetch 60 s. +- Requests send `User-Agent: hive-inbox/<version>`. +- `HTTP(S)_PROXY` from the environment is not used (a proxy would resolve + the name again). When `network.upstream_proxy` matches the host, the hive + asks the proxy to `CONNECT` to the checked IP and port. + ## Limits All limits have defaults; `inbox.uploads` in `hive.yaml` overrides them and @@ -111,6 +148,11 @@ and its share history stay. Text documents are kept unless ## API +`POST /api/inbox/uploads/url` (owner or read-write) takes `{"url": "..."}` +and answers like a single-file upload (201, or 400 `link_refused`, 413, 415, +422, 502 `fetch_failed`, 507); refusals and additions are audited as +`inbox_upload` with `origin=link` and the host, never the full URL. + `POST /api/inbox/uploads` (owner or read-write) takes `multipart/form-data` with one or more `file` fields. One file answers `201` with `{results: [{filename, ok: true, source: "upload", item_id, title, kind}]}`, @@ -123,4 +165,4 @@ files answer `200` with one result per file unless all of them failed. See `GET /api/inbox` reports the limits and current usage under `settings.uploads` (`enabled`, `max_file_bytes`, `max_files_per_drop`, `quota_bytes`, `used_bytes`, `media_retention_days`, -`document_retention_days`, `extensions`). +`document_retention_days`, `extensions`, `links`). diff --git a/src/docs/security-model.md b/src/docs/security-model.md index 08caeb0dcb..daf1b52907 100644 --- a/src/docs/security-model.md +++ b/src/docs/security-model.md @@ -168,7 +168,12 @@ timeout and size caps; the original, the extracted text and the meta are stored 0600 in a 0700 directory per upload, never executed or served raw, and bounded by a per-hive quota. The extracted text is fenced in the same `<untrusted-inbox-data>` envelope in its Knowledge fact and before any agent -prompt ([inbox-uploads.md](inbox-uploads.md)). +prompt ([inbox-uploads.md](inbox-uploads.md)). Links dropped onto the Inbox are +fetched by the hive under an SSRF guard: http/https only, no credentials in +the URL, the hive's own DNS lookup with the connection pinned to the checked +address (any private, loopback, link-local, CGNAT, multicast, ULA or metadata +answer refuses the link), every redirect re-checked (at most 3), the body +size- and time-capped, and the environment proxy never used. ## 5. Network and credentials diff --git a/src/pkg/dashboard/api_inbox_uploads.go b/src/pkg/dashboard/api_inbox_uploads.go index 931b2855c6..d7eca3b5fe 100644 --- a/src/pkg/dashboard/api_inbox_uploads.go +++ b/src/pkg/dashboard/api_inbox_uploads.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "strconv" + "strings" "time" "github.com/hivecommons/hive/pkg/inbox" @@ -24,6 +25,7 @@ const ( // registerInboxUploadRoutes adds the drag-and-drop ingest routes. func (s *Server) registerInboxUploadRoutes() { s.mux.HandleFunc("POST /api/inbox/uploads", s.handleInboxUpload) + s.mux.HandleFunc("POST /api/inbox/uploads/url", s.handleInboxUploadURL) } // InboxUploadResult is one file of POST /api/inbox/uploads. @@ -58,8 +60,10 @@ func inboxUploadStatus(code string) int { return http.StatusUnprocessableEntity case inbox.UploadErrQuota: return http.StatusInsufficientStorage - case inbox.UploadErrEmpty, inbox.UploadErrTooMany: + case inbox.UploadErrEmpty, inbox.UploadErrTooMany, inbox.UploadErrLinkRefused: return http.StatusBadRequest + case inbox.UploadErrFetch: + return http.StatusBadGateway } return http.StatusInternalServerError } @@ -184,3 +188,54 @@ func (s *Server) inboxUploadRefused(r *http.Request, name string, size int64, co s.auditFromRequest(r, "inbox_upload", auditDetail("source", inbox.UploadType, "bytes", strconv.FormatInt(size, 10), "outcome", "refused", "code", code), "") return InboxUploadResult{Filename: name, Code: code, Error: msg} } + +type inboxUploadURLRequest struct { + URL string `json:"url"` +} + +// handleInboxUploadURL (owner or read-write) adds a link: the hive fetches it +// with the SSRF guard (http/https only, no credentials, its own DNS lookup +// with private, loopback, link-local, CGNAT, multicast, ULA and metadata +// ranges refused, every redirect re-checked, at most 3, size and time +// capped) and runs the body through the same type pipeline as a dropped +// file. 201 with one result, or 400 link_refused, 413, 415, 422, 502 +// fetch_failed, 507. Audited as inbox_upload with origin=link and the host. +func (s *Server) handleInboxUploadURL(w http.ResponseWriter, r *http.Request) { + if !requireInboxAddRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + if svc.Uploads() == nil { + jsonError(w, inbox.ErrUploadsDisabled.Error(), http.StatusNotFound) + return + } + var req inboxUploadURLRequest + if err := decodeBody(r, &req); err != nil || strings.TrimSpace(req.URL) == "" { + jsonError(w, "url is required", http.StatusBadRequest) + return + } + host := "" + if u, err := inbox.ValidateLinkURL(req.URL); err == nil { + host = u.Hostname() + } + ctx, cancel := context.WithTimeout(r.Context(), inboxUploadTimeout) + defer cancel() + it, err := svc.AddLink(ctx, inbox.LinkInput{URL: req.URL, AddedBy: requestUser(r)}) + if err != nil { + code, msg := inbox.UploadErrInternal, "adding the link failed" + var ue *inbox.UploadError + if errors.As(err, &ue) { + code, msg = ue.Code, ue.Message + } else if s.logger != nil { + s.logger.Warn("inbox: link failed", "error", err) + } + s.auditFromRequest(r, "inbox_upload", auditDetail("source", inbox.UploadType, "origin", inbox.UploadOriginLink, "host", host, "outcome", "refused", "code", code), "") + jsonStatusResponse(w, inboxUploadStatus(code), InboxUploadResponse{Results: []InboxUploadResult{{Filename: host, Code: code, Error: msg}}}) + return + } + s.auditFromRequest(r, "inbox_upload", auditDetail("source", it.Source, "item", it.ID, "origin", inbox.UploadOriginLink, "host", host, "kind", it.UploadKind, "outcome", "added"), "") + jsonStatusResponse(w, http.StatusCreated, InboxUploadResponse{Results: []InboxUploadResult{{Filename: host, OK: true, Source: it.Source, ItemID: it.ID, Title: it.Title, Kind: it.UploadKind}}}) +} diff --git a/src/pkg/dashboard/api_inbox_uploads_test.go b/src/pkg/dashboard/api_inbox_uploads_test.go index 5564d8518e..18e7bc628e 100644 --- a/src/pkg/dashboard/api_inbox_uploads_test.go +++ b/src/pkg/dashboard/api_inbox_uploads_test.go @@ -2,10 +2,15 @@ package dashboard import ( "bytes" + "context" "encoding/json" + "io" "mime/multipart" + "net" "net/http" "net/http/httptest" + "net/netip" + "net/url" "path/filepath" "strings" "testing" @@ -201,3 +206,74 @@ func TestInboxUploadRequestErrors(t *testing.T) { } } } + +func TestInboxUploadURL(t *testing.T) { + site := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = io.WriteString(w, "<title>Linked page

linked text

") + })) + t.Cleanup(site.Close) + s, svc := newUploadTestServer(t, config.InboxUploads{}) + svc.Uploads().Fetcher = &inbox.LinkFetcher{ + Resolve: func(context.Context, string) ([]netip.Addr, error) { + return []netip.Addr{netip.MustParseAddr("93.184.216.34")}, nil + }, + Dial: func(ctx context.Context, network, _ string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, network, site.Listener.Addr().String()) + }, + Proxy: func(string) *url.URL { return nil }, + } + path := "/api/inbox/uploads/url" + body := map[string]string{"url": "https://docs.example/page"} + for _, role := range []string{"", "read-only", "read"} { + if rec := doPostRole(s, path, role, false, body); rec.Code != http.StatusForbidden { + t.Fatalf("role %q = %d", role, rec.Code) + } + } + // The test site speaks plain http. + body["url"] = "http://docs.example/page" + rec := doPostRole(s, path, "read-write", false, body) + if rec.Code != http.StatusCreated { + t.Fatalf("link = %d %s", rec.Code, rec.Body.String()) + } + res := decodeUploadResponse(t, rec).Results + if len(res) != 1 || !res[0].OK || res[0].Kind != inbox.UploadKindHTML || res[0].Title != "Linked page" || res[0].Filename != "docs.example" { + t.Fatalf("results = %+v", res) + } + for raw, want := range map[string]int{ + "http://169.254.169.254/latest/meta-data/": http.StatusBadRequest, + "http://user:pw@docs.example/": http.StatusBadRequest, + "ftp://docs.example/x": http.StatusBadRequest, + } { + rec := doPostRole(s, path, "owner", true, map[string]string{"url": raw}) + if rec.Code != want { + t.Fatalf("%s = %d %s", raw, rec.Code, rec.Body.String()) + } + if r := decodeUploadResponse(t, rec).Results; len(r) != 1 || r[0].Code != inbox.UploadErrLinkRefused { + t.Fatalf("%s results = %+v", raw, r) + } + } + if rec := doPostRole(s, path, "owner", true, map[string]string{}); rec.Code != http.StatusBadRequest { + t.Fatalf("no url = %d", rec.Code) + } + var added, refused int + for _, e := range s.audit.Recent(50) { + if e.Action != "inbox_upload" || !strings.Contains(e.Detail, "origin=link") { + continue + } + if strings.Contains(e.Detail, "pw") || strings.Contains(e.Detail, "meta-data") { + t.Fatalf("audit leaks the URL: %q", e.Detail) + } + if strings.Contains(e.Detail, "outcome=added") { + added++ + } else { + refused++ + } + } + if added != 1 || refused != 3 { + t.Fatalf("audits added=%d refused=%d", added, refused) + } + if inboxUploadStatus(inbox.UploadErrFetch) != http.StatusBadGateway { + t.Fatal("fetch_failed must be 502") + } +} diff --git a/src/pkg/dashboard/inbox_uploads.test.js b/src/pkg/dashboard/inbox_uploads.test.js index 24bca5dc8c..c22b61110c 100644 --- a/src/pkg/dashboard/inbox_uploads.test.js +++ b/src/pkg/dashboard/inbox_uploads.test.js @@ -29,24 +29,31 @@ const lineSrc = prefix => { const escapeHtml = s => String(s == null ? '' : s).replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"'); const MB = 1048576; -const UPLOADS = { enabled: true, max_file_bytes: 25 * MB, max_files_per_drop: 3, extensions: ['.csv', '.json', '.md', '.txt'] }; +const UPLOADS = { enabled: true, links: true, max_file_bytes: 25 * MB, max_files_per_drop: 3, extensions: ['.csv', '.json', '.md', '.txt'] }; const FNS = ['inboxViewerCanPlay', 'inboxUploadSettings', 'inboxCanAdd', 'inboxFmtBytes', 'inboxUploadPlan', 'inboxUploadRowHtml', 'inboxUploadsHtml', 'inboxAddPanelHtml', 'inboxAnnounce', 'inboxUploadRowUpdate', 'inboxAddFiles', 'inboxUploadSend', 'inboxDragHasFiles', 'inboxInSection', 'inboxSetDragOver', 'inboxOnDragOver', 'inboxOnDragLeave', 'inboxOnDrop', 'inboxOnPaste', - 'inboxPickFiles', 'inboxClearUploads']; + 'inboxPickFiles', 'inboxClearUploads', 'inboxAddUrl', 'inboxAddUrlFromField']; // harness builds the upload functions with a fake document, XHR and the // Inbox cache. replies is a list of {status, body} answered in order. -const harness = ({ role = 'owner', uploads = UPLOADS, replies = [] } = {}) => { - const calls = { sends: [], renders: 0, refreshes: 0, rows: {}, classes: {} }; +const harness = ({ role = 'owner', uploads = UPLOADS, replies = [], fetchReplies = [] } = {}) => { + const calls = { sends: [], fetches: [], renders: 0, refreshes: 0, rows: {}, classes: {} }; + const urlField = { value: '' }; + const fetch = async (url, opts) => { + calls.fetches.push({ url, opts }); + const r = fetchReplies.shift() || { status: 201, body: { results: [{ ok: true, source: 'upload', item_id: 'link1', title: 'Page' }] } }; + if (r.network) throw new Error('offline'); + return { ok: r.status >= 200 && r.status < 300, status: r.status, json: async () => r.body }; + }; const status = { textContent: '' }; const section = { classList: { toggle: (c, on) => { calls.classes[c] = on; } } }; const inside = { closest: sel => (sel === '#inbox-section' ? section : null) }; const outside = { closest: () => null }; const document = { activeElement: outside, - getElementById: id => (id === 'inbox-upload-status' ? status : id === 'inbox-section' ? section : null), + getElementById: id => (id === 'inbox-upload-status' ? status : id === 'inbox-section' ? section : id === 'inbox-add-url' ? urlField : null), querySelector: sel => { const m = /data-inbox-upload="(\d+)"/.exec(sel); if (!m) return null; @@ -69,16 +76,16 @@ const harness = ({ role = 'owner', uploads = UPLOADS, replies = [] } = {}) => { } } const ctx = { _inboxCache: { settings: { uploads } } }; - const api = new Function('ctx', 'window', 'document', 'escapeHtml', 'renderInbox', 'fetchInbox', 'calls', 'FakeXHR', + const api = new Function('ctx', 'window', 'document', 'escapeHtml', 'renderInbox', 'fetchInbox', 'calls', 'FakeXHR', 'fetch', 'let _inboxUploads = []; let _inboxUploadSeq = 0; let _inboxAddOpen = true; let _inboxUploadQueue = Promise.resolve(); let _inboxPointerInside = false;\n' + 'const _inboxCache = ctx._inboxCache;\n' + - lineSrc('const INBOX_UPLOAD_LABELS = ') + '\n' + + lineSrc('const INBOX_UPLOAD_LABELS = ') + '\n' + lineSrc('const INBOX_URL_RE = ') + '\n' + 'function inboxNewXHR() { return new FakeXHR(); }\n' + 'function inboxUploadForm(row) { return { file: row.file, name: row.name }; }\n' + FNS.map(fnSrc).join('\n') + '\n' + 'return { ' + FNS.join(', ') + ', state: () => ({ uploads: _inboxUploads }), queue: () => _inboxUploadQueue, setPointerInside: v => { _inboxPointerInside = v; } };', - )(ctx, { _hiveRole: role }, document, escapeHtml, () => { calls.renders++; }, async () => { calls.refreshes++; }, calls, FakeXHR); - return { api, calls, status, inside, outside, document }; + )(ctx, { _hiveRole: role }, document, escapeHtml, () => { calls.renders++; }, async () => { calls.refreshes++; }, calls, FakeXHR, fetch); + return { api, calls, status, inside, outside, document, urlField }; }; const file = (name, size) => ({ name, size }); @@ -234,3 +241,52 @@ test('+ Add panel: keyboard-operable picker, limits and privacy note', () => { assert.strictEqual(api.state().uploads.length, 1); assert.match(fnSrc('inboxToggleAdd'), /aria-expanded|focus\(\)/); }); + +test('links: the Add field and a pasted URL are fetched by the hive, errors stay per row', async () => { + const { api, calls, inside, urlField, status } = harness({ fetchReplies: [ + { status: 201, body: { results: [{ ok: true, source: 'upload', item_id: 'l1', title: 'Docs' }] } }, + { status: 400, body: { results: [{ ok: false, code: 'link_refused', error: 'link refused: 10.0.0.5 is a private address' }] } }, + { network: true }, + ] }); + urlField.value = ' https://docs.example/page '; + await api.inboxAddUrlFromField(); + assert.strictEqual(urlField.value, '', 'the field is cleared'); + assert.strictEqual(calls.fetches[0].url, '/api/inbox/uploads/url'); + assert.deepStrictEqual(JSON.parse(calls.fetches[0].opts.body), { url: 'https://docs.example/page' }); + let rows = api.state().uploads; + assert.strictEqual(rows[0].status, 'done'); + assert.strictEqual(rows[0].key, 'upload/l1'); + assert.strictEqual(rows[0].name, 'docs.example'); + assert.match(status.textContent, /docs\.example added to the Inbox/); + assert.strictEqual(calls.refreshes, 1); + // Pasting a URL while the Inbox has focus adds it; in a text field it stays a paste. + const paste = (target, text) => { + const e = { target, clipboardData: { files: [], getData: () => text }, prevented: false }; + e.preventDefault = () => { e.prevented = true; }; + api.inboxOnPaste(e); + return e; + }; + const field = { closest: sel => (sel === '#inbox-section' ? {} : sel.startsWith('input') ? {} : null) }; + assert.strictEqual(paste(field, 'https://x.example/').prevented, false, 'paste into a field is left alone'); + assert.strictEqual(paste(inside, 'not a link').prevented, false); + assert.strictEqual(paste(inside, 'https://a.example/ and more').prevented, false, 'only a single URL'); + assert.strictEqual(paste(inside, 'http://intranet.example/').prevented, true); + await api.queue(); + rows = api.state().uploads; + assert.strictEqual(rows[1].status, 'error'); + assert.strictEqual(rows[1].code, 'link_refused'); + assert.match(api.inboxUploadRowHtml(rows[1]), /role="alert">link refused: 10\.0\.0\.5 is a private address/); + assert.doesNotMatch(api.inboxUploadRowHtml(rows[1]), /]*data-enter-action="inboxAddUrlFromField"/); + assert.match(api.inboxUploadRowHtml({ id: 9, name: 'e.example', link: 'https://e.example/', status: 'uploading' }), /Fetching/); +}); diff --git a/src/pkg/dashboard/static-src/css/inbox.css b/src/pkg/dashboard/static-src/css/inbox.css index fcb7e2fc8d..3aa9804ec3 100644 --- a/src/pkg/dashboard/static-src/css/inbox.css +++ b/src/pkg/dashboard/static-src/css/inbox.css @@ -41,6 +41,7 @@ .inbox-upload { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); font-size: var(--fs-sm); } .inbox-upload-name { font-weight: 600; overflow-wrap: anywhere; } .inbox-upload-bar { flex: 1 1 120px; max-width: 240px; } + .inbox-add-url { flex: 1 1 16rem; min-width: 0; } .inbox-upload .kc-error { flex-basis: 100%; } @media (max-width: 720px) { .inbox-settings-grid, .inbox-rule { grid-template-columns: 1fr; } diff --git a/src/pkg/dashboard/static-src/js/inbox.js b/src/pkg/dashboard/static-src/js/inbox.js index 77e33b8564..557ffbf22f 100644 --- a/src/pkg/dashboard/static-src/js/inbox.js +++ b/src/pkg/dashboard/static-src/js/inbox.js @@ -1055,11 +1055,11 @@ function inboxUploadRowHtml(row) { const pct = Math.max(0, Math.min(100, Math.round((Number(row.progress) || 0) * 100))); const pill = row.status === 'done' ? 'ok' : (row.status === 'error' ? 'error' : 'syncing'); - const label = (INBOX_UPLOAD_LABELS[row.status] || row.status) + (row.status === 'uploading' ? ' ' + pct + '%' : ''); + const label = row.link && row.status === 'uploading' ? 'Fetching' : (INBOX_UPLOAD_LABELS[row.status] || row.status) + (row.status === 'uploading' ? ' ' + pct + '%' : ''); let html = '
  • ' + - '' + escapeHtml(row.name) + ' ' + escapeHtml(inboxFmtBytes(row.size)) + ' ' + + '' + escapeHtml(row.name) + ' ' + escapeHtml(row.link ? 'link' : inboxFmtBytes(row.size)) + ' ' + '' + escapeHtml(label) + ''; - if (row.status === 'uploading' || row.status === 'queued') html += ''; + if (!row.link && (row.status === 'uploading' || row.status === 'queued')) html += ''; if (row.status === 'done' && row.key) html += ' show item'; if (row.status === 'error' && row.error) html += ' ' + escapeHtml(row.error) + ''; return html + '
  • '; @@ -1079,7 +1079,10 @@ return '
    ' + '

    Drop files anywhere on the Inbox, paste them with Ctrl/Cmd+V, or choose them here: ' + escapeHtml(limit) + '. Supported: ' + escapeHtml(exts.join(' ')) + '. Files stay on this hive; their text reaches an LLM only when you share the item.

    ' + '
    ' + - '
    '; + '' + + (u.links ? '
    ' + + '
    ' + + '

    The hive fetches the link itself (public http/https addresses only, at most 3 redirects) and reads it like a dropped file.

    ' : '') + ''; } function inboxToggleAdd() { _inboxAddOpen = !_inboxAddOpen; renderInbox(); if (_inboxAddOpen) { const b = document.querySelector('#inbox-add [data-action="inboxChooseFiles"]'); if (b && b.focus) b.focus(); } } function inboxChooseFiles() { const el = document.getElementById('inbox-file-input'); if (el) el.click(); } @@ -1164,9 +1167,53 @@ if (!inboxCanAdd()) return; if (!(inboxInSection(e.target) || inboxInSection(document.activeElement) || _inboxPointerInside)) return; const files = Array.from((e.clipboardData && e.clipboardData.files) || []); - if (!files.length) return; + if (files.length) { e.preventDefault(); inboxAddFiles(files); return; } + // A pasted link (one http/https URL) outside a text field is added + // like a dropped one; pasting into a field stays a normal paste. + const text = String((e.clipboardData && e.clipboardData.getData && e.clipboardData.getData('text')) || '').trim(); + const field = e.target && e.target.closest && e.target.closest('input, textarea, [contenteditable="true"]'); + if (field || !inboxUploadSettings().links || !INBOX_URL_RE.test(text)) return; e.preventDefault(); - inboxAddFiles(files); + inboxAddUrl(text); + } + const INBOX_URL_RE = /^https?:\/\/[^\s]+$/i; + function inboxAddUrlFromField() { + const el = document.getElementById('inbox-add-url'); + const raw = el ? String(el.value || '').trim() : ''; + if (!raw) return Promise.resolve(null); + if (el) el.value = ''; + return inboxAddUrl(raw); + } + // inboxAddUrl shows a row for one link and asks the hive to fetch it + // (POST /api/inbox/uploads/url); the hive applies the SSRF rules. + function inboxAddUrl(raw) { + if (!inboxCanAdd() || !inboxUploadSettings().links) return Promise.resolve(null); + let host = raw; + try { host = new URL(raw).host || raw; } catch (e) { host = raw; } + const row = { id: ++_inboxUploadSeq, name: host, size: 0, link: raw, status: 'queued', progress: 0 }; + if (!INBOX_URL_RE.test(raw)) { row.status = 'error'; row.code = 'link_refused'; row.error = 'Only http and https links can be added.'; } + _inboxUploads = _inboxUploads.concat([row]); + renderInbox(); + if (row.status === 'error') return Promise.resolve(row); + inboxAnnounce('Fetching ' + host + ' for the Inbox'); + _inboxUploadQueue = _inboxUploadQueue.then(async () => { + row.status = 'uploading'; + inboxUploadRowUpdate(row); + try { + const r = await fetch('/api/inbox/uploads/url', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ url: raw }) }); + const data = await r.json().catch(() => ({})); + const res = (Array.isArray(data.results) ? data.results : [])[0]; + if (r.ok && res && res.ok) { row.status = 'done'; row.key = res.source + '/' + res.item_id; row.title = res.title || host; } + else { row.status = 'error'; row.code = (res && res.code) || ''; row.error = (res && res.error) || data.error || ('HTTP ' + r.status); } + } catch (e) { + row.status = 'error'; row.error = 'The request did not reach the hive (network error).'; + } + inboxUploadRowUpdate(row); + inboxAnnounce(row.status === 'done' ? host + ' added to the Inbox' : host + ' not added: ' + row.error); + if (row.status === 'done') await fetchInbox(); + return row; + }); + return _inboxUploadQueue; } document.addEventListener('dragenter', inboxOnDragOver); document.addEventListener('dragover', inboxOnDragOver); diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 791b5a3fd0..de0c0c7255 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -6097,6 +6097,7 @@ .inbox-upload { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); font-size: var(--fs-sm); } .inbox-upload-name { font-weight: 600; overflow-wrap: anywhere; } .inbox-upload-bar { flex: 1 1 120px; max-width: 240px; } + .inbox-add-url { flex: 1 1 16rem; min-width: 0; } .inbox-upload .kc-error { flex-basis: 100%; } @media (max-width: 720px) { .inbox-settings-grid, .inbox-rule { grid-template-columns: 1fr; } @@ -48409,11 +48410,11 @@ function inboxUploadRowHtml(row) { const pct = Math.max(0, Math.min(100, Math.round((Number(row.progress) || 0) * 100))); const pill = row.status === 'done' ? 'ok' : (row.status === 'error' ? 'error' : 'syncing'); - const label = (INBOX_UPLOAD_LABELS[row.status] || row.status) + (row.status === 'uploading' ? ' ' + pct + '%' : ''); + const label = row.link && row.status === 'uploading' ? 'Fetching' : (INBOX_UPLOAD_LABELS[row.status] || row.status) + (row.status === 'uploading' ? ' ' + pct + '%' : ''); let html = '
  • ' + - '' + escapeHtml(row.name) + ' ' + escapeHtml(inboxFmtBytes(row.size)) + ' ' + + '' + escapeHtml(row.name) + ' ' + escapeHtml(row.link ? 'link' : inboxFmtBytes(row.size)) + ' ' + '' + escapeHtml(label) + ''; - if (row.status === 'uploading' || row.status === 'queued') html += ''; + if (!row.link && (row.status === 'uploading' || row.status === 'queued')) html += ''; if (row.status === 'done' && row.key) html += ' show item'; if (row.status === 'error' && row.error) html += ' ' + escapeHtml(row.error) + ''; return html + '
  • '; @@ -48433,7 +48434,10 @@ return '
    ' + '

    Drop files anywhere on the Inbox, paste them with Ctrl/Cmd+V, or choose them here: ' + escapeHtml(limit) + '. Supported: ' + escapeHtml(exts.join(' ')) + '. Files stay on this hive; their text reaches an LLM only when you share the item.

    ' + '
    ' + - '
    '; + '' + + (u.links ? '
    ' + + '
    ' + + '

    The hive fetches the link itself (public http/https addresses only, at most 3 redirects) and reads it like a dropped file.

    ' : '') + ''; } function inboxToggleAdd() { _inboxAddOpen = !_inboxAddOpen; renderInbox(); if (_inboxAddOpen) { const b = document.querySelector('#inbox-add [data-action="inboxChooseFiles"]'); if (b && b.focus) b.focus(); } } function inboxChooseFiles() { const el = document.getElementById('inbox-file-input'); if (el) el.click(); } @@ -48518,9 +48522,53 @@ if (!inboxCanAdd()) return; if (!(inboxInSection(e.target) || inboxInSection(document.activeElement) || _inboxPointerInside)) return; const files = Array.from((e.clipboardData && e.clipboardData.files) || []); - if (!files.length) return; + if (files.length) { e.preventDefault(); inboxAddFiles(files); return; } + // A pasted link (one http/https URL) outside a text field is added + // like a dropped one; pasting into a field stays a normal paste. + const text = String((e.clipboardData && e.clipboardData.getData && e.clipboardData.getData('text')) || '').trim(); + const field = e.target && e.target.closest && e.target.closest('input, textarea, [contenteditable="true"]'); + if (field || !inboxUploadSettings().links || !INBOX_URL_RE.test(text)) return; e.preventDefault(); - inboxAddFiles(files); + inboxAddUrl(text); + } + const INBOX_URL_RE = /^https?:\/\/[^\s]+$/i; + function inboxAddUrlFromField() { + const el = document.getElementById('inbox-add-url'); + const raw = el ? String(el.value || '').trim() : ''; + if (!raw) return Promise.resolve(null); + if (el) el.value = ''; + return inboxAddUrl(raw); + } + // inboxAddUrl shows a row for one link and asks the hive to fetch it + // (POST /api/inbox/uploads/url); the hive applies the SSRF rules. + function inboxAddUrl(raw) { + if (!inboxCanAdd() || !inboxUploadSettings().links) return Promise.resolve(null); + let host = raw; + try { host = new URL(raw).host || raw; } catch (e) { host = raw; } + const row = { id: ++_inboxUploadSeq, name: host, size: 0, link: raw, status: 'queued', progress: 0 }; + if (!INBOX_URL_RE.test(raw)) { row.status = 'error'; row.code = 'link_refused'; row.error = 'Only http and https links can be added.'; } + _inboxUploads = _inboxUploads.concat([row]); + renderInbox(); + if (row.status === 'error') return Promise.resolve(row); + inboxAnnounce('Fetching ' + host + ' for the Inbox'); + _inboxUploadQueue = _inboxUploadQueue.then(async () => { + row.status = 'uploading'; + inboxUploadRowUpdate(row); + try { + const r = await fetch('/api/inbox/uploads/url', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ url: raw }) }); + const data = await r.json().catch(() => ({})); + const res = (Array.isArray(data.results) ? data.results : [])[0]; + if (r.ok && res && res.ok) { row.status = 'done'; row.key = res.source + '/' + res.item_id; row.title = res.title || host; } + else { row.status = 'error'; row.code = (res && res.code) || ''; row.error = (res && res.error) || data.error || ('HTTP ' + r.status); } + } catch (e) { + row.status = 'error'; row.error = 'The request did not reach the hive (network error).'; + } + inboxUploadRowUpdate(row); + inboxAnnounce(row.status === 'done' ? host + ' added to the Inbox' : host + ' not added: ' + row.error); + if (row.status === 'done') await fetchInbox(); + return row; + }); + return _inboxUploadQueue; } document.addEventListener('dragenter', inboxOnDragOver); document.addEventListener('dragover', inboxOnDragOver); diff --git a/src/pkg/inbox/link_fetch.go b/src/pkg/inbox/link_fetch.go new file mode 100644 index 0000000000..93fbbcf5be --- /dev/null +++ b/src/pkg/inbox/link_fetch.go @@ -0,0 +1,399 @@ +package inbox + +import ( + "context" + "errors" + "fmt" + "io" + "mime" + "net" + "net/http" + "net/netip" + "net/url" + "path" + "strings" + "time" + + "github.com/hivecommons/hive/crux/upstreamproxy" +) + +// Link fetch limits. A link dropped onto the Inbox is fetched by the hive +// itself, so every hop is checked against the SSRF rules below. +const ( + // LinkMaxRedirects is the most redirects one fetch follows. + LinkMaxRedirects = 3 + // linkMaxURLBytes bounds the URL the owner may submit. + linkMaxURLBytes = 2048 + // linkDialTimeout bounds one TCP connect (to the site or the proxy). + linkDialTimeout = 10 * time.Second + // linkTLSTimeout bounds the TLS handshake. + linkTLSTimeout = 10 * time.Second + // linkHeaderTimeout bounds the wait for response headers. + linkHeaderTimeout = 15 * time.Second + // linkTotalTimeout bounds the whole fetch, body included. + linkTotalTimeout = 60 * time.Second + // DefaultLinkUserAgent is sent when no version is known. + DefaultLinkUserAgent = "hive-inbox/dev" + // linkAccept is the Accept header of a link fetch. + linkAccept = "text/html,application/xhtml+xml,text/markdown,text/plain,application/json,application/yaml,text/csv;q=0.9,*/*;q=0.5" +) + +// ErrLinkRefused marks a link the SSRF rules refuse; errors.Is reports it for +// every refusal so the dashboard answers 400. +var ErrLinkRefused = errors.New("link refused") + +func linkRefused(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrLinkRefused, fmt.Sprintf(format, args...)) +} + +// linkBlockedPrefixes are the address ranges a link may never reach: +// loopback, private, link-local, CGNAT, multicast, reserved and +// documentation ranges, IPv6 ULA and link-local, and the cloud metadata +// addresses (169.254.169.254 and fd00:ec2::254 fall inside link-local and +// ULA; 100.100.100.200 inside CGNAT; 192.0.0.192 inside 192.0.0.0/24). +var linkBlockedPrefixes = mustPrefixes( + "0.0.0.0/8", // "this network" + "10.0.0.0/8", // RFC 1918 + "100.64.0.0/10", // CGNAT (RFC 6598) + "127.0.0.0/8", // loopback + "169.254.0.0/16", // link-local, cloud metadata + "172.16.0.0/12", // RFC 1918 + "192.0.0.0/24", // IETF protocol assignments + "192.0.2.0/24", // TEST-NET-1 + "192.88.99.0/24", // 6to4 relay anycast + "192.168.0.0/16", // RFC 1918 + "198.18.0.0/15", // benchmarking + "198.51.100.0/24", // TEST-NET-2 + "203.0.113.0/24", // TEST-NET-3 + "224.0.0.0/4", // multicast + "240.0.0.0/4", // reserved, broadcast + "::/128", // unspecified + "::1/128", // loopback + "100::/64", // discard-only + "2001::/32", // Teredo + "2001:db8::/32", // documentation + "fc00::/7", // unique local, fd00:ec2::254 metadata + "fe80::/10", // link-local + "fec0::/10", // site-local (deprecated) + "ff00::/8", // multicast +) + +// linkBlockedHosts are names refused before any lookup. +var linkBlockedHosts = map[string]bool{ + "localhost": true, + "metadata": true, + "metadata.google.internal": true, + "instance-data": true, +} + +func mustPrefixes(list ...string) []netip.Prefix { + out := make([]netip.Prefix, len(list)) + for i, s := range list { + out[i] = netip.MustParsePrefix(s) + } + return out +} + +// nat64Prefix and sixToFour embed an IPv4 address that is checked too. +var ( + nat64Prefix = netip.MustParsePrefix("64:ff9b::/96") + sixToFour = netip.MustParsePrefix("2002::/16") +) + +// LinkAddrAllowed reports whether a link may connect to addr. IPv4-mapped, +// NAT64 and 6to4 addresses are judged by the IPv4 address they carry. +func LinkAddrAllowed(addr netip.Addr) bool { + if !addr.IsValid() { + return false + } + addr = addr.Unmap() + if addr.Is6() { + b := addr.As16() + switch { + case nat64Prefix.Contains(addr): + if !LinkAddrAllowed(netip.AddrFrom4([4]byte{b[12], b[13], b[14], b[15]})) { + return false + } + case sixToFour.Contains(addr): + if !LinkAddrAllowed(netip.AddrFrom4([4]byte{b[2], b[3], b[4], b[5]})) { + return false + } + } + } + if addr.IsLoopback() || addr.IsPrivate() || addr.IsLinkLocalUnicast() || addr.IsLinkLocalMulticast() || addr.IsMulticast() || addr.IsUnspecified() || addr.IsInterfaceLocalMulticast() { + return false + } + for _, p := range linkBlockedPrefixes { + if p.Contains(addr) { + return false + } + } + return true +} + +// ValidateLinkURL parses a submitted link and applies the rules that need no +// lookup: http or https only, no user name or password, a host, a bounded +// length, and no blocked host name or IP literal. +func ValidateLinkURL(raw string) (*url.URL, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, linkRefused("the link is empty") + } + if len(raw) > linkMaxURLBytes { + return nil, linkRefused("the link is longer than %d characters", linkMaxURLBytes) + } + u, err := url.Parse(raw) + if err != nil { + return nil, linkRefused("the link is not a valid URL") + } + return u, checkLinkURL(u) +} + +func checkLinkURL(u *url.URL) error { + switch strings.ToLower(u.Scheme) { + case "http", "https": + default: + return linkRefused("only http and https links can be added (got %q)", u.Scheme) + } + if u.User != nil { + return linkRefused("links with a user name or password are refused: remove the credentials from the link") + } + host := strings.TrimSuffix(strings.ToLower(u.Hostname()), ".") + if host == "" { + return linkRefused("the link has no host") + } + if linkBlockedHosts[host] || strings.HasSuffix(host, ".localhost") || strings.HasSuffix(host, ".internal") || strings.HasSuffix(host, ".local") { + return linkRefused("%s is a local or internal host name", host) + } + if addr, err := netip.ParseAddr(strings.Trim(host, "[]")); err == nil && !LinkAddrAllowed(addr) { + return linkRefused("%s is a private, loopback, link-local or metadata address", host) + } + return nil +} + +// LinkFetcher fetches one link for the Inbox. It resolves the host itself, +// refuses the fetch when any resolved address is blocked, and connects to +// the checked address (so a second, rebinding answer is never used). Every +// redirect hop goes through the same checks. The environment's HTTP(S)_PROXY +// is never used, because a proxy would resolve the name again; a configured +// network.upstream_proxy that matches the host is reached with CONNECT to the +// checked IP and port. +type LinkFetcher struct { + // Resolve looks up a host; nil uses the system resolver. + Resolve func(ctx context.Context, host string) ([]netip.Addr, error) + // Dial connects to an address ("ip:port"); nil uses a net.Dialer. + Dial func(ctx context.Context, network, addr string) (net.Conn, error) + // Proxy returns the upstream proxy for a host, or nil; nil uses the + // hive's network.upstream_proxy selector. + Proxy func(host string) *url.URL + // UserAgent is sent with every request (hive-inbox/). + UserAgent string +} + +// FetchedLink is a fetched body with what the response said about it. +type FetchedLink struct { + // URL is the final URL after redirects. + URL *url.URL + // ContentType is the response media type without parameters. + ContentType string + // Filename is the Content-Disposition file name or the last path + // segment, when it has an extension. + Filename string + Body []byte +} + +func (f *LinkFetcher) resolve(ctx context.Context, host string) ([]netip.Addr, error) { + if f.Resolve != nil { + return f.Resolve(ctx, host) + } + ips, err := net.DefaultResolver.LookupNetIP(ctx, "ip", host) + if err != nil { + return nil, err + } + return ips, nil +} + +func (f *LinkFetcher) dialRaw(ctx context.Context, network, addr string) (net.Conn, error) { + if f.Dial != nil { + return f.Dial(ctx, network, addr) + } + d := &net.Dialer{Timeout: linkDialTimeout} + return d.DialContext(ctx, network, addr) +} + +func (f *LinkFetcher) proxyFor(host string) *url.URL { + if f.Proxy != nil { + return f.Proxy(host) + } + return upstreamproxy.Default().ProxyFor(host) +} + +// dial resolves addr's host once, refuses it when any answer is blocked, and +// connects to the first checked address, directly or through the upstream +// proxy. +func (f *LinkFetcher) dial(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + var ips []netip.Addr + if ip, perr := netip.ParseAddr(host); perr == nil { + ips = []netip.Addr{ip} + } else { + ips, err = f.resolve(ctx, host) + if err != nil { + return nil, fmt.Errorf("looking up %s: %w", host, err) + } + } + if len(ips) == 0 { + return nil, fmt.Errorf("looking up %s: no addresses", host) + } + for _, ip := range ips { + if !LinkAddrAllowed(ip) { + return nil, linkRefused("%s resolves to a private, loopback, link-local or metadata address", host) + } + } + target := net.JoinHostPort(ips[0].Unmap().String(), port) + if proxy := f.proxyFor(host); proxy != nil { + return upstreamproxy.Dial(ctx, f.dialRaw, proxy, target) + } + return f.dialRaw(ctx, network, target) +} + +func (f *LinkFetcher) client() *http.Client { + tr := &http.Transport{ + Proxy: nil, + DialContext: f.dial, + TLSHandshakeTimeout: linkTLSTimeout, + ResponseHeaderTimeout: linkHeaderTimeout, + DisableKeepAlives: true, + MaxResponseHeaderBytes: 64 << 10, + } + return &http.Client{ + Transport: tr, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > LinkMaxRedirects { + return linkRefused("the link redirected more than %d times", LinkMaxRedirects) + } + if err := checkLinkURL(req.URL); err != nil { + return err + } + // A redirect never carries credentials to another host. + req.Header.Del("Authorization") + req.Header.Del("Cookie") + return nil + }, + } +} + +// Fetch downloads raw, at most maxBytes of body. +func (f *LinkFetcher) Fetch(ctx context.Context, raw string, maxBytes int64) (FetchedLink, error) { + u, err := ValidateLinkURL(raw) + if err != nil { + return FetchedLink{}, err + } + ctx, cancel := context.WithTimeout(ctx, linkTotalTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return FetchedLink{}, linkRefused("the link is not a valid URL") + } + ua := f.UserAgent + if ua == "" { + ua = DefaultLinkUserAgent + } + req.Header.Set("User-Agent", ua) + req.Header.Set("Accept", linkAccept) + resp, err := f.client().Do(req) + if err != nil { + if errors.Is(err, ErrLinkRefused) { + return FetchedLink{}, unwrapURLError(err) + } + return FetchedLink{}, fmt.Errorf("fetching the link: %w", unwrapURLError(err)) + } + defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return FetchedLink{}, fmt.Errorf("the site answered %s", resp.Status) + } + if resp.ContentLength > maxBytes { + return FetchedLink{}, uploadErr(UploadErrTooLarge, "the linked file is too large (%s; the limit is %s)", humanBytes(resp.ContentLength), humanBytes(maxBytes)) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxBytes+1)) + if err != nil { + return FetchedLink{}, fmt.Errorf("reading the link: %w", err) + } + if int64(len(body)) > maxBytes { + return FetchedLink{}, uploadErr(UploadErrTooLarge, "the linked file is larger than the limit of %s", humanBytes(maxBytes)) + } + ct, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type")) + return FetchedLink{URL: resp.Request.URL, ContentType: strings.ToLower(ct), Filename: linkFilename(resp), Body: body}, nil +} + +// unwrapURLError drops the *url.Error wrapper so messages read as the cause +// (the URL is not repeated: it may carry a query the owner did not mean to log). +func unwrapURLError(err error) error { + var ue *url.Error + if errors.As(err, &ue) { + return ue.Err + } + return err +} + +// linkFilename is the Content-Disposition file name, else the final path +// segment when it has an extension. +func linkFilename(resp *http.Response) string { + if _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")); err == nil && params["filename"] != "" { + return CleanUploadName(params["filename"]) + } + base := path.Base(resp.Request.URL.Path) + if base == "." || base == "/" || path.Ext(base) == "" { + return "" + } + if unescaped, err := url.PathUnescape(base); err == nil { + base = unescaped + } + return CleanUploadName(base) +} + +// linkTypeExtensions maps a response media type to the extension the type +// pipeline reads it as. +var linkTypeExtensions = map[string]string{ + "text/html": ".html", + "application/xhtml+xml": ".html", + "text/markdown": ".md", + "text/x-markdown": ".md", + "text/plain": ".txt", + "application/json": ".json", + "application/ld+json": ".json", + "application/x-ndjson": ".jsonl", + "application/jsonl": ".jsonl", + "application/yaml": ".yaml", + "application/x-yaml": ".yaml", + "text/yaml": ".yaml", + "text/x-yaml": ".yaml", + "text/csv": ".csv", + "text/tab-separated-values": ".tsv", +} + +// linkPipelineName picks the file name the type pipeline sniffs a fetched +// body under: a file name whose extension the pipeline reads wins (a .md +// served as text/plain stays markdown), else the extension of the declared +// Content-Type; the content must still pass the sniff and the parse. +func linkPipelineName(l FetchedLink) string { + name := l.Filename + ext := strings.ToLower(path.Ext(name)) + if _, ok := uploadTextExtensions[ext]; ok { + return name + } + if typeExt, ok := linkTypeExtensions[l.ContentType]; ok { + base := strings.TrimSuffix(name, path.Ext(name)) + if base == "" { + base = "link" + } + return base + typeExt + } + if name != "" { + return name + } + return "link" +} diff --git a/src/pkg/inbox/link_fetch_test.go b/src/pkg/inbox/link_fetch_test.go new file mode 100644 index 0000000000..541752361a --- /dev/null +++ b/src/pkg/inbox/link_fetch_test.go @@ -0,0 +1,338 @@ +package inbox + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "net/url" + "strings" + "sync" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// fakePublicIP is what the fake resolver answers for public test hosts; the +// fake dialer routes it to the local test server, so the real SSRF rules run +// on a public-looking address. +const fakePublicIP = "93.184.216.34" + +type linkTestEnv struct { + srv *httptest.Server + fetcher *LinkFetcher + mu sync.Mutex + dialed []string + proxy *url.URL +} + +func newLinkTestEnv(t *testing.T, mux *http.ServeMux) *linkTestEnv { + t.Helper() + e := &linkTestEnv{srv: httptest.NewServer(mux)} + t.Cleanup(e.srv.Close) + answers := map[string][]string{ + "public.example": {fakePublicIP}, + "rebind.example": {"10.0.0.5"}, + "v6ula.example": {"fd00::1"}, + "mixed.example": {fakePublicIP, "127.0.0.1"}, + "meta.example": {"169.254.169.254"}, + "cgnat.example": {"100.64.1.1"}, + "0x7f000001": {"127.0.0.1"}, + } + e.fetcher = &LinkFetcher{ + UserAgent: "hive-inbox/test", + Resolve: func(_ context.Context, host string) ([]netip.Addr, error) { + list, ok := answers[host] + if !ok { + return nil, fmt.Errorf("no such host %s", host) + } + out := make([]netip.Addr, len(list)) + for i, a := range list { + out[i] = netip.MustParseAddr(a) + } + return out, nil + }, + Dial: func(ctx context.Context, network, addr string) (net.Conn, error) { + e.mu.Lock() + e.dialed = append(e.dialed, addr) + e.mu.Unlock() + host, _, _ := net.SplitHostPort(addr) + if e.proxy != nil && addr == e.proxy.Host { + return (&net.Dialer{}).DialContext(ctx, network, addr) + } + if host != fakePublicIP { + return nil, fmt.Errorf("test dialer: refusing %s", addr) + } + return (&net.Dialer{}).DialContext(ctx, network, e.srv.Listener.Addr().String()) + }, + Proxy: func(string) *url.URL { return e.proxy }, + } + return e +} + +func (e *linkTestEnv) url(path string) string { + _, port, _ := net.SplitHostPort(e.srv.Listener.Addr().String()) + return "http://public.example:" + port + path +} + +func linkMux(t *testing.T) *http.ServeMux { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/page", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = io.WriteString(w, `Launch notes +

    Plan

    Ship the drop zone first.

    +
    • docs
    • tests
    c
    `) + }) + mux.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"ua":"`+r.Header.Get("User-Agent")+`"}`) + }) + mux.HandleFunc("/files/data.csv", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + _, _ = io.WriteString(w, "a,b\n1,2\n") + }) + mux.HandleFunc("/bin", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write([]byte("MZ\x90\x00binary")) + }) + mux.HandleFunc("/audio", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "audio/mpeg") + _, _ = w.Write([]byte("ID3\x04")) + }) + mux.HandleFunc("/missing", http.NotFound) + mux.HandleFunc("/big", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + w.(http.Flusher).Flush() + _, _ = io.WriteString(w, strings.Repeat("x", 2048)) + }) + mux.HandleFunc("/to-private", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://rebind.example/secret", http.StatusFound) + }) + mux.HandleFunc("/to-metadata", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://169.254.169.254/latest/meta-data/", http.StatusFound) + }) + mux.HandleFunc("/to-creds", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "http://user:pw@public.example/json", http.StatusFound) + }) + mux.HandleFunc("/to-file", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "file:///etc/passwd", http.StatusFound) + }) + mux.HandleFunc("/hop/", func(w http.ResponseWriter, r *http.Request) { + var n int + _, _ = fmt.Sscanf(strings.TrimPrefix(r.URL.Path, "/hop/"), "%d", &n) + if n <= 0 { + http.Redirect(w, r, "/json", http.StatusFound) + return + } + http.Redirect(w, r, fmt.Sprintf("/hop/%d", n-1), http.StatusFound) + }) + return mux +} + +func TestLinkAddrAllowedTable(t *testing.T) { + blocked := []string{"127.0.0.1", "10.1.2.3", "172.16.0.1", "192.168.1.1", "169.254.169.254", "100.64.0.1", "100.100.100.200", + "0.0.0.0", "224.0.0.1", "239.255.255.250", "255.255.255.255", "192.0.0.192", "198.18.0.1", "::1", "::", "fd00:ec2::254", + "fc00::1", "fe80::1", "ff02::1", "::ffff:127.0.0.1", "::ffff:10.0.0.1", "64:ff9b::a00:1", "2002:a00:1::", "2001:db8::1", "2001::1"} + for _, a := range blocked { + if LinkAddrAllowed(netip.MustParseAddr(a)) { + t.Errorf("%s must be blocked", a) + } + } + for _, a := range []string{fakePublicIP, "1.1.1.1", "2606:4700:4700::1111", "64:ff9b::808:808", "2002:808:808::"} { + if !LinkAddrAllowed(netip.MustParseAddr(a)) { + t.Errorf("%s must be allowed", a) + } + } + if LinkAddrAllowed(netip.Addr{}) { + t.Error("the zero address must be blocked") + } +} + +func TestValidateLinkURLRefusals(t *testing.T) { + for _, raw := range []string{"", "ftp://public.example/x", "file:///etc/passwd", "javascript:alert(1)", "gopher://x", + "http://user:pw@public.example/", "https://token@public.example/", "http://127.0.0.1/", "http://[::1]:8080/", + "http://169.254.169.254/latest/meta-data/", "http://[fd00:ec2::254]/", "http://10.1.2.3/", "http://100.64.0.1/", + "http://224.0.0.1/", "http://localhost:8080/", "http://app.localhost/", "http://metadata.google.internal/", + "http://printer.local/", "http://[::ffff:127.0.0.1]/", "http:///nohost", "http://" + strings.Repeat("a", linkMaxURLBytes) + ".example/", "::not a url"} { + if _, err := ValidateLinkURL(raw); !errors.Is(err, ErrLinkRefused) { + t.Errorf("ValidateLinkURL(%q) = %v, want refused", raw, err) + } + } + if u, err := ValidateLinkURL(" https://public.example/a?b=c "); err != nil || u.Host != "public.example" { + t.Fatalf("public URL = %v, %v", u, err) + } +} + +func TestLinkFetchSSRF(t *testing.T) { + e := newLinkTestEnv(t, linkMux(t)) + ctx := context.Background() + _, port, _ := net.SplitHostPort(e.srv.Listener.Addr().String()) + refused := []string{ + "http://rebind.example:" + port + "/", // resolves private + "http://v6ula.example:" + port + "/", // resolves to IPv6 ULA + "http://mixed.example:" + port + "/", // one public, one loopback answer + "http://meta.example:" + port + "/", // resolves to metadata + "http://cgnat.example:" + port + "/", // CGNAT + "http://0x7f000001:" + port + "/", // a numeric name that resolves to loopback + e.url("/to-private"), // DNS-rebinding style redirect to a private IP + e.url("/to-metadata"), // redirect to the metadata literal + e.url("/to-creds"), // redirect adds credentials + e.url("/to-file"), // redirect to a non-http scheme + e.url("/hop/4"), // four redirects + } + for _, raw := range refused { + if _, err := e.fetcher.Fetch(ctx, raw, 1<<20); !errors.Is(err, ErrLinkRefused) { + t.Errorf("Fetch(%s) = %v, want refused", raw, err) + } + } + for _, d := range e.dialed { + if host, _, _ := net.SplitHostPort(d); host != fakePublicIP { + t.Fatalf("dialed a refused address: %s", d) + } + } + // Three redirects are fine; the user agent is sent. + fl, err := e.fetcher.Fetch(ctx, e.url("/hop/2"), 1<<20) + if err != nil || fl.ContentType != "application/json" || !strings.Contains(string(fl.Body), "hive-inbox/test") || fl.URL.Path != "/json" { + t.Fatalf("three hops = %+v, %v", fl, err) + } + // Oversize body, streamed without a length. + if _, err := e.fetcher.Fetch(ctx, e.url("/big"), 1024); err == nil || !strings.Contains(err.Error(), "larger than the limit") { + t.Fatalf("oversize = %v", err) + } + if _, err := e.fetcher.Fetch(ctx, e.url("/missing"), 1024); err == nil || errors.Is(err, ErrLinkRefused) || !strings.Contains(err.Error(), "404") { + t.Fatalf("404 = %v", err) + } + if _, err := e.fetcher.Fetch(ctx, "http://unknown.example/", 1024); err == nil || errors.Is(err, ErrLinkRefused) { + t.Fatalf("unknown host = %v", err) + } +} + +// TestLinkFetchThroughUpstreamProxy pins that a configured upstream proxy is +// asked to CONNECT to the checked IP, never to the name (which the proxy +// would resolve again). +func TestLinkFetchThroughUpstreamProxy(t *testing.T) { + e := newLinkTestEnv(t, linkMux(t)) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + targets := make(chan string, 4) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func(c net.Conn) { + defer c.Close() + br := bufio.NewReader(c) + req, err := http.ReadRequest(br) + if err != nil { + return + } + targets <- req.Host + up, err := net.Dial("tcp", e.srv.Listener.Addr().String()) + if err != nil { + return + } + defer up.Close() + _, _ = io.WriteString(c, "HTTP/1.1 200 Connection established\r\n\r\n") + go func() { _, _ = io.Copy(up, br) }() + _, _ = io.Copy(c, up) + }(c) + } + }() + e.proxy = &url.URL{Scheme: "http", Host: ln.Addr().String()} + fl, err := e.fetcher.Fetch(context.Background(), e.url("/json"), 1<<20) + if err != nil || fl.ContentType != "application/json" { + t.Fatalf("via proxy = %+v, %v", fl, err) + } + if got := <-targets; !strings.HasPrefix(got, fakePublicIP+":") { + t.Fatalf("CONNECT target = %q, want the checked IP", got) + } +} + +func TestAddLinkRunsTheTypePipeline(t *testing.T) { + e := newLinkTestEnv(t, linkMux(t)) + env := newUploadEnv(t, config.InboxConfig{Uploads: config.InboxUploads{MaxFileBytes: 1024}}) + env.src.Fetcher = e.fetcher + ctx := context.Background() + + it, err := env.svc.AddLink(ctx, LinkInput{URL: e.url("/page"), AddedBy: "ana"}) + if err != nil { + t.Fatal(err) + } + if it.Title != "Launch notes" || it.UploadKind != UploadKindHTML || !strings.HasPrefix(it.Origin, "link · public.example") || it.AddedBy != "ana" { + t.Fatalf("page item = %+v", it) + } + c, err := env.svc.Content(ctx, it) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"## Plan", "Ship the **drop zone** first.", "- docs\n- tests"} { + if !strings.Contains(c.Markdown, want) { + t.Errorf("page markdown missing %q:\n%s", want, c.Markdown) + } + } + for _, gone := range []string{"alert(1)", "evil", "Home", "p{}"} { + if strings.Contains(c.Markdown, gone) { + t.Errorf("page markdown kept %q:\n%s", gone, c.Markdown) + } + } + if c.Provenance != e.url("/page") { + t.Fatalf("provenance = %q", c.Provenance) + } + + // A linked CSV served as text/plain is read by its file name. + it, err = env.svc.AddLink(ctx, LinkInput{URL: e.url("/files/data.csv")}) + if err != nil || it.UploadKind != UploadKindCSV || it.Title != "data.csv" { + t.Fatalf("csv link = %+v, %v", it, err) + } + code := func(err error) string { + var ue *UploadError + if errors.As(err, &ue) { + return ue.Code + } + return "?" + } + for path, want := range map[string]string{"/bin": UploadErrUnsupported, "/audio": UploadErrUnsupported, "/big": UploadErrTooLarge, "/missing": UploadErrFetch, "/to-private": UploadErrLinkRefused} { + if _, err := env.svc.AddLink(ctx, LinkInput{URL: e.url(path)}); code(err) != want { + t.Errorf("%s = %v, want %s", path, err, want) + } + } + if _, err := env.svc.AddLink(ctx, LinkInput{URL: "http://user:pw@public.example/"}); code(err) != UploadErrLinkRefused { + t.Fatalf("credentials = %v", err) + } + env.src.Fetcher = nil + if _, err := env.svc.AddLink(ctx, LinkInput{URL: e.url("/page")}); code(err) != UploadErrLinkRefused { + t.Fatalf("no fetcher = %v", err) + } + if env.svc.Settings().Uploads.Links { + t.Fatal("links must be off without a fetcher") + } +} + +func TestLinkPipelineName(t *testing.T) { + cases := []struct { + fl FetchedLink + want string + }{ + {FetchedLink{Filename: "notes.md", ContentType: "text/plain"}, "notes.md"}, + {FetchedLink{Filename: "", ContentType: "text/html"}, "link.html"}, + {FetchedLink{Filename: "report.php", ContentType: "application/json"}, "report.json"}, + {FetchedLink{Filename: "x.bin", ContentType: "application/octet-stream"}, "x.bin"}, + {FetchedLink{}, "link"}, + } + for _, tc := range cases { + if got := linkPipelineName(tc.fl); got != tc.want { + t.Errorf("linkPipelineName(%+v) = %q, want %q", tc.fl, got, tc.want) + } + } +} diff --git a/src/pkg/inbox/service.go b/src/pkg/inbox/service.go index 2f7e10c864..cc61b28e39 100644 --- a/src/pkg/inbox/service.go +++ b/src/pkg/inbox/service.go @@ -171,6 +171,19 @@ func (s *Service) AddUpload(ctx context.Context, in UploadInput) (Item, error) { return u.item(m), nil } +// AddLink fetches one URL and returns its Inbox item. +func (s *Service) AddLink(ctx context.Context, in LinkInput) (Item, error) { + u := s.Uploads() + if u == nil { + return Item{}, ErrUploadsDisabled + } + m, err := u.AddLink(ctx, in) + if err != nil { + return Item{}, err + } + return u.item(m), nil +} + // ErrUploadsDisabled is returned when the hive has no upload store. var ErrUploadsDisabled = errors.New("uploads are not available on this hive") diff --git a/src/pkg/inbox/upload_extract.go b/src/pkg/inbox/upload_extract.go index 6beafe0c30..ddc67d939f 100644 --- a/src/pkg/inbox/upload_extract.go +++ b/src/pkg/inbox/upload_extract.go @@ -98,6 +98,8 @@ func extractUpload(ctx context.Context, kind string, data []byte) (Extracted, *U return extractTable(ctx, text, ',', "CSV") case UploadKindTSV: return extractTable(ctx, text, '\t', "TSV") + case UploadKindHTML: + return extractHTML(ctx, text) } return Extracted{}, uploadErr(UploadErrUnsupported, "unsupported file type %q", kind) } diff --git a/src/pkg/inbox/upload_html.go b/src/pkg/inbox/upload_html.go new file mode 100644 index 0000000000..ee54683340 --- /dev/null +++ b/src/pkg/inbox/upload_html.go @@ -0,0 +1,351 @@ +package inbox + +import ( + "context" + "strconv" + "strings" + + "golang.org/x/net/html" + "golang.org/x/net/html/atom" +) + +// HTML extraction limits. The page is parsed with golang.org/x/net/html (a +// spec-compliant tokenizer; nothing is executed, no resource is loaded) and +// reduced to readable markdown: scripts, styles, forms and page chrome are +// dropped; headings, paragraphs, lists, quotes, code and tables are kept. +const ( + // htmlMaxDepth stops descending into absurdly nested markup. + htmlMaxDepth = 256 + // htmlMaxNodes bounds the nodes one page may render. + htmlMaxNodes = 200000 + // htmlHeadingLevels is the deepest markdown heading. + htmlHeadingLevels = 6 +) + +// htmlSkipped are elements whose content never reaches the text. +var htmlSkipped = map[atom.Atom]bool{ + atom.Script: true, atom.Style: true, atom.Noscript: true, atom.Template: true, + atom.Iframe: true, atom.Object: true, atom.Embed: true, atom.Svg: true, + atom.Canvas: true, atom.Form: true, atom.Button: true, atom.Input: true, + atom.Select: true, atom.Textarea: true, atom.Nav: true, atom.Footer: true, + atom.Aside: true, atom.Head: true, atom.Dialog: true, atom.Audio: true, + atom.Video: true, atom.Picture: true, atom.Source: true, atom.Track: true, +} + +// htmlBlocks are rendered as their own paragraph. +var htmlBlocks = map[atom.Atom]bool{ + atom.P: true, atom.Div: true, atom.Section: true, atom.Article: true, + atom.Main: true, atom.Body: true, atom.Header: true, atom.Figure: true, + atom.Figcaption: true, atom.Dl: true, atom.Dt: true, atom.Dd: true, + atom.Address: true, atom.Details: true, atom.Summary: true, atom.Center: true, +} + +type htmlConv struct { + ctx context.Context + nodes int +} + +// extractHTML turns an HTML page into markdown with its . +func extractHTML(ctx context.Context, text string) (Extracted, *UploadError) { + doc, err := html.Parse(strings.NewReader(text)) + if err != nil { + return Extracted{}, uploadErr(UploadErrParse, "the page is not readable HTML (%s)", err) + } + title := truncateRunes(strings.Join(strings.Fields(htmlText(htmlFind(doc, atom.Title))), " "), uploadTitleRunes) + root := htmlFind(doc, atom.Article) + if root == nil { + root = htmlFind(doc, atom.Main) + } + if root == nil { + root = htmlFind(doc, atom.Body) + } + if root == nil { + root = doc + } + c := &htmlConv{ctx: ctx} + md := c.children(root, 0) + if ctx.Err() != nil { + return Extracted{}, uploadErr(UploadErrParse, "reading the page took too long") + } + md = tidyMarkdown(md) + if md == "" { + return Extracted{}, uploadErr(UploadErrEmpty, "the page has no readable text") + } + if title == "" { + title = htmlFirstHeading(root) + } + return Extracted{Markdown: md, Title: title}, nil +} + +// htmlFind returns the first element of type a, depth first. +func htmlFind(n *html.Node, a atom.Atom) *html.Node { + if n == nil { + return nil + } + if n.Type == html.ElementNode && n.DataAtom == a { + return n + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if f := htmlFind(c, a); f != nil { + return f + } + } + return nil +} + +// htmlText is the raw text content of n. +func htmlText(n *html.Node) string { + if n == nil { + return "" + } + var b strings.Builder + var walk func(*html.Node, int) + walk = func(n *html.Node, depth int) { + if depth > htmlMaxDepth { + return + } + if n.Type == html.TextNode { + b.WriteString(n.Data) + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + walk(c, depth+1) + } + } + walk(n, 0) + return b.String() +} + +func htmlHasAttr(n *html.Node, key string) bool { + for _, a := range n.Attr { + if a.Key == key { + return true + } + } + return false +} + +// htmlFirstHeading is the text of the first h1, h2 or h3 under n. +func htmlFirstHeading(n *html.Node) string { + for _, a := range []atom.Atom{atom.H1, atom.H2, atom.H3} { + if h := htmlFind(n, a); h != nil { + if t := oneLine(htmlText(h)); t != "" { + return truncateRunes(t, uploadTitleRunes) + } + } + } + return "" +} + +func htmlAttr(n *html.Node, key string) string { + for _, a := range n.Attr { + if a.Key == key { + return a.Val + } + } + return "" +} + +func (c *htmlConv) children(n *html.Node, depth int) string { + var b strings.Builder + for ch := n.FirstChild; ch != nil; ch = ch.NextSibling { + b.WriteString(c.node(ch, depth+1)) + } + return b.String() +} + +func (c *htmlConv) node(n *html.Node, depth int) string { + c.nodes++ + if depth > htmlMaxDepth || c.nodes > htmlMaxNodes || (c.nodes%1024 == 0 && c.ctx.Err() != nil) { + return "" + } + switch n.Type { + case html.TextNode: + return collapseSpace(n.Data) + case html.ElementNode: + default: + return c.children(n, depth) + } + if htmlSkipped[n.DataAtom] || htmlHasAttr(n, "hidden") || strings.EqualFold(htmlAttr(n, "aria-hidden"), "true") { + return "" + } + switch n.DataAtom { + case atom.H1, atom.H2, atom.H3, atom.H4, atom.H5, atom.H6: + level := int(n.Data[1] - '0') + text := oneLine(c.children(n, depth)) + if text == "" { + return "" + } + return "\n\n" + strings.Repeat("#", min(level, htmlHeadingLevels)) + " " + text + "\n\n" + case atom.Br: + return "\n" + case atom.Hr: + return "\n\n---\n\n" + case atom.Pre: + return "\n\n" + fenced("", strings.Trim(htmlText(n), "\n")) + "\n\n" + case atom.Blockquote: + inner := tidyMarkdown(c.children(n, depth)) + if inner == "" { + return "" + } + return "\n\n> " + strings.ReplaceAll(inner, "\n", "\n> ") + "\n\n" + case atom.Ul, atom.Ol: + return c.list(n, depth) + case atom.Table: + return c.table(n, depth) + case atom.Strong, atom.B: + return wrapInline("**", c.children(n, depth)) + case atom.Em, atom.I: + return wrapInline("_", c.children(n, depth)) + case atom.Code: + if t := strings.TrimSpace(htmlText(n)); t != "" { + return "`" + strings.ReplaceAll(t, "`", "'") + "`" + } + return "" + case atom.A: + text := c.children(n, depth) + href := htmlAttr(n, "href") + if t := strings.TrimSpace(text); t != "" && (strings.HasPrefix(href, "https://") || strings.HasPrefix(href, "http://")) { + return "[" + oneLine(t) + "](" + strings.ReplaceAll(href, ")", "%29") + ")" + } + return text + case atom.Img: + if alt := strings.TrimSpace(htmlAttr(n, "alt")); alt != "" { + return "[image: " + oneLine(alt) + "]" + } + return "" + } + inner := c.children(n, depth) + if htmlBlocks[n.DataAtom] || n.DataAtom == atom.Li { + return "\n\n" + inner + "\n\n" + } + return inner +} + +func (c *htmlConv) list(n *html.Node, depth int) string { + var b strings.Builder + num := 0 + for li := n.FirstChild; li != nil; li = li.NextSibling { + if li.Type != html.ElementNode || li.DataAtom != atom.Li { + continue + } + num++ + item := tidyMarkdown(c.children(li, depth+1)) + // An item is one block: its paragraphs and sub-lists stay together. + for strings.Contains(item, "\n\n") { + item = strings.ReplaceAll(item, "\n\n", "\n") + } + if item == "" { + continue + } + marker := "- " + if n.DataAtom == atom.Ol { + marker = strconv.Itoa(num) + ". " + } + b.WriteString(marker + strings.ReplaceAll(item, "\n", "\n"+strings.Repeat(" ", len(marker))) + "\n") + } + if b.Len() == 0 { + return "" + } + return "\n\n" + b.String() + "\n" +} + +func (c *htmlConv) table(n *html.Node, depth int) string { + var rows [][]string + var walk func(*html.Node, int) + walk = func(x *html.Node, d int) { + if d > htmlMaxDepth { + return + } + for ch := x.FirstChild; ch != nil; ch = ch.NextSibling { + if ch.Type != html.ElementNode { + continue + } + if ch.DataAtom == atom.Tr { + var row []string + for cell := ch.FirstChild; cell != nil; cell = cell.NextSibling { + if cell.Type == html.ElementNode && (cell.DataAtom == atom.Td || cell.DataAtom == atom.Th) { + row = append(row, oneLine(c.children(cell, d+1))) + } + } + if len(row) > 0 { + rows = append(rows, row) + } + continue + } + if ch.DataAtom != atom.Table { + walk(ch, d+1) + } + } + } + walk(n, depth) + if len(rows) == 0 { + return "" + } + cols := 0 + for _, r := range rows { + cols = max(cols, len(r)) + } + if len(rows) > uploadTablePreviewRows { + rows = rows[:uploadTablePreviewRows] + } + return "\n\n" + markdownTable(rows, cols) + "\n\n" +} + +// wrapInline puts marks around trimmed inline text, keeping outer spaces. +func wrapInline(mark, s string) string { + t := strings.TrimSpace(s) + if t == "" { + return s + } + lead, trail := "", "" + if strings.HasPrefix(s, " ") { + lead = " " + } + if strings.HasSuffix(s, " ") { + trail = " " + } + return lead + mark + t + mark + trail +} + +// collapseSpace folds runs of whitespace into one space. +func collapseSpace(s string) string { + var b strings.Builder + space := false + for _, r := range s { + if r == ' ' || r == '\n' || r == '\t' || r == '\r' || r == '\f' { + if !space { + b.WriteByte(' ') + } + space = true + continue + } + space = false + b.WriteRune(r) + } + return b.String() +} + +func oneLine(s string) string { return strings.Join(strings.Fields(s), " ") } + +// tidyMarkdown trims every line and squeezes blank-line runs. +func tidyMarkdown(s string) string { + lines := strings.Split(s, "\n") + inFence := false + for i, l := range lines { + if strings.HasPrefix(strings.TrimSpace(l), "```") { + inFence = !inFence + lines[i] = strings.TrimSpace(l) + continue + } + if !inFence { + l = strings.TrimRight(l, " ") + // One leading space is a whitespace artefact; two or more + // indent a list continuation. + if strings.HasPrefix(l, " ") && !strings.HasPrefix(l, " ") { + l = l[1:] + } + lines[i] = l + } + } + return collapseBlankLines(strings.Join(lines, "\n")) +} diff --git a/src/pkg/inbox/upload_html_test.go b/src/pkg/inbox/upload_html_test.go new file mode 100644 index 0000000000..71a394d524 --- /dev/null +++ b/src/pkg/inbox/upload_html_test.go @@ -0,0 +1,69 @@ +package inbox + +import ( + "context" + "strings" + "testing" +) + +func TestExtractHTMLReadableMarkdown(t *testing.T) { + page := `<html><head><title> The Title +
    Site header
    +
    +

    Main heading

    +

    Read the doc and relative, now.

    + +
    1. first
    2. second x()
    +

    quoted line

    +
      code
    +    indented
    +
    kv
    a|b2
    +diagram
    +
    form text
    +
    Footer
    ` + ex, err := ExtractUpload(context.Background(), UploadKindHTML, []byte(page)) + if err != nil { + t.Fatal(err) + } + if ex.Title != "The Title" { + t.Fatalf("title = %q", ex.Title) + } + for _, want := range []string{"# Main heading", "Read [the doc](https://example.org/doc) and relative, _now_.", "1. first\n2. second `x()`", + "> quoted line", "```\n code\n indented\n```", "| k | v |", `| a\|b | 2 |`, "[image: diagram]", "---"} { + if !strings.Contains(ex.Markdown, want) { + t.Errorf("markdown missing %q:\n%s", want, ex.Markdown) + } + } + for _, gone := range []string{"steal", "Menu", "Footer", "form text", "secret", ".x{}", "Site header"} { + if strings.Contains(ex.Markdown, gone) { + t.Errorf("markdown kept %q:\n%s", gone, ex.Markdown) + } + } +} + +func TestExtractHTMLEdgeCases(t *testing.T) { + // No : the first heading names it; body without main/article. + ex, err := ExtractUpload(context.Background(), UploadKindHTML, []byte("<body><h2>Only heading</h2><ul><li><p>nested</p><ul><li>deep</li></ul></li></ul></body>")) + if err != nil || ex.Title != "Only heading" || !strings.Contains(ex.Markdown, "- nested\n - deep") { + t.Fatalf("no title = %+v, %v", ex, err) + } + if _, err := ExtractUpload(context.Background(), UploadKindHTML, []byte("<html><script>x</script></html>")); err == nil || err.Code != UploadErrEmpty { + t.Fatalf("empty page = %v", err) + } + // Absurd nesting is cut off instead of recursing without bound. + deep := strings.Repeat("<div>", htmlMaxDepth*4) + "bottom" + strings.Repeat("</div>", htmlMaxDepth*4) + // The parser itself refuses absurd nesting; either way nothing recurses + // without bound. + if _, err := ExtractUpload(context.Background(), UploadKindHTML, []byte("<p>top</p>"+deep)); err != nil && err.Code != UploadErrParse { + t.Fatalf("deep nesting = %v", err) + } + if ex, err := ExtractUpload(context.Background(), UploadKindHTML, []byte("<p>top</p>"+strings.Repeat("<span>", htmlMaxDepth+20)+"bottom")); err != nil || !strings.Contains(ex.Markdown, "top") || strings.Contains(ex.Markdown, "bottom") { + t.Fatalf("depth cap = %+v, %v", ex, err) + } + if got := wrapInline("**", " "); got != " " { + t.Fatalf("empty inline = %q", got) + } + if k, err := SniffUpload("page.htm", []byte("<p>hi</p>")); err != nil || k != UploadKindHTML { + t.Fatalf("sniff .htm = %q, %v", k, err) + } +} diff --git a/src/pkg/inbox/upload_source.go b/src/pkg/inbox/upload_source.go index e127bf4764..67a8e76931 100644 --- a/src/pkg/inbox/upload_source.go +++ b/src/pkg/inbox/upload_source.go @@ -64,8 +64,10 @@ type UploadSource struct { // FactWritten runs after a fact was written (the hive reindexes the // vault so knowledge search sees it at once). FactWritten func(dir string) - Logger *slog.Logger - Now func() time.Time + // Fetcher fetches links; nil refuses links. + Fetcher *LinkFetcher + Logger *slog.Logger + Now func() time.Time factMu sync.Mutex } @@ -239,6 +241,70 @@ func (u *UploadSource) Add(ctx context.Context, in UploadInput) (UploadMeta, err return u.store(m, in.Data, ex.Markdown) } +// LinkInput is one dropped or pasted URL. +type LinkInput struct { + URL string + AddedBy string +} + +// AddLink fetches a URL through the SSRF-guarded fetcher and runs the body +// through the same type pipeline as a dropped file: an HTML page becomes +// text, a linked JSON or CSV file becomes that kind. The URL is kept as the +// item's provenance only. +func (u *UploadSource) AddLink(ctx context.Context, in LinkInput) (UploadMeta, error) { + if u.Fetcher == nil { + return UploadMeta{}, uploadErr(UploadErrLinkRefused, "links cannot be added on this hive") + } + src, err := ValidateLinkURL(in.URL) + if err != nil { + return UploadMeta{}, uploadErr(UploadErrLinkRefused, "%s", err.Error()) + } + lim := u.limits() + fl, err := u.Fetcher.Fetch(ctx, src.String(), lim.EffectiveMaxFileBytes()) + if err != nil { + var ue *UploadError + switch { + case errors.As(err, &ue): + return UploadMeta{}, ue + case errors.Is(err, ErrLinkRefused): + return UploadMeta{}, uploadErr(UploadErrLinkRefused, "%s", err.Error()) + } + return UploadMeta{}, uploadErr(UploadErrFetch, "%s", err.Error()) + } + if strings.HasPrefix(fl.ContentType, "audio/") || strings.HasPrefix(fl.ContentType, "video/") { + return UploadMeta{}, uploadErr(UploadErrUnsupported, "audio and video links are not supported yet: download the file and drop it instead") + } + name := linkPipelineName(fl) + kind, uerr := SniffUpload(name, fl.Body) + if uerr != nil { + return UploadMeta{}, uerr + } + ex, uerr := ExtractUpload(ctx, kind, fl.Body) + if uerr != nil { + return UploadMeta{}, uerr + } + id, err := u.Store.NewID() + if err != nil { + return UploadMeta{}, err + } + sum := sha256.Sum256(fl.Body) + m := UploadMeta{ + ID: id, + Kind: kind, + Origin: UploadOriginLink, + Filename: fl.Filename, + URL: src.String(), + Host: strings.ToLower(src.Hostname()), + Title: firstNonEmpty(ex.Title, fl.Filename, src.String()), + Summary: ex.Summary, + Size: int64(len(fl.Body)), + SHA256: hex.EncodeToString(sum[:]), + AddedBy: firstNonEmpty(in.AddedBy, "local"), + AddedAt: u.now(), + } + return u.store(m, fl.Body, ex.Markdown) +} + // store persists an upload under quota and writes its fact. func (u *UploadSource) store(m UploadMeta, original []byte, markdown string) (UploadMeta, error) { if err := u.Store.Put(m, original, markdown, u.limits().EffectiveQuotaBytes()); err != nil { @@ -333,6 +399,8 @@ type UploadSettings struct { MediaRetentionDays int `json:"media_retention_days"` DocumentRetentionDays int `json:"document_retention_days"` Extensions []string `json:"extensions"` + // Links reports that URLs can be added (the hive has a link fetcher). + Links bool `json:"links"` } // Settings returns the effective limits and current usage. @@ -348,6 +416,7 @@ func (u *UploadSource) Settings() UploadSettings { MediaRetentionDays: lim.EffectiveMediaRetentionDays(), DocumentRetentionDays: lim.EffectiveDocumentRetentionDays(), Extensions: UploadSupportedExtensions(), + Links: u.Fetcher != nil, } } diff --git a/src/pkg/inbox/upload_types.go b/src/pkg/inbox/upload_types.go index 901dac1b95..99e883796d 100644 --- a/src/pkg/inbox/upload_types.go +++ b/src/pkg/inbox/upload_types.go @@ -23,6 +23,7 @@ const ( UploadKindYAML = "yaml" UploadKindCSV = "csv" UploadKindTSV = "tsv" + UploadKindHTML = "html" // UploadKindAudio and UploadKindVideo are media uploads (their original // binary is deleted after inbox.uploads.media_retention_days). UploadKindAudio = "audio" @@ -38,6 +39,11 @@ const ( UploadErrQuota = "quota_exceeded" UploadErrTooMany = "too_many_files" UploadErrInternal = "internal" + // UploadErrLinkRefused: the link breaks the fetch rules (scheme, + // credentials, a private or metadata address, too many redirects). + UploadErrLinkRefused = "link_refused" + // UploadErrFetch: the site could not be reached or answered an error. + UploadErrFetch = "fetch_failed" ) // UploadError is a per-file rejection with a message the owner can act on. @@ -68,6 +74,8 @@ var uploadTextExtensions = map[string]string{ ".csv": UploadKindCSV, ".tsv": UploadKindTSV, ".tab": UploadKindTSV, + ".html": UploadKindHTML, + ".htm": UploadKindHTML, } // uploadUnsupportedExtensions names formats people commonly drop that this @@ -94,7 +102,7 @@ func UploadSupportedExtensions() []string { } func uploadSupportedList() string { - return "md, txt, json, jsonl, yaml, csv, tsv" + return "md, txt, json, jsonl, yaml, csv, tsv, html" } // uploadMagic is a binary signature that is never a text document.