From 60a75b8774a66ea661ab1ecbcb1ad758ef8b0b7d Mon Sep 17 00:00:00 2001
From: Jay Baywatch <jay.baywatch@gmail.com>
Date: Thu, 29 Oct 2026 20:23:56 -0400
Subject: [PATCH 1/5] Add pluggable fetchers with headless-browser support for
 SPAs

Introduce a Fetcher abstraction so the transport step of a crawl can be
swapped without touching robots.txt handling, rate limiting, retries,
caching, or link parsing:

- HttpFetcher (default) reproduces the existing aiohttp behaviour.
- BrowserFetcher renders pages in headless Chromium via Playwright
  (optional `browser` extra; imported lazily).
- RoutingFetcher fetches over HTTP first and escalates to the browser
  when the response looks like an un-rendered SPA shell, or when the URL
  matches render_patterns.

Spider and Crawler accept fetcher= and keep_hash_routes= (preserve
"#/..." fragments for hash-router apps). Document.fetcher records which
transport produced a page and is exported by the serializers. Cache
entries for non-default fetchers are namespaced so legacy caches keep
working.

Memory: Spider(flush_to="crawl.jsonl") streams each page to disk as it is
crawled, and retain_documents controls whether Documents stay in memory.
Streaming callbacks now release documents as the docs always claimed.

Also close three documentation/code gaps: add Spider.run(), retry 5xx
responses with backoff, and populate Document.broken_internal_links
after the crawl. Docs updated throughout; 53 new tests (175 total), with
browser tests running against a local page and skipping when Playwright
is absent.
---
 README.md                                    |  84 +++-
 docs/api-reference.md                        | 199 ++++++++-
 docs/core-concepts.md                        | 163 ++++++-
 docs/examples.md                             | 176 +++++++-
 docs/getting-started.md                      |  45 ++
 docs/index.md                                |   2 +
 docs/troubleshooting.md                      | 132 +++++-
 linktrace/Crawler.py                         | 184 +++++---
 linktrace/Serializers.py                     |  48 +-
 linktrace/Spider.py                          | 101 ++++-
 linktrace/__init__.py                        |  20 +
 linktrace/cache.py                           |  46 +-
 linktrace/fetchers/__init__.py               |  24 +
 linktrace/fetchers/base.py                   | 113 +++++
 linktrace/fetchers/browser.py                | 217 +++++++++
 linktrace/fetchers/http.py                   |  64 +++
 linktrace/fetchers/routing.py                | 116 +++++
 pyproject.toml                               |   4 +
 tests/conftest.py                            |  44 ++
 tests/test_browser_fetcher.py                | 111 +++++
 tests/test_fetchers.py                       | 447 +++++++++++++++++++
 tests/test_rate_limiting_and_broken_links.py |   7 +-
 tests/test_spider_memory.py                  | 154 +++++++
 uv.lock                                      | 106 ++++-
 24 files changed, 2443 insertions(+), 164 deletions(-)
 create mode 100644 linktrace/fetchers/__init__.py
 create mode 100644 linktrace/fetchers/base.py
 create mode 100644 linktrace/fetchers/browser.py
 create mode 100644 linktrace/fetchers/http.py
 create mode 100644 linktrace/fetchers/routing.py
 create mode 100644 tests/test_browser_fetcher.py
 create mode 100644 tests/test_fetchers.py
 create mode 100644 tests/test_spider_memory.py

diff --git a/README.md b/README.md
index 5c53fc3..5764e0a 100644
--- a/README.md
+++ b/README.md
@@ -20,6 +20,7 @@ LinkTrace is a document-oriented crawler. Every crawled page becomes a rich Docu
 - 🎯 **URL filtering rules** — Include/exclude by regex, path prefix, extension, query param, or domain
 - 🎚️ **Configurable concurrency** — Tune `max_concurrency` and connection-pool limits for throughput
 - 🗺️ **Sitemap discovery** — Seed from `sitemap.xml` / robots.txt `Sitemap:`, including nested indexes
+- 🧭 **Pluggable fetchers** — Plain HTTP by default; optional headless-browser rendering (Playwright) for JavaScript SPAs, or route per URL with automatic escalation
 - 📄 **Rich document model** — Full HTML source, parsed links, metadata, headers
 - 🔄 **Persistent sessions** — Connection pooling for 10-100x faster same-domain crawls
 - 🔁 **Retries + backoff** — Exponential backoff for transient errors (timeouts, 5xx)
@@ -32,6 +33,7 @@ LinkTrace is a document-oriented crawler. Every crawled page becomes a rich Docu
 - 🔀 **Traversal strategies** — BFS (broad) or DFS (deep) crawling
 - 📊 **Multi-format export** — JSON, Pandas, Polars, PyArrow for data analysis
 - 📍 **Callbacks & streaming** — Process results as crawled without memory buildup
+- 🗂️ **Flush to disk** — `flush_to="crawl.jsonl"` streams every page to a JSONL file for crawls too big for RAM
 
 ## Quick Start
 
@@ -63,6 +65,12 @@ pip install linktrace[serializers]  # pandas + polars + pyarrow
 pip install linktrace[pandas]       # Just pandas
 ```
 
+**Optional JavaScript rendering (SPAs):**
+```bash
+pip install linktrace[browser]      # Playwright
+playwright install chromium         # one-time browser download
+```
+
 ## Core Concepts
 
 ### Spider
@@ -86,6 +94,9 @@ Declarative include/exclude policy for which discovered URLs the Spider follows
 ### SitemapParser
 Fetches and parses XML sitemaps (urlsets and nested indexes). Used when `Spider(use_sitemaps=True)`, or directly for sitemap inspection.
 
+### Fetchers
+The pluggable "get me the page" step. `HttpFetcher` (default) uses aiohttp; `BrowserFetcher` renders pages in headless Chromium; `RoutingFetcher` tries HTTP first and escalates to the browser only for pages that look like un-rendered SPA shells. Passed as `Spider(fetcher=...)`.
+
 See [Core Concepts](https://github.com/JayBaywatch/linktrace/blob/main/docs/core-concepts.md) for more.
 
 ## Configuration
@@ -183,6 +194,34 @@ Sitemap discovery is best-effort — if none exists, ordinary link-following sti
 runs. You can also inspect sitemaps directly via `SitemapParser` or
 `Crawler.discover_sitemap_urls(base_url)`.
 
+### JavaScript-Rendered Sites (SPAs)
+
+Plain HTTP only sees the empty shell of a React/Vue/Angular app. Swap the
+fetcher to render pages in a headless browser (requires `linktrace[browser]`):
+
+```python
+from linktrace import Spider, BrowserFetcher, RoutingFetcher
+
+# Render everything in headless Chromium
+spider = Spider(
+    start_url="https://app.example.com/",
+    fetcher=BrowserFetcher(wait_until="networkidle", max_pages=4),
+)
+
+# Cheaper: plain HTTP first, browser only for pages that look like SPA shells
+spider = Spider(
+    start_url="https://app.example.com/",
+    fetcher=RoutingFetcher(),
+)
+documents = await spider.run_async()
+print({doc.url: doc.fetcher for doc in documents})   # "http" or "browser"
+```
+
+`RoutingFetcher` also takes `render_patterns=[r"/search"]` to send matching
+URLs straight to the browser. Robots.txt, rate limiting, retries and caching
+apply identically whichever fetcher is used. For hash-router apps
+(`/#/listings/42`) add `keep_hash_routes=True` so each route is its own URL.
+
 ### SSL & Corporate Proxies
 
 ```python
@@ -243,9 +282,27 @@ results = await spider.run_async()  # Returns list of URLs
 
 **Return Logic:**
 - No callback → returns all documents (default)
-- Callback + `accumulate_results=False` → returns [] (streaming mode)
+- Callback + `accumulate_results=False` → returns [] (streaming mode; documents are not retained)
 - Callback + `accumulate_results=True` → returns callback results
 
+### Large Crawls: Flush to Disk
+
+For crawls that won't fit in memory, stream each page to a JSONL file as soon
+as it's crawled. Nothing is retained in RAM beyond the URL bookkeeping:
+
+```python
+spider = Spider(
+    start_url="https://example.com",
+    max_depth=5,
+    flush_to="crawl.jsonl",   # one JSON object per line, same schema as to_json()
+    flush_html=False,         # True to include page source
+)
+await spider.run_async()      # returns []
+```
+
+Streaming and flushing default to `retain_documents=False`; pass
+`retain_documents=True` to keep the `Document` list as well.
+
 ### Traversal Strategies
 
 **BFS (Breadth-First) — Default**
@@ -305,6 +362,11 @@ for doc in error_pages:
 # Monitor disallowed pages (403 from robots.txt)
 disallowed = [doc for doc in documents if doc.status_code == 403]
 print(f"Disallowed by robots.txt: {len(disallowed)} pages")
+
+# Which pages link to broken internal pages (populated after the crawl)
+for doc in documents:
+    for broken in doc.broken_internal_links:
+        print(f"{doc.url} -> {broken.url} (HTTP {broken.status_code})")
 ```
 
 Stream crawl status in real-time:
@@ -388,6 +450,9 @@ requests, and consider adding a `request_delay`. Defaults are conservative.
 ### "Crawler hits timeout on deep sites"
 Try DFS traversal instead of BFS, or increase `request_timeout`.
 
+### "No links found on a JavaScript app"
+Use `fetcher=RoutingFetcher()` or `fetcher=BrowserFetcher()` (requires `pip install linktrace[browser]` and `playwright install chromium`).
+
 See [Troubleshooting](https://github.com/JayBaywatch/linktrace/blob/main/docs/troubleshooting.md) for more.
 
 ## Performance
@@ -403,14 +468,17 @@ With persistent sessions + connection pooling, same-domain requests are 10-100x
 
 ```
 Spider (orchestrator)
-  └─ Crawler (persistent session)
-      ├─ aiohttp (HTTP requests + connection pooling)
+  └─ Crawler (robots.txt, rate limiting, retries, cache, parsing)
+      ├─ Fetcher (pluggable transport)
+      │   ├─ HttpFetcher     — aiohttp (default)
+      │   ├─ BrowserFetcher  — Playwright / headless Chromium
+      │   └─ RoutingFetcher  — HTTP first, browser on escalation
+      ├─ aiohttp session (connection pooling, cookies, SSL; also robots + sitemaps)
       ├─ lxml (HTML parsing)
-      ├─ ResponseCache (optional disk caching)
-      └─ CookieJar (automatic cookie handling)
+      └─ ResponseCache (optional disk caching)
 ```
 
-Spider manages the crawl queue and traversal. Crawler handles individual document fetching/parsing. All requests share one persistent aiohttp session per Spider instance, so connection pooling, cookies, SSL configuration, and DNS caching are reused across the crawl.
+Spider manages the crawl queue and traversal. Crawler handles individual document fetching/parsing and delegates the transport to a Fetcher. All HTTP requests share one persistent aiohttp session per Spider instance, so connection pooling, cookies, SSL configuration, and DNS caching are reused across the crawl.
 
 ## Why linktrace?
 
@@ -428,7 +496,7 @@ That makes `linktrace` useful when your goal is to inspect site structure, trace
 
 **vs requests + BeautifulSoup:** Built-in async concurrency, automatic session reuse, retries, caching, rate limiting, and structured document objects. Better for crawling multiple pages.
 
-**vs Selenium:** Pure HTTP crawler (no JS execution). Faster, lighter, but can't handle dynamic sites.
+**vs Selenium / bare Playwright:** HTTP by default (fast, light), with an optional headless-browser fetcher for JavaScript-rendered sites. You get rendering *plus* the crawl machinery (queue, robots.txt, rate limiting, retries, caching, exports) instead of scripting a browser by hand.
 
 ## Testing
 
@@ -437,7 +505,7 @@ just test          # Run all tests
 just test-cov      # Run with coverage report
 ```
 
-All 122 tests pass. 100% of core crawling paths tested (rate limiting, broken link tracking, robots.txt, callbacks, URL filtering rules, sitemap discovery).
+All 175 tests pass. 100% of core crawling paths tested (rate limiting, broken link tracking, robots.txt, callbacks, URL filtering rules, sitemap discovery, fetchers, flushing). Browser tests run against a local page in headless Chromium and skip automatically when Playwright is not installed.
 
 ## Contributing
 
diff --git a/docs/api-reference.md b/docs/api-reference.md
index ca910e3..af78c99 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -30,7 +30,12 @@ Spider(
     max_connections: int = 100,
     max_connections_per_host: int = 10,
     rules: CrawlRules = None,
-    use_sitemaps: bool = False
+    use_sitemaps: bool = False,
+    fetcher: Fetcher = None,
+    keep_hash_routes: bool = False,
+    flush_to: str = None,
+    flush_html: bool = False,
+    retain_documents: bool = None
 )
 ```
 
@@ -61,6 +66,11 @@ Spider(
 | `max_connections_per_host` | int | 10 | Max simultaneous connections to a single host |
 | `rules` | CrawlRules | None | URL include/exclude policy for discovered links (None = allow all) |
 | `use_sitemaps` | bool | False | Seed the queue from the site's sitemaps before link-following |
+| `fetcher` | Fetcher | None | Transport used to fetch pages (None = `HttpFetcher()`). See [Fetchers](#fetchers) |
+| `keep_hash_routes` | bool | False | Keep `#/...` and `#!...` fragments as distinct URLs (hash-router SPAs) |
+| `flush_to` | str | None | Append each crawled page as one JSON line to this file as soon as it is crawled |
+| `flush_html` | bool | False | Include page source in flushed lines |
+| `retain_documents` | bool | None | Keep `Document` objects in memory. Defaults to False when `flush_to` is set or a callback is used without `accumulate_results`, else True |
 
 **Raises:** `ValueError` if `max_concurrency < 1`.
 
@@ -82,7 +92,8 @@ documents = await spider.run_async()
 
 #### `run() -> List[Document]`
 
-Crawl synchronously (blocking). Use `asyncio.run()` wrapper for async context.
+Crawl synchronously (blocking). A convenience wrapper around `run_async()` for
+scripts.
 
 ```python
 spider = Spider(start_url="https://example.com")
@@ -91,6 +102,9 @@ documents = spider.run()  # Blocks until complete
 
 **Returns:** List of Document objects
 
+**Raises:** `RuntimeError` if called from inside a running event loop (Jupyter,
+an async application). Use `await spider.run_async()` there.
+
 ### Attributes
 
 | Attribute | Type | Description |
@@ -115,6 +129,10 @@ documents = spider.run()  # Blocks until complete
 | `max_connections_per_host` | int | Max connections to a single host |
 | `rules` | CrawlRules | URL filtering policy (always a CrawlRules; empty allows all) |
 | `use_sitemaps` | bool | Whether the queue is seeded from sitemaps |
+| `fetcher` | Fetcher \| None | Fetcher passed to the Crawler (None = default HTTP) |
+| `keep_hash_routes` | bool | Whether hash-router fragments are preserved |
+| `flush_to` | str \| None | JSONL path pages are streamed to |
+| `retain_documents` | bool | Whether Documents are kept in memory (resolved from the constructor args) |
 
 ### Callback Signature
 
@@ -140,11 +158,15 @@ async def on_crawl_complete() -> None:
 
 ### Return Behavior
 
-| Callback | `accumulate_results` | Return Value |
-|----------|----------------------|--------------|
-| None | Any | List of all Documents |
-| Provided | False | Empty list [] |
-| Provided | True | List of accumulated callback returns |
+| Callback | `accumulate_results` | Return Value | Documents retained? |
+|----------|----------------------|--------------|---------------------|
+| None | Any | List of all Documents | Yes |
+| Provided | False | Empty list [] | No (streaming) |
+| Provided | True | List of accumulated callback returns | Yes |
+
+`flush_to` returns `[]` and retains nothing unless `retain_documents=True`.
+When documents are not retained, `source` and `dom` are cleared after the
+callback runs and `broken_internal_links` is not computed.
 
 ---
 
@@ -168,7 +190,9 @@ Crawler(
     user_agent: str = "linktrace/0.1.0",
     respect_robots_txt: bool = True,
     max_connections: int = 100,
-    max_connections_per_host: int = 10
+    max_connections_per_host: int = 10,
+    fetcher: Fetcher = None,
+    keep_hash_routes: bool = False
 )
 ```
 
@@ -182,12 +206,15 @@ Same as Spider, plus:
 | `backoff_factor` | int | 2 | Exponential backoff multiplier (wait_time = 2^attempt * factor) |
 | `max_connections` | int | 100 | Total size of the aiohttp connection pool |
 | `max_connections_per_host` | int | 10 | Max simultaneous connections to a single host |
+| `fetcher` | Fetcher | None | Transport used to fetch pages (None = `HttpFetcher()`) |
+| `keep_hash_routes` | bool | False | Keep `#/...` and `#!...` fragments as distinct URLs |
 
 ### Methods
 
 #### `async __aenter__() -> Crawler`
 
-Enter async context manager. Creates persistent aiohttp session.
+Enter async context manager. Creates the persistent aiohttp session, then
+calls `fetcher.open(FetchContext)`.
 
 ```python
 async with Crawler(...) as crawler:
@@ -196,7 +223,7 @@ async with Crawler(...) as crawler:
 
 #### `async __aexit__(exc_type, exc_val, exc_tb)`
 
-Exit async context manager. Closes session.
+Exit async context manager. Calls `fetcher.close()`, then closes the session.
 
 #### `async crawl_document_async(url: str) -> Document`
 
@@ -216,9 +243,10 @@ async with Crawler() as crawler:
 
 **Notes:**
 - Checks cache first if `cache_dir` configured
-- Retries transient errors up to `max_retries` times
-- Returns Document even on 4xx/5xx (check `status_code`)
+- Retries `TransientFetchError` and 5xx responses up to `max_retries` times with exponential backoff
+- Returns Document even on 4xx/5xx (check `status_code`); returns `None` when every attempt failed with an error
 - Automatic cookie handling
+- `Document.fetcher` names the fetcher that produced the page
 
 #### `async discover_sitemap_urls(base_url: str) -> List[str]`
 
@@ -238,6 +266,8 @@ async with Crawler() as crawler:
 | Attribute | Type | Description |
 |-----------|------|-------------|
 | `session` | aiohttp.ClientSession | HTTP session (None until context entered) |
+| `fetcher` | Fetcher | The fetcher in use (an `HttpFetcher` by default) |
+| `keep_hash_routes` | bool | Whether hash-router fragments are preserved |
 | `cache` | ResponseCache | Cache object (None if cache_dir not set) |
 | `ssl_verify` | bool\|str | SSL verification setting |
 | `verify_hostname` | bool | Hostname verification setting |
@@ -249,6 +279,125 @@ async with Crawler() as crawler:
 
 ---
 
+## Fetchers
+
+The pluggable transport used by the Crawler. All fetchers live in
+`linktrace.fetchers` and are re-exported from `linktrace`.
+
+### Fetcher (base class)
+
+```python
+class Fetcher(ABC):
+    name: str = "custom"
+    async def open(self, ctx: FetchContext) -> None: ...   # optional
+    async def close(self) -> None: ...                     # optional
+    async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult: ...
+```
+
+Subclass it and implement `fetch`. `open`/`close` are called once per crawl
+by the Crawler's context manager. Raise `TransientFetchError` for failures the
+Crawler should retry and `FetchError` for those it should not. `name` is
+recorded on `Document.fetcher` and namespaces cache entries.
+
+### FetchResult
+
+```python
+FetchResult(url: str, status: int, headers: dict = {}, body: str | None = None, fetcher: str = "http")
+```
+
+`body` should be `None` for non-200 responses.
+
+### FetchContext
+
+Passed to `Fetcher.open`:
+
+| Attribute | Type | Description |
+|-----------|------|-------------|
+| `session` | aiohttp.ClientSession | The Crawler's shared session (pooling, cookies, SSL) |
+| `request_timeout` | float | Per-request timeout in seconds |
+| `user_agent` | str | User-Agent for the crawl |
+| `ssl_verify` | bool \| str | The Crawler's SSL setting |
+
+### HttpFetcher
+
+```python
+HttpFetcher(session: aiohttp.ClientSession = None)
+```
+
+The default. One `GET` per URL through the Crawler's session (or the one
+supplied). `name = "http"`.
+
+### BrowserFetcher
+
+```python
+BrowserFetcher(
+    browser: "chromium" | "firefox" | "webkit" = "chromium",
+    headless: bool = True,
+    wait_until: "commit" | "domcontentloaded" | "load" | "networkidle" = "networkidle",
+    wait_for_selector: str = None,
+    settle_ms: int = 0,
+    max_pages: int = 4,
+    timeout: float = None,
+    block_resources: Iterable[str] = ("image", "media", "font"),
+    viewport: dict[str, int] = None,
+    launch_args: list[str] = None,
+    ignore_https_errors: bool = None,
+)
+```
+
+Renders pages with Playwright. Requires `pip install linktrace[browser]` and
+`playwright install chromium`. `name = "browser"`.
+
+| Parameter | Description |
+|-----------|-------------|
+| `wait_until` | When navigation counts as done. `"networkidle"` suits SPAs; use `"domcontentloaded"` + `wait_for_selector` for long-polling sites |
+| `wait_for_selector` | CSS selector to wait for after navigation (e.g. `"a[href]"`) |
+| `settle_ms` | Extra fixed wait after the above |
+| `max_pages` | Max concurrent tabs, independent of the Spider's `max_concurrency` |
+| `timeout` | Navigation timeout in seconds (defaults to the Crawler's `request_timeout`) |
+| `block_resources` | Playwright resource types to abort; `()` loads everything |
+| `ignore_https_errors` | Defaults to `True` only when the Crawler has `ssl_verify=False` |
+
+**Raises:** `ImportError` from `open()` if Playwright is missing;
+`RuntimeError` if the browser binary is not installed. Navigation timeouts are
+raised as `TransientFetchError` (retried); other Playwright errors as
+`FetchError`.
+
+### RoutingFetcher
+
+```python
+RoutingFetcher(
+    default: Fetcher = None,          # HttpFetcher()
+    fallback: Fetcher = None,         # BrowserFetcher()
+    render_patterns: Iterable[str] = (),
+    route: Callable[[str], bool] = None,
+    escalate: Callable[[FetchResult], bool] = looks_like_spa_shell,
+)
+```
+
+Fetches with `default`, then re-fetches with `fallback` if `escalate(result)`
+is true. URLs matching `render_patterns` or for which `route(url)` is true skip
+`default`. `escalate=None` disables escalation. The `escalations` attribute
+counts re-fetches. `name = "routing"`.
+
+#### `needs_render(url: str) -> bool`
+
+True if `url` is routed straight to `fallback`.
+
+### `looks_like_spa_shell(result: FetchResult) -> bool`
+
+Cheap regex heuristic for un-rendered SPA shells: a 200 body with a
+`<noscript>` mentioning JavaScript, an empty mount element
+(`<div id="root"></div>`, `#app`, `#__next`, ...), or no `<a href>` but an
+external `<script src>`.
+
+### Exceptions
+
+- `FetchError` — non-retryable fetch failure.
+- `TransientFetchError(FetchError)` — retried by the Crawler with backoff.
+
+---
+
 ## CrawlRules
 
 Declarative include/exclude policy for which discovered URLs the Spider follows.
@@ -377,12 +526,13 @@ Document(url: str, source: str = None)
 | `title` | str | HTML `<title>` tag content |
 | `status_code` | int | HTTP status code |
 | `response_headers` | dict | HTTP response headers |
+| `fetcher` | str | Name of the fetcher that produced the page (`"http"`, `"browser"`, ...) |
 | `domain` | str | Domain extracted from URL (read-only property) |
 | `internal_links` | List[HtmlLink] | Links to same domain (successful crawl) |
 | `external_links` | List[HtmlLink] | Links to other domains (successful crawl) |
 | `links` | List[HtmlLink] | All links (internal + external) |
-| `broken_internal_links` | List[BrokenLink] | Internal links with HTTP error (4xx, 5xx) |
-| `broken_external_links` | List[BrokenLink] | External links with HTTP error (4xx, 5xx) |
+| `broken_internal_links` | List[BrokenLink] | Internal links whose target page returned 4xx/5xx. Filled in by the Spider after the crawl, for retained documents |
+| `broken_external_links` | List[BrokenLink] | Always empty: external links are not fetched |
 
 ### Properties
 
@@ -476,6 +626,8 @@ for doc in documents:
 
 ---
 
+## Serializers
+
 Export documents to multiple formats.
 
 ### Constructor
@@ -510,6 +662,7 @@ serializer.to_json("output.json", include_html=False)
     "title": "Example",
     "status_code": 200,
     "domain": "example",
+    "fetcher": "http",
     "response_headers": {...},
     "internal_links": [{"url": "...", "text": "..."}],
     "external_links": [...]
@@ -529,10 +682,15 @@ print(df[["url", "title", "link_url", "link_type"]].head())
 **Returns:** pandas DataFrame
 
 **Columns:**
-- `url`, `title`, `status_code`, `domain`
+- `url`, `title`, `status_code`, `domain`, `fetcher`
 - `link_url`, `link_text`, `link_type` (internal/external/None)
 - `html` (if include_html=True)
 
+#### `static document_to_dict(doc: Document, include_html: bool = False) -> dict`
+
+The per-document structure used by `to_json` and by `Spider(flush_to=...)`,
+one object per JSONL line.
+
 **Notes:**
 - One row per link
 - Documents without links have one row with NULL link fields
@@ -572,13 +730,16 @@ ResponseCache(cache_dir: str, ttl_seconds: int = 86400)
 
 ### Methods
 
-#### `async get(url: str) -> CachedResponse | None`
+#### `async get(url: str, *, namespace: str = None) -> CachedResponse | None`
 
-Retrieve cached response if not expired.
+Retrieve cached response if not expired. `namespace` selects a separate entry
+for the same URL; the Crawler passes the fetcher name for non-default fetchers
+and `None` for plain HTTP, so existing cache directories keep working.
 
-#### `async set(url: str, status_code: int, headers: dict, content: str) -> None`
+#### `async set(url: str, status_code: int, headers: dict, content: str, *, namespace: str = None, fetcher: str = "http") -> None`
 
-Store response in cache.
+Store response in cache. `fetcher` is recorded and restored onto
+`Document.fetcher` on a cache hit.
 
 #### `async clear() -> None`
 
diff --git a/docs/core-concepts.md b/docs/core-concepts.md
index fd3f123..1f77a5a 100644
--- a/docs/core-concepts.md
+++ b/docs/core-concepts.md
@@ -2,7 +2,7 @@
 
 ## Architecture Overview
 
-linktrace has three main components:
+linktrace has three main components plus a pluggable transport:
 
 ```
 ┌─────────────────────────────────────────────────┐
@@ -10,27 +10,27 @@ linktrace has three main components:
 │ - Manages crawl queue (to_visit)                │
 │ - Tracks visited pages                          │
 │ - Implements BFS or DFS traversal               │
-│ - Returns collection of Documents               │
+│ - Returns Documents, or flushes them to disk    │
 └──────────────────┬──────────────────────────────┘
                    │
                    │ creates one persistent
                    │
 ┌──────────────────▼──────────────────────────────┐
-│ Crawler (HTTP Engine)                           │
-│ - Persistent aiohttp session                    │
-│ - Connection pooling                            │
+│ Crawler (Crawl Engine)                          │
+│ - robots.txt + per-domain rate limiting         │
 │ - Retries with exponential backoff              │
 │ - Caching (optional)                            │
-│ - SSL verification                              │
-│ - Cookie jar                                    │
+│ - Link parsing / classification                 │
+│ - Persistent aiohttp session (SSL, cookies)     │
 └──────────────────┬──────────────────────────────┘
+                   │ delegates "get the page" to
                    │
-        ┌──────────┼──────────┐
-        │          │          │
-    ┌───▼──┐  ┌───▼──┐  ┌───▼──┐
-    │aiohttp  │  lxml   │  Cache  │
-    │ (HTTP)  │ (parse) │ (disk)  │
-    └────────┘  └────────┘  └────────┘
+┌──────────────────▼──────────────────────────────┐
+│ Fetcher (Transport)                             │
+│ - HttpFetcher    → aiohttp (default)            │
+│ - BrowserFetcher → Playwright headless Chromium │
+│ - RoutingFetcher → HTTP, escalate to browser    │
+└─────────────────────────────────────────────────┘
 ```
 
 ## Spider
@@ -88,7 +88,7 @@ async with Crawler(ssl_verify=True, cache_dir=".cache") as crawler:
 
 ### Retry Logic
 
-Transient errors (timeouts, connection errors, 5xx responses) are retried with exponential backoff:
+Transient errors (timeouts, connection errors, 5xx responses) are retried with exponential backoff. If every attempt returns 5xx, the last response is returned as a Document with that `status_code`:
 
 ```
 wait_time = 2^attempt * backoff_factor
@@ -127,6 +127,109 @@ crawler = Crawler(cache_dir=".webcrawler_cache")
 - Default TTL: 1 day
 - Expired entries auto-deleted on retrieval
 - Highly effective for repeat crawls (2-50x speedup)
+- Non-default fetchers get their own namespace, so a cached raw HTTP shell is
+  never served to a crawl that asked for a rendered page
+
+## Fetchers
+
+The **Fetcher** is the one step of a crawl that talks to the outside world:
+given a URL, return a status, headers and body. Everything the Crawler does
+around it (robots.txt, rate limiting, retries, caching, parsing) is transport
+agnostic, so swapping the fetcher is the only change needed to crawl sites
+plain HTTP cannot handle.
+
+```python
+from linktrace import Spider, HttpFetcher, BrowserFetcher, RoutingFetcher
+
+Spider(url)                               # HttpFetcher() — the default
+Spider(url, fetcher=BrowserFetcher())     # render every page in headless Chromium
+Spider(url, fetcher=RoutingFetcher())     # HTTP first, browser when it looks like an SPA
+```
+
+### HttpFetcher (default)
+
+One `GET` per URL through the Crawler's shared aiohttp session. Timeouts and
+connection errors are raised as `TransientFetchError` so the Crawler retries
+them; other client errors are `FetchError` and are not retried.
+
+### BrowserFetcher
+
+Renders each page in a headless browser via [Playwright](https://playwright.dev)
+and returns the resulting DOM. Requires `pip install linktrace[browser]` and a
+one-time `playwright install chromium`. Playwright is imported only when the
+fetcher is opened, so the core package never depends on it.
+
+- One browser and one browser context are shared for the whole crawl; each
+  fetch opens a tab, navigates, waits, serialises the DOM and closes the tab.
+- `max_pages` (default 4) caps concurrent tabs **independently** of the
+  Spider's `max_concurrency`, because a rendered page costs far more than an
+  HTTP request.
+- `wait_until="networkidle"` (default) is the most reliable choice for SPAs.
+  Sites with long-polling connections never go idle; use
+  `wait_until="domcontentloaded"` plus `wait_for_selector="a[href]"` there.
+- Images, media and fonts are blocked by default (`block_resources`) so pages
+  render faster. Pass `block_resources=()` to load everything.
+- `ignore_https_errors` follows the Crawler's `ssl_verify=False`.
+
+Rendered pages have real anchors when the app uses a router (React Router,
+Vue Router, Angular Router, Next.js). Apps that navigate only through click
+handlers expose nothing to extract and remain out of reach.
+
+### RoutingFetcher
+
+Sends every URL to `default` (plain HTTP) first, then re-fetches with
+`fallback` (the browser) when the response looks like an un-rendered SPA
+shell. The built-in `looks_like_spa_shell` heuristic checks for a `<noscript>`
+mentioning JavaScript, an empty mount element such as `<div id="root"></div>`,
+or a page with no anchors but a script bundle.
+
+```python
+RoutingFetcher(
+    render_patterns=[r"/search", r"/map"],   # always render these
+    route=lambda url: "?q=" in url,          # or decide in code
+    escalate=looks_like_spa_shell,           # None disables escalation
+)
+```
+
+`routing.escalations` counts how many pages were re-fetched. The browser is
+only launched when the fetcher is opened, but Playwright must still be
+installed if any URL may reach it.
+
+### Which fetcher produced a Document
+
+`Document.fetcher` records the name (`"http"`, `"browser"`, or a custom
+fetcher's name) and the serializers export it as a `fetcher` column, so a
+DataFrame can tell rendered rows from raw ones. With `BrowserFetcher`,
+`Document.source` is the rendered DOM rather than the raw response.
+
+### Hash-router apps
+
+URL normalization strips fragments, so `/#/about` and `/#/contact` would
+collapse into one page. Pass `keep_hash_routes=True` to keep fragments that
+start with `#/` or `#!` as distinct URLs (plain in-page anchors like `#top`
+are still dropped).
+
+### Writing your own fetcher
+
+Subclass `Fetcher` and implement `fetch`. `open`/`close` are optional hooks
+called once per crawl with a `FetchContext` (the shared aiohttp session,
+timeout, user agent and SSL setting).
+
+```python
+from linktrace import Fetcher, FetchResult, TransientFetchError
+
+class CurlImpersonateFetcher(Fetcher):
+    name = "curl"
+
+    async def fetch(self, url, headers):
+        status, body = await my_client.get(url, headers)   # your transport
+        if status == 429:
+            raise TransientFetchError("rate limited")      # Crawler retries
+        return FetchResult(url, status, body=body if status == 200 else None,
+                           fetcher=self.name)
+```
+
+Robots.txt and sitemap fetches always use plain HTTP regardless of fetcher.
 
 ## Document
 
@@ -139,12 +242,18 @@ class Document:
     source: str                 # Raw HTML
     status_code: int            # HTTP status (200, 404, etc)
     response_headers: dict      # HTTP response headers
+    fetcher: str                # "http", "browser", or a custom fetcher name
     domain: str                 # Domain extracted via tldextract
     internal_links: List[Link]  # Links to same domain
     external_links: List[Link]  # Links to other domains
     links: List[Link]           # All links (internal + external)
+    broken_internal_links: List[BrokenLink]  # Internal links whose target returned 4xx/5xx
 ```
 
+`broken_internal_links` is filled in once the crawl finishes, from the status
+codes of the pages actually visited. External links are never fetched, so
+`broken_external_links` stays empty.
+
 ### HtmlLink
 
 ```python
@@ -311,14 +420,30 @@ Failed requests log errors but don't crash the crawl. Document is returned with
 ## Memory Model
 
 Memory usage scales with:
-- Number of pages (each Document ~100KB with HTML)
+- Number of pages (each Document ~100KB with HTML; rendered DOMs from
+  `BrowserFetcher` are typically several times larger)
 - Queue size (larger for BFS on wide sites)
 - Cache size (if enabled)
 
 Typical: ~1MB per 100 pages.
 
-To reduce memory:
-- Disable HTML in Document: don't store `doc.source`
+By default every `Document` (including its HTML) is kept until the crawl
+finishes. Two options change that:
+
+- **Streaming callbacks.** With `on_page_crawled` set and
+  `accumulate_results=False`, documents are handed to your callback and then
+  released: `source` and `dom` are cleared and nothing is appended to
+  `spider.documents`.
+- **Flush to disk.** `flush_to="crawl.jsonl"` appends each page as one JSON
+  object per line (the same schema as `Serializers.to_json`; add
+  `flush_html=True` to include the source) as soon as it is crawled, and
+  likewise retains nothing.
+
+Both set `retain_documents=False` implicitly; pass `retain_documents=True` to
+keep the in-memory list as well. `broken_internal_links` is only computed for
+retained documents.
+
+To reduce memory further:
 - Use DFS instead of BFS (smaller queue)
 - Disable caching
 - Lower max_depth
@@ -364,3 +489,7 @@ spider = Spider(
 Raise `max_concurrency` and `max_connections` to go faster across **many**
 hosts; keep `max_connections_per_host` modest (and pair with `request_delay` or
 `respect_robots_txt=True`) to avoid overwhelming a single server.
+
+With `BrowserFetcher`, rendered fetches are additionally capped by its own
+`max_pages` (default 4), so a batch of 50 URLs still opens at most 4 tabs at a
+time.
diff --git a/docs/examples.md b/docs/examples.md
index 1b62f67..402b991 100644
--- a/docs/examples.md
+++ b/docs/examples.md
@@ -249,7 +249,9 @@ asyncio.run(main())
 
 ## 10. Find Broken Internal Links
 
-Identify 404s and other errors on internal links.
+Identify 404s and other errors on internal links. After the crawl, every
+retained Document carries `broken_internal_links`: the internal links whose
+target page came back with a 4xx/5xx status.
 
 ```python
 from linktrace import Spider
@@ -257,27 +259,14 @@ from linktrace import Spider
 async def main():
     spider = Spider(start_url="https://example.com", max_depth=2)
     documents = await spider.run_async()
-    
-    broken = []
+
+    total = 0
     for doc in documents:
-        for link in doc.internal_links:
-            # Find if this link was also crawled
-            linked_doc = next(
-                (d for d in documents if d.url == link.url),
-                None
-            )
-            
-            if linked_doc and linked_doc.status_code != 200:
-                broken.append({
-                    'from': doc.url,
-                    'to': link.url,
-                    'status': linked_doc.status_code,
-                    'text': link.text
-                })
-    
-    print(f"Found {len(broken)} broken internal links:")
-    for item in broken:
-        print(f"  {item['from']} → {item['to']} ({item['status']})")
+        for broken in doc.broken_internal_links:
+            total += 1
+            print(f"  {doc.url} → {broken.url} (HTTP {broken.status_code})")
+
+    print(f"Found {total} broken internal links")
 
 asyncio.run(main())
 ```
@@ -751,3 +740,148 @@ async def main():
 
 asyncio.run(main())
 ```
+
+## 24. Crawl a JavaScript-Rendered Site (SPA)
+
+React, Vue and Angular apps ship an empty HTML shell and build their links in
+the browser, so plain HTTP finds nothing. `BrowserFetcher` renders every page
+in headless Chromium before linktrace parses it. Requires
+`pip install linktrace[browser]` and a one-time `playwright install chromium`.
+
+```python
+import asyncio
+from linktrace import Spider, BrowserFetcher, CrawlRules
+
+async def main():
+    spider = Spider(
+        start_url="https://listings.example.com/",
+        max_depth=2,
+        fetcher=BrowserFetcher(
+            wait_until="networkidle",      # let the app finish its XHR calls
+            wait_for_selector="a[href]",   # ...and actually render links
+            max_pages=4,                   # concurrent tabs
+        ),
+        rules=CrawlRules(include_path_prefixes=["/listing/", "/"]),
+        request_delay=0.5,                 # rate limiting still applies
+    )
+    documents = await spider.run_async()
+
+    for doc in documents:
+        print(f"[{doc.fetcher}] {doc.url}: {len(doc.internal_links)} links")
+
+asyncio.run(main())
+```
+
+!!! tip "Hash-router apps"
+    If the app's routes look like `/#/listing/42`, add
+    `keep_hash_routes=True` to the Spider. Without it every route normalizes
+    to the same bare URL and the crawl stops after one page.
+
+## 25. HTTP First, Browser Only When Needed
+
+Rendering is 10-50x slower than an HTTP request. `RoutingFetcher` fetches
+each URL with plain HTTP first and only re-fetches in the browser when the
+response looks like an un-rendered SPA shell (a `<noscript>` telling you to
+enable JavaScript, an empty `<div id="root">`, or no links but a script
+bundle). Mixed sites, where a server-rendered marketing site wraps a
+client-rendered app, get the cheap path for most pages.
+
+```python
+import asyncio
+from linktrace import Spider, RoutingFetcher, BrowserFetcher
+
+async def main():
+    routing = RoutingFetcher(
+        fallback=BrowserFetcher(max_pages=2),
+        render_patterns=[r"/search", r"/map"],   # always render these
+    )
+    spider = Spider(
+        start_url="https://realty.example.com/",
+        max_depth=3,
+        fetcher=routing,
+    )
+    documents = await spider.run_async()
+
+    rendered = [d for d in documents if d.fetcher == "browser"]
+    print(f"{len(documents)} pages, {len(rendered)} rendered "
+          f"({routing.escalations} escalated automatically)")
+
+asyncio.run(main())
+```
+
+Pass `escalate=None` to route purely by pattern, or your own
+`escalate=lambda result: ...` for site-specific signals.
+
+## 26. Stream a Huge Crawl to Disk
+
+Documents hold their full HTML, so a deep crawl can exhaust memory. `flush_to`
+appends each page to a JSONL file the moment it is crawled and keeps nothing
+in RAM beyond the URL bookkeeping. The line format matches
+`Serializers.to_json`, so it loads straight into pandas or polars.
+
+```python
+import asyncio
+import polars as pl
+from linktrace import Spider
+
+async def main():
+    spider = Spider(
+        start_url="https://example.com/",
+        max_depth=6,
+        flush_to="crawl.jsonl",
+        flush_html=False,          # True to keep page source too
+        show_progress=True,
+    )
+    await spider.run_async()       # returns []
+
+    df = pl.read_ndjson("crawl.jsonl")
+    print(df.select("url", "status_code", "fetcher").head())
+
+asyncio.run(main())
+```
+
+`flush_to` combines with a callback or with `retain_documents=True` if you
+also want the objects in memory.
+
+## 27. Write Your Own Fetcher
+
+Any transport can plug into the crawl: subclass `Fetcher`, return a
+`FetchResult`, and raise `TransientFetchError` for failures the Crawler should
+retry. Robots.txt, rate limiting, retries, caching and parsing all keep
+working unchanged.
+
+```python
+import asyncio
+import httpx
+from linktrace import Spider, Fetcher, FetchResult, TransientFetchError
+
+class HttpxFetcher(Fetcher):
+    name = "httpx"
+
+    async def open(self, ctx):
+        self.client = httpx.AsyncClient(
+            http2=True, timeout=ctx.request_timeout,
+            headers={"User-Agent": ctx.user_agent},
+        )
+
+    async def close(self):
+        await self.client.aclose()
+
+    async def fetch(self, url, headers):
+        try:
+            r = await self.client.get(url, headers=headers, follow_redirects=True)
+        except httpx.TimeoutException as e:
+            raise TransientFetchError(str(e)) from e
+        return FetchResult(
+            url, r.status_code, dict(r.headers),
+            body=r.text if r.status_code == 200 else None,
+            fetcher=self.name,
+        )
+
+async def main():
+    spider = Spider("https://example.com/", fetcher=HttpxFetcher())
+    documents = await spider.run_async()
+    print({d.fetcher for d in documents})   # {"httpx"}
+
+asyncio.run(main())
+```
diff --git a/docs/getting-started.md b/docs/getting-started.md
index 09389cd..7cd8f1a 100644
--- a/docs/getting-started.md
+++ b/docs/getting-started.md
@@ -20,6 +20,15 @@ pip install linktrace[polars]
 pip install linktrace[pyarrow]
 ```
 
+### With JavaScript Rendering
+
+For single-page apps that build their links in the browser:
+
+```bash
+pip install linktrace[browser]
+playwright install chromium     # one-time download of the browser build
+```
+
 ### Development Installation
 
 ```bash
@@ -191,6 +200,42 @@ spider = Spider(
 documents = await spider.run_async()
 ```
 
+### Crawl a JavaScript-Rendered Site
+
+Plain HTTP only sees the empty shell of a React/Vue/Angular app. Swap the
+fetcher and the same crawl renders each page in headless Chromium first:
+
+```python
+from linktrace import Spider, RoutingFetcher
+
+spider = Spider(
+    start_url="https://app.example.com/",
+    max_depth=2,
+    fetcher=RoutingFetcher(),   # HTTP first; browser only for SPA-looking pages
+)
+documents = await spider.run_async()
+for doc in documents:
+    print(doc.fetcher, doc.url, len(doc.internal_links))   # "http" or "browser"
+```
+
+Use `fetcher=BrowserFetcher()` to render every page, and
+`keep_hash_routes=True` for hash-router apps (`/#/listings/42`). See
+[Core Concepts](core-concepts.md#fetchers) for how the pieces fit.
+
+### Stream a Large Crawl to Disk
+
+When a crawl won't fit in memory, write each page to a JSONL file as it is
+crawled instead of collecting `Document` objects:
+
+```python
+spider = Spider(
+    start_url="https://example.com/",
+    max_depth=5,
+    flush_to="crawl.jsonl",   # one JSON object per line
+)
+await spider.run_async()      # returns []; results are in crawl.jsonl
+```
+
 ## Jupyter Notebooks
 
 See the `notebooks/` directory for interactive examples:
diff --git a/docs/index.md b/docs/index.md
index 2ca2f8d..9b00cf9 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -8,6 +8,8 @@ LinkTrace is a document-oriented crawler. Every crawled page becomes a rich Docu
 
 **Targeted crawling:** Scope a crawl with [URL filtering rules](getting-started.md#focus-the-crawl-with-filtering-rules) (regex, path, extension, query-param, domain), tune throughput with [configurable concurrency](getting-started.md#tune-concurrency), and seed from a site's [sitemaps](getting-started.md#seed-from-sitemaps).
 
+**JavaScript sites:** Plain HTTP by default, with an optional [headless-browser fetcher](getting-started.md#crawl-a-javascript-rendered-site) for React/Vue/Angular apps. Big crawls can [stream straight to disk](getting-started.md#stream-a-large-crawl-to-disk).
+
 ## Quick Start
 
 ```bash
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 8ddfe03..891b3fe 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -133,11 +133,37 @@ for link in doc.internal_links[:3]:
 
 **Common causes:**
 - Relative links not resolved correctly
-- JavaScript-generated links (linktrace doesn't execute JS)
+- JavaScript-generated links (the default HTTP fetcher doesn't execute JS)
 - Links hidden behind `onclick` or `data-href`
+- Hash-router URLs (`/#/page`) collapsing to one URL
 - Page returned error status (check `doc.status_code`)
 
-**Solution for JS-heavy sites:** Use Selenium or Playwright instead.
+**Solution for JS-heavy sites:** render in a headless browser.
+
+```python
+from linktrace import Spider, RoutingFetcher
+
+spider = Spider(
+    start_url="https://app.example.com/",
+    fetcher=RoutingFetcher(),      # HTTP first, browser when the page looks like an SPA
+    keep_hash_routes=True,         # only if routes look like /#/...
+)
+```
+
+Requires `pip install linktrace[browser]` and `playwright install chromium`.
+Quick check that rendering is the problem: `looks_like_spa_shell` on the
+raw response.
+
+```python
+from linktrace import Crawler, looks_like_spa_shell
+
+async with Crawler() as crawler:
+    result = await crawler.fetcher.fetch("https://app.example.com/", {})
+    print(looks_like_spa_shell(result))   # True → you need BrowserFetcher
+```
+
+Links driven purely by `onclick` handlers have no `href` and stay out of
+reach even when rendered.
 
 ### "Wrong internal/external classification"
 
@@ -167,6 +193,89 @@ for link in doc.links:
 
 ---
 
+## Browser Fetcher Issues
+
+### "BrowserFetcher requires Playwright"
+
+**Problem:** The `browser` extra is not installed.
+
+```bash
+pip install "linktrace[browser]"
+playwright install chromium
+```
+
+### "Could not launch chromium" / "Executable doesn't exist"
+
+**Problem:** Playwright is installed but its browser build is not.
+
+```bash
+playwright install chromium
+```
+
+In Docker use the `mcr.microsoft.com/playwright/python` base image, or run
+`playwright install --with-deps chromium` to pull system libraries too.
+
+### "Browser timeout" on every page
+
+**Problem:** `wait_until="networkidle"` never fires because the site keeps a
+long-polling or websocket connection open.
+
+**Solution:** Wait for the DOM and a selector instead:
+
+```python
+BrowserFetcher(wait_until="domcontentloaded", wait_for_selector="a[href]")
+```
+
+Timeouts are retried like any transient error; lower `max_retries` if a site
+is consistently slow rather than flaky.
+
+### Rendered pages still have no links
+
+- Confirm the crawl actually used the browser: `doc.fetcher == "browser"`.
+  With `RoutingFetcher`, a page that doesn't trip the SPA heuristic stays on
+  HTTP; add it to `render_patterns`.
+- The app may render links only after an XHR completes; add
+  `wait_for_selector` for an element that appears with the data, or
+  `settle_ms=500`.
+- A stale cache entry from an HTTP crawl is never served to a browser crawl
+  (entries are namespaced by fetcher), but clearing `cache_dir` rules it out.
+
+### Browser crawl is slow
+
+- Keep `max_pages` small (default 4); more tabs rarely helps on one host.
+- Prefer `RoutingFetcher` so only SPA pages are rendered.
+- Leave `block_resources` at its default (images, media, fonts are skipped).
+- Enable `cache_dir` so repeat crawls skip rendering entirely.
+
+---
+
+## Memory Issues
+
+### "Crawl runs out of memory"
+
+**Problem:** Every `Document` holds its HTML (and rendered DOMs are larger),
+and by default all of them are kept until the crawl finishes.
+
+**Solutions:**
+
+1. **Flush to disk** and keep nothing in memory:
+   ```python
+   spider = Spider(start_url="...", flush_to="crawl.jsonl")
+   await spider.run_async()   # returns []; read crawl.jsonl afterwards
+   ```
+
+2. **Stream through a callback** (documents are released after it runs):
+   ```python
+   spider = Spider(start_url="...", on_page_crawled=handle, accumulate_results=False)
+   ```
+
+3. Lower `max_depth`, tighten `CrawlRules`, or use DFS for a smaller queue.
+
+Both streaming modes set `retain_documents=False`; `broken_internal_links` is
+only computed for retained documents.
+
+---
+
 ## Caching Issues
 
 ### "Cache file corrupted" warning
@@ -448,14 +557,25 @@ spider = Spider(start_url="...", traversal_strategy="dfs-fast")
 spider = Spider(start_url="...", traversal_strategy="dfs")  # or "bfs"
 ```
 
-### ❌ Expecting JavaScript execution
+### ❌ Expecting JavaScript execution from the default fetcher
 
 ```python
-# Won't work - linktrace doesn't execute JavaScript
+# Finds nothing - the default HTTP fetcher doesn't execute JavaScript
 spider = Spider(start_url="https://react-app.example.com")
-documents = await spider.run_async()
 
-# Solution: Use Playwright or Selenium for JS-heavy sites
+# Correct: render in a headless browser (pip install linktrace[browser])
+from linktrace import RoutingFetcher
+spider = Spider(start_url="https://react-app.example.com", fetcher=RoutingFetcher())
+```
+
+### ❌ Calling `run()` inside Jupyter
+
+```python
+# Wrong - Jupyter already runs an event loop
+documents = spider.run()   # RuntimeError
+
+# Correct
+documents = await spider.run_async()
 ```
 
 ---
diff --git a/linktrace/Crawler.py b/linktrace/Crawler.py
index 91d5984..b45d732 100755
--- a/linktrace/Crawler.py
+++ b/linktrace/Crawler.py
@@ -9,6 +9,14 @@
 import tldextract
 
 from linktrace.cache import ResponseCache
+from linktrace.fetchers.base import (
+    FetchContext,
+    Fetcher,
+    FetchError,
+    FetchResult,
+    TransientFetchError,
+)
+from linktrace.fetchers.http import HttpFetcher
 from linktrace.robots import RobotsManager
 
 
@@ -59,6 +67,9 @@ def __init__(self, url: str, source: str | None) -> None:
         self.status_code: int = 0
         self.response_headers: dict[str, str] = {}
         self.dom: object = None
+        # Which fetcher produced ``source``: "http" (raw response) or
+        # "browser" (rendered DOM), or a custom fetcher's name.
+        self.fetcher: str = "http"
 
     @property
     def domain(self) -> str:
@@ -94,9 +105,18 @@ def __init__(
         respect_robots_txt: bool = True,
         max_connections: int = 100,
         max_connections_per_host: int = 10,
+        fetcher: Fetcher | None = None,
+        keep_hash_routes: bool = False,
     ) -> None:
         self._logger = logging.getLogger(log_name if log_name else __name__)
         self._logger.setLevel(log_level)
+
+        # Pluggable transport. HttpFetcher reproduces the classic behaviour;
+        # BrowserFetcher / RoutingFetcher add JavaScript rendering.
+        self.fetcher: Fetcher = fetcher if fetcher is not None else HttpFetcher()
+        # Keep "#/..." and "#!..." fragments (hash-router SPAs) instead of
+        # stripping every fragment during URL normalization.
+        self.keep_hash_routes = keep_hash_routes
         self.visited_urls: set[str] = set()
         self._queue: list[str] = []
         self._links: list[HtmlLink] = []
@@ -129,8 +149,21 @@ def __init__(
         self._cookie_jar: aiohttp.CookieJar | None = None
 
     async def __aenter__(self) -> "Crawler":
-        """Enter async context manager: create and setup session."""
+        """Enter async context manager: create session, then open the fetcher."""
         self.session = await self._create_session()
+        try:
+            await self.fetcher.open(
+                FetchContext(
+                    session=self.session,
+                    request_timeout=self.request_timeout,
+                    user_agent=self.user_agent,
+                    ssl_verify=self.ssl_verify,
+                )
+            )
+        except Exception:
+            await self.session.close()
+            self.session = None
+            raise
         return self
 
     async def __aexit__(
@@ -139,9 +172,12 @@ async def __aexit__(
         exc_val: BaseException | None,
         exc_tb: object,
     ) -> None:
-        """Exit async context manager: cleanup session."""
-        if self.session:
-            await self.session.close()
+        """Exit async context manager: close the fetcher, then the session."""
+        try:
+            await self.fetcher.close()
+        finally:
+            if self.session:
+                await self.session.close()
 
     async def _create_session(self) -> aiohttp.ClientSession:
         """Create persistent aiohttp session with SSL context, connector, timeouts."""
@@ -246,14 +282,17 @@ async def crawl_document_async(self, url: str) -> Document | None:
                 "Use 'async with Crawler(...) as crawler:' context manager."
             )
 
-        # Check cache first
+        # Check cache first. Non-default fetchers get their own namespace so a
+        # cached raw HTTP shell never satisfies a request for a rendered page.
+        cache_ns = None if self.fetcher.name == HttpFetcher.name else self.fetcher.name
         if self.cache:
-            cached = await self.cache.get(url)
+            cached = await self.cache.get(url, namespace=cache_ns)
             if cached:
                 self._logger.debug(f"Cache hit for {url}")
                 doc = self.parse_document(url, cached.content)
                 doc.status_code = cached.status_code
                 doc.response_headers = cached.response_headers
+                doc.fetcher = cached.fetcher
                 return doc
 
         # Check robots.txt before fetching
@@ -273,59 +312,73 @@ async def crawl_document_async(self, url: str) -> Document | None:
         # Enforce per-domain rate limiting
         await self._rate_limit_domain(url, delay)
 
-        # Implement retry logic
+        # Fetch with retries. Transient errors and 5xx responses are retried
+        # with exponential backoff; other failures return immediately.
+        headers = {"User-Agent": self.user_agent}
+        result: FetchResult | None = None
         for attempt in range(self.max_retries):
+            last_attempt = attempt >= self.max_retries - 1
             try:
-                headers = {"User-Agent": self.user_agent}
-                async with self.session.get(url, headers=headers) as response:
-                    self._logger.debug(f"Fetching {url} (attempt {attempt + 1})")
-                    status = response.status
-                    response_headers = dict(response.headers)
-
-                    if status != 200:
-                        self._logger.error(f"Failed to fetch {url}: HTTP {status}")
-                        doc = Document(url, None)
-                        doc.status_code = status
-                        doc.response_headers = response_headers
-                        return doc
-
-                    html = await response.text()
-
-                doc = self.parse_document(url, html)
-                doc.status_code = status
-                doc.response_headers = response_headers
-
-                # Cache successful responses
-                if self.cache:
-                    await self.cache.set(url, status, response_headers, html)
-
-                return doc
-
-            except (
-                TimeoutError,
-                aiohttp.ClientConnectorError,
-                aiohttp.ServerTimeoutError,
-            ) as e:
-                if attempt < self.max_retries - 1:
-                    wait_time = 2**attempt * self.backoff_factor
-                    self._logger.warning(
-                        f"Transient error fetching {url} "
-                        f"(attempt {attempt + 1}/{self.max_retries}): {e}. "
-                        f"Retrying in {wait_time}s..."
-                    )
-                    await asyncio.sleep(wait_time)
-                else:
+                self._logger.debug(f"Fetching {url} (attempt {attempt + 1})")
+                result = await self.fetcher.fetch(url, headers)
+            except TransientFetchError as e:
+                if last_attempt:
                     self._logger.error(
                         f"Failed to fetch {url} after {self.max_retries} attempts: {e}"
                     )
-            except aiohttp.ClientError as e:
+                    return None
+                await self._backoff(url, attempt, f"Transient error: {e}")
+                continue
+            except FetchError as e:
                 self._logger.error(f"Client error while fetching {url}: {e}")
-                break
+                return None
             except Exception as e:
                 self._logger.error(f"Unexpected error while fetching {url}: {e}")
                 raise
 
-        return None
+            if 500 <= result.status < 600 and not last_attempt:
+                await self._backoff(url, attempt, f"HTTP {result.status}")
+                continue
+            break
+
+        if result is None:  # pragma: no cover - max_retries < 1
+            return None
+
+        if result.status != 200:
+            self._logger.error(f"Failed to fetch {url}: HTTP {result.status}")
+            doc = Document(url, None)
+            doc.status_code = result.status
+            doc.response_headers = result.headers
+            doc.fetcher = result.fetcher
+            return doc
+
+        doc = self.parse_document(url, result.body)
+        doc.status_code = result.status
+        doc.response_headers = result.headers
+        doc.fetcher = result.fetcher
+
+        # Cache successful responses
+        if self.cache and result.body is not None:
+            await self.cache.set(
+                url,
+                result.status,
+                result.headers,
+                result.body,
+                namespace=cache_ns,
+                fetcher=result.fetcher,
+            )
+
+        return doc
+
+    async def _backoff(self, url: str, attempt: int, reason: str) -> None:
+        """Log and sleep before retrying ``url`` after ``attempt`` (0-based)."""
+        wait_time = 2**attempt * self.backoff_factor
+        self._logger.warning(
+            f"{reason} fetching {url} "
+            f"(attempt {attempt + 1}/{self.max_retries}). "
+            f"Retrying in {wait_time}s..."
+        )
+        await asyncio.sleep(wait_time)
 
     async def discover_sitemap_urls(self, base_url: str) -> list[str]:
         """Discover page URLs from the site's sitemaps.
@@ -371,14 +424,20 @@ def parse_document(self, url: str, source: str | None) -> Document:
             links = [
                 link
                 for link in dom.xpath("//a/@href/..")
-                if link.attrib["href"] and link.attrib["href"][:1] not in skip_words
+                if link.attrib["href"]
+                and (
+                    link.attrib["href"][:1] not in skip_words
+                    or self._is_hash_route(link.attrib["href"])
+                )
             ]
 
             # Use local list per document to avoid state leakage
             found_links: list[HtmlLink] = []
             for link in links:
                 link_url = link.attrib["href"]
-                link_url = self.normalize_url(urljoin(url, link_url))
+                link_url = self.normalize_url(
+                    urljoin(url, link_url), keep_hash_routes=self.keep_hash_routes
+                )
                 if urlparse(link_url).scheme not in ["http", "https", "ftp"]:
                     continue
 
@@ -426,14 +485,25 @@ def queue_link(self, link: str) -> None:
         if link not in self.visited_urls and link not in self._queue:
             self._queue.append(link)
 
+    def _is_hash_route(self, href: str) -> bool:
+        """True if ``href`` is a hash-router path we have been told to keep."""
+        return self.keep_hash_routes and href[:2] in ("#/", "#!")
+
     @staticmethod
-    def normalize_url(url: str) -> str:
-        """Strip fragments and normalize an empty path to '/'."""
-        url = urldefrag(url).url
-        parsed = urlparse(url)
+    def normalize_url(url: str, keep_hash_routes: bool = False) -> str:
+        """Strip fragments and normalize an empty path to '/'.
+
+        With ``keep_hash_routes=True``, fragments that look like client-side
+        routes (``#/about``, ``#!/about``) are preserved so hash-router SPAs
+        keep one URL per route instead of collapsing to the bare page.
+        """
+        base, fragment = urldefrag(url)
+        parsed = urlparse(base)
         if parsed.netloc and not parsed.path:
-            url = parsed._replace(path="/").geturl()
-        return url
+            parsed = parsed._replace(path="/")
+        if keep_hash_routes and fragment[:1] in ("/", "!"):
+            parsed = parsed._replace(fragment=fragment)
+        return parsed.geturl()
 
     @staticmethod
     def get_domain_parts(url: str) -> str:
diff --git a/linktrace/Serializers.py b/linktrace/Serializers.py
index 2cd9af8..2d3906e 100644
--- a/linktrace/Serializers.py
+++ b/linktrace/Serializers.py
@@ -24,29 +24,40 @@ def to_json(self, output_path: str, include_html: bool = False) -> None:
             output_path: Path to write JSON file
             include_html: Include raw HTML source in output (default False)
         """
-        data = []
-        for doc in self.documents:
-            doc_data = {
-                "url": doc.url,
-                "title": doc.title,
-                "status_code": doc.status_code,
-                "domain": doc.domain,
-                "response_headers": doc.response_headers,
-                "internal_links": [
-                    {"url": link.url, "text": link.text} for link in doc.internal_links
-                ],
-                "external_links": [
-                    {"url": link.url, "text": link.text} for link in doc.external_links
-                ],
-            }
-            if include_html and doc.source:
-                doc_data["html"] = doc.source
-            data.append(doc_data)
+        data = [self.document_to_dict(doc, include_html) for doc in self.documents]
 
         Path(output_path).parent.mkdir(parents=True, exist_ok=True)
         with open(output_path, "w") as f:
             json.dump(data, f, indent=2)
 
+    @staticmethod
+    def document_to_dict(doc: Document, include_html: bool = False) -> dict:
+        """Convert one Document to the nested JSON structure used by ``to_json``.
+
+        Also used by ``Spider(flush_to=...)`` to write one JSON line per page.
+
+        Args:
+            doc: Document to convert
+            include_html: Include raw HTML source under the ``html`` key
+        """
+        doc_data: dict = {
+            "url": doc.url,
+            "title": doc.title,
+            "status_code": doc.status_code,
+            "domain": doc.domain,
+            "fetcher": doc.fetcher,
+            "response_headers": doc.response_headers,
+            "internal_links": [
+                {"url": link.url, "text": link.text} for link in doc.internal_links
+            ],
+            "external_links": [
+                {"url": link.url, "text": link.text} for link in doc.external_links
+            ],
+        }
+        if include_html and doc.source:
+            doc_data["html"] = doc.source
+        return doc_data
+
     def to_pandas(self, include_html: bool = False):
         """Export documents to pandas DataFrame with flattened links.
 
@@ -139,6 +150,7 @@ def _flatten_documents(self, include_html: bool = False) -> list[dict]:
                 "title": doc.title,
                 "status_code": doc.status_code,
                 "domain": doc.domain,
+                "fetcher": doc.fetcher,
             }
 
             if include_html:
diff --git a/linktrace/Spider.py b/linktrace/Spider.py
index ddc0fb2..16a8a49 100755
--- a/linktrace/Spider.py
+++ b/linktrace/Spider.py
@@ -1,10 +1,13 @@
 import asyncio
+import json
 import logging
 from collections.abc import Callable
 from typing import Any, Literal
 
-from linktrace.Crawler import Crawler, Document
+from linktrace.Crawler import BrokenLink, Crawler, Document
+from linktrace.fetchers.base import Fetcher
 from linktrace.rules import CrawlRules
+from linktrace.Serializers import Serializers
 
 
 class Spider:
@@ -37,8 +40,29 @@ def __init__(
         max_connections_per_host: int = 10,
         rules: CrawlRules | None = None,
         use_sitemaps: bool = False,
+        fetcher: Fetcher | None = None,
+        keep_hash_routes: bool = False,
+        flush_to: str | None = None,
+        flush_html: bool = False,
+        retain_documents: bool | None = None,
     ) -> None:
-        start_url = Crawler.normalize_url(start_url)
+        self.fetcher = fetcher
+        self.keep_hash_routes = keep_hash_routes
+
+        # Memory management. ``flush_to`` streams each Document to a JSONL
+        # file as soon as it is crawled. Documents are retained in memory
+        # unless the caller opted into a streaming output (flush_to, or a
+        # callback without accumulate_results) or said so explicitly.
+        self.flush_to = flush_to
+        self.flush_html = flush_html
+        streaming = flush_to is not None or (
+            on_page_crawled is not None and not accumulate_results
+        )
+        self.retain_documents = (
+            retain_documents if retain_documents is not None else not streaming
+        )
+        self._flush_file: Any = None
+        start_url = Crawler.normalize_url(start_url, keep_hash_routes=keep_hash_routes)
         self.start_url = start_url
         self.max_depth = max_depth
         self.ssl_verify = ssl_verify  # bool or str(path to CA cert)
@@ -114,6 +138,22 @@ async def _invoke_callback(
         else:
             return callback(*args)
 
+    def run(self) -> list[Document]:
+        """Run the spider synchronously (blocking) and return the documents.
+
+        Convenience wrapper around :meth:`run_async` for scripts. Inside an
+        already-running event loop (Jupyter, an async app) use
+        ``await spider.run_async()`` instead.
+        """
+        try:
+            asyncio.get_running_loop()
+        except RuntimeError:
+            return asyncio.run(self.run_async())
+        raise RuntimeError(
+            "Spider.run() cannot be called from a running event loop "
+            "(e.g. Jupyter). Use 'await spider.run_async()' instead."
+        )
+
     async def run_async(self) -> list[Document]:
         """Run the spider with persistent session."""
         if self.show_progress:
@@ -128,6 +168,13 @@ async def run_async(self) -> list[Document]:
             )
 
         try:
+            if self.flush_to is not None:
+                import aiofiles
+
+                self._flush_file = await aiofiles.open(
+                    self.flush_to, "a", encoding="utf-8"
+                )
+
             async with Crawler(
                 log_name=self._logger.name,
                 ssl_verify=self.ssl_verify,
@@ -140,6 +187,8 @@ async def run_async(self) -> list[Document]:
                 respect_robots_txt=self.respect_robots_txt,
                 max_connections=self.max_connections,
                 max_connections_per_host=self.max_connections_per_host,
+                fetcher=self.fetcher,
+                keep_hash_routes=self.keep_hash_routes,
             ) as crawler:
                 if self.use_sitemaps:
                     await self._seed_from_sitemaps(crawler)
@@ -168,7 +217,13 @@ async def run_async(self) -> list[Document]:
                             for url, depth in batch
                         ]
                         await asyncio.gather(*tasks)
+
+            self._mark_broken_links()
         finally:
+            if self._flush_file is not None:
+                await self._flush_file.close()
+                self._flush_file = None
+
             if self._pbar is not None:
                 self._pbar.close()
                 self._pbar = None
@@ -177,7 +232,8 @@ async def run_async(self) -> list[Document]:
             if self.on_crawl_complete is not None:
                 await self._invoke_callback(self.on_crawl_complete)
 
-        # Return logic: Option B
+        # Return logic: documents when retained, else the callback results
+        # (streaming / flushed crawls return an empty list).
         if self.on_page_crawled is None:
             return self.documents
         elif self.accumulate_results:
@@ -185,6 +241,23 @@ async def run_async(self) -> list[Document]:
         else:
             return []
 
+    def _mark_broken_links(self) -> None:
+        """Fill ``Document.broken_internal_links`` from crawled status codes.
+
+        Runs once after the crawl. An internal link is broken when the page
+        it points to was crawled and came back with a 4xx/5xx status. External
+        links are never fetched, so ``broken_external_links`` stays empty.
+        Only retained documents are marked; with ``retain_documents=False``
+        (streaming or ``flush_to`` crawls) there is nothing to mark.
+        """
+        status_by_url = {doc.url: doc.status_code for doc in self.documents}
+        for doc in self.documents:
+            doc.broken_internal_links = [
+                BrokenLink(link.url, status_by_url[link.url])
+                for link in doc.internal_links
+                if status_by_url.get(link.url, 0) >= 400
+            ]
+
     async def _seed_from_sitemaps(self, crawler: Crawler) -> None:
         """Seed the queue with URLs discovered from the site's sitemaps.
 
@@ -201,7 +274,9 @@ async def _seed_from_sitemaps(self, crawler: Crawler) -> None:
         seeded = 0
         async with self.lock:
             for url in urls:
-                norm = Crawler.normalize_url(url)
+                norm = Crawler.normalize_url(
+                    url, keep_hash_routes=self.keep_hash_routes
+                )
                 if norm in self.discovered:
                     continue
                 self.discovered.add(norm)
@@ -239,6 +314,15 @@ async def crawl_and_collect(
                     if self.accumulate_results:
                         self.accumulated_results.append(result)
 
+                if self._flush_file is not None:
+                    await self._flush_document(doc)
+
+                if not self.retain_documents:
+                    # Free the heavy parts now; the Document object itself is
+                    # only referenced by the caller's callback (if any).
+                    doc.source = None
+                    doc.dom = None
+
                 # Add internal links to the queue
                 for link in doc.internal_links:
                     async with self.lock:
@@ -260,9 +344,16 @@ async def crawl_and_collect(
             self.in_progress.discard(url)
 
     def track_visits(self, url: str, doc: Document) -> None:
-        self.documents.append(doc)
+        if self.retain_documents:
+            self.documents.append(doc)
         self.visited.add(url)
 
+    async def _flush_document(self, doc: Document) -> None:
+        """Append ``doc`` as one JSON line to the ``flush_to`` file."""
+        line = json.dumps(Serializers.document_to_dict(doc, self.flush_html))
+        async with self.lock:
+            await self._flush_file.write(line + "\n")
+
 
 if __name__ == "__main__":
     test_uri = "https://cnn.com/"
diff --git a/linktrace/__init__.py b/linktrace/__init__.py
index bc85d4c..8788a90 100644
--- a/linktrace/__init__.py
+++ b/linktrace/__init__.py
@@ -2,6 +2,17 @@
 
 from linktrace.cache import ResponseCache
 from linktrace.Crawler import BrokenLink, Crawler, CrawlException, Document, HtmlLink
+from linktrace.fetchers import (
+    BrowserFetcher,
+    FetchContext,
+    Fetcher,
+    FetchError,
+    FetchResult,
+    HttpFetcher,
+    RoutingFetcher,
+    TransientFetchError,
+    looks_like_spa_shell,
+)
 from linktrace.rules import CrawlRules
 from linktrace.Serializers import Serializers
 from linktrace.sitemap import SitemapParser
@@ -9,13 +20,22 @@
 
 __all__ = [
     "BrokenLink",
+    "BrowserFetcher",
     "Crawler",
     "CrawlException",
     "CrawlRules",
     "Document",
+    "FetchContext",
+    "Fetcher",
+    "FetchError",
+    "FetchResult",
     "HtmlLink",
+    "HttpFetcher",
     "ResponseCache",
+    "RoutingFetcher",
     "Serializers",
     "SitemapParser",
     "Spider",
+    "TransientFetchError",
+    "looks_like_spa_shell",
 ]
diff --git a/linktrace/cache.py b/linktrace/cache.py
index b1a2bfc..8d36ec3 100644
--- a/linktrace/cache.py
+++ b/linktrace/cache.py
@@ -10,11 +10,16 @@ class CachedResponse:
     """Cached HTTP response data."""
 
     def __init__(
-        self, status_code: int, response_headers: dict[str, str], content: str
+        self,
+        status_code: int,
+        response_headers: dict[str, str],
+        content: str,
+        fetcher: str = "http",
     ) -> None:
         self.status_code = status_code
         self.response_headers = response_headers
         self.content = content
+        self.fetcher = fetcher
 
 
 class ResponseCache:
@@ -31,24 +36,34 @@ def __init__(self, cache_dir: str, ttl_seconds: int = 86400) -> None:
         self.cache_dir.mkdir(parents=True, exist_ok=True)
         self.ttl = ttl_seconds
 
-    def _url_hash(self, url: str) -> str:
-        """Generate cache file name from URL."""
-        return hashlib.sha256(url.encode()).hexdigest()[:16]
+    def _url_hash(self, url: str, namespace: str | None = None) -> str:
+        """Generate cache file name from URL (and optional namespace).
 
-    def _cache_path(self, url: str) -> Path:
+        With no namespace the hash is of the bare URL, so cache directories
+        written by earlier versions keep working. A namespace (e.g. the name of
+        a non-default fetcher) yields a distinct entry for the same URL.
+        """
+        key = url if namespace is None else f"{namespace}:{url}"
+        return hashlib.sha256(key.encode()).hexdigest()[:16]
+
+    def _cache_path(self, url: str, namespace: str | None = None) -> Path:
         """Get cache file path for URL."""
-        return self.cache_dir / f"{self._url_hash(url)}.json"
+        return self.cache_dir / f"{self._url_hash(url, namespace)}.json"
 
-    async def get(self, url: str) -> CachedResponse | None:
+    async def get(
+        self, url: str, *, namespace: str | None = None
+    ) -> CachedResponse | None:
         """Retrieve cached response if not expired.
 
         Args:
             url: URL to retrieve from cache
+            namespace: Optional namespace (e.g. fetcher name) the entry was
+                stored under
 
         Returns:
             CachedResponse if found and not expired, None otherwise
         """
-        cache_file = self._cache_path(url)
+        cache_file = self._cache_path(url, namespace)
         if not cache_file.exists():
             return None
 
@@ -66,6 +81,7 @@ async def get(self, url: str) -> CachedResponse | None:
                 status_code=data["status_code"],
                 response_headers=data["response_headers"],
                 content=data["content"],
+                fetcher=data.get("fetcher", "http"),
             )
         except (json.JSONDecodeError, KeyError):
             # Cache file corrupted, remove it
@@ -73,7 +89,14 @@ async def get(self, url: str) -> CachedResponse | None:
             return None
 
     async def set(
-        self, url: str, status_code: int, headers: dict[str, str], content: str
+        self,
+        url: str,
+        status_code: int,
+        headers: dict[str, str],
+        content: str,
+        *,
+        namespace: str | None = None,
+        fetcher: str = "http",
     ) -> None:
         """Store response in cache.
 
@@ -82,8 +105,10 @@ async def set(
             status_code: HTTP status code
             headers: Response headers dict
             content: Response body text
+            namespace: Optional namespace (e.g. fetcher name) to store under
+            fetcher: Name of the fetcher that produced ``content``
         """
-        cache_file = self._cache_path(url)
+        cache_file = self._cache_path(url, namespace)
 
         data = {
             "url": url,
@@ -91,6 +116,7 @@ async def set(
             "status_code": status_code,
             "response_headers": headers,
             "content": content,
+            "fetcher": fetcher,
         }
 
         try:
diff --git a/linktrace/fetchers/__init__.py b/linktrace/fetchers/__init__.py
new file mode 100644
index 0000000..1b5bf8b
--- /dev/null
+++ b/linktrace/fetchers/__init__.py
@@ -0,0 +1,24 @@
+"""Pluggable page fetchers: plain HTTP, headless browser, or routed per URL."""
+
+from linktrace.fetchers.base import (
+    FetchContext,
+    Fetcher,
+    FetchError,
+    FetchResult,
+    TransientFetchError,
+)
+from linktrace.fetchers.browser import BrowserFetcher
+from linktrace.fetchers.http import HttpFetcher
+from linktrace.fetchers.routing import RoutingFetcher, looks_like_spa_shell
+
+__all__ = [
+    "BrowserFetcher",
+    "FetchContext",
+    "FetchError",
+    "FetchResult",
+    "Fetcher",
+    "HttpFetcher",
+    "RoutingFetcher",
+    "TransientFetchError",
+    "looks_like_spa_shell",
+]
diff --git a/linktrace/fetchers/base.py b/linktrace/fetchers/base.py
new file mode 100644
index 0000000..6dd6c2e
--- /dev/null
+++ b/linktrace/fetchers/base.py
@@ -0,0 +1,113 @@
+"""The pluggable "get me the page" step of a crawl.
+
+A :class:`Fetcher` turns a URL into a :class:`FetchResult` (status, headers,
+body). Everything else the :class:`~linktrace.Crawler.Crawler` does around that
+step (robots.txt, per-domain rate limiting, retries, caching, link parsing)
+is transport-agnostic, so swapping the fetcher is the only change needed to
+crawl sites that plain HTTP cannot handle, such as JavaScript-rendered SPAs.
+
+Built-in fetchers:
+
+- :class:`~linktrace.fetchers.http.HttpFetcher` (default) uses aiohttp.
+- :class:`~linktrace.fetchers.browser.BrowserFetcher` renders pages in a
+  headless browser via Playwright.
+- :class:`~linktrace.fetchers.routing.RoutingFetcher` tries HTTP first and
+  escalates to the browser only for pages that look like SPA shells.
+
+Writing your own is a matter of subclassing :class:`Fetcher` and implementing
+:meth:`Fetcher.fetch`.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import aiohttp
+
+
+@dataclass
+class FetchResult:
+    """What a fetcher hands back to the Crawler for one URL.
+
+    Attributes:
+        url: The URL that was requested.
+        status: HTTP status code of the final response.
+        headers: Response headers.
+        body: Page body as text, or ``None`` when the status was not 200.
+        fetcher: Name of the fetcher that produced the body (``"http"``,
+            ``"browser"``, ...). Recorded on the resulting ``Document``.
+    """
+
+    url: str
+    status: int
+    headers: dict[str, str] = field(default_factory=dict)
+    body: str | None = None
+    fetcher: str = "http"
+
+
+@dataclass
+class FetchContext:
+    """Crawler-owned resources and settings handed to a fetcher on ``open``.
+
+    Attributes:
+        session: The Crawler's shared aiohttp session (connection pool, cookie
+            jar, SSL context). Fetchers that speak plain HTTP should use it.
+        request_timeout: Per-request timeout in seconds.
+        user_agent: User-Agent string the crawl identifies itself with.
+        ssl_verify: The Crawler's SSL setting (``True``, ``False`` or a CA path).
+    """
+
+    session: aiohttp.ClientSession
+    request_timeout: float = 30
+    user_agent: str = "linktrace/0.1.0"
+    ssl_verify: bool | str = True
+
+
+class FetchError(Exception):
+    """A fetch failed in a way that retrying will not fix."""
+
+
+class TransientFetchError(FetchError):
+    """A fetch failed in a way that may succeed on retry (timeout, reset).
+
+    The Crawler retries these with exponential backoff up to ``max_retries``.
+    """
+
+
+class Fetcher(ABC):
+    """Base class for fetchers.
+
+    Subclasses implement :meth:`fetch`. :meth:`open` and :meth:`close` are
+    called once per crawl by the Crawler's async context manager and default
+    to no-ops; override them to acquire and release resources (a browser, a
+    connection pool, ...).
+
+    Attributes:
+        name: Short identifier recorded on ``FetchResult.fetcher`` and used to
+            namespace cache entries. The default HTTP fetcher is ``"http"``.
+    """
+
+    name: str = "custom"
+
+    async def open(self, ctx: FetchContext) -> None:  # noqa: B027 - opt-in hook
+        """Acquire resources. Called once when the Crawler is entered."""
+
+    async def close(self) -> None:  # noqa: B027 - opt-in hook
+        """Release resources. Called once when the Crawler is exited."""
+
+    @abstractmethod
+    async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
+        """Fetch ``url`` and return a :class:`FetchResult`.
+
+        Args:
+            url: Absolute URL to fetch.
+            headers: Request headers the Crawler wants sent (at minimum
+                ``User-Agent``).
+
+        Raises:
+            TransientFetchError: for failures worth retrying.
+            FetchError: for failures that are not.
+        """
diff --git a/linktrace/fetchers/browser.py b/linktrace/fetchers/browser.py
new file mode 100644
index 0000000..77aaf94
--- /dev/null
+++ b/linktrace/fetchers/browser.py
@@ -0,0 +1,217 @@
+"""Headless-browser fetcher built on Playwright, for JavaScript-rendered sites.
+
+Requires the optional ``browser`` extra::
+
+    pip install "linktrace[browser]"
+    playwright install chromium
+
+Playwright is imported lazily inside :meth:`BrowserFetcher.open`, so importing
+this module (or ``linktrace`` itself) never requires it.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Iterable
+from typing import Any, Literal
+
+from linktrace.fetchers.base import (
+    FetchContext,
+    Fetcher,
+    FetchError,
+    FetchResult,
+    TransientFetchError,
+)
+
+WaitUntil = Literal["commit", "domcontentloaded", "load", "networkidle"]
+
+_INSTALL_HINT = (
+    "BrowserFetcher requires Playwright. Install it with:\n"
+    "    pip install 'linktrace[browser]'\n"
+    "    playwright install chromium"
+)
+
+
+class BrowserFetcher(Fetcher):
+    """Render each page in a headless browser and return the resulting DOM.
+
+    One browser and one browser context are shared for the whole crawl; each
+    fetch opens a fresh page (tab), navigates, waits for the page to settle,
+    serialises ``document.documentElement.outerHTML`` and closes the tab.
+    Concurrency is capped independently of the Spider's ``max_concurrency``
+    because rendered pages are far more expensive than HTTP requests.
+
+    Args:
+        browser: Playwright browser type: ``"chromium"`` (default),
+            ``"firefox"`` or ``"webkit"``.
+        headless: Run without a visible window (default ``True``).
+        wait_until: When to consider navigation finished. ``"networkidle"``
+            (default) waits until no network requests for 500ms, which is the
+            most reliable choice for SPAs. Use ``"domcontentloaded"`` plus
+            ``wait_for_selector`` for sites with long-polling connections that
+            never go idle.
+        wait_for_selector: Optional CSS selector to wait for after navigation,
+            e.g. ``"a[href]"`` or ``"#listings .card"``.
+        settle_ms: Extra fixed delay after the waits above, for apps that
+            render in a second pass. Default ``0``.
+        max_pages: Maximum tabs open at once (default ``4``).
+        timeout: Navigation timeout in seconds. Defaults to the Crawler's
+            ``request_timeout``.
+        block_resources: Playwright resource types to abort so pages render
+            faster. Default blocks images, media and fonts. Pass an empty
+            tuple to load everything.
+        viewport: ``{"width": ..., "height": ...}`` for the browser context.
+        launch_args: Extra command-line args for the browser binary.
+        ignore_https_errors: Skip TLS verification. Defaults to ``True`` only
+            when the Crawler was created with ``ssl_verify=False``.
+    """
+
+    name = "browser"
+
+    def __init__(
+        self,
+        *,
+        browser: Literal["chromium", "firefox", "webkit"] = "chromium",
+        headless: bool = True,
+        wait_until: WaitUntil = "networkidle",
+        wait_for_selector: str | None = None,
+        settle_ms: int = 0,
+        max_pages: int = 4,
+        timeout: float | None = None,
+        block_resources: Iterable[str] = ("image", "media", "font"),
+        viewport: dict[str, int] | None = None,
+        launch_args: list[str] | None = None,
+        ignore_https_errors: bool | None = None,
+    ) -> None:
+        if max_pages < 1:
+            raise ValueError(f"max_pages must be >= 1, got {max_pages}.")
+        self.browser_type = browser
+        self.headless = headless
+        self.wait_until: WaitUntil = wait_until
+        self.wait_for_selector = wait_for_selector
+        self.settle_ms = settle_ms
+        self.max_pages = max_pages
+        self.timeout = timeout
+        self.block_resources = frozenset(block_resources)
+        self.viewport = viewport
+        self.launch_args = launch_args or []
+        self.ignore_https_errors = ignore_https_errors
+
+        self._playwright: Any = None
+        self._browser: Any = None
+        self._context: Any = None
+        self._semaphore: asyncio.Semaphore | None = None
+        self._timeout_ms: float = 30_000
+        self._timeout_error: type[Exception] = Exception
+        self._playwright_error: type[Exception] = Exception
+
+    async def open(self, ctx: FetchContext) -> None:
+        try:
+            from playwright.async_api import Error as PlaywrightError
+            from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+            from playwright.async_api import async_playwright
+        except ImportError as e:  # pragma: no cover - exercised only w/o extra
+            raise ImportError(_INSTALL_HINT) from e
+
+        self._timeout_error = PlaywrightTimeoutError
+        self._playwright_error = PlaywrightError
+        self._timeout_ms = (self.timeout or ctx.request_timeout) * 1000
+
+        self._playwright = await async_playwright().start()
+        launcher = getattr(self._playwright, self.browser_type)
+        try:
+            self._browser = await launcher.launch(
+                headless=self.headless, args=self.launch_args
+            )
+        except PlaywrightError as e:
+            await self._playwright.stop()
+            self._playwright = None
+            raise RuntimeError(
+                f"Could not launch {self.browser_type}: {e}\n"
+                f"Run 'playwright install {self.browser_type}' to download it."
+            ) from e
+
+        ignore_https = (
+            self.ignore_https_errors
+            if self.ignore_https_errors is not None
+            else ctx.ssl_verify is False
+        )
+        context_kwargs: dict[str, Any] = {
+            "user_agent": ctx.user_agent,
+            "ignore_https_errors": ignore_https,
+        }
+        if self.viewport:
+            context_kwargs["viewport"] = self.viewport
+        self._context = await self._browser.new_context(**context_kwargs)
+
+        if self.block_resources:
+            blocked = self.block_resources
+
+            async def _route(route: Any) -> None:
+                if route.request.resource_type in blocked:
+                    await route.abort()
+                else:
+                    await route.continue_()
+
+            await self._context.route("**/*", _route)
+
+        self._semaphore = asyncio.Semaphore(self.max_pages)
+
+    async def close(self) -> None:
+        for closer in (self._context, self._browser):
+            if closer is not None:
+                try:
+                    await closer.close()
+                except Exception:  # pragma: no cover - best-effort teardown
+                    pass
+        if self._playwright is not None:
+            await self._playwright.stop()
+        self._context = self._browser = self._playwright = None
+        self._semaphore = None
+
+    async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
+        if self._context is None or self._semaphore is None:
+            raise RuntimeError(
+                "BrowserFetcher is not open. Use it through "
+                "'async with Crawler(...)' so open()/close() are called."
+            )
+
+        async with self._semaphore:
+            page = await self._context.new_page()
+            try:
+                # User-Agent is fixed at context level; forward anything else.
+                extra = {k: v for k, v in headers.items() if k.lower() != "user-agent"}
+                if extra:
+                    await page.set_extra_http_headers(extra)
+
+                try:
+                    response = await page.goto(
+                        url, wait_until=self.wait_until, timeout=self._timeout_ms
+                    )
+                    if self.wait_for_selector:
+                        await page.wait_for_selector(
+                            self.wait_for_selector, timeout=self._timeout_ms
+                        )
+                    if self.settle_ms:
+                        await page.wait_for_timeout(self.settle_ms)
+                    body = await page.content()
+                except self._timeout_error as e:
+                    raise TransientFetchError(f"Browser timeout: {e}") from e
+                except self._playwright_error as e:
+                    raise FetchError(f"Browser error: {e}") from e
+
+                # goto() returns None for same-document navigations; treat as OK.
+                status = response.status if response is not None else 200
+                response_headers = (
+                    await response.all_headers() if response is not None else {}
+                )
+            finally:
+                await page.close()
+
+        return FetchResult(
+            url=url,
+            status=status,
+            headers=dict(response_headers),
+            body=body if status == 200 else None,
+            fetcher=self.name,
+        )
diff --git a/linktrace/fetchers/http.py b/linktrace/fetchers/http.py
new file mode 100644
index 0000000..629e5a3
--- /dev/null
+++ b/linktrace/fetchers/http.py
@@ -0,0 +1,64 @@
+"""Plain HTTP fetcher built on aiohttp. This is the Crawler's default."""
+
+from __future__ import annotations
+
+import aiohttp
+
+from linktrace.fetchers.base import (
+    FetchContext,
+    Fetcher,
+    FetchError,
+    FetchResult,
+    TransientFetchError,
+)
+
+
+class HttpFetcher(Fetcher):
+    """Fetch pages with the Crawler's shared aiohttp session.
+
+    This reproduces the crawler's original behaviour exactly: one GET per URL,
+    body read only for 200 responses, timeouts and connection errors raised as
+    :class:`~linktrace.fetchers.base.TransientFetchError` so the Crawler retries
+    them with backoff.
+
+    Args:
+        session: Optional session to use instead of the Crawler's. Mostly
+            useful in tests; normally leave it ``None``.
+    """
+
+    name = "http"
+
+    def __init__(self, session: aiohttp.ClientSession | None = None) -> None:
+        self._session = session
+
+    async def open(self, ctx: FetchContext) -> None:
+        if self._session is None:
+            self._session = ctx.session
+
+    async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
+        if self._session is None:
+            raise RuntimeError(
+                "HttpFetcher has no session. Use it through "
+                "'async with Crawler(...)' or pass a session explicitly."
+            )
+        try:
+            async with self._session.get(url, headers=headers) as response:
+                status = response.status
+                response_headers = dict(response.headers)
+                body = await response.text() if status == 200 else None
+        except (
+            TimeoutError,
+            aiohttp.ClientConnectorError,
+            aiohttp.ServerTimeoutError,
+        ) as e:
+            raise TransientFetchError(f"{type(e).__name__}: {e}") from e
+        except aiohttp.ClientError as e:
+            raise FetchError(f"{type(e).__name__}: {e}") from e
+
+        return FetchResult(
+            url=url,
+            status=status,
+            headers=response_headers,
+            body=body,
+            fetcher=self.name,
+        )
diff --git a/linktrace/fetchers/routing.py b/linktrace/fetchers/routing.py
new file mode 100644
index 0000000..d9111d2
--- /dev/null
+++ b/linktrace/fetchers/routing.py
@@ -0,0 +1,116 @@
+"""Route each URL to a fetcher: cheap HTTP by default, a browser when needed."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Iterable
+
+from linktrace.fetchers.base import FetchContext, Fetcher, FetchResult
+from linktrace.fetchers.http import HttpFetcher
+
+# "You need to enable JavaScript to run this app." and friends.
+_NOSCRIPT_JS = re.compile(r"<noscript[^>]*>.*?javascript.*?</noscript>", re.I | re.S)
+_ANCHOR = re.compile(r"<a\s[^>]*href\s*=", re.I)
+_SCRIPT_SRC = re.compile(r"<script[^>]*\ssrc\s*=", re.I)
+# An empty mount point: <div id="root"></div>, <div id="app"></div>, ...
+_EMPTY_ROOT = re.compile(
+    r"<(?:div|main|section)\s[^>]*id\s*=\s*[\"'](?:root|app|__next|___gatsby|"
+    r"__nuxt|svelte|ember)[\"'][^>]*>\s*</(?:div|main|section)>",
+    re.I,
+)
+
+
+def looks_like_spa_shell(result: FetchResult) -> bool:
+    """Heuristic: does this HTTP response look like an un-rendered SPA shell?
+
+    Returns ``True`` when the body is a successful (200) response that shows
+    any of the classic signs of a client-rendered app:
+
+    - a ``<noscript>`` block mentioning JavaScript,
+    - an empty mount element such as ``<div id="root"></div>``,
+    - no ``<a href>`` at all but at least one external ``<script src>``.
+
+    Cheap regex checks only; no DOM parsing.
+    """
+    if result.status != 200 or not result.body:
+        return False
+    body = result.body
+    if _NOSCRIPT_JS.search(body):
+        return True
+    if _EMPTY_ROOT.search(body):
+        return True
+    if not _ANCHOR.search(body) and _SCRIPT_SRC.search(body):
+        return True
+    return False
+
+
+class RoutingFetcher(Fetcher):
+    """Pick a fetcher per URL, with optional escalation after the fact.
+
+    Every URL goes to ``default`` (plain HTTP unless told otherwise) except:
+
+    - URLs matching any regex in ``render_patterns`` or for which
+      ``route(url)`` returns ``True`` go straight to ``fallback``.
+    - After a ``default`` fetch, if ``escalate(result)`` returns ``True`` the
+      URL is fetched again with ``fallback`` and that result wins. The default
+      predicate is :func:`looks_like_spa_shell`; pass ``None`` to disable.
+
+    ``fallback`` is created lazily as a
+    :class:`~linktrace.fetchers.browser.BrowserFetcher` if not supplied, so
+    Playwright is only required when a URL actually reaches it.
+
+    Args:
+        default: Fetcher used first. Defaults to :class:`HttpFetcher`.
+        fallback: Fetcher used for routed or escalated URLs. Defaults to
+            ``BrowserFetcher()``.
+        render_patterns: Regexes; matching URLs skip ``default`` entirely.
+        route: Callable ``(url) -> bool``; ``True`` skips ``default``.
+        escalate: Callable ``(FetchResult) -> bool`` evaluated on ``default``
+            results. ``None`` disables escalation.
+    """
+
+    name = "routing"
+
+    def __init__(
+        self,
+        default: Fetcher | None = None,
+        fallback: Fetcher | None = None,
+        *,
+        render_patterns: Iterable[str] = (),
+        route: Callable[[str], bool] | None = None,
+        escalate: Callable[[FetchResult], bool] | None = looks_like_spa_shell,
+    ) -> None:
+        self.default: Fetcher = default if default is not None else HttpFetcher()
+        if fallback is None:
+            from linktrace.fetchers.browser import BrowserFetcher
+
+            fallback = BrowserFetcher()
+        self.fallback: Fetcher = fallback
+        self._render_re = [re.compile(p) for p in render_patterns]
+        self.route = route
+        self.escalate = escalate
+        self.escalations = 0
+
+    async def open(self, ctx: FetchContext) -> None:
+        await self.default.open(ctx)
+        await self.fallback.open(ctx)
+
+    async def close(self) -> None:
+        await self.fallback.close()
+        await self.default.close()
+
+    def needs_render(self, url: str) -> bool:
+        """True if ``url`` is routed directly to the fallback fetcher."""
+        if any(r.search(url) for r in self._render_re):
+            return True
+        return bool(self.route and self.route(url))
+
+    async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
+        if self.needs_render(url):
+            return await self.fallback.fetch(url, headers)
+
+        result = await self.default.fetch(url, headers)
+        if self.escalate is not None and self.escalate(result):
+            self.escalations += 1
+            return await self.fallback.fetch(url, headers)
+        return result
diff --git a/pyproject.toml b/pyproject.toml
index f118cad..5c5a97d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -41,6 +41,9 @@ pandas = ["pandas>=2.0"]
 polars = ["polars>=1.0"]
 pyarrow = ["pyarrow>=14.0"]
 serializers = ["pandas>=2.0", "polars>=1.0", "pyarrow>=14.0"]
+# Headless-browser fetcher for JavaScript-rendered sites (SPAs).
+# After installing, run: playwright install chromium
+browser = ["playwright>=1.45"]
 
 [project.urls]
 Homepage = "https://github.com/JayBaywatch/linktrace"
@@ -62,6 +65,7 @@ dev = [
     "pandas>=2.0",
     "polars>=1.0",
     "pyarrow>=14.0",
+    "playwright>=1.45",
 ]
 
 # Distribution name is "linktrace"; the import package is "linktrace" (the
diff --git a/tests/conftest.py b/tests/conftest.py
index 9cbda59..ebefa78 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,6 +1,50 @@
 """Pytest fixtures and configuration."""
 
+from collections.abc import AsyncIterator, Awaitable, Callable
+
 import pytest
+import pytest_asyncio
+from aiohttp import web
+
+SiteFactory = Callable[[dict[str, str | tuple[int, str]]], Awaitable[str]]
+
+
+@pytest_asyncio.fixture
+async def local_site() -> AsyncIterator[SiteFactory]:
+    """Factory that serves a dict of ``path -> html`` on a local port.
+
+    A value may also be ``(status, html)`` to serve a non-200 response.
+    Returns the base URL (``http://127.0.0.1:<port>``). Used to exercise the
+    real HTTP and browser fetchers without touching the network.
+    """
+    runners: list[web.AppRunner] = []
+
+    async def serve(pages: dict[str, str | tuple[int, str]]) -> str:
+        app = web.Application()
+
+        def make_handler(value: str | tuple[int, str]):
+            status, body = value if isinstance(value, tuple) else (200, value)
+
+            async def handler(request: web.Request) -> web.Response:
+                return web.Response(text=body, status=status, content_type="text/html")
+
+            return handler
+
+        for path, value in pages.items():
+            app.router.add_get(path, make_handler(value))
+
+        runner = web.AppRunner(app)
+        await runner.setup()
+        site = web.TCPSite(runner, "127.0.0.1", 0)
+        await site.start()
+        runners.append(runner)
+        port = runner.addresses[0][1]
+        return f"http://127.0.0.1:{port}"
+
+    yield serve
+
+    for runner in runners:
+        await runner.cleanup()
 
 
 @pytest.fixture
diff --git a/tests/test_browser_fetcher.py b/tests/test_browser_fetcher.py
new file mode 100644
index 0000000..2e655f8
--- /dev/null
+++ b/tests/test_browser_fetcher.py
@@ -0,0 +1,111 @@
+"""Tests for BrowserFetcher against a local JavaScript-rendered page.
+
+Skipped automatically when Playwright or its Chromium build is not installed
+(``pip install 'linktrace[browser]' && playwright install chromium``).
+"""
+
+import pytest
+
+from linktrace import BrowserFetcher, Crawler, RoutingFetcher, Spider
+
+pytest.importorskip("playwright", reason="playwright not installed")
+
+# An SPA-style page: no anchors in the HTML, links injected by script.
+SPA_HOME = """
+<html><head><title>Listings</title></head>
+<body>
+  <noscript>You need to enable JavaScript to run this app.</noscript>
+  <div id="root"></div>
+  <script>
+    document.getElementById('root').innerHTML =
+      '<a href="/listing/1">One</a><a href="/listing/2">Two</a>' +
+      '<a href="https://other.example/x">Ext</a>';
+    document.title = 'Listings (rendered)';
+  </script>
+</body></html>
+"""
+LISTING = "<html><head><title>Listing</title></head><body><p>house</p></body></html>"
+PLAIN = '<html><head><title>Plain</title></head><body><a href="/listing/1">1</a>'
+
+
+@pytest.fixture
+async def chromium_available():
+    """Skip when the Chromium build has not been downloaded."""
+    from playwright.async_api import Error, async_playwright
+
+    async with async_playwright() as pw:
+        try:
+            browser = await pw.chromium.launch(headless=True)
+        except Error as e:
+            pytest.skip(f"Chromium not installed: {str(e).splitlines()[0]}")
+        await browser.close()
+
+
+@pytest.mark.usefixtures("chromium_available")
+class TestBrowserFetcher:
+    async def test_renders_javascript_links(self, local_site):
+        base = await local_site({"/": SPA_HOME, "/listing/1": LISTING})
+
+        # Plain HTTP sees an empty shell...
+        async with Crawler(respect_robots_txt=False) as crawler:
+            plain = await crawler.crawl_document_async(f"{base}/")
+        assert plain is not None
+        assert plain.internal_links == []
+        assert plain.fetcher == "http"
+
+        # ...the browser sees the rendered DOM.
+        async with Crawler(
+            fetcher=BrowserFetcher(), respect_robots_txt=False
+        ) as crawler:
+            doc = await crawler.crawl_document_async(f"{base}/")
+        assert doc is not None
+        assert doc.fetcher == "browser"
+        assert doc.status_code == 200
+        assert doc.title == "Listings (rendered)"
+        assert [link.url for link in doc.internal_links] == [
+            f"{base}/listing/1",
+            f"{base}/listing/2",
+        ]
+        assert [link.url for link in doc.external_links] == ["https://other.example/x"]
+
+    async def test_non_200_status_propagates(self, local_site):
+        base = await local_site({"/missing": (404, "<h1>nope</h1>")})
+        async with Crawler(
+            fetcher=BrowserFetcher(), respect_robots_txt=False, max_retries=1
+        ) as crawler:
+            doc = await crawler.crawl_document_async(f"{base}/missing")
+        assert doc is not None
+        assert doc.status_code == 404
+        assert doc.source is None
+
+    async def test_wait_for_selector(self, local_site):
+        base = await local_site({"/": SPA_HOME})
+        fetcher = BrowserFetcher(
+            wait_until="domcontentloaded", wait_for_selector="a[href]"
+        )
+        async with Crawler(fetcher=fetcher, respect_robots_txt=False) as crawler:
+            doc = await crawler.crawl_document_async(f"{base}/")
+        assert doc is not None and len(doc.internal_links) == 2
+
+    async def test_routing_escalates_only_spa_pages(self, local_site):
+        base = await local_site(
+            {"/": SPA_HOME, "/listing/1": LISTING, "/listing/2": LISTING}
+        )
+        routing = RoutingFetcher()
+        spider = Spider(
+            f"{base}/", max_depth=1, fetcher=routing, respect_robots_txt=False
+        )
+        docs = await spider.run_async()
+        by_url = {d.url: d for d in docs}
+        assert by_url[f"{base}/"].fetcher == "browser"
+        assert by_url[f"{base}/listing/1"].fetcher == "http"
+        assert by_url[f"{base}/listing/2"].fetcher == "http"
+        assert routing.escalations == 1
+
+    async def test_unopened_fetcher_raises(self):
+        with pytest.raises(RuntimeError, match="not open"):
+            await BrowserFetcher().fetch("https://x.test/", {})
+
+    def test_max_pages_validated(self):
+        with pytest.raises(ValueError):
+            BrowserFetcher(max_pages=0)
diff --git a/tests/test_fetchers.py b/tests/test_fetchers.py
new file mode 100644
index 0000000..3d71545
--- /dev/null
+++ b/tests/test_fetchers.py
@@ -0,0 +1,447 @@
+"""Tests for the pluggable fetcher layer (HTTP, routing, custom fetchers)."""
+
+from unittest.mock import patch
+
+import pytest
+
+from linktrace import (
+    Crawler,
+    FetchContext,
+    Fetcher,
+    FetchError,
+    FetchResult,
+    HttpFetcher,
+    RoutingFetcher,
+    Spider,
+    TransientFetchError,
+    looks_like_spa_shell,
+)
+
+SPA_SHELL = """
+<html><head><title>App</title></head>
+<body>
+  <noscript>You need to enable JavaScript to run this app.</noscript>
+  <div id="root"></div>
+  <script src="/static/bundle.js"></script>
+</body></html>
+"""
+
+RENDERED = """
+<html><head><title>App</title></head>
+<body><div id="root"><a href="/listing/1">One</a><a href="/listing/2">Two</a></div>
+</body></html>
+"""
+
+PLAIN = '<html><head><title>Plain</title></head><body><a href="/b">B</a></body></html>'
+
+
+class FakeFetcher(Fetcher):
+    """Scripted fetcher: ``responses`` maps url -> FetchResult or Exception."""
+
+    name = "fake"
+
+    def __init__(self, responses: dict, *, default_status: int = 404) -> None:
+        self.responses = responses
+        self.default_status = default_status
+        self.calls: list[str] = []
+        self.opened = False
+        self.closed = False
+
+    async def open(self, ctx: FetchContext) -> None:
+        self.opened = True
+        self.ctx = ctx
+
+    async def close(self) -> None:
+        self.closed = True
+
+    async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
+        self.calls.append(url)
+        value = self.responses.get(url)
+        if isinstance(value, Exception):
+            raise value
+        if isinstance(value, list):  # sequence of outcomes, one per call
+            value = value.pop(0)
+            if isinstance(value, Exception):
+                raise value
+        if value is None:
+            return FetchResult(url=url, status=self.default_status, fetcher=self.name)
+        if isinstance(value, FetchResult):
+            return value
+        return FetchResult(url=url, status=200, body=value, fetcher=self.name)
+
+
+# --------------------------------------------------------------------------
+# Crawler <-> Fetcher contract
+# --------------------------------------------------------------------------
+
+
+class TestCrawlerUsesFetcher:
+    def test_default_fetcher_is_http(self):
+        assert isinstance(Crawler().fetcher, HttpFetcher)
+        assert Crawler().fetcher.name == "http"
+
+    async def test_open_and_close_called_with_context(self):
+        fetcher = FakeFetcher({})
+        async with Crawler(
+            fetcher=fetcher, user_agent="Bot/2", request_timeout=7, ssl_verify=False
+        ) as crawler:
+            assert fetcher.opened
+            assert fetcher.ctx.session is crawler.session
+            assert fetcher.ctx.user_agent == "Bot/2"
+            assert fetcher.ctx.request_timeout == 7
+            assert fetcher.ctx.ssl_verify is False
+        assert fetcher.closed
+
+    async def test_session_closed_if_open_fails(self):
+        class Broken(FakeFetcher):
+            async def open(self, ctx: FetchContext) -> None:
+                raise RuntimeError("no browser")
+
+        crawler = Crawler(fetcher=Broken({}))
+        with pytest.raises(RuntimeError, match="no browser"):
+            async with crawler:
+                pass
+        assert crawler.session is None
+
+    async def test_document_built_from_fetch_result(self):
+        fetcher = FakeFetcher({"https://x.test/": PLAIN})
+        async with Crawler(fetcher=fetcher, respect_robots_txt=False) as crawler:
+            doc = await crawler.crawl_document_async("https://x.test/")
+        assert doc is not None
+        assert doc.title == "Plain"
+        assert doc.status_code == 200
+        assert doc.fetcher == "fake"
+        assert [link.url for link in doc.internal_links] == ["https://x.test/b"]
+        assert fetcher.calls == ["https://x.test/"]
+
+    async def test_user_agent_header_passed_to_fetcher(self):
+        seen: dict[str, dict[str, str]] = {}
+
+        class Recording(FakeFetcher):
+            async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
+                seen[url] = headers
+                return await super().fetch(url, headers)
+
+        fetcher = Recording({"https://x.test/": PLAIN})
+        async with Crawler(
+            fetcher=fetcher, user_agent="Bot/9", respect_robots_txt=False
+        ) as crawler:
+            await crawler.crawl_document_async("https://x.test/")
+        assert seen["https://x.test/"]["User-Agent"] == "Bot/9"
+
+    async def test_non_200_returns_document_without_source(self):
+        fetcher = FakeFetcher({}, default_status=404)
+        async with Crawler(fetcher=fetcher, respect_robots_txt=False) as crawler:
+            doc = await crawler.crawl_document_async("https://x.test/missing")
+        assert doc is not None
+        assert doc.status_code == 404
+        assert doc.source is None
+        assert doc.fetcher == "fake"
+
+    async def test_transient_error_is_retried_then_succeeds(self):
+        fetcher = FakeFetcher(
+            {"https://x.test/": [TransientFetchError("reset"), PLAIN]}
+        )
+        with patch("asyncio.sleep") as sleep:
+            async with Crawler(
+                fetcher=fetcher, respect_robots_txt=False, max_retries=3
+            ) as crawler:
+                doc = await crawler.crawl_document_async("https://x.test/")
+        assert doc is not None and doc.title == "Plain"
+        assert len(fetcher.calls) == 2
+        assert sleep.call_count == 1
+
+    async def test_transient_error_exhausts_retries_returns_none(self):
+        fetcher = FakeFetcher({"https://x.test/": TransientFetchError("timeout")})
+        with patch("asyncio.sleep") as sleep:
+            async with Crawler(
+                fetcher=fetcher, respect_robots_txt=False, max_retries=3
+            ) as crawler:
+                doc = await crawler.crawl_document_async("https://x.test/")
+        assert doc is None
+        assert len(fetcher.calls) == 3
+        assert sleep.call_count == 2
+
+    async def test_fetch_error_is_not_retried(self):
+        fetcher = FakeFetcher({"https://x.test/": FetchError("bad request")})
+        async with Crawler(
+            fetcher=fetcher, respect_robots_txt=False, max_retries=3
+        ) as crawler:
+            doc = await crawler.crawl_document_async("https://x.test/")
+        assert doc is None
+        assert len(fetcher.calls) == 1
+
+    async def test_5xx_is_retried_with_backoff(self):
+        """Docs promise 5xx retries; the last 5xx is returned if none succeed."""
+        fetcher = FakeFetcher(
+            {
+                "https://x.test/": [
+                    FetchResult("https://x.test/", 503, fetcher="fake"),
+                    FetchResult("https://x.test/", 503, fetcher="fake"),
+                    PLAIN,
+                ]
+            }
+        )
+        with patch("asyncio.sleep") as sleep:
+            async with Crawler(
+                fetcher=fetcher, respect_robots_txt=False, max_retries=3
+            ) as crawler:
+                doc = await crawler.crawl_document_async("https://x.test/")
+        assert doc is not None and doc.status_code == 200
+        assert len(fetcher.calls) == 3
+        assert sleep.call_count == 2
+
+    async def test_5xx_after_all_retries_returns_last_status(self):
+        fetcher = FakeFetcher({}, default_status=500)
+        with patch("asyncio.sleep"):
+            async with Crawler(
+                fetcher=fetcher, respect_robots_txt=False, max_retries=2
+            ) as crawler:
+                doc = await crawler.crawl_document_async("https://x.test/")
+        assert doc is not None and doc.status_code == 500
+        assert len(fetcher.calls) == 2
+
+
+# --------------------------------------------------------------------------
+# Cache namespacing
+# --------------------------------------------------------------------------
+
+
+class TestCacheNamespacing:
+    async def test_http_fetcher_uses_legacy_cache_key(self, tmp_path):
+        """Default fetcher keeps the pre-existing on-disk key layout."""
+        from linktrace.cache import ResponseCache
+
+        legacy = ResponseCache(str(tmp_path))
+        await legacy.set("https://x.test/", 200, {}, PLAIN)
+
+        fetcher = FakeFetcher({})
+        fetcher.name = "http"  # pretend to be the default fetcher
+        async with Crawler(
+            fetcher=fetcher, cache_dir=str(tmp_path), respect_robots_txt=False
+        ) as crawler:
+            doc = await crawler.crawl_document_async("https://x.test/")
+        assert doc is not None and doc.title == "Plain"
+        assert fetcher.calls == []  # served from cache
+
+    async def test_non_default_fetcher_does_not_read_http_cache(self, tmp_path):
+        from linktrace.cache import ResponseCache
+
+        legacy = ResponseCache(str(tmp_path))
+        await legacy.set("https://x.test/", 200, {}, SPA_SHELL)
+
+        fetcher = FakeFetcher({"https://x.test/": RENDERED})
+        async with Crawler(
+            fetcher=fetcher, cache_dir=str(tmp_path), respect_robots_txt=False
+        ) as crawler:
+            doc = await crawler.crawl_document_async("https://x.test/")
+            assert doc is not None and len(doc.internal_links) == 2
+            assert fetcher.calls == ["https://x.test/"]
+
+            # Second request is served from the fetcher's own namespace and
+            # remembers which fetcher produced it.
+            doc2 = await crawler.crawl_document_async("https://x.test/")
+        assert doc2 is not None and doc2.fetcher == "fake"
+        assert fetcher.calls == ["https://x.test/"]
+
+
+# --------------------------------------------------------------------------
+# HttpFetcher against a real local server
+# --------------------------------------------------------------------------
+
+
+class TestHttpFetcher:
+    async def test_fetch_success(self, local_site):
+        base = await local_site({"/": PLAIN})
+        async with Crawler(respect_robots_txt=False) as crawler:
+            result = await crawler.fetcher.fetch(f"{base}/", {"User-Agent": "t"})
+        assert result.status == 200
+        assert result.fetcher == "http"
+        assert result.body is not None and "Plain" in result.body
+        assert "Content-Type" in result.headers
+
+    async def test_fetch_non_200_has_no_body(self, local_site):
+        base = await local_site({"/gone": (410, "<h1>gone</h1>")})
+        async with Crawler(respect_robots_txt=False) as crawler:
+            result = await crawler.fetcher.fetch(f"{base}/gone", {})
+        assert result.status == 410
+        assert result.body is None
+
+    async def test_connection_refused_is_transient(self):
+        async with Crawler(respect_robots_txt=False, max_retries=1) as crawler:
+            with pytest.raises(TransientFetchError):
+                await crawler.fetcher.fetch("http://127.0.0.1:9/", {})
+
+    async def test_unopened_fetcher_raises(self):
+        with pytest.raises(RuntimeError, match="no session"):
+            await HttpFetcher().fetch("https://x.test/", {})
+
+
+# --------------------------------------------------------------------------
+# SPA-shell heuristic
+# --------------------------------------------------------------------------
+
+
+class TestLooksLikeSpaShell:
+    def _res(self, body, status=200):
+        return FetchResult("https://x.test/", status, body=body)
+
+    def test_noscript_javascript_hint(self):
+        body = "<html><body><noscript>Enable JavaScript</noscript><a href='/a'>a</a>"
+        assert looks_like_spa_shell(self._res(body))
+
+    def test_empty_root_div(self):
+        assert looks_like_spa_shell(self._res('<div id="root"></div>'))
+        assert looks_like_spa_shell(self._res("<div id='app'>\n  </div>"))
+        assert looks_like_spa_shell(self._res('<div id="__next"></div>'))
+
+    def test_no_anchors_but_script_bundle(self):
+        assert looks_like_spa_shell(self._res('<script src="/app.js"></script>'))
+
+    def test_plain_html_is_not_a_shell(self):
+        assert not looks_like_spa_shell(self._res(PLAIN))
+
+    def test_populated_root_is_not_a_shell(self):
+        assert not looks_like_spa_shell(self._res(RENDERED))
+
+    def test_inline_script_without_bundle_is_not_a_shell(self):
+        assert not looks_like_spa_shell(self._res("<script>var x=1</script><p>hi"))
+
+    def test_non_200_or_empty_is_not_a_shell(self):
+        assert not looks_like_spa_shell(self._res(SPA_SHELL, status=404))
+        assert not looks_like_spa_shell(self._res(None))
+        assert not looks_like_spa_shell(self._res(""))
+
+
+# --------------------------------------------------------------------------
+# RoutingFetcher
+# --------------------------------------------------------------------------
+
+
+class TestRoutingFetcher:
+    def _pair(self):
+        http = FakeFetcher({"https://x.test/": SPA_SHELL, "https://x.test/p": PLAIN})
+        http.name = "http"
+        browser = FakeFetcher(
+            {"https://x.test/": RENDERED, "https://x.test/p": RENDERED}
+        )
+        browser.name = "browser"
+        return http, browser
+
+    async def test_opens_and_closes_both(self):
+        http, browser = self._pair()
+        routing = RoutingFetcher(http, browser)
+        async with Crawler(fetcher=routing, respect_robots_txt=False):
+            assert http.opened and browser.opened
+        assert http.closed and browser.closed
+
+    async def test_escalates_spa_shell_to_fallback(self):
+        http, browser = self._pair()
+        routing = RoutingFetcher(http, browser)
+        async with Crawler(fetcher=routing, respect_robots_txt=False) as crawler:
+            doc = await crawler.crawl_document_async("https://x.test/")
+        assert doc is not None
+        assert doc.fetcher == "browser"
+        assert len(doc.internal_links) == 2
+        assert http.calls == ["https://x.test/"]
+        assert browser.calls == ["https://x.test/"]
+        assert routing.escalations == 1
+
+    async def test_plain_page_stays_on_default(self):
+        http, browser = self._pair()
+        routing = RoutingFetcher(http, browser)
+        async with Crawler(fetcher=routing, respect_robots_txt=False) as crawler:
+            doc = await crawler.crawl_document_async("https://x.test/p")
+        assert doc is not None and doc.fetcher == "http"
+        assert browser.calls == []
+        assert routing.escalations == 0
+
+    async def test_render_patterns_skip_default(self):
+        http, browser = self._pair()
+        routing = RoutingFetcher(http, browser, render_patterns=[r"/p$"])
+        async with Crawler(fetcher=routing, respect_robots_txt=False) as crawler:
+            doc = await crawler.crawl_document_async("https://x.test/p")
+        assert doc is not None and doc.fetcher == "browser"
+        assert http.calls == []
+
+    async def test_route_callable_skips_default(self):
+        http, browser = self._pair()
+        routing = RoutingFetcher(http, browser, route=lambda url: url.endswith("/p"))
+        assert routing.needs_render("https://x.test/p")
+        assert not routing.needs_render("https://x.test/")
+
+    async def test_escalation_can_be_disabled(self):
+        http, browser = self._pair()
+        routing = RoutingFetcher(http, browser, escalate=None)
+        async with Crawler(fetcher=routing, respect_robots_txt=False) as crawler:
+            doc = await crawler.crawl_document_async("https://x.test/")
+        assert doc is not None and doc.fetcher == "http"
+        assert browser.calls == []
+
+    def test_defaults_to_http_and_browser(self):
+        from linktrace import BrowserFetcher
+
+        routing = RoutingFetcher()
+        assert isinstance(routing.default, HttpFetcher)
+        assert isinstance(routing.fallback, BrowserFetcher)
+
+    async def test_spider_end_to_end_with_routing(self):
+        http = FakeFetcher(
+            {"https://x.test/": SPA_SHELL, "https://x.test/listing/1": PLAIN}
+        )
+        http.name = "http"
+        browser = FakeFetcher({"https://x.test/": RENDERED})
+        browser.name = "browser"
+        spider = Spider(
+            "https://x.test/",
+            max_depth=1,
+            fetcher=RoutingFetcher(http, browser),
+            respect_robots_txt=False,
+        )
+        docs = await spider.run_async()
+        by_url = {d.url: d for d in docs}
+        assert by_url["https://x.test/"].fetcher == "browser"
+        assert by_url["https://x.test/listing/1"].fetcher == "http"
+        assert by_url["https://x.test/listing/2"].status_code == 404
+
+
+# --------------------------------------------------------------------------
+# Hash-router SPAs
+# --------------------------------------------------------------------------
+
+
+class TestHashRoutes:
+    def test_normalize_strips_fragments_by_default(self):
+        assert Crawler.normalize_url("https://x.test/#/about") == "https://x.test/"
+        assert Crawler.normalize_url("https://x.test/p#top") == "https://x.test/p"
+
+    def test_normalize_keeps_route_fragments_when_asked(self):
+        assert (
+            Crawler.normalize_url("https://x.test/#/about", keep_hash_routes=True)
+            == "https://x.test/#/about"
+        )
+        assert (
+            Crawler.normalize_url("https://x.test#!/a", keep_hash_routes=True)
+            == "https://x.test/#!/a"
+        )
+        # Plain in-page anchors are still dropped.
+        assert (
+            Crawler.normalize_url("https://x.test/p#top", keep_hash_routes=True)
+            == "https://x.test/p"
+        )
+
+    def test_parse_document_extracts_hash_routes(self):
+        html = '<a href="#/about">About</a><a href="#top">Top</a><a href="/x">X</a>'
+        default = Crawler().parse_document("https://x.test/", html)
+        assert [link.url for link in default.internal_links] == ["https://x.test/x"]
+
+        keep = Crawler(keep_hash_routes=True).parse_document("https://x.test/", html)
+        assert [link.url for link in keep.internal_links] == [
+            "https://x.test/#/about",
+            "https://x.test/x",
+        ]
+
+    def test_spider_threads_option_through(self):
+        spider = Spider("https://x.test/#/home", keep_hash_routes=True)
+        assert spider.start_url == "https://x.test/#/home"
+        assert Spider("https://x.test/#/home").start_url == "https://x.test/"
diff --git a/tests/test_rate_limiting_and_broken_links.py b/tests/test_rate_limiting_and_broken_links.py
index ff2d811..3725a98 100644
--- a/tests/test_rate_limiting_and_broken_links.py
+++ b/tests/test_rate_limiting_and_broken_links.py
@@ -165,8 +165,11 @@ async def test_http_4xx_tracked_as_broken(self):
 
     @pytest.mark.asyncio
     async def test_http_5xx_tracked_as_broken(self):
-        """HTTP 5xx responses should return Document with broken link."""
-        crawler = Crawler()
+        """HTTP 5xx responses should return Document with broken link.
+
+        5xx responses are retried with backoff, so use a single attempt here.
+        """
+        crawler = Crawler(max_retries=1)
         async with crawler:
             with patch("aiohttp.ClientSession.get") as mock_get:
                 mock_response = AsyncMock()
diff --git a/tests/test_spider_memory.py b/tests/test_spider_memory.py
new file mode 100644
index 0000000..8744a2f
--- /dev/null
+++ b/tests/test_spider_memory.py
@@ -0,0 +1,154 @@
+"""Tests for Spider.run(), broken-link marking, flushing and memory retention."""
+
+import asyncio
+import json
+
+import pytest
+
+from linktrace import FetchContext, Fetcher, FetchResult, Spider
+
+HOME = '<title>Home</title><a href="/ok">ok</a><a href="/gone">gone</a>'
+OK = "<title>OK</title>"
+
+
+class SiteFetcher(Fetcher):
+    name = "http"
+
+    def __init__(self) -> None:
+        self.pages = {
+            "https://x.test/": HOME,
+            "https://x.test/ok": OK,
+        }
+
+    async def open(self, ctx: FetchContext) -> None:
+        pass
+
+    async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
+        body = self.pages.get(url)
+        if body is None:
+            return FetchResult(url, 404, fetcher=self.name)
+        return FetchResult(url, 200, body=body, fetcher=self.name)
+
+
+def make_spider(**kw) -> Spider:
+    return Spider(
+        "https://x.test/",
+        max_depth=1,
+        fetcher=SiteFetcher(),
+        respect_robots_txt=False,
+        **kw,
+    )
+
+
+class TestRunSync:
+    def test_run_returns_documents(self):
+        docs = make_spider().run()
+        assert {d.url for d in docs} == {
+            "https://x.test/",
+            "https://x.test/ok",
+            "https://x.test/gone",
+        }
+
+    async def test_run_inside_event_loop_raises(self):
+        with pytest.raises(RuntimeError, match="run_async"):
+            make_spider().run()
+
+
+class TestBrokenLinks:
+    async def test_broken_internal_links_marked_after_crawl(self):
+        docs = await make_spider().run_async()
+        home = next(d for d in docs if d.url == "https://x.test/")
+        assert [(b.url, b.status_code) for b in home.broken_internal_links] == [
+            ("https://x.test/gone", 404)
+        ]
+        assert home.broken_external_links == []
+        ok = next(d for d in docs if d.url == "https://x.test/ok")
+        assert ok.broken_internal_links == []
+
+
+class TestRetention:
+    def test_defaults(self):
+        assert make_spider().retain_documents is True
+        assert make_spider(on_page_crawled=lambda d: None).retain_documents is False
+        assert (
+            make_spider(
+                on_page_crawled=lambda d: None, accumulate_results=True
+            ).retain_documents
+            is True
+        )
+        assert make_spider(flush_to="x.jsonl").retain_documents is False
+        assert (
+            make_spider(flush_to="x.jsonl", retain_documents=True).retain_documents
+            is True
+        )
+
+    async def test_streaming_mode_does_not_retain_documents(self):
+        seen = []
+        spider = make_spider(on_page_crawled=seen.append)
+        result = await spider.run_async()
+        assert result == []
+        assert spider.documents == []
+        assert len(seen) == 3
+        assert spider.visited_count == 3
+        # Heavy fields were released after the callback ran.
+        assert all(d.source is None and d.dom is None for d in seen)
+
+    async def test_callback_sees_source_before_release(self):
+        sources = []
+
+        async def cb(doc):
+            await asyncio.sleep(0)
+            sources.append(doc.source)
+
+        await make_spider(on_page_crawled=cb).run_async()
+        assert any(s and "Home" in s for s in sources)
+
+    async def test_explicit_retain_with_callback(self):
+        spider = make_spider(on_page_crawled=lambda d: None, retain_documents=True)
+        await spider.run_async()
+        assert len(spider.documents) == 3
+        assert any(d.source for d in spider.documents)
+
+
+class TestFlushToDisk:
+    async def test_flush_writes_one_json_line_per_document(self, tmp_path):
+        out = tmp_path / "crawl.jsonl"
+        spider = make_spider(flush_to=str(out))
+        result = await spider.run_async()
+
+        assert result == []
+        assert spider.documents == []
+        lines = [json.loads(line) for line in out.read_text().splitlines()]
+        assert {row["url"] for row in lines} == {
+            "https://x.test/",
+            "https://x.test/ok",
+            "https://x.test/gone",
+        }
+        home = next(row for row in lines if row["url"] == "https://x.test/")
+        assert home["title"] == "Home"
+        assert home["fetcher"] == "http"
+        assert {link["url"] for link in home["internal_links"]} == {
+            "https://x.test/ok",
+            "https://x.test/gone",
+        }
+        assert "html" not in home
+
+    async def test_flush_html_includes_source(self, tmp_path):
+        out = tmp_path / "crawl.jsonl"
+        await make_spider(flush_to=str(out), flush_html=True).run_async()
+        rows = [json.loads(line) for line in out.read_text().splitlines()]
+        home = next(row for row in rows if row["url"] == "https://x.test/")
+        assert "Home" in home["html"]
+
+    async def test_flush_appends_across_runs(self, tmp_path):
+        out = tmp_path / "crawl.jsonl"
+        await make_spider(flush_to=str(out)).run_async()
+        await make_spider(flush_to=str(out)).run_async()
+        assert len(out.read_text().splitlines()) == 6
+
+    async def test_flush_and_retain_together(self, tmp_path):
+        out = tmp_path / "crawl.jsonl"
+        spider = make_spider(flush_to=str(out), retain_documents=True)
+        docs = await spider.run_async()
+        assert len(docs) == 3
+        assert len(out.read_text().splitlines()) == 3
diff --git a/uv.lock b/uv.lock
index a5f1a82..59c9779 100644
--- a/uv.lock
+++ b/uv.lock
@@ -607,6 +607,73 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" },
 ]
 
+[[package]]
+name = "greenlet"
+version = "3.5.6"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/3e/6e/0091f175ccd02b02bc8811bbcbcc6ac2e980be116e3b2f7a736ca322bf84/greenlet-3.5.6.tar.gz", hash = "sha256:8e67c43bdfc88d5fee6db0d3e40175b362fc95fb85f0412d233b9b203c53a575", size = 207653, upload-time = "2026-09-14T15:42:51.806Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/72/18/3fc6d951466ae9a2a688edcddde3b2e388da0a8244e0caf7117bbeb0eb95/greenlet-3.5.6-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:a5876d0a60355af98d535c47f6cd6eb0f8a432396dab26845d380b92f8412422", size = 295668, upload-time = "2026-09-14T14:22:33.241Z" },
+    { url = "https://files.pythonhosted.org/packages/27/89/366d2af5061eeefa5012f510d95a99c8620dcc457609838db4d538820318/greenlet-3.5.6-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e85880b538e59a59f55117b81f208a6660ad5ac328aad9305f812d9b8bc67a0f", size = 611700, upload-time = "2026-09-14T15:12:01.962Z" },
+    { url = "https://files.pythonhosted.org/packages/54/1c/07f133f865fd58ae593dd2bbec3144acaee9b04ffe2eb48c6e121747ceef/greenlet-3.5.6-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f0ba7c2a329d650628f4c8572fd1db29f0a59dd70a3e3e0710dcf18a35cce9d8", size = 624223, upload-time = "2026-09-14T15:20:42.459Z" },
+    { url = "https://files.pythonhosted.org/packages/a7/f2/844dc823ff2752ad049caa6b59d57e4572f9c445934b02d3518f4c67197c/greenlet-3.5.6-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee7d9da3bf493909cf811a3f038840cb34fab5ae2956b8a263919f6e289ab188", size = 629529, upload-time = "2026-09-14T15:25:06.354Z" },
+    { url = "https://files.pythonhosted.org/packages/66/6a/1594f3869c57c149abdb380492529e04d4c0229b5e4d79572c5bd0aaa673/greenlet-3.5.6-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:975736b002ed080d124cf81a79cb7e05cb26d6b3f5c7a7b651c0fcce70353aa1", size = 621404, upload-time = "2026-09-14T14:35:59.027Z" },
+    { url = "https://files.pythonhosted.org/packages/c0/42/b1f8dbc89a53b9e77859fc1ad1627d106fc361daa3ea4bdf43a91ebb4338/greenlet-3.5.6-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:71890d5247020c25c21a6b65202782bfc281d4e6e244842419d30e3492bb6dcc", size = 432385, upload-time = "2026-09-14T15:28:37.369Z" },
+    { url = "https://files.pythonhosted.org/packages/a2/f5/33e5c9e48178b9259fd000f8f45caa4a65036f65d3d0c06a602f570f025d/greenlet-3.5.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0616b8f878098c5681fd8f0dc92d887551717402342a70f0abcbfea5f5ad8a44", size = 1584998, upload-time = "2026-09-14T15:10:06.653Z" },
+    { url = "https://files.pythonhosted.org/packages/ef/31/9b4e140bc24d0ad7927ebd651f5608b0acc2334d061748c3b6ad19085cfa/greenlet-3.5.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3dbb4596a6a4e5d47121a33ff20533a81e60f302d9e67b69909a8bc21a43f0a7", size = 1647568, upload-time = "2026-09-14T14:35:49.787Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/71/d79f1791f824f8ff15c2978746640467ae932a2365e0201069f7f272395f/greenlet-3.5.6-cp312-cp312-win_amd64.whl", hash = "sha256:7ac4abb3877c43af320392c664774eef6fa2cc063c79a55fc02d844a3cbe7395", size = 324203, upload-time = "2026-09-14T14:22:54.504Z" },
+    { url = "https://files.pythonhosted.org/packages/63/af/42aca4d56e8cb321912203069d8d34734cb288222f10ad2ae102718cc577/greenlet-3.5.6-cp312-cp312-win_arm64.whl", hash = "sha256:301102a49120b095e72a7838792b41233975fc1c155daec6d98f81c00c9280e0", size = 308310, upload-time = "2026-09-14T14:24:03.008Z" },
+    { url = "https://files.pythonhosted.org/packages/f1/a1/e720a38852366c589e1a46cf570b886507ad2cf591050c203365638baab0/greenlet-3.5.6-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:f96f0e30b5a95c7631b12bfe214cbc90ec8fe8cfa36920596c10514a65743519", size = 294627, upload-time = "2026-09-14T14:24:40.102Z" },
+    { url = "https://files.pythonhosted.org/packages/eb/c3/58187858df41354a11e6a55b421e7af9059798abdab3a384cc51b8567c38/greenlet-3.5.6-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c75116c9de79949de23006e2d9b35ee82874c594fcf5c0311b439acaa14b8441", size = 614356, upload-time = "2026-09-14T15:12:03.399Z" },
+    { url = "https://files.pythonhosted.org/packages/ce/b9/3a7e67d5f05c9760b1ad411fa52264bd69cc08e22a2ebfb4018b90628ced/greenlet-3.5.6-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cad5782f93f7f738b62c6527b6f32a60694d924029f299a8b524758cfa53d815", size = 626756, upload-time = "2026-09-14T15:20:44.269Z" },
+    { url = "https://files.pythonhosted.org/packages/c6/7c/40400455f5b5a65bb83e94fde66d1be9e5ec518638113f8083ace746c309/greenlet-3.5.6-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a93ee7c6e8fd0f8a83525a51bd777be57ee17787e91d805bd8d6faf9dcada18e", size = 632632, upload-time = "2026-09-14T15:25:07.813Z" },
+    { url = "https://files.pythonhosted.org/packages/85/cb/ab0c123c514ed4e94c0dc9ee2e86362633e6b998cfc05de7fc9ac2eb9690/greenlet-3.5.6-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f98e8215e172f567ce80eeaed9107fb4d32b6c44f26983d9b8334658136a205a", size = 623779, upload-time = "2026-09-14T14:36:01.104Z" },
+    { url = "https://files.pythonhosted.org/packages/f9/67/1f35cff30a6c51c3f23b63d4afcc7313ab4f97490ba3676fa78178984b27/greenlet-3.5.6-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:7f731ebac68ea06d628658295cb2d217b10186329fcf9a3b6a149045059bf92e", size = 434933, upload-time = "2026-09-14T15:28:38.858Z" },
+    { url = "https://files.pythonhosted.org/packages/a5/26/fda8a5a06e7073333ccb038133c5893b9e0c4fe29d5992a17e83c241bc6e/greenlet-3.5.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:df19e2d0b1620039af5102563fbd96e8938c7f5c3f5828528d641d9fc585525e", size = 1584930, upload-time = "2026-09-14T15:10:08.234Z" },
+    { url = "https://files.pythonhosted.org/packages/2f/37/50f8813163148d6234e08b23dcad6a9e37f01d148c8ec976e4c44ea2d918/greenlet-3.5.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:06c0e933290fba8ffe53ead4ae1b8044b0e9754b75cebf381aa2bc3e50d82fac", size = 1647590, upload-time = "2026-09-14T14:35:51.173Z" },
+    { url = "https://files.pythonhosted.org/packages/86/da/b7669b09586365654083a62bd0724cf06cb74bd5085a15cdd161271f992f/greenlet-3.5.6-cp313-cp313-win_amd64.whl", hash = "sha256:5b602b4201b965a8354d74e232364a66ff243dd142e350d035f46169bb36e13d", size = 324086, upload-time = "2026-09-14T14:23:48.428Z" },
+    { url = "https://files.pythonhosted.org/packages/e5/5d/c9663cfe84a2a9e0aa96f066f5b0594c227ea4c647511e087e2e11d4ac0a/greenlet-3.5.6-cp313-cp313-win_arm64.whl", hash = "sha256:876077e7ebb8c84ed068e2b23d4c62ebb010d60df84b9591af1be2f39010ffb2", size = 308211, upload-time = "2026-09-14T14:28:01.634Z" },
+    { url = "https://files.pythonhosted.org/packages/66/c0/d254544ae2b8bdd311aef000fafc02828c2771b17d994b3075620ea7cc6e/greenlet-3.5.6-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8cddea1b8339451c2fb3388e138347b6126744f33b611bdb55b7357361cfef46", size = 295221, upload-time = "2026-09-14T14:25:11.583Z" },
+    { url = "https://files.pythonhosted.org/packages/18/18/eb54be16b9cc3971e09ca5b73334e1b8c804a4630d9addaaf218a4fe300f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c59acfa8eb73a1e0d484392dc002bdf001fd4ce73394e0132df3d1ab6093d7cb", size = 660992, upload-time = "2026-09-14T15:12:04.876Z" },
+    { url = "https://files.pythonhosted.org/packages/8f/b4/e193efe65671dcf294bc51fcc59efb52d154adf8612c4ea016da0d2c486c/greenlet-3.5.6-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3b4a01c6da07ef9f80d4fe8933b994bc99747bcea3eab0330a9c34d3c12655b", size = 673428, upload-time = "2026-09-14T15:20:45.756Z" },
+    { url = "https://files.pythonhosted.org/packages/fd/21/631bb45fafde1dca782152377c0676d182ec924820064047f533a3627b28/greenlet-3.5.6-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dd0b83bed3405b586a3133629f1d1a5bc7bfd64822a3b7ab342bdc68e6dbc61b", size = 677688, upload-time = "2026-09-14T15:25:09.279Z" },
+    { url = "https://files.pythonhosted.org/packages/45/ac/28fa7a9e50f2859466214c4ac584d776db52c1604ad4dd158960a5af2a1f/greenlet-3.5.6-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9a09d59bef1db94f384b5bcc2d523694d338f3df6b757aeeaf7baca5d0c0be88", size = 670773, upload-time = "2026-09-14T14:36:02.577Z" },
+    { url = "https://files.pythonhosted.org/packages/40/30/2b0a73e68e1e18e30b601d0d183cfdfc2beca4de5a6843c630f0fc9fb90c/greenlet-3.5.6-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:fdacf26402389bdd89857ad3c045a26fe8f3314f9a8b28226f82f88463a65b77", size = 480475, upload-time = "2026-09-14T15:28:40.741Z" },
+    { url = "https://files.pythonhosted.org/packages/c3/cd/fb7d6cdd86ff3427c1494854f0e35437eba05142be91f530f6da75e09e19/greenlet-3.5.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8b7c73d1cef3d9ae963e9ff03f6222df43efbb9054ffd2f1969c935b7fc84c02", size = 1631900, upload-time = "2026-09-14T15:10:09.745Z" },
+    { url = "https://files.pythonhosted.org/packages/f6/40/143bdbb20a516628cb15074ae52ed17d850b450292609c7a6fccac6dbece/greenlet-3.5.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8b27df301f56e3b3d2298095c8f7d6b68f2521f6b1693e901fa039bdbae34424", size = 1693740, upload-time = "2026-09-14T14:35:52.959Z" },
+    { url = "https://files.pythonhosted.org/packages/c9/9e/019642432e6ae283301df1361227d47610709d2dc69a38f95edef266d713/greenlet-3.5.6-cp314-cp314-win_amd64.whl", hash = "sha256:f8f0bd690e1a41294ac87905e8121c81a3761ec2583c768f13467428606c8c7a", size = 327473, upload-time = "2026-09-14T14:28:12.948Z" },
+    { url = "https://files.pythonhosted.org/packages/e9/7f/8aafc7bf70c948786dba7221d0dc0838e5329bebc6d434ef2208b4f0e760/greenlet-3.5.6-cp314-cp314-win_arm64.whl", hash = "sha256:8cda13494d86a4f12429641117cb6ac4bbbc9c30a33f711f7d3a2e5fbe4b0b7e", size = 311095, upload-time = "2026-09-14T14:28:00.7Z" },
+    { url = "https://files.pythonhosted.org/packages/14/7e/7a205688a5b3074933b18a906608d46d106e9a79d776bdab5a4abf4b4feb/greenlet-3.5.6-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:97c5a53e8c1754df58e73f047a99e287d4da1bdfe64b0072fb25c87000897951", size = 305352, upload-time = "2026-09-14T14:21:31.962Z" },
+    { url = "https://files.pythonhosted.org/packages/78/cb/9c4a57a9d9dd0256e20b8f7f4f06554c2c92badebf0ab73ce344321b78b9/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fea4427d1ffdb3b523d7daa6712038428a4c16c450b9777bdd1221cfee0eab49", size = 672671, upload-time = "2026-09-14T15:12:06.347Z" },
+    { url = "https://files.pythonhosted.org/packages/97/52/c6729681ebbd298f4decd28746815acc8a0b0a0fde21d2df33776fd4d042/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73a29b5ba642e35433166a03a3e02935e7238c4b3467fbd77523b99edea23e5b", size = 679489, upload-time = "2026-09-14T15:20:47.291Z" },
+    { url = "https://files.pythonhosted.org/packages/71/76/3c11c21e0716b1f1dc7c1a4b3d690abb1d3b448c69a9d32049fecb64010a/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:61a61b4a95a4f97922c3a6f5606d3e360851584bd47e500a5161373c53810e3d", size = 681303, upload-time = "2026-09-14T15:25:11.088Z" },
+    { url = "https://files.pythonhosted.org/packages/58/c5/2b6c721ba8b8963da42d5a0f57f25b8aaeb1fe9bdd156875e57f3be648a2/greenlet-3.5.6-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:460e70b033aba8ed47e2ac9b5d0d2157b05a34fbfa30a241400aef4118902cdc", size = 676608, upload-time = "2026-09-14T14:36:03.959Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/26/3ae402202452cd5941bbbd483e5a74297e2397e7aa3182c2a5e3ab7d5666/greenlet-3.5.6-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:fe3170a69fe039b18ad18171e66faa9a75f6fe9d78f968fd9b54e09fbd714d81", size = 510112, upload-time = "2026-09-14T15:28:42.112Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/04/0d018e0d05bcdde19a0fcb907834155f1fc853a9bedd3f3f5e6acadcae19/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ca80a49b53ed1d22f7282da7255f7bb2fd1935fd0f623d8613fda38745f18961", size = 1641479, upload-time = "2026-09-14T15:10:11.216Z" },
+    { url = "https://files.pythonhosted.org/packages/59/bb/f02ef9073919158f6403fe3701d4ed4403d646720e7201dfc6e9d264bac3/greenlet-3.5.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:916f92f2a8db10508f739d0b5e00b83defe5d1115a997c54532a6d7cf8c95404", size = 1698758, upload-time = "2026-09-14T14:35:54.336Z" },
+    { url = "https://files.pythonhosted.org/packages/08/a5/1f48fe647473a2dcccfd1839b2ff2c78eb57009be776b4da071e901c9bff/greenlet-3.5.6-cp314-cp314t-win_amd64.whl", hash = "sha256:886bcf1870af74c32bc310fd00a6b803445e17e51b7d5a107c7b35c0f362cc16", size = 331574, upload-time = "2026-09-14T14:27:18.451Z" },
+    { url = "https://files.pythonhosted.org/packages/cd/72/3882855a75838faeb54a58aeef4fd77d20b2a86d4bad570c70d41b565dcf/greenlet-3.5.6-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:3ac3494c381dab876cad7d0b22f3a722f3e0c8deb3a65b9e7f35ad7f58b8fcb3", size = 295926, upload-time = "2026-09-14T14:27:21.16Z" },
+    { url = "https://files.pythonhosted.org/packages/10/1f/be4d957d8a9b90bcbe8db206548a42134d96222d43e5ed3fc4708fb6e24b/greenlet-3.5.6-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:602024dae6d77e161f4b89491b62ca1d4f19949d79d47b2db057e476d21179d6", size = 666910, upload-time = "2026-09-14T15:12:07.901Z" },
+    { url = "https://files.pythonhosted.org/packages/a1/af/60d62571a7d6de961e4ce7625d6c2faf359345659fc782d2cdf517c34577/greenlet-3.5.6-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8e63209c3e1e828ee6a457529b4a6d8b05d050fe0ae03a7ae49e967c5d312e0", size = 677415, upload-time = "2026-09-14T15:20:48.817Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/41/b3114c97c10e796010f00a30f51c81470072bca4b53e396ccca87484fcf7/greenlet-3.5.6-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9133d68624b1f2e89ec2f554d56aea8a5b0d7168cd9320200ba58d4d794845a4", size = 681337, upload-time = "2026-09-14T15:25:12.812Z" },
+    { url = "https://files.pythonhosted.org/packages/fb/16/ac9e547b611539aaed1870eb1d6ddc57abdd5924b3a99bb9b5f0b44176b8/greenlet-3.5.6-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccadce0130fd813ec86ebfe969a6c58b42acc1d0fe55a47525375b740e07b605", size = 675927, upload-time = "2026-09-14T14:36:05.34Z" },
+    { url = "https://files.pythonhosted.org/packages/48/1b/d41861c2fa00968e39e467a495ca8db9ce9b6310a5d9b57561b3d0dc48fa/greenlet-3.5.6-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:5adcbbfe78bdc242c71740a02e0991cc1b2f34d33c8bb15ca45eee8fd1140942", size = 487339, upload-time = "2026-09-14T15:28:43.497Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/b1/b7ba08d6431121741f1d30be0d5d292e76873325179a63586cd9217b62f6/greenlet-3.5.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9297fb9c39b9a2c039dbcd306c410bd6906b95244dec3bba4318d36c718c164c", size = 1637236, upload-time = "2026-09-14T15:10:12.442Z" },
+    { url = "https://files.pythonhosted.org/packages/af/c5/3b1cbc68f0c082022fc8717f7fe4b8b13b8d583c52352be37f4e9f55bcd2/greenlet-3.5.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:b374e79ffa7511afc11773aef40a4ccea6191fba1c856ea2f9c56738dca69d7a", size = 1698526, upload-time = "2026-09-14T14:35:56.039Z" },
+    { url = "https://files.pythonhosted.org/packages/de/56/12941ed2711400451c89d544e10f831800a2770f19dd55eac8f0f7f2003b/greenlet-3.5.6-cp315-cp315-win_amd64.whl", hash = "sha256:7969bffa322c097bd46ae595ada6a931cefda613f18ba64587e9cff4cb320756", size = 327739, upload-time = "2026-09-14T14:23:55.768Z" },
+    { url = "https://files.pythonhosted.org/packages/c5/3b/576b9ed5ac929252e340cf60b4bcb6a8515350dc20797064b1922dc4ea75/greenlet-3.5.6-cp315-cp315-win_arm64.whl", hash = "sha256:8dba0129b93e7091dfefaf4cf7000172741bff7f47bf6326fcf17f32fbb54d6b", size = 311715, upload-time = "2026-09-14T14:28:25.154Z" },
+    { url = "https://files.pythonhosted.org/packages/16/c2/86cfc5555a98e12b86966ddbd24fd39af32f71f2f785c6595b7feb2db156/greenlet-3.5.6-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:de3de000d459402cda015068fd135aa50c0bf6f2477a80d4da1e646f123b4e78", size = 306244, upload-time = "2026-09-14T14:27:57.565Z" },
+    { url = "https://files.pythonhosted.org/packages/14/6d/83ffc9d05a75a80ab3a7595dbb1d9604e5d4fc2996d73a8ae2dbd1284900/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:45663c01a4de48b9a64a2ee1509d92d1dfd3afb02b2ccfc9333029d11aef996a", size = 676578, upload-time = "2026-09-14T15:12:09.468Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/d6/c2cf684810e5caded075970aaadea654ecb58b8382b9aecf1d231b936894/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3deccbb57a481e3a408fe61cdfd5c13e0678fc0a30fdd09597917ca87b4be877", size = 684203, upload-time = "2026-09-14T15:20:50.261Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/d1/039c353d5593a97a89699e989324c9bc86af499e6c6152fe0180f5742204/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:63aff70fe5aac59c72215f42ec39fcb59ff46774fa966e717f8ecb6ee2273577", size = 686023, upload-time = "2026-09-14T15:25:14.528Z" },
+    { url = "https://files.pythonhosted.org/packages/62/19/00e1bee5d2af890dc8f400b54d0b0f9b489965f92bc12b407ff72cc6f469/greenlet-3.5.6-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:311018b46472fb26ee85870847fb89eb64cc8aaddb617400789d87076f7cfeec", size = 681253, upload-time = "2026-09-14T14:36:06.742Z" },
+    { url = "https://files.pythonhosted.org/packages/8a/62/97ceb8e0b2ea96046cdf8e95b042715020ebb12d83ea0690db80a8f03d23/greenlet-3.5.6-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:520648db8fb92eef7b3e6013f5a6f901cdf0d6685f639c2f7a245879f865bef7", size = 517058, upload-time = "2026-09-14T15:28:44.924Z" },
+    { url = "https://files.pythonhosted.org/packages/89/58/c9275fd0ca195d1d3402931bcce8cfcc74726ff76efb1883d229e6e1a3d7/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:7f924a5a9d5890649566f2f6682e0d8ad8ca23028bacffbbac36dbd7fd680176", size = 1645988, upload-time = "2026-09-14T15:10:13.758Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/36/b35747582fa4f1a5453f8f3002405dbac788e450cec7674dc2d204b6ccb5/greenlet-3.5.6-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:de9923832f2d8c1a5ecd8d7260465a6ca5a86888a0d129e3bd5cf0406d2fc5bf", size = 1702383, upload-time = "2026-09-14T14:35:58.143Z" },
+    { url = "https://files.pythonhosted.org/packages/ed/69/6ec22ac9351e474d2a134d0ff9400dc80362d1c20f0721088ffffdfc205b/greenlet-3.5.6-cp315-cp315t-win_amd64.whl", hash = "sha256:2ab5f42ac6c238eb71770715e6e909ad9a1a92b6c681ccb64cd5a0f07edb953f", size = 331845, upload-time = "2026-09-14T14:27:41.723Z" },
+    { url = "https://files.pythonhosted.org/packages/30/cf/697c051fd534e223461fb8b523890e21a24eeca229cd50624cff6f02fabd/greenlet-3.5.6-cp315-cp315t-win_arm64.whl", hash = "sha256:f9fe868463ec7e1363733af77e38a5fda3e9b63940337048c945d69e0c80ff24", size = 315070, upload-time = "2026-09-14T14:22:21.476Z" },
+]
+
 [[package]]
 name = "identify"
 version = "2.6.19"
@@ -807,6 +874,9 @@ dependencies = [
 ]
 
 [package.optional-dependencies]
+browser = [
+    { name = "playwright" },
+]
 pandas = [
     { name = "pandas" },
 ]
@@ -827,6 +897,7 @@ dev = [
     { name = "ipykernel" },
     { name = "mypy" },
     { name = "pandas" },
+    { name = "playwright" },
     { name = "polars" },
     { name = "pre-commit" },
     { name = "pyarrow" },
@@ -844,6 +915,7 @@ requires-dist = [
     { name = "lxml", specifier = ">=5.0" },
     { name = "pandas", marker = "extra == 'pandas'", specifier = ">=2.0" },
     { name = "pandas", marker = "extra == 'serializers'", specifier = ">=2.0" },
+    { name = "playwright", marker = "extra == 'browser'", specifier = ">=1.45" },
     { name = "polars", marker = "extra == 'polars'", specifier = ">=1.0" },
     { name = "polars", marker = "extra == 'serializers'", specifier = ">=1.0" },
     { name = "pyarrow", marker = "extra == 'pyarrow'", specifier = ">=14.0" },
@@ -851,13 +923,14 @@ requires-dist = [
     { name = "tenacity", specifier = ">=8.2.3" },
     { name = "tldextract", specifier = ">=5.0" },
 ]
-provides-extras = ["pandas", "polars", "pyarrow", "serializers"]
+provides-extras = ["pandas", "polars", "pyarrow", "serializers", "browser"]
 
 [package.metadata.requires-dev]
 dev = [
     { name = "ipykernel", specifier = ">=7.2.0" },
     { name = "mypy", specifier = ">=1.14" },
     { name = "pandas", specifier = ">=2.0" },
+    { name = "playwright", specifier = ">=1.45" },
     { name = "polars", specifier = ">=1.0" },
     { name = "pre-commit", specifier = ">=4.6.0" },
     { name = "pyarrow", specifier = ">=14.0" },
@@ -1291,6 +1364,25 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/81/e6/cd9575ac904136b3cbf7aa7ee819ef86eedb7274e46f230e94ea4342e729/platformdirs-4.10.0-py3-none-any.whl", hash = "sha256:fb516cdb12eb0d857d0cd85a7c57cea4d060bee4578d6cf5a14dfdf8cbf8784a", size = 22743, upload-time = "2026-05-28T03:32:52.175Z" },
 ]
 
+[[package]]
+name = "playwright"
+version = "1.63.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "greenlet" },
+    { name = "pyee" },
+]
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/19/dd/fbb3d34228ad753bc14464d5f2252585b73367539775a88d0a3949cf5a45/playwright-1.63.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:84c540759e8e7f01e690e197e04060f48899d37cea322299f255843273d3385d", size = 44278407, upload-time = "2026-09-15T16:49:06.045Z" },
+    { url = "https://files.pythonhosted.org/packages/f5/9a/948b930b1a8c4ee869e5a139a2b7747caa06aab56a3f09a2f0abdcbda221/playwright-1.63.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fd1aa00631d44d55e56e0975bf3f3f285fac4a9fd2813183f0c12a488f1a1b24", size = 42944679, upload-time = "2026-09-15T16:49:10.039Z" },
+    { url = "https://files.pythonhosted.org/packages/94/11/dc5c13fa1602371603acd461be47529c1b3513815d3a0dc98f642c291a10/playwright-1.63.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:c89fc4736502a1f0fac2c8ca5d10c0cbc1c669f1f4774a2d8507a43140e4d53f", size = 44278410, upload-time = "2026-09-15T16:49:13.861Z" },
+    { url = "https://files.pythonhosted.org/packages/27/9c/103a5037789062bdab27c7dca53f3ca6b075b572ab2cd96eec825b3aec4e/playwright-1.63.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:ad21bc07516b187965a7521c5cf0df0bd657b17482eaad74335272d35a2b07de", size = 48217159, upload-time = "2026-09-15T16:49:17.404Z" },
+    { url = "https://files.pythonhosted.org/packages/f3/82/3d85505284c5a210f2da6c07b8f757524e79d1fba9cfdafe1eafb766ae59/playwright-1.63.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:354e15b29503565fc598b89f16fbe070459343bef9d7498a93e304864000c6a7", size = 47902708, upload-time = "2026-09-15T16:49:21.055Z" },
+    { url = "https://files.pythonhosted.org/packages/69/8d/f74ff6b52751859f4b69caf66aa7d4a3c8d6c1f0c7dc9920da103612ee39/playwright-1.63.0-py3-none-win32.whl", hash = "sha256:660c00c62639e31b16700ba5456b351ddba55bb766b7ce8261223aa34928e482", size = 38606075, upload-time = "2026-09-15T16:49:29.546Z" },
+    { url = "https://files.pythonhosted.org/packages/76/eb/d6b8d92658038e260dbc7dd69fb3fdbe245aac283953de8b50b02bfe5f61/playwright-1.63.0-py3-none-win_amd64.whl", hash = "sha256:2f9a707a6c6c91157ed77bff2b8caeb04b3c8d46e70d585fc134298cbe4b5cc6", size = 38606082, upload-time = "2026-09-15T16:49:32.838Z" },
+    { url = "https://files.pythonhosted.org/packages/be/75/2432b3e3c7c62103b72d4c4cc8b16a56383ada372bbb0d1278591e988f71/playwright-1.63.0-py3-none-win_arm64.whl", hash = "sha256:1e4a3a838ce22fb68ad17193fcd142a19610d9d70fb9966d2239f5dddc0cc05b", size = 34523554, upload-time = "2026-09-15T16:49:36.455Z" },
+]
+
 [[package]]
 name = "pluggy"
 version = "1.6.0"
@@ -1548,6 +1640,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" },
 ]
 
+[[package]]
+name = "pyee"
+version = "13.0.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" },
+]
+
 [[package]]
 name = "pygments"
 version = "2.20.0"

From 3af337d0f64ffa73beaca0e026bb8a3322728014 Mon Sep 17 00:00:00 2001
From: Jay Baywatch <jay.baywatch@gmail.com>
Date: Thu, 29 Oct 2026 20:36:25 -0400
Subject: [PATCH 2/5] Harden the crawl loop and position linktrace for AI data
 projects

Production guard rails raised in review:

- Frontier is a collections.deque: O(1) pops for both BFS and DFS instead
  of list.pop(0).
- Retry backoff uses equal jitter (uniform in [base/2, base]) so concurrent
  retries against one host spread out; backoff_jitter=False restores
  deterministic waits.
- max_response_bytes (default 10 MiB) bounds every fetcher. HttpFetcher
  refuses an oversize Content-Length before reading and otherwise stops at
  the limit; BrowserFetcher checks the rendered DOM. Oversize pages raise
  ResponseTooLarge and are dropped, not retried.
- Redirects are tracked: FetchResult.final_url / Document.final_url,
  logged at INFO, relative links resolve against the served URL, links are
  classified by the served host, and the Spider marks redirect targets
  visited so they are not fetched twice. Cache entries carry final_url.
- ResponseCache.prune() removes expired entries and enforces an optional
  cache_max_bytes cap (oldest first); the Crawler runs it on enter.
  cache_ttl and cache_max_bytes are exposed on Spider and Crawler.
- Empty 200 bodies parse to an empty Document instead of raising.

Docs: README "Why linktrace?" rewritten around document-centric crawling
for AI/data work with an honest comparison table against Scrapy; API
reference, core concepts, troubleshooting and index updated for every new
option. The real-estate notebook gains sections on crawling a
JavaScript-rendered SPA with RoutingFetcher and flushing a crawl to JSONL
(both executed against live sites).

Tests: 21 new (196 total) covering frontier order, jitter bounds, size
limits over Content-Length and chunked bodies, redirect resolution and
dedupe, and cache pruning.
---
 README.md                                    |  58 +++-
 docs/api-reference.md                        |  76 ++++-
 docs/core-concepts.md                        |  39 ++-
 docs/getting-started.md                      |   3 +-
 docs/index.md                                |   2 +
 docs/troubleshooting.md                      |  13 +
 linktrace/Crawler.py                         |  85 ++++-
 linktrace/Spider.py                          |  24 +-
 linktrace/__init__.py                        |   2 +
 linktrace/cache.py                           |  59 +++-
 linktrace/fetchers/__init__.py               |   2 +
 linktrace/fetchers/base.py                   |  24 ++
 linktrace/fetchers/browser.py                |   8 +
 linktrace/fetchers/http.py                   |  59 +++-
 notebooks/config_rules_sitemaps.ipynb        | 154 ++++++++-
 tests/test_hardening.py                      | 328 +++++++++++++++++++
 tests/test_rate_limiting_and_broken_links.py |  61 ++--
 17 files changed, 896 insertions(+), 101 deletions(-)
 create mode 100644 tests/test_hardening.py

diff --git a/README.md b/README.md
index 5764e0a..714bcc9 100644
--- a/README.md
+++ b/README.md
@@ -23,11 +23,12 @@ LinkTrace is a document-oriented crawler. Every crawled page becomes a rich Docu
 - 🧭 **Pluggable fetchers** — Plain HTTP by default; optional headless-browser rendering (Playwright) for JavaScript SPAs, or route per URL with automatic escalation
 - 📄 **Rich document model** — Full HTML source, parsed links, metadata, headers
 - 🔄 **Persistent sessions** — Connection pooling for 10-100x faster same-domain crawls
-- 🔁 **Retries + backoff** — Exponential backoff for transient errors (timeouts, 5xx)
+- 🔁 **Retries + backoff** — Jittered exponential backoff for timeouts, connection errors and 5xx responses
+- 🛡️ **Guard rails** — Response size limit, redirect tracking (links resolve against the served URL), O(1) frontier, pruned cache with a size cap
 - ⏱️ **Rate limiting** — Per-domain rate limiting with asyncio.Lock, no thundering herd
 - 🤖 **robots.txt support** — Automatically respect Crawl-delay directives and Disallow rules per domain
 - 🔍 **Broken link tracking** — Audit 404s and 5xx errors for site structure validation
-- 💾 **Optional caching** — Disk-based cache (1-day TTL) for repeat crawls
+- 💾 **Optional caching** — Disk-based cache (configurable TTL and size cap, pruned on start) for repeat crawls
 - 🔐 **SSL verification** — Secure by default, with corporate proxy support
 - 🍪 **Automatic cookies** — Set-Cookie extraction and sending built-in
 - 🔀 **Traversal strategies** — BFS (broad) or DFS (deep) crawling
@@ -432,7 +433,7 @@ See [Examples](https://github.com/JayBaywatch/linktrace/blob/main/docs/examples.
 Interactive examples in `notebooks/`:
 - `crawl_cnn.ipynb` — Crawls CNN.com, analyzes link structure, demonstrates all export formats
 - `crawl_tax_assessor.ipynb` — Crawls a municipal property/GIS site, with callbacks and aggregation
-- `config_rules_sitemaps.ipynb` — URL filtering rules, configurable concurrency, and sitemap discovery
+- `config_rules_sitemaps.ipynb` — URL filtering rules, configurable concurrency, sitemap discovery, crawling a JavaScript-rendered SPA with `RoutingFetcher`, and flushing a crawl to JSONL
 
 ## API Reference
 
@@ -482,17 +483,44 @@ Spider manages the crawl queue and traversal. Crawler handles individual documen
 
 ## Why linktrace?
 
-Scrapy is an excellent full crawling and extraction framework. `linktrace` is designed for a narrower job: fast async link analysis with minimal setup.
-
-Instead of building a Scrapy project around spiders, requests, responses, callbacks, items, pipelines, middleware, and settings, `linktrace` gives you a direct document-centric API. Each crawled URL becomes a `Document` object containing the page source, title, status code, response headers, domain, internal links, external links, and crawl status metadata.
-
-That makes `linktrace` useful when your goal is to inspect site structure, trace links, audit crawl status, or export crawl results to dataframe-oriented tools without creating a larger scraping project.
-
-`linktrace` also reuses a persistent `aiohttp` session during a crawl. Connection pooling, cookie reuse, SSL configuration, request timeouts, per-host limits, and DNS caching are carried across requests, which can make repeated same-domain crawls much faster than creating a fresh client/session per URL.
-
-**Use Scrapy when:** you need a mature scraping framework with item pipelines, middleware, schedulers, broad ecosystem support, and complex extraction workflows.
-
-**Use linktrace when:** you want a focused async crawler that turns URLs into analyzable `Document` objects with automatic link classification and simple exports.
+Scrapy is an excellent full crawling and extraction framework, built around a
+job that was common ten years ago: pull specific fields out of pages and push
+them through item pipelines. Most crawling in AI and data projects today is a
+different job. You want the **pages themselves** as documents (for a RAG corpus,
+an underwriting model's property data, an LLM agent's research step), you want
+them in a notebook or a dataframe or a JSONL file in minutes, and the sources
+are increasingly JavaScript apps.
+
+`linktrace` is built for that job:
+
+| | linktrace | Scrapy |
+|---|---|---|
+| **First crawl** | One constructor call, `await spider.run_async()` | Project scaffold, a Spider class, `settings.py`, a CLI run |
+| **Output** | `Document` objects, JSONL, pandas / polars / arrow | `Item`s through pipelines you write |
+| **JavaScript sites** | `fetcher=RoutingFetcher()`; renders only pages that need it | `scrapy-playwright` plugin, reactor swap, per-request `meta` |
+| **Runs inside your app** | Native asyncio; `await` it from a FastAPI handler, an agent loop, or Jupyter | Twisted reactor; embedding in an existing event loop takes care |
+| **Memory on big crawls** | `flush_to="crawl.jsonl"` streams to disk, nothing retained | Items stream through pipelines |
+| **Politeness** | robots.txt (incl. `Crawl-delay`), per-domain rate limit, pool limits | `ROBOTSTXT_OBEY`, `DOWNLOAD_DELAY`, AutoThrottle |
+| **Guard rails** | Size limit, jittered retries, redirect-aware dedupe, pruned cache | Mature equivalents via settings and middleware |
+| **Learning surface** | Two classes and a handful of keyword arguments | Spiders, requests, responses, items, loaders, pipelines, middleware, settings |
+
+Each crawled URL becomes a `Document` with the page source (raw or rendered),
+title, status, headers, domain, classified links, which fetcher produced it,
+and where redirects actually landed. Everything you need to decide whether a
+page belongs in a corpus is on the object, and every export carries it along.
+
+One persistent `aiohttp` session per crawl carries connection pooling,
+cookies, SSL configuration, timeouts, per-host limits and DNS caching across
+requests, so same-domain crawls stay fast without tuning.
+
+**Use Scrapy when:** you need a mature extraction framework with item
+pipelines, middleware, a scheduler, distributed back ends and a large plugin
+ecosystem, or when your deliverable is structured fields rather than pages.
+
+**Use linktrace when:** your deliverable is a set of documents and their
+links, you want results in a notebook or a data pipeline today, the sources
+may be JavaScript apps, and you would rather await a crawl from your own code
+than run a framework around it.
 
 **vs requests + BeautifulSoup:** Built-in async concurrency, automatic session reuse, retries, caching, rate limiting, and structured document objects. Better for crawling multiple pages.
 
@@ -505,7 +533,7 @@ just test          # Run all tests
 just test-cov      # Run with coverage report
 ```
 
-All 175 tests pass. 100% of core crawling paths tested (rate limiting, broken link tracking, robots.txt, callbacks, URL filtering rules, sitemap discovery, fetchers, flushing). Browser tests run against a local page in headless Chromium and skip automatically when Playwright is not installed.
+All 196 tests pass. 100% of core crawling paths tested (rate limiting, broken link tracking, robots.txt, callbacks, URL filtering rules, sitemap discovery, fetchers, flushing, size limits, redirects, cache pruning). Browser tests run against a local page in headless Chromium and skip automatically when Playwright is not installed.
 
 ## Contributing
 
diff --git a/docs/api-reference.md b/docs/api-reference.md
index af78c99..9e3d4bf 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -35,7 +35,10 @@ Spider(
     keep_hash_routes: bool = False,
     flush_to: str = None,
     flush_html: bool = False,
-    retain_documents: bool = None
+    retain_documents: bool = None,
+    max_response_bytes: int = 10_485_760,
+    cache_ttl: int = 86400,
+    cache_max_bytes: int = None
 )
 ```
 
@@ -71,6 +74,9 @@ Spider(
 | `flush_to` | str | None | Append each crawled page as one JSON line to this file as soon as it is crawled |
 | `flush_html` | bool | False | Include page source in flushed lines |
 | `retain_documents` | bool | None | Keep `Document` objects in memory. Defaults to False when `flush_to` is set or a callback is used without `accumulate_results`, else True |
+| `max_response_bytes` | int | 10 MiB | Largest page body a fetcher may buffer; bigger responses are dropped (not retried). `None` disables |
+| `cache_ttl` | int | 86400 | Seconds a cache entry stays valid |
+| `cache_max_bytes` | int | None | Cap on total cache size on disk; oldest entries are evicted when the Crawler starts |
 
 **Raises:** `ValueError` if `max_concurrency < 1`.
 
@@ -111,8 +117,8 @@ an async application). Use `await spider.run_async()` there.
 |-----------|------|-------------|
 | `start_url` | str | Starting URL |
 | `max_depth` | int | Max crawl depth |
-| `visited` | set | Set of visited URLs |
-| `to_visit` | list | Queue of (url, depth) tuples |
+| `visited` | set | Set of visited URLs (includes redirect targets) |
+| `to_visit` | deque | Frontier of (url, depth) tuples; BFS pops the left end, DFS the right |
 | `documents` | list | Results (accumulated Documents) |
 | `visited_count` | int | Number of pages successfully fetched |
 | `traversal_strategy` | str | "bfs" or "dfs" |
@@ -133,6 +139,9 @@ an async application). Use `await spider.run_async()` there.
 | `keep_hash_routes` | bool | Whether hash-router fragments are preserved |
 | `flush_to` | str \| None | JSONL path pages are streamed to |
 | `retain_documents` | bool | Whether Documents are kept in memory (resolved from the constructor args) |
+| `max_response_bytes` | int \| None | Body size limit passed to the Crawler |
+| `cache_ttl` | int | Cache TTL passed to the Crawler |
+| `cache_max_bytes` | int \| None | Cache size cap passed to the Crawler |
 
 ### Callback Signature
 
@@ -192,7 +201,11 @@ Crawler(
     max_connections: int = 100,
     max_connections_per_host: int = 10,
     fetcher: Fetcher = None,
-    keep_hash_routes: bool = False
+    keep_hash_routes: bool = False,
+    max_response_bytes: int = 10_485_760,
+    backoff_jitter: bool = True,
+    cache_ttl: int = 86400,
+    cache_max_bytes: int = None
 )
 ```
 
@@ -203,7 +216,11 @@ Same as Spider, plus:
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
 | `log_level` | int | logging.DEBUG | Logging level |
-| `backoff_factor` | int | 2 | Exponential backoff multiplier (wait_time = 2^attempt * factor) |
+| `backoff_factor` | int | 2 | Exponential backoff multiplier (base wait = 2^attempt * factor) |
+| `backoff_jitter` | bool | True | Draw each wait uniformly from [base/2, base] so concurrent retries spread out |
+| `max_response_bytes` | int | 10 MiB | Largest body a fetcher may buffer; `ResponseTooLarge` drops bigger pages. `None` disables |
+| `cache_ttl` | int | 86400 | Seconds a cache entry stays valid |
+| `cache_max_bytes` | int | None | Cap on total cache size; `prune()` runs on enter |
 | `max_connections` | int | 100 | Total size of the aiohttp connection pool |
 | `max_connections_per_host` | int | 10 | Max simultaneous connections to a single host |
 | `fetcher` | Fetcher | None | Transport used to fetch pages (None = `HttpFetcher()`) |
@@ -213,8 +230,8 @@ Same as Spider, plus:
 
 #### `async __aenter__() -> Crawler`
 
-Enter async context manager. Creates the persistent aiohttp session, then
-calls `fetcher.open(FetchContext)`.
+Enter async context manager. Prunes the cache (if enabled), creates the
+persistent aiohttp session, then calls `fetcher.open(FetchContext)`.
 
 ```python
 async with Crawler(...) as crawler:
@@ -243,11 +260,18 @@ async with Crawler() as crawler:
 
 **Notes:**
 - Checks cache first if `cache_dir` configured
-- Retries `TransientFetchError` and 5xx responses up to `max_retries` times with exponential backoff
-- Returns Document even on 4xx/5xx (check `status_code`); returns `None` when every attempt failed with an error
+- Retries `TransientFetchError` and 5xx responses up to `max_retries` times with jittered exponential backoff
+- Returns Document even on 4xx/5xx (check `status_code`); returns `None` when every attempt failed with an error or the body exceeded `max_response_bytes`
+- Follows redirects; `Document.final_url` holds the served URL and relative links are resolved against it
 - Automatic cookie handling
 - `Document.fetcher` names the fetcher that produced the page
 
+#### `parse_document(url: str, source: str | None, base_url: str = None) -> Document`
+
+Parse HTML into a Document without fetching. `base_url` (default `url`) is the
+URL the content was served from: relative links resolve against it and the
+internal/external split uses its host.
+
 #### `async discover_sitemap_urls(base_url: str) -> List[str]`
 
 Discover page URLs from the site's sitemaps. Reads `Sitemap:` declarations from
@@ -274,6 +298,8 @@ async with Crawler() as crawler:
 | `request_timeout` | int | Request timeout in seconds |
 | `max_retries` | int | Max retry attempts |
 | `backoff_factor` | int | Exponential backoff multiplier |
+| `backoff_jitter` | bool | Whether retry waits are jittered |
+| `max_response_bytes` | int \| None | Body size limit handed to the fetcher |
 | `max_connections` | int | Total aiohttp connection pool size |
 | `max_connections_per_host` | int | Max connections to a single host |
 
@@ -302,10 +328,12 @@ recorded on `Document.fetcher` and namespaces cache entries.
 ### FetchResult
 
 ```python
-FetchResult(url: str, status: int, headers: dict = {}, body: str | None = None, fetcher: str = "http")
+FetchResult(url: str, status: int, headers: dict = {}, body: str | None = None, fetcher: str = "http", final_url: str | None = None)
 ```
 
-`body` should be `None` for non-200 responses.
+`body` should be `None` for non-200 responses. `final_url` is the URL actually
+served after redirects; the `redirected` property is true when it differs from
+`url`. Fetchers that cannot tell leave it `None`.
 
 ### FetchContext
 
@@ -317,15 +345,20 @@ Passed to `Fetcher.open`:
 | `request_timeout` | float | Per-request timeout in seconds |
 | `user_agent` | str | User-Agent for the crawl |
 | `ssl_verify` | bool \| str | The Crawler's SSL setting |
+| `max_response_bytes` | int \| None | Body limit a fetcher must enforce by raising `ResponseTooLarge` |
 
 ### HttpFetcher
 
 ```python
-HttpFetcher(session: aiohttp.ClientSession = None)
+HttpFetcher(session: aiohttp.ClientSession = None, *, max_response_bytes=<inherit>)
 ```
 
 The default. One `GET` per URL through the Crawler's session (or the one
-supplied). `name = "http"`.
+supplied). Rejects bodies over the size limit before buffering them: a
+`Content-Length` above the limit is refused outright, otherwise the body is
+read up to limit + 1 bytes. Reports the served URL as `final_url` and logs
+redirect chains at DEBUG. `max_response_bytes` overrides the Crawler's limit
+for this fetcher (`None` disables). `name = "http"`.
 
 ### BrowserFetcher
 
@@ -395,6 +428,7 @@ external `<script src>`.
 
 - `FetchError` — non-retryable fetch failure.
 - `TransientFetchError(FetchError)` — retried by the Crawler with backoff.
+- `ResponseTooLarge(FetchError)` — body exceeded `max_response_bytes`; carries `url`, `limit` and (when known) `size`.
 
 ---
 
@@ -527,6 +561,8 @@ Document(url: str, source: str = None)
 | `status_code` | int | HTTP status code |
 | `response_headers` | dict | HTTP response headers |
 | `fetcher` | str | Name of the fetcher that produced the page (`"http"`, `"browser"`, ...) |
+| `final_url` | str | URL actually served after redirects (equals `url` when none) |
+| `redirected` | bool | Read-only: `final_url != url` |
 | `domain` | str | Domain extracted from URL (read-only property) |
 | `internal_links` | List[HtmlLink] | Links to same domain (successful crawl) |
 | `external_links` | List[HtmlLink] | Links to other domains (successful crawl) |
@@ -721,12 +757,13 @@ Disk-based response caching (used internally by Crawler).
 ### Constructor
 
 ```python
-ResponseCache(cache_dir: str, ttl_seconds: int = 86400)
+ResponseCache(cache_dir: str, ttl_seconds: int = 86400, max_bytes: int = None)
 ```
 
 **Parameters:**
 - `cache_dir` (str) — Directory to store cache files
 - `ttl_seconds` (int) — Time-to-live for entries (default: 1 day)
+- `max_bytes` (int) — Optional cap on total size on disk, enforced by `prune()`
 
 ### Methods
 
@@ -738,8 +775,15 @@ and `None` for plain HTTP, so existing cache directories keep working.
 
 #### `async set(url: str, status_code: int, headers: dict, content: str, *, namespace: str = None, fetcher: str = "http") -> None`
 
-Store response in cache. `fetcher` is recorded and restored onto
-`Document.fetcher` on a cache hit.
+Store response in cache. `fetcher` and `final_url` are recorded and restored
+onto the Document on a cache hit.
+
+#### `async prune() -> int`
+
+Delete entries older than `ttl_seconds` and, if `max_bytes` is set, the oldest
+entries until the directory fits. Uses file modification times, so no entry is
+parsed. The Crawler calls this once on enter. Returns the number of files
+removed.
 
 #### `async clear() -> None`
 
diff --git a/docs/core-concepts.md b/docs/core-concepts.md
index 1f77a5a..c6fcc0b 100644
--- a/docs/core-concepts.md
+++ b/docs/core-concepts.md
@@ -61,7 +61,7 @@ documents = await spider.run_async()
 ### Internal Logic
 
 1. Pop URL from `to_visit` queue (FIFO for BFS, LIFO for DFS)
-2. Skip if visited or exceeds max_depth
+2. Skip if visited or exceeds max_depth (the frontier is a `deque`: O(1) pops from either end)
 3. Tell Crawler to fetch + parse the URL
 4. Add returned Document to results
 5. Extract internal links, add to queue
@@ -91,15 +91,35 @@ async with Crawler(ssl_verify=True, cache_dir=".cache") as crawler:
 Transient errors (timeouts, connection errors, 5xx responses) are retried with exponential backoff. If every attempt returns 5xx, the last response is returned as a Document with that `status_code`:
 
 ```
-wait_time = 2^attempt * backoff_factor
+base      = 2^attempt * backoff_factor
+wait_time = uniform(base / 2, base)        # "equal jitter"
 
-Attempt 1: wait 2s
-Attempt 2: wait 4s  
-Attempt 3: wait 8s
+Attempt 1: base 2s  → wait 1–2s
+Attempt 2: base 4s  → wait 2–4s
+Attempt 3: base 8s  → wait 4–8s
 Attempt 4: fail
 ```
 
-Default: 3 retries, backoff_factor=2.
+Default: 3 retries, backoff_factor=2. The jitter keeps a batch of concurrent
+retries against one host from landing at the same instant; set
+`backoff_jitter=False` for deterministic waits.
+
+### Redirects
+
+aiohttp follows redirects. The Crawler records the served URL as
+`Document.final_url`, logs the hop at INFO, resolves relative links against the
+served URL, and classifies internal vs external by the served host (so a
+`www` → apex redirect does not turn the whole site "external"). The Spider
+marks the redirect target as visited so it is not fetched again under its own
+name.
+
+### Response size limit
+
+`max_response_bytes` (default 10 MiB) bounds how much of a body any fetcher
+will buffer. The HTTP fetcher refuses a larger `Content-Length` before reading
+anything and otherwise stops reading at the limit; the browser fetcher checks
+the rendered DOM. An oversize page raises `ResponseTooLarge`, is logged, and is
+not retried. Set `max_response_bytes=None` to disable.
 
 ### SSL Verification
 
@@ -124,8 +144,10 @@ crawler = Crawler(cache_dir=".webcrawler_cache")
 ```
 
 - Responses stored as `{cache_dir}/{url_hash}.json`
-- Default TTL: 1 day
-- Expired entries auto-deleted on retrieval
+- Default TTL: 1 day (`cache_ttl`)
+- Expired entries auto-deleted on retrieval, and the whole directory is
+  pruned when a Crawler starts; `cache_max_bytes` evicts the oldest entries
+  beyond a size cap
 - Highly effective for repeat crawls (2-50x speedup)
 - Non-default fetchers get their own namespace, so a cached raw HTTP shell is
   never served to a crawl that asked for a rendered page
@@ -414,6 +436,7 @@ linktrace reuses one persistent session across all requests in a single crawl. S
 - 4xx responses (invalid URL, etc)
 - DNS failures
 - SSL errors
+- Responses over `max_response_bytes`
 
 Failed requests log errors but don't crash the crawl. Document is returned with status_code and empty links.
 
diff --git a/docs/getting-started.md b/docs/getting-started.md
index 7cd8f1a..f428f41 100644
--- a/docs/getting-started.md
+++ b/docs/getting-started.md
@@ -247,7 +247,8 @@ jupyter notebook notebooks/crawl_cnn.ipynb
 - **`crawl_cnn.ipynb`** — basic crawling, link-structure analysis, and exporting
   to JSON / Pandas / Polars / PyArrow
 - **`config_rules_sitemaps.ipynb`** — URL filtering rules, configurable
-  concurrency, and sitemap discovery
+  concurrency, sitemap discovery, crawling a JavaScript-rendered SPA with
+  `RoutingFetcher`, and flushing a crawl to JSONL
 
 ## Next Steps
 
diff --git a/docs/index.md b/docs/index.md
index 9b00cf9..b9c5a7d 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -10,6 +10,8 @@ LinkTrace is a document-oriented crawler. Every crawled page becomes a rich Docu
 
 **JavaScript sites:** Plain HTTP by default, with an optional [headless-browser fetcher](getting-started.md#crawl-a-javascript-rendered-site) for React/Vue/Angular apps. Big crawls can [stream straight to disk](getting-started.md#stream-a-large-crawl-to-disk).
 
+**Built to run for hours:** jittered retries, a response size limit, redirect-aware de-duplication, an O(1) frontier and a self-pruning cache. See [Core Concepts](core-concepts.md#retry-logic).
+
 ## Quick Start
 
 ```bash
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 891b3fe..0841c1b 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -251,6 +251,19 @@ is consistently slow rather than flaky.
 
 ## Memory Issues
 
+### "Response for ... is N bytes; limit is 10485760 bytes"
+
+**Problem:** A page exceeded `max_response_bytes` (default 10 MiB). It is
+logged, dropped (`crawl_document_async` returns `None`) and not retried.
+
+**Solutions:** Raise the limit for a site with genuinely large pages, or
+disable it when you trust the source:
+
+```python
+spider = Spider(start_url="...", max_response_bytes=50 * 1024 * 1024)
+spider = Spider(start_url="...", max_response_bytes=None)   # no limit
+```
+
 ### "Crawl runs out of memory"
 
 **Problem:** Every `Document` holds its HTML (and rendered DOMs are larger),
diff --git a/linktrace/Crawler.py b/linktrace/Crawler.py
index b45d732..aa1eda8 100755
--- a/linktrace/Crawler.py
+++ b/linktrace/Crawler.py
@@ -1,5 +1,6 @@
 import asyncio
 import logging
+import random
 import ssl
 from urllib.parse import urldefrag, urljoin, urlparse
 
@@ -70,6 +71,13 @@ def __init__(self, url: str, source: str | None) -> None:
         # Which fetcher produced ``source``: "http" (raw response) or
         # "browser" (rendered DOM), or a custom fetcher's name.
         self.fetcher: str = "http"
+        # URL actually served after redirects (equals ``url`` when none).
+        self.final_url: str = url
+
+    @property
+    def redirected(self) -> bool:
+        """True if the server redirected this request to a different URL."""
+        return self.final_url != self.url
 
     @property
     def domain(self) -> str:
@@ -107,10 +115,18 @@ def __init__(
         max_connections_per_host: int = 10,
         fetcher: Fetcher | None = None,
         keep_hash_routes: bool = False,
+        max_response_bytes: int | None = 10 * 1024 * 1024,
+        backoff_jitter: bool = True,
+        cache_ttl: int = 86400,
+        cache_max_bytes: int | None = None,
     ) -> None:
         self._logger = logging.getLogger(log_name if log_name else __name__)
         self._logger.setLevel(log_level)
 
+        # Guard rails for long or adversarial crawls.
+        self.max_response_bytes = max_response_bytes
+        self.backoff_jitter = backoff_jitter
+
         # Pluggable transport. HttpFetcher reproduces the classic behaviour;
         # BrowserFetcher / RoutingFetcher add JavaScript rendering.
         self.fetcher: Fetcher = fetcher if fetcher is not None else HttpFetcher()
@@ -142,7 +158,9 @@ def __init__(
 
         # Caching (opt-in)
         self.cache: ResponseCache | None = (
-            ResponseCache(cache_dir) if cache_dir else None
+            ResponseCache(cache_dir, ttl_seconds=cache_ttl, max_bytes=cache_max_bytes)
+            if cache_dir
+            else None
         )
 
         # Cookies handled automatically via CookieJar in session (created lazily)
@@ -150,6 +168,11 @@ def __init__(
 
     async def __aenter__(self) -> "Crawler":
         """Enter async context manager: create session, then open the fetcher."""
+        if self.cache:
+            removed = await self.cache.prune()
+            if removed:
+                self._logger.debug(f"Pruned {removed} expired/excess cache entries")
+
         self.session = await self._create_session()
         try:
             await self.fetcher.open(
@@ -158,6 +181,7 @@ async def __aenter__(self) -> "Crawler":
                     request_timeout=self.request_timeout,
                     user_agent=self.user_agent,
                     ssl_verify=self.ssl_verify,
+                    max_response_bytes=self.max_response_bytes,
                 )
             )
         except Exception:
@@ -289,10 +313,12 @@ async def crawl_document_async(self, url: str) -> Document | None:
             cached = await self.cache.get(url, namespace=cache_ns)
             if cached:
                 self._logger.debug(f"Cache hit for {url}")
-                doc = self.parse_document(url, cached.content)
+                final_url = cached.final_url or url
+                doc = self.parse_document(url, cached.content, base_url=final_url)
                 doc.status_code = cached.status_code
                 doc.response_headers = cached.response_headers
                 doc.fetcher = cached.fetcher
+                doc.final_url = final_url
                 return doc
 
         # Check robots.txt before fetching
@@ -344,18 +370,28 @@ async def crawl_document_async(self, url: str) -> Document | None:
         if result is None:  # pragma: no cover - max_retries < 1
             return None
 
+        final_url = url
+        if result.final_url:
+            final_url = self.normalize_url(
+                result.final_url, keep_hash_routes=self.keep_hash_routes
+            )
+            if final_url != url:
+                self._logger.info(f"Redirected: {url} -> {final_url}")
+
         if result.status != 200:
             self._logger.error(f"Failed to fetch {url}: HTTP {result.status}")
             doc = Document(url, None)
             doc.status_code = result.status
             doc.response_headers = result.headers
             doc.fetcher = result.fetcher
+            doc.final_url = final_url
             return doc
 
-        doc = self.parse_document(url, result.body)
+        doc = self.parse_document(url, result.body, base_url=final_url)
         doc.status_code = result.status
         doc.response_headers = result.headers
         doc.fetcher = result.fetcher
+        doc.final_url = final_url
 
         # Cache successful responses
         if self.cache and result.body is not None:
@@ -366,17 +402,30 @@ async def crawl_document_async(self, url: str) -> Document | None:
                 result.body,
                 namespace=cache_ns,
                 fetcher=result.fetcher,
+                final_url=final_url if final_url != url else None,
             )
 
         return doc
 
+    def _backoff_delay(self, attempt: int) -> float:
+        """Exponential backoff for ``attempt`` (0-based), with equal jitter.
+
+        Base delay is ``2**attempt * backoff_factor``. With jitter enabled the
+        actual wait is uniformly drawn from [base/2, base], so concurrent
+        retries against one host spread out instead of landing together.
+        """
+        base = float(2**attempt * self.backoff_factor)
+        if not self.backoff_jitter:
+            return base
+        return base / 2 + random.uniform(0, base / 2)
+
     async def _backoff(self, url: str, attempt: int, reason: str) -> None:
         """Log and sleep before retrying ``url`` after ``attempt`` (0-based)."""
-        wait_time = 2**attempt * self.backoff_factor
+        wait_time = self._backoff_delay(attempt)
         self._logger.warning(
             f"{reason} fetching {url} "
             f"(attempt {attempt + 1}/{self.max_retries}). "
-            f"Retrying in {wait_time}s..."
+            f"Retrying in {wait_time:.1f}s..."
         )
         await asyncio.sleep(wait_time)
 
@@ -401,11 +450,22 @@ async def discover_sitemap_urls(self, base_url: str) -> list[str]:
         parser = SitemapParser(self.session, self.user_agent, self._logger)
         return await parser.discover(base_url, robots_sitemaps or None)
 
-    def parse_document(self, url: str, source: str | None) -> Document:
-        self._protocol_ = urlparse(url).scheme if not urlparse(url).scheme else "http"
-        self.base_uri = urlparse(url).netloc
+    def parse_document(
+        self, url: str, source: str | None, base_url: str | None = None
+    ) -> Document:
+        """Parse ``source`` into a Document for ``url``.
+
+        ``base_url`` is the URL the content was actually served from (after
+        redirects). Relative links are resolved against it and internal vs
+        external classification uses its host, so a ``www`` -> apex redirect
+        does not turn every link on the site into an "external" one.
+        """
+        base = base_url or url
+        self._protocol_ = urlparse(base).scheme or "http"
+        self.base_uri = urlparse(base).netloc
 
         doc = Document(url, source)
+        doc.final_url = base
         if source is None:
             return doc
 
@@ -436,7 +496,7 @@ def parse_document(self, url: str, source: str | None) -> Document:
             for link in links:
                 link_url = link.attrib["href"]
                 link_url = self.normalize_url(
-                    urljoin(url, link_url), keep_hash_routes=self.keep_hash_routes
+                    urljoin(base, link_url), keep_hash_routes=self.keep_hash_routes
                 )
                 if urlparse(link_url).scheme not in ["http", "https", "ftp"]:
                     continue
@@ -452,7 +512,7 @@ def parse_document(self, url: str, source: str | None) -> Document:
             doc.internal_links = [
                 link
                 for link in found_links
-                if self.get_domain_parts(link.url) == self.get_domain_parts(doc.url)
+                if self.get_domain_parts(link.url) == self.get_domain_parts(base)
                 or link.url[:1] == "/"
                 or link.url[:1] == ""
             ]
@@ -461,7 +521,7 @@ def parse_document(self, url: str, source: str | None) -> Document:
             doc.external_links = [
                 link
                 for link in found_links
-                if self.get_domain_parts(link.url) != self.get_domain_parts(doc.url)
+                if self.get_domain_parts(link.url) != self.get_domain_parts(base)
                 and urlparse(link.url).scheme != ""
             ]
             doc.external_links = list(dict.fromkeys(doc.external_links))
@@ -470,6 +530,9 @@ def parse_document(self, url: str, source: str | None) -> Document:
 
             for link in doc.links:
                 self.queue_link(link.url)
+        except lxml.etree.ParserError:
+            # Empty or whitespace-only body: a valid page with nothing to parse.
+            self._logger.debug(f"Empty document at {url}")
         except lxml.etree.XMLSyntaxError:
             self._logger.error(
                 f"XMLSyntaxError: Invalid source document or truncated source at {url}"
diff --git a/linktrace/Spider.py b/linktrace/Spider.py
index 16a8a49..effa1cb 100755
--- a/linktrace/Spider.py
+++ b/linktrace/Spider.py
@@ -1,6 +1,7 @@
 import asyncio
 import json
 import logging
+from collections import deque
 from collections.abc import Callable
 from typing import Any, Literal
 
@@ -45,9 +46,15 @@ def __init__(
         flush_to: str | None = None,
         flush_html: bool = False,
         retain_documents: bool | None = None,
+        max_response_bytes: int | None = 10 * 1024 * 1024,
+        cache_ttl: int = 86400,
+        cache_max_bytes: int | None = None,
     ) -> None:
         self.fetcher = fetcher
         self.keep_hash_routes = keep_hash_routes
+        self.max_response_bytes = max_response_bytes
+        self.cache_ttl = cache_ttl
+        self.cache_max_bytes = cache_max_bytes
 
         # Memory management. ``flush_to`` streams each Document to a JSONL
         # file as soon as it is crawled. Documents are retained in memory
@@ -95,7 +102,8 @@ def __init__(
         self.visited: set[str] = set()
         self.discovered: set[str] = {start_url}
         self.in_progress: set[str] = set()
-        self.to_visit = [(start_url, 0)]
+        # Frontier: appended at the right; BFS pops left, DFS pops right.
+        self.to_visit: deque[tuple[str, int]] = deque([(start_url, 0)])
         self.documents: list[Document] = []
         self._logger = logging.getLogger(log_name if log_name else __name__)
         self._logger.setLevel(logging.DEBUG)
@@ -189,6 +197,9 @@ async def run_async(self) -> list[Document]:
                 max_connections_per_host=self.max_connections_per_host,
                 fetcher=self.fetcher,
                 keep_hash_routes=self.keep_hash_routes,
+                max_response_bytes=self.max_response_bytes,
+                cache_ttl=self.cache_ttl,
+                cache_max_bytes=self.cache_max_bytes,
             ) as crawler:
                 if self.use_sitemaps:
                     await self._seed_from_sitemaps(crawler)
@@ -200,8 +211,10 @@ async def run_async(self) -> list[Document]:
                         for _ in range(batch_size):
                             if not self.to_visit:
                                 break
-                            idx = 0 if self.traversal_strategy == "bfs" else -1
-                            url, current_depth = self.to_visit.pop(idx)
+                            if self.traversal_strategy == "bfs":
+                                url, current_depth = self.to_visit.popleft()
+                            else:
+                                url, current_depth = self.to_visit.pop()
                             if (
                                 url in self.visited
                                 or url in self.in_progress
@@ -295,6 +308,11 @@ async def crawl_and_collect(
             if doc is not None:
                 self.track_visits(url, doc)
                 async with self.lock:
+                    if doc.redirected:
+                        # The served URL is now known content; never fetch it
+                        # again under its own name.
+                        self.discovered.add(doc.final_url)
+                        self.visited.add(doc.final_url)
                     self.visited_count += 1
                     self._logger.info(
                         f"Visited: {url} (Total Visited: {self.visited_count})"
diff --git a/linktrace/__init__.py b/linktrace/__init__.py
index 8788a90..2b878f9 100644
--- a/linktrace/__init__.py
+++ b/linktrace/__init__.py
@@ -9,6 +9,7 @@
     FetchError,
     FetchResult,
     HttpFetcher,
+    ResponseTooLarge,
     RoutingFetcher,
     TransientFetchError,
     looks_like_spa_shell,
@@ -32,6 +33,7 @@
     "HtmlLink",
     "HttpFetcher",
     "ResponseCache",
+    "ResponseTooLarge",
     "RoutingFetcher",
     "Serializers",
     "SitemapParser",
diff --git a/linktrace/cache.py b/linktrace/cache.py
index 8d36ec3..0199f21 100644
--- a/linktrace/cache.py
+++ b/linktrace/cache.py
@@ -15,26 +15,36 @@ def __init__(
         response_headers: dict[str, str],
         content: str,
         fetcher: str = "http",
+        final_url: str | None = None,
     ) -> None:
         self.status_code = status_code
         self.response_headers = response_headers
         self.content = content
         self.fetcher = fetcher
+        self.final_url = final_url
 
 
 class ResponseCache:
     """Disk-based response cache with TTL support."""
 
-    def __init__(self, cache_dir: str, ttl_seconds: int = 86400) -> None:
+    def __init__(
+        self,
+        cache_dir: str,
+        ttl_seconds: int = 86400,
+        max_bytes: int | None = None,
+    ) -> None:
         """Initialize cache.
 
         Args:
             cache_dir: Directory to store cache files
             ttl_seconds: Time-to-live for cached responses (default 1 day)
+            max_bytes: Optional cap on total cache size on disk. ``prune()``
+                evicts the oldest entries beyond it.
         """
         self.cache_dir = Path(cache_dir)
         self.cache_dir.mkdir(parents=True, exist_ok=True)
         self.ttl = ttl_seconds
+        self.max_bytes = max_bytes
 
     def _url_hash(self, url: str, namespace: str | None = None) -> str:
         """Generate cache file name from URL (and optional namespace).
@@ -82,6 +92,7 @@ async def get(
                 response_headers=data["response_headers"],
                 content=data["content"],
                 fetcher=data.get("fetcher", "http"),
+                final_url=data.get("final_url"),
             )
         except (json.JSONDecodeError, KeyError):
             # Cache file corrupted, remove it
@@ -97,6 +108,7 @@ async def set(
         *,
         namespace: str | None = None,
         fetcher: str = "http",
+        final_url: str | None = None,
     ) -> None:
         """Store response in cache.
 
@@ -107,6 +119,7 @@ async def set(
             content: Response body text
             namespace: Optional namespace (e.g. fetcher name) to store under
             fetcher: Name of the fetcher that produced ``content``
+            final_url: URL served after redirects, if different
         """
         cache_file = self._cache_path(url, namespace)
 
@@ -117,6 +130,7 @@ async def set(
             "response_headers": headers,
             "content": content,
             "fetcher": fetcher,
+            "final_url": final_url,
         }
 
         try:
@@ -126,6 +140,49 @@ async def set(
             # Silently skip cache write if filesystem issues
             pass
 
+    async def prune(self) -> int:
+        """Delete expired entries and enforce ``max_bytes``.
+
+        Expiry uses the file's modification time, which ``set`` writes at the
+        same moment as the embedded timestamp, so no JSON has to be parsed.
+        Oldest entries go first when the directory exceeds ``max_bytes``.
+        The Crawler calls this once when it is entered, so a long-running or
+        repeated crawl cannot grow the cache without bound.
+
+        Returns:
+            Number of files removed
+        """
+        now = time.time()
+        removed = 0
+        entries: list[tuple[float, int, Path]] = []
+        for path in self.cache_dir.glob("*.json"):
+            try:
+                st = path.stat()
+            except OSError:
+                continue
+            if now - st.st_mtime > self.ttl:
+                try:
+                    path.unlink()
+                    removed += 1
+                except OSError:
+                    pass
+                continue
+            entries.append((st.st_mtime, st.st_size, path))
+
+        if self.max_bytes is not None:
+            total = sum(size for _, size, _ in entries)
+            for _, size, path in sorted(entries):  # oldest first
+                if total <= self.max_bytes:
+                    break
+                try:
+                    path.unlink()
+                    removed += 1
+                    total -= size
+                except OSError:
+                    pass
+
+        return removed
+
     async def clear(self) -> None:
         """Clear all cached responses."""
         import shutil
diff --git a/linktrace/fetchers/__init__.py b/linktrace/fetchers/__init__.py
index 1b5bf8b..b03d089 100644
--- a/linktrace/fetchers/__init__.py
+++ b/linktrace/fetchers/__init__.py
@@ -5,6 +5,7 @@
     Fetcher,
     FetchError,
     FetchResult,
+    ResponseTooLarge,
     TransientFetchError,
 )
 from linktrace.fetchers.browser import BrowserFetcher
@@ -18,6 +19,7 @@
     "FetchResult",
     "Fetcher",
     "HttpFetcher",
+    "ResponseTooLarge",
     "RoutingFetcher",
     "TransientFetchError",
     "looks_like_spa_shell",
diff --git a/linktrace/fetchers/base.py b/linktrace/fetchers/base.py
index 6dd6c2e..7ada468 100644
--- a/linktrace/fetchers/base.py
+++ b/linktrace/fetchers/base.py
@@ -39,6 +39,9 @@ class FetchResult:
         body: Page body as text, or ``None`` when the status was not 200.
         fetcher: Name of the fetcher that produced the body (``"http"``,
             ``"browser"``, ...). Recorded on the resulting ``Document``.
+        final_url: URL actually served after any redirects, or ``None`` if
+            the fetcher cannot tell. The Crawler logs redirects and resolves
+            relative links against this URL.
     """
 
     url: str
@@ -46,6 +49,12 @@ class FetchResult:
     headers: dict[str, str] = field(default_factory=dict)
     body: str | None = None
     fetcher: str = "http"
+    final_url: str | None = None
+
+    @property
+    def redirected(self) -> bool:
+        """True when the response came from a different URL than requested."""
+        return self.final_url is not None and self.final_url != self.url
 
 
 @dataclass
@@ -58,12 +67,16 @@ class FetchContext:
         request_timeout: Per-request timeout in seconds.
         user_agent: User-Agent string the crawl identifies itself with.
         ssl_verify: The Crawler's SSL setting (``True``, ``False`` or a CA path).
+        max_response_bytes: Upper bound on a page body. Fetchers must raise
+            :class:`ResponseTooLarge` instead of buffering more than this.
+            ``None`` disables the limit.
     """
 
     session: aiohttp.ClientSession
     request_timeout: float = 30
     user_agent: str = "linktrace/0.1.0"
     ssl_verify: bool | str = True
+    max_response_bytes: int | None = 10 * 1024 * 1024
 
 
 class FetchError(Exception):
@@ -77,6 +90,17 @@ class TransientFetchError(FetchError):
     """
 
 
+class ResponseTooLarge(FetchError):
+    """The body exceeded ``FetchContext.max_response_bytes``; not retried."""
+
+    def __init__(self, url: str, limit: int, size: int | None = None) -> None:
+        self.url = url
+        self.limit = limit
+        self.size = size
+        seen = f"{size} bytes" if size is not None else f"more than {limit} bytes"
+        super().__init__(f"Response for {url} is {seen}; limit is {limit} bytes")
+
+
 class Fetcher(ABC):
     """Base class for fetchers.
 
diff --git a/linktrace/fetchers/browser.py b/linktrace/fetchers/browser.py
index 77aaf94..1e8a864 100644
--- a/linktrace/fetchers/browser.py
+++ b/linktrace/fetchers/browser.py
@@ -20,6 +20,7 @@
     Fetcher,
     FetchError,
     FetchResult,
+    ResponseTooLarge,
     TransientFetchError,
 )
 
@@ -101,6 +102,7 @@ def __init__(
         self._browser: Any = None
         self._context: Any = None
         self._semaphore: asyncio.Semaphore | None = None
+        self._max_bytes: int | None = None
         self._timeout_ms: float = 30_000
         self._timeout_error: type[Exception] = Exception
         self._playwright_error: type[Exception] = Exception
@@ -116,6 +118,7 @@ async def open(self, ctx: FetchContext) -> None:
         self._timeout_error = PlaywrightTimeoutError
         self._playwright_error = PlaywrightError
         self._timeout_ms = (self.timeout or ctx.request_timeout) * 1000
+        self._max_bytes = ctx.max_response_bytes
 
         self._playwright = await async_playwright().start()
         launcher = getattr(self._playwright, self.browser_type)
@@ -205,13 +208,18 @@ async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
                 response_headers = (
                     await response.all_headers() if response is not None else {}
                 )
+                final_url = page.url
             finally:
                 await page.close()
 
+        if self._max_bytes is not None and len(body) > self._max_bytes:
+            raise ResponseTooLarge(url, self._max_bytes, len(body))
+
         return FetchResult(
             url=url,
             status=status,
             headers=dict(response_headers),
             body=body if status == 200 else None,
             fetcher=self.name,
+            final_url=final_url,
         )
diff --git a/linktrace/fetchers/http.py b/linktrace/fetchers/http.py
index 629e5a3..cae9451 100644
--- a/linktrace/fetchers/http.py
+++ b/linktrace/fetchers/http.py
@@ -2,6 +2,8 @@
 
 from __future__ import annotations
 
+import logging
+
 import aiohttp
 
 from linktrace.fetchers.base import (
@@ -9,31 +11,51 @@
     Fetcher,
     FetchError,
     FetchResult,
+    ResponseTooLarge,
     TransientFetchError,
 )
 
+logger = logging.getLogger(__name__)
+
+_INHERIT = object()  # sentinel: take max_response_bytes from the FetchContext
+
 
 class HttpFetcher(Fetcher):
     """Fetch pages with the Crawler's shared aiohttp session.
 
-    This reproduces the crawler's original behaviour exactly: one GET per URL,
-    body read only for 200 responses, timeouts and connection errors raised as
-    :class:`~linktrace.fetchers.base.TransientFetchError` so the Crawler retries
-    them with backoff.
+    One GET per URL; the body is read only for 200 responses and never beyond
+    ``FetchContext.max_response_bytes`` (a ``Content-Length`` above the limit
+    is rejected before any body is read). Timeouts and connection errors are
+    raised as :class:`~linktrace.fetchers.base.TransientFetchError` so the
+    Crawler retries them with backoff. Redirects are followed by aiohttp; the
+    served URL is reported as ``FetchResult.final_url`` and logged.
 
     Args:
         session: Optional session to use instead of the Crawler's. Mostly
             useful in tests; normally leave it ``None``.
+        max_response_bytes: Override the Crawler's body limit for this fetcher
+            (``None`` disables it). Left unset, the Crawler's value applies.
     """
 
     name = "http"
 
-    def __init__(self, session: aiohttp.ClientSession | None = None) -> None:
+    def __init__(
+        self,
+        session: aiohttp.ClientSession | None = None,
+        *,
+        max_response_bytes: int | None | object = _INHERIT,
+    ) -> None:
         self._session = session
+        self._max_bytes_override = max_response_bytes
+        self._max_bytes: int | None = (
+            None if max_response_bytes is _INHERIT else max_response_bytes  # type: ignore[assignment]
+        )
 
     async def open(self, ctx: FetchContext) -> None:
         if self._session is None:
             self._session = ctx.session
+        if self._max_bytes_override is _INHERIT:
+            self._max_bytes = ctx.max_response_bytes
 
     async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
         if self._session is None:
@@ -45,7 +67,11 @@ async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
             async with self._session.get(url, headers=headers) as response:
                 status = response.status
                 response_headers = dict(response.headers)
-                body = await response.text() if status == 200 else None
+                final_url = str(response.url)
+                if response.history:
+                    chain = " -> ".join(str(r.url) for r in response.history)
+                    logger.debug(f"Redirect: {chain} -> {final_url}")
+                body = await self._read_body(url, response) if status == 200 else None
         except (
             TimeoutError,
             aiohttp.ClientConnectorError,
@@ -61,4 +87,25 @@ async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
             headers=response_headers,
             body=body,
             fetcher=self.name,
+            final_url=final_url,
         )
+
+    async def _read_body(self, url: str, response: aiohttp.ClientResponse) -> str:
+        """Read the body as text without buffering more than the size limit."""
+        limit = self._max_bytes
+        if limit is None:
+            return await response.text()
+
+        declared = response.headers.get("Content-Length")
+        if declared and declared.isdigit() and int(declared) > limit:
+            raise ResponseTooLarge(url, limit, int(declared))
+
+        raw = await response.content.read(limit + 1)
+        if len(raw) > limit:
+            raise ResponseTooLarge(url, limit)
+
+        try:
+            encoding = response.get_encoding()
+        except RuntimeError:
+            encoding = "utf-8"
+        return raw.decode(encoding, errors="replace")
diff --git a/notebooks/config_rules_sitemaps.ipynb b/notebooks/config_rules_sitemaps.ipynb
index c45d8e6..8c13db6 100644
--- a/notebooks/config_rules_sitemaps.ipynb
+++ b/notebooks/config_rules_sitemaps.ipynb
@@ -4,10 +4,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Real Estate Crawling: Filtering, Concurrency & Sitemaps\n",
+    "# Real Estate Crawling: Filtering, Concurrency, Sitemaps & SPAs\n",
     "\n",
-    "This notebook demonstrates three features for turning a broad link-follower\n",
-    "into a focused **document-inventory** crawler:\n",
+    "This notebook demonstrates the features that turn a broad link-follower into a\n",
+    "focused **document-inventory** crawler for real-estate and municipal data:\n",
     "\n",
     "1. **Configurable concurrency** — tune throughput with `max_concurrency` and\n",
     "   the connection-pool limits.\n",
@@ -16,10 +16,14 @@
     "   off-site domains).\n",
     "3. **Sitemap discovery** — seed the queue from a site's own published\n",
     "   document inventory before link-following begins.\n",
+    "4. **JavaScript-rendered sites** — swap in a headless-browser fetcher for\n",
+    "   listing portals built as single-page apps, where plain HTTP sees an empty\n",
+    "   shell.\n",
+    "5. **Flushing to disk** — stream a crawl too big for memory to a JSONL file.\n",
     "\n",
     "We use a municipal property/GIS site whose street pages link to many\n",
     "`?Letter=` query-parameter variants — a classic crawl trap that filtering\n",
-    "rules are designed to tame."
+    "rules are designed to tame — and a public React demo app for the SPA section."
    ]
   },
   {
@@ -33,6 +37,7 @@
     "from urllib.parse import urlparse, parse_qs\n",
     "\n",
     "from linktrace import Spider, CrawlRules, Crawler, SitemapParser\n",
+    "from linktrace import RoutingFetcher, BrowserFetcher, looks_like_spa_shell\n",
     "\n",
     "logging.basicConfig(\n",
     "    level=logging.INFO,\n",
@@ -273,6 +278,143 @@
     "print(f\"Pages with a Letter= param (should be 0): {len(trap)}\")"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 5. JavaScript-rendered listings (SPAs)\n",
+    "\n",
+    "Many listing portals and GIS front ends are single-page apps: the server\n",
+    "returns a near-empty HTML shell and React/Vue/Angular builds the page — and\n",
+    "every link on it — in the browser. The default HTTP fetcher gets the shell and\n",
+    "finds nothing to follow.\n",
+    "\n",
+    "linktrace's transport is pluggable. `BrowserFetcher` renders each page in\n",
+    "headless Chromium via Playwright; `RoutingFetcher` tries plain HTTP first and\n",
+    "escalates to the browser only when the response looks like an un-rendered\n",
+    "shell, so server-rendered pages stay on the cheap path.\n",
+    "\n",
+    "This section needs the optional browser extra:\n",
+    "\n",
+    "```bash\n",
+    "pip install \"linktrace[browser]\"\n",
+    "playwright install chromium\n",
+    "```\n",
+    "\n",
+    "First, look at what plain HTTP sees on a public React demo app."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SPA_URL = \"https://react-shopping-cart-67954.firebaseapp.com/\"\n",
+    "\n",
+    "async with Crawler(respect_robots_txt=False) as crawler:\n",
+    "    raw = await crawler.fetcher.fetch(SPA_URL, {\"User-Agent\": crawler.user_agent})\n",
+    "\n",
+    "print(f\"HTTP {raw.status}, {len(raw.body or '')} bytes\")\n",
+    "print(f\"anchors in raw HTML : {(raw.body or '').lower().count('<a ')}\")\n",
+    "print(f\"looks_like_spa_shell: {looks_like_spa_shell(raw)}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Zero anchors and the heuristic flags it: a `<noscript>` asking you to enable\n",
+    "JavaScript plus an empty `<div id=\"root\">`. Now crawl the same URL with\n",
+    "`RoutingFetcher`. Everything else — rules, caching, rate limiting, exports —\n",
+    "is unchanged; only the transport differs."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "routing = RoutingFetcher(fallback=BrowserFetcher(max_pages=2))\n",
+    "\n",
+    "spa_spider = Spider(\n",
+    "    start_url=SPA_URL,\n",
+    "    max_depth=1,\n",
+    "    fetcher=routing,\n",
+    "    rules=CrawlRules(allowed_domains=[\"react-shopping-cart-67954.firebaseapp.com\"]),\n",
+    "    cache_dir=CACHE,\n",
+    ")\n",
+    "\n",
+    "spa_docs = await spa_spider.run_async()\n",
+    "\n",
+    "print(f\"Crawled {len(spa_docs)} page(s); {routing.escalations} escalated to the browser\\n\")\n",
+    "for d in spa_docs:\n",
+    "    print(f\"[{d.fetcher:7s}] {d.url}\")\n",
+    "    print(f\"           {len(d.internal_links)} internal / {len(d.external_links)} external links\")\n",
+    "    for link in d.external_links[:3]:\n",
+    "        print(f\"             -> {link.url}\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The start page was fetched over HTTP, recognised as a shell, re-fetched in the\n",
+    "browser and parsed from the rendered DOM — `doc.fetcher == \"browser\"` records\n",
+    "that. Pages that render fine over HTTP keep `fetcher == \"http\"`, and the\n",
+    "`fetcher` column comes through in every export so downstream code can tell\n",
+    "them apart.\n",
+    "\n",
+    "Two knobs matter on real portals:\n",
+    "\n",
+    "- `render_patterns=[r\"/search\", r\"/map\"]` sends matching URLs straight to the\n",
+    "  browser without the HTTP probe.\n",
+    "- `keep_hash_routes=True` on the Spider keeps `/#/listing/42`-style routes as\n",
+    "  distinct URLs instead of collapsing them to the bare page."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 6. Big crawls: flush to disk\n",
+    "\n",
+    "A deep crawl of a listing portal can run to tens of thousands of pages, and\n",
+    "rendered DOMs are several times larger than raw HTML. `flush_to` appends each\n",
+    "page to a JSONL file the moment it is crawled and keeps nothing in memory\n",
+    "beyond URL bookkeeping. The line format is the same as `Serializers.to_json`,\n",
+    "so it loads straight into pandas or polars."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "OUT = \"focused_crawl.jsonl\"\n",
+    "\n",
+    "streamed = Spider(\n",
+    "    start_url=START_URL,\n",
+    "    max_depth=1,\n",
+    "    rules=CrawlRules(allowed_domains=[\"gis.vgsi.com\"], exclude_query_params=[\"Letter\"]),\n",
+    "    cache_dir=CACHE,\n",
+    "    flush_to=OUT,          # one JSON object per line, written as each page completes\n",
+    "    flush_html=False,      # True to keep page source as well\n",
+    ")\n",
+    "\n",
+    "result = await streamed.run_async()\n",
+    "print(f\"run_async() returned {len(result)} documents (nothing retained in memory)\")\n",
+    "\n",
+    "with open(OUT) as f:\n",
+    "    rows = [json.loads(line) for line in f]\n",
+    "print(f\"{len(rows)} pages in {OUT}\")\n",
+    "print(rows[0][\"url\"], \"->\", len(rows[0][\"internal_links\"]), \"internal links\")"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -285,6 +427,10 @@
     "  document crawler; `allows()` is pure and unit-testable.\n",
     "- **`use_sitemaps=True`** discovers pages link-following would miss, and the\n",
     "  same rules keep that inventory on-topic.\n",
+    "- **`fetcher=RoutingFetcher()`** makes JavaScript-rendered portals crawlable\n",
+    "  with one argument, paying the browser cost only where it is needed.\n",
+    "- **`flush_to=\"crawl.jsonl\"`** keeps memory flat on crawls that would never\n",
+    "  fit in RAM.\n",
     "\n",
     "See the [Examples](https://linktrace.readthedocs.io/examples/) and\n",
     "[API Reference](https://linktrace.readthedocs.io/api-reference/) docs for the\n",
diff --git a/tests/test_hardening.py b/tests/test_hardening.py
new file mode 100644
index 0000000..5e871f3
--- /dev/null
+++ b/tests/test_hardening.py
@@ -0,0 +1,328 @@
+"""Tests for production guard rails: frontier, jitter, size limits, redirects,
+cache pruning."""
+
+import os
+import time
+from collections import deque
+from unittest.mock import patch
+
+import pytest
+from aiohttp import web
+
+from linktrace import (
+    BrowserFetcher,
+    Crawler,
+    FetchContext,
+    Fetcher,
+    FetchResult,
+    HttpFetcher,
+    ResponseTooLarge,
+    Spider,
+)
+from linktrace.cache import ResponseCache
+
+PAGE = '<html><head><title>P</title></head><body><a href="/x">x</a></body></html>'
+
+
+class TestFrontier:
+    def test_to_visit_is_a_deque(self):
+        spider = Spider("https://x.test/")
+        assert isinstance(spider.to_visit, deque)
+        assert list(spider.to_visit) == [("https://x.test/", 0)]
+
+    async def test_bfs_and_dfs_order(self):
+        order: dict[str, list[str]] = {"bfs": [], "dfs": []}
+
+        class Recorder(Fetcher):
+            name = "http"
+
+            def __init__(self, bucket: list[str]) -> None:
+                self.bucket = bucket
+
+            async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
+                self.bucket.append(url)
+                body = {
+                    "https://x.test/": '<a href="/a">a</a><a href="/b">b</a>',
+                    "https://x.test/a": '<a href="/a/1">a1</a>',
+                    "https://x.test/b": '<a href="/b/1">b1</a>',
+                }.get(url, "")
+                return FetchResult(url, 200, body=body, fetcher="http")
+
+        for strategy in ("bfs", "dfs"):
+            spider = Spider(
+                "https://x.test/",
+                max_depth=3,
+                max_concurrency=1,
+                traversal_strategy=strategy,  # type: ignore[arg-type]
+                fetcher=Recorder(order[strategy]),
+                respect_robots_txt=False,
+            )
+            await spider.run_async()
+
+        assert order["bfs"] == [
+            "https://x.test/",
+            "https://x.test/a",
+            "https://x.test/b",
+            "https://x.test/a/1",
+            "https://x.test/b/1",
+        ]
+        # DFS pops the most recently discovered link first.
+        assert order["dfs"][:3] == [
+            "https://x.test/",
+            "https://x.test/b",
+            "https://x.test/b/1",
+        ]
+
+
+class TestBackoffJitter:
+    def test_jittered_delay_within_equal_jitter_band(self):
+        crawler = Crawler(backoff_factor=2)
+        for attempt in range(4):
+            base = 2**attempt * 2
+            samples = {crawler._backoff_delay(attempt) for _ in range(200)}
+            assert all(base / 2 <= s <= base for s in samples)
+            assert len(samples) > 1, "jitter should vary"
+
+    def test_jitter_can_be_disabled(self):
+        crawler = Crawler(backoff_factor=2, backoff_jitter=False)
+        assert [crawler._backoff_delay(a) for a in range(3)] == [2.0, 4.0, 8.0]
+
+    async def test_sleep_receives_jittered_value(self):
+        from linktrace import TransientFetchError
+
+        class Flaky(Fetcher):
+            name = "flaky"
+
+            async def fetch(self, url, headers):
+                raise TransientFetchError("nope")
+
+        with patch("asyncio.sleep") as sleep:
+            async with Crawler(
+                fetcher=Flaky(), respect_robots_txt=False, max_retries=2
+            ) as crawler:
+                await crawler.crawl_document_async("https://x.test/")
+        (delay,), _ = sleep.call_args
+        assert 1.0 <= delay <= 2.0
+
+
+class TestResponseSizeLimit:
+    async def test_default_limit_propagates_to_fetcher(self):
+        async with Crawler(respect_robots_txt=False) as crawler:
+            assert isinstance(crawler.fetcher, HttpFetcher)
+            assert crawler.fetcher._max_bytes == 10 * 1024 * 1024
+        async with Crawler(respect_robots_txt=False, max_response_bytes=None) as c:
+            assert c.fetcher._max_bytes is None
+
+    async def test_fetcher_override_wins(self):
+        fetcher = HttpFetcher(max_response_bytes=5)
+        async with Crawler(fetcher=fetcher, respect_robots_txt=False):
+            assert fetcher._max_bytes == 5
+
+    async def test_content_length_over_limit_rejected_before_read(self, local_site):
+        base = await local_site({"/big": "x" * 1000})
+        async with Crawler(respect_robots_txt=False, max_response_bytes=100) as crawler:
+            with pytest.raises(ResponseTooLarge) as exc:
+                await crawler.fetcher.fetch(f"{base}/big", {})
+        assert exc.value.size == 1000
+        assert exc.value.limit == 100
+
+    async def test_streamed_body_over_limit_rejected(self):
+        """A chunked response without Content-Length is cut off at the limit."""
+
+        async def handler(request: web.Request) -> web.StreamResponse:
+            resp = web.StreamResponse(headers={"Content-Type": "text/html"})
+            resp.enable_chunked_encoding()
+            await resp.prepare(request)
+            for _ in range(20):
+                await resp.write(b"y" * 100)
+            await resp.write_eof()
+            return resp
+
+        app = web.Application()
+        app.router.add_get("/stream", handler)
+        runner = web.AppRunner(app)
+        await runner.setup()
+        site = web.TCPSite(runner, "127.0.0.1", 0)
+        await site.start()
+        port = runner.addresses[0][1]
+        try:
+            async with Crawler(
+                respect_robots_txt=False, max_response_bytes=500
+            ) as crawler:
+                with pytest.raises(ResponseTooLarge):
+                    await crawler.fetcher.fetch(f"http://127.0.0.1:{port}/stream", {})
+        finally:
+            await runner.cleanup()
+
+    async def test_under_limit_is_fine_and_decoded(self, local_site):
+        base = await local_site({"/": PAGE})
+        async with Crawler(respect_robots_txt=False, max_response_bytes=10_000) as c:
+            doc = await c.crawl_document_async(f"{base}/")
+        assert doc is not None and doc.title == "P"
+
+    async def test_too_large_is_not_retried_and_returns_none(self, local_site):
+        base = await local_site({"/big": "x" * 1000})
+        with patch("asyncio.sleep") as sleep:
+            async with Crawler(
+                respect_robots_txt=False, max_response_bytes=100, max_retries=3
+            ) as crawler:
+                doc = await crawler.crawl_document_async(f"{base}/big")
+        assert doc is None
+        assert sleep.call_count == 0
+
+    async def test_browser_fetcher_honours_limit(self, local_site):
+        pytest.importorskip("playwright")
+        base = await local_site({"/": PAGE})
+        try:
+            async with Crawler(
+                fetcher=BrowserFetcher(),
+                respect_robots_txt=False,
+                max_response_bytes=10,
+                max_retries=1,
+            ) as crawler:
+                doc = await crawler.crawl_document_async(f"{base}/")
+        except RuntimeError as e:  # Chromium not installed
+            pytest.skip(str(e))
+        assert doc is None
+
+
+class TestRedirects:
+    async def test_final_url_recorded_and_links_resolve_against_it(self, local_site):
+        pages = {
+            "/old": (302, ""),
+            "/new/index": '<title>New</title><a href="sibling">s</a>',
+        }
+        base = await local_site(pages)
+
+        # local_site serves a plain 302 without Location; build a real redirect.
+        async def redirect(request: web.Request) -> web.Response:
+            raise web.HTTPFound(location=f"{base}/new/index")
+
+        app = web.Application()
+        app.router.add_get("/old", redirect)
+        runner = web.AppRunner(app)
+        await runner.setup()
+        site = web.TCPSite(runner, "127.0.0.1", 0)
+        await site.start()
+        rport = runner.addresses[0][1]
+        try:
+            async with Crawler(respect_robots_txt=False) as crawler:
+                doc = await crawler.crawl_document_async(
+                    f"http://127.0.0.1:{rport}/old"
+                )
+        finally:
+            await runner.cleanup()
+
+        assert doc is not None
+        assert doc.url == f"http://127.0.0.1:{rport}/old"
+        assert doc.final_url == f"{base}/new/index"
+        assert doc.redirected
+        assert doc.title == "New"
+        # Relative link resolved against the served URL, not the requested one.
+        assert [link.url for link in doc.links] == [f"{base}/new/sibling"]
+        # ...and classified internal relative to the served host.
+        assert doc.internal_links and not doc.external_links
+
+    def test_not_redirected_by_default(self):
+        doc = Crawler().parse_document("https://x.test/", PAGE)
+        assert doc.final_url == "https://x.test/"
+        assert not doc.redirected
+
+    async def test_spider_marks_redirect_target_visited(self):
+        class Redirecting(Fetcher):
+            name = "http"
+
+            def __init__(self) -> None:
+                self.calls: list[str] = []
+
+            async def fetch(self, url, headers):
+                self.calls.append(url)
+                if url == "https://x.test/":
+                    return FetchResult(
+                        url,
+                        200,
+                        body='<a href="/home">h</a><a href="/other">o</a>',
+                        fetcher="http",
+                        final_url="https://x.test/home",
+                    )
+                return FetchResult(url, 200, body="", fetcher="http")
+
+        fetcher = Redirecting()
+        spider = Spider(
+            "https://x.test/", max_depth=2, fetcher=fetcher, respect_robots_txt=False
+        )
+        docs = await spider.run_async()
+        # /home was served as the redirect target of "/", so it is not refetched.
+        assert fetcher.calls == ["https://x.test/", "https://x.test/other"]
+        assert "https://x.test/home" in spider.visited
+        assert {d.url for d in docs} == {"https://x.test/", "https://x.test/other"}
+
+    async def test_cache_round_trips_final_url(self, tmp_path):
+        class Redirecting(Fetcher):
+            name = "http"
+            calls = 0
+
+            async def fetch(self, url, headers):
+                Redirecting.calls += 1
+                return FetchResult(
+                    url, 200, body=PAGE, fetcher="http", final_url="https://x.test/b"
+                )
+
+        for _ in range(2):
+            async with Crawler(
+                fetcher=Redirecting(), cache_dir=str(tmp_path), respect_robots_txt=False
+            ) as crawler:
+                doc = await crawler.crawl_document_async("https://x.test/a")
+            assert doc is not None and doc.final_url == "https://x.test/b"
+        assert Redirecting.calls == 1
+
+
+class TestCachePrune:
+    async def test_prune_removes_expired_entries(self, tmp_path):
+        cache = ResponseCache(str(tmp_path), ttl_seconds=60)
+        await cache.set("https://x.test/fresh", 200, {}, "f")
+        await cache.set("https://x.test/stale", 200, {}, "s")
+        stale = cache._cache_path("https://x.test/stale")
+        old = time.time() - 120
+        os.utime(stale, (old, old))
+
+        removed = await cache.prune()
+        assert removed == 1
+        assert not stale.exists()
+        assert await cache.get("https://x.test/fresh") is not None
+
+    async def test_prune_enforces_max_bytes_oldest_first(self, tmp_path):
+        cache = ResponseCache(str(tmp_path), ttl_seconds=3600, max_bytes=1)
+        for i, url in enumerate(["https://x.test/1", "https://x.test/2"]):
+            await cache.set(url, 200, {}, "body" * 50)
+            path = cache._cache_path(url)
+            os.utime(path, (time.time() - 100 + i, time.time() - 100 + i))
+
+        removed = await cache.prune()
+        assert removed == 2  # both exceed a 1-byte cap; oldest evicted first
+
+        cache.max_bytes = None
+        await cache.set("https://x.test/3", 200, {}, "x")
+        assert await cache.prune() == 0
+
+    async def test_crawler_prunes_on_enter(self, tmp_path):
+        cache = ResponseCache(str(tmp_path), ttl_seconds=60)
+        await cache.set("https://x.test/stale", 200, {}, "s")
+        stale = cache._cache_path("https://x.test/stale")
+        os.utime(stale, (time.time() - 120, time.time() - 120))
+
+        async with Crawler(
+            cache_dir=str(tmp_path), cache_ttl=60, respect_robots_txt=False
+        ):
+            pass
+        assert not stale.exists()
+
+    def test_cache_options_thread_through_spider(self):
+        spider = Spider("https://x.test/", cache_ttl=5, cache_max_bytes=10)
+        assert spider.cache_ttl == 5 and spider.cache_max_bytes == 10
+
+
+class TestFetchContextDefaults:
+    def test_context_carries_limit(self):
+        ctx = FetchContext(session=None)  # type: ignore[arg-type]
+        assert ctx.max_response_bytes == 10 * 1024 * 1024
diff --git a/tests/test_rate_limiting_and_broken_links.py b/tests/test_rate_limiting_and_broken_links.py
index 3725a98..5ac71f1 100644
--- a/tests/test_rate_limiting_and_broken_links.py
+++ b/tests/test_rate_limiting_and_broken_links.py
@@ -9,6 +9,27 @@
 from linktrace import Crawler, Document, Spider
 
 
+def mock_aiohttp_get(mock_get, status: int, body: str = "") -> None:
+    """Point a patched ``ClientSession.get`` at a canned response.
+
+    The response carries ``.url`` and a readable ``.content`` as well as
+    ``.text`` so it works with the HttpFetcher's size-limited body read.
+    """
+    mock_response = AsyncMock()
+    mock_response.status = status
+    mock_response.headers = {}
+    mock_response.url = "https://example.com/"
+    mock_response.history = []
+    mock_response.text = AsyncMock(return_value=body)
+    mock_response.content.read = AsyncMock(return_value=body.encode())
+    mock_response.get_encoding = lambda: "utf-8"
+
+    mock_cm = AsyncMock()
+    mock_cm.__aenter__.return_value = mock_response
+    mock_cm.__aexit__.return_value = None
+    mock_get.return_value = mock_cm
+
+
 @pytest.mark.asyncio
 class TestRateLimiting:
     """Test per-domain rate limiting."""
@@ -92,15 +113,7 @@ async def test_user_agent_header_set_in_requests(self):
         crawler = Crawler(user_agent="TestBot/1.0")
         async with crawler:
             with patch("aiohttp.ClientSession.get") as mock_get:
-                mock_response = AsyncMock()
-                mock_response.status = 200
-                mock_response.headers = {}
-                mock_response.text = AsyncMock(return_value="<html></html>")
-
-                mock_cm = AsyncMock()
-                mock_cm.__aenter__.return_value = mock_response
-                mock_cm.__aexit__.return_value = None
-                mock_get.return_value = mock_cm
+                mock_aiohttp_get(mock_get, 200, "<html></html>")
 
                 try:
                     await crawler.crawl_document_async("https://example.com")
@@ -149,14 +162,7 @@ async def test_http_4xx_tracked_as_broken(self):
         crawler = Crawler()
         async with crawler:
             with patch("aiohttp.ClientSession.get") as mock_get:
-                mock_response = AsyncMock()
-                mock_response.status = 404
-                mock_response.headers = {}
-
-                mock_cm = AsyncMock()
-                mock_cm.__aenter__.return_value = mock_response
-                mock_cm.__aexit__.return_value = None
-                mock_get.return_value = mock_cm
+                mock_aiohttp_get(mock_get, 404)
 
                 doc = await crawler.crawl_document_async("https://example.com/notfound")
 
@@ -172,14 +178,7 @@ async def test_http_5xx_tracked_as_broken(self):
         crawler = Crawler(max_retries=1)
         async with crawler:
             with patch("aiohttp.ClientSession.get") as mock_get:
-                mock_response = AsyncMock()
-                mock_response.status = 500
-                mock_response.headers = {}
-
-                mock_cm = AsyncMock()
-                mock_cm.__aenter__.return_value = mock_response
-                mock_cm.__aexit__.return_value = None
-                mock_get.return_value = mock_cm
+                mock_aiohttp_get(mock_get, 500)
 
                 doc = await crawler.crawl_document_async("https://example.com/error")
 
@@ -230,17 +229,7 @@ async def test_existing_crawl_unaffected(self):
         crawler = Crawler()
         async with crawler:
             with patch("aiohttp.ClientSession.get") as mock_get:
-                mock_response = AsyncMock()
-                mock_response.status = 200
-                mock_response.headers = {}
-                mock_response.text = AsyncMock(
-                    return_value="<html><title>Test</title></html>"
-                )
-
-                mock_cm = AsyncMock()
-                mock_cm.__aenter__.return_value = mock_response
-                mock_cm.__aexit__.return_value = None
-                mock_get.return_value = mock_cm
+                mock_aiohttp_get(mock_get, 200, "<html><title>Test</title></html>")
 
                 doc = await crawler.crawl_document_async("https://example.com")
 

From ff223bfe977fdbae62e3a88b2ba726085079fe8c Mon Sep 17 00:00:00 2001
From: Jay Baywatch <jay.baywatch@gmail.com>
Date: Thu, 29 Oct 2026 20:41:08 -0400
Subject: [PATCH 3/5] Complete documentation coverage for fetchers, flushing
 and guard rails

Audit of public API against the docs turned up the remaining gaps:

- README: quick start points at run(), RoutingFetcher and flush_to; Document
  section lists final_url, fetcher, HtmlLink and BrokenLink; new "Bring Your
  Own Transport" and "Guard Rails for Long Crawls" sections cover the custom
  Fetcher contract, looks_like_spa_shell, max_response_bytes, cache_ttl,
  cache_max_bytes, show_progress and redirect handling; notebook list adds
  crawl_cnn_callbacks.
- Getting started: fix stale "cd webcrawler", mention run() vs run_async().
- Core concepts: to_visit is a deque; documents may be empty when not
  retained; visited includes redirect targets.
- API reference: document Crawler.normalize_url and keep_hash_routes.
- Examples: add "Audit Redirects".
- CONTRIBUTING: one-time playwright install for the browser tests.
- crawl_cnn_callbacks notebook: closing cell on flush_to and
  retain_documents.
---
 CONTRIBUTING.md                     |  4 ++
 README.md                           | 73 +++++++++++++++++++++++++++--
 docs/api-reference.md               | 11 +++++
 docs/core-concepts.md               |  6 +--
 docs/examples.md                    | 25 ++++++++++
 docs/getting-started.md             |  6 ++-
 notebooks/crawl_cnn_callbacks.ipynb | 25 ++++++++++
 7 files changed, 141 insertions(+), 9 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 61f4e8f..b4bca37 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -12,6 +12,10 @@ cd linktrace
 # Create a virtual environment and install in editable mode
 uv sync
 uv pip install -e ".[serializers]"
+
+# Browser (SPA) tests render a local page in headless Chromium.
+# They skip automatically if this one-time download has not been run.
+uv run playwright install chromium
 ```
 
 ## Running Tests
diff --git a/README.md b/README.md
index 714bcc9..eaf6f74 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,11 @@ async def main():
 asyncio.run(main())
 ```
 
+In a plain script you can skip the event loop boilerplate with
+`documents = Spider("https://example.com").run()`. For a JavaScript-rendered
+site add `fetcher=RoutingFetcher()`; for a crawl too big for memory add
+`flush_to="crawl.jsonl"`. Both are covered below.
+
 ## Installation
 
 ```bash
@@ -82,11 +87,12 @@ Low-level engine that fetches and parses individual documents. Handles retries,
 
 ### Document
 Rich object containing:
-- `url` — page URL
+- `url` — page URL as requested; `final_url` — where it was actually served after redirects
 - `title` — HTML title tag
-- `source` — raw HTML
-- `internal_links` — links to same domain
-- `external_links` — links to other domains
+- `source` — raw HTML, or the rendered DOM when a browser fetcher produced it
+- `fetcher` — `"http"`, `"browser"`, or your own fetcher's name
+- `internal_links` / `external_links` — `HtmlLink` objects (`url`, `text`), classified by domain
+- `broken_internal_links` — `BrokenLink` objects for internal links whose target returned 4xx/5xx (filled after the crawl)
 - `status_code`, `response_headers`, `domain` — metadata
 
 ### CrawlRules
@@ -96,7 +102,7 @@ Declarative include/exclude policy for which discovered URLs the Spider follows
 Fetches and parses XML sitemaps (urlsets and nested indexes). Used when `Spider(use_sitemaps=True)`, or directly for sitemap inspection.
 
 ### Fetchers
-The pluggable "get me the page" step. `HttpFetcher` (default) uses aiohttp; `BrowserFetcher` renders pages in headless Chromium; `RoutingFetcher` tries HTTP first and escalates to the browser only for pages that look like un-rendered SPA shells. Passed as `Spider(fetcher=...)`.
+The pluggable "get me the page" step. `HttpFetcher` (default) uses aiohttp; `BrowserFetcher` renders pages in headless Chromium; `RoutingFetcher` tries HTTP first and escalates to the browser only for pages that look like un-rendered SPA shells (the `looks_like_spa_shell` heuristic). Passed as `Spider(fetcher=...)`. Subclass `Fetcher` and return a `FetchResult` to plug in any other transport; raise `TransientFetchError` for failures the crawler should retry.
 
 See [Core Concepts](https://github.com/JayBaywatch/linktrace/blob/main/docs/core-concepts.md) for more.
 
@@ -223,6 +229,62 @@ URLs straight to the browser. Robots.txt, rate limiting, retries and caching
 apply identically whichever fetcher is used. For hash-router apps
 (`/#/listings/42`) add `keep_hash_routes=True` so each route is its own URL.
 
+Not sure whether a site needs rendering? Probe it:
+
+```python
+from linktrace import Crawler, looks_like_spa_shell
+
+async with Crawler() as crawler:
+    result = await crawler.fetcher.fetch("https://app.example.com/", {})
+    print(looks_like_spa_shell(result))   # True → plain HTTP only sees the shell
+```
+
+### Bring Your Own Transport
+
+Any client can plug in. Subclass `Fetcher`, return a `FetchResult`, and raise
+`TransientFetchError` for failures the crawler should retry. Robots.txt,
+rate limiting, retries, caching and parsing all keep working:
+
+```python
+from linktrace import Spider, Fetcher, FetchResult, TransientFetchError
+
+class MyFetcher(Fetcher):
+    name = "mine"
+
+    async def open(self, ctx):          # optional: ctx has the shared aiohttp
+        self.client = make_client(ctx)  # session, timeout, user agent, SSL setting
+
+    async def fetch(self, url, headers):
+        status, body = await self.client.get(url, headers)
+        if status == 429:
+            raise TransientFetchError("rate limited")
+        return FetchResult(url, status, body=body if status == 200 else None,
+                           fetcher=self.name)
+
+spider = Spider("https://example.com", fetcher=MyFetcher())
+```
+
+### Guard Rails for Long Crawls
+
+Defaults are tuned to run unattended against large or badly behaved sites.
+Every knob is adjustable:
+
+```python
+spider = Spider(
+    start_url="https://example.com",
+    max_response_bytes=10 * 1024 * 1024,  # drop bodies over 10 MiB (default); None = no limit
+    max_retries=3,                        # jittered exponential backoff on timeouts and 5xx
+    cache_dir=".linktrace_cache",
+    cache_ttl=86400,                      # seconds an entry stays valid (default 1 day)
+    cache_max_bytes=2 * 1024**3,          # evict oldest entries past 2 GiB, on start
+    show_progress=True,                   # tqdm bar with visited/pending counts
+)
+```
+
+Redirects are followed and recorded: `doc.final_url` is where the page was
+served, relative links resolve against it, and the redirect target is marked
+visited so it is never fetched twice.
+
 ### SSL & Corporate Proxies
 
 ```python
@@ -432,6 +494,7 @@ See [Examples](https://github.com/JayBaywatch/linktrace/blob/main/docs/examples.
 
 Interactive examples in `notebooks/`:
 - `crawl_cnn.ipynb` — Crawls CNN.com, analyzes link structure, demonstrates all export formats
+- `crawl_cnn_callbacks.ipynb` — Memory-efficient streaming with callbacks, and the one-argument `flush_to` alternative
 - `crawl_tax_assessor.ipynb` — Crawls a municipal property/GIS site, with callbacks and aggregation
 - `config_rules_sitemaps.ipynb` — URL filtering rules, configurable concurrency, sitemap discovery, crawling a JavaScript-rendered SPA with `RoutingFetcher`, and flushing a crawl to JSONL
 
diff --git a/docs/api-reference.md b/docs/api-reference.md
index 9e3d4bf..630291c 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -272,6 +272,17 @@ Parse HTML into a Document without fetching. `base_url` (default `url`) is the
 URL the content was served from: relative links resolve against it and the
 internal/external split uses its host.
 
+#### `static normalize_url(url: str, keep_hash_routes: bool = False) -> str`
+
+Canonical form used for de-duplication: strips the fragment and normalizes an
+empty path to `/`. With `keep_hash_routes=True`, fragments beginning with `#/`
+or `#!` are kept so hash-router SPA routes stay distinct.
+
+```python
+Crawler.normalize_url("https://example.com#top")                        # "https://example.com/"
+Crawler.normalize_url("https://example.com/#/about", keep_hash_routes=True)  # "https://example.com/#/about"
+```
+
 #### `async discover_sitemap_urls(base_url: str) -> List[str]`
 
 Discover page URLs from the site's sitemaps. Reads `Sitemap:` declarations from
diff --git a/docs/core-concepts.md b/docs/core-concepts.md
index c6fcc0b..e3d96e3 100644
--- a/docs/core-concepts.md
+++ b/docs/core-concepts.md
@@ -50,9 +50,9 @@ documents = await spider.run_async()
 - `start_url` — Starting URL for crawl
 - `max_depth` — How many levels deep to follow links
 - `traversal_strategy` — "bfs" (breadth-first) or "dfs" (depth-first)
-- `visited` — Set of visited URLs
-- `to_visit` — Queue of (url, depth) tuples waiting to be crawled
-- `documents` — List of Document objects (results)
+- `visited` — Set of visited URLs (redirect targets included)
+- `to_visit` — `deque` of (url, depth) tuples waiting to be crawled
+- `documents` — List of Document objects (empty when `retain_documents=False`)
 
 ### Methods
 - `run_async()` — Async crawl, returns list of Documents
diff --git a/docs/examples.md b/docs/examples.md
index 402b991..cf52a15 100644
--- a/docs/examples.md
+++ b/docs/examples.md
@@ -885,3 +885,28 @@ async def main():
 
 asyncio.run(main())
 ```
+
+## 28. Audit Redirects
+
+Redirects are followed automatically, and each Document records where it was
+actually served. Use that to find moved pages, `www`/apex mismatches, or links
+that bounce through a tracking redirect.
+
+```python
+import asyncio
+from linktrace import Spider
+
+async def main():
+    spider = Spider(start_url="https://example.com/", max_depth=2)
+    documents = await spider.run_async()
+
+    moved = [d for d in documents if d.redirected]
+    print(f"{len(moved)} of {len(documents)} pages were redirected")
+    for d in moved:
+        print(f"  {d.url}  ->  {d.final_url}  (HTTP {d.status_code})")
+
+asyncio.run(main())
+```
+
+Relative links on a redirected page are resolved against `final_url`, and the
+target URL is marked visited so the crawl never fetches it a second time.
diff --git a/docs/getting-started.md b/docs/getting-started.md
index f428f41..7619f72 100644
--- a/docs/getting-started.md
+++ b/docs/getting-started.md
@@ -33,7 +33,7 @@ playwright install chromium     # one-time download of the browser build
 
 ```bash
 git clone https://github.com/JayBaywatch/linktrace
-cd webcrawler
+cd linktrace
 pip install -e .
 pip install -e ".[serializers]"  # Optional formats
 ```
@@ -54,6 +54,10 @@ async def main():
 asyncio.run(main())
 ```
 
+In a plain script, `Spider(...).run()` does the same without the `asyncio`
+boilerplate. Inside Jupyter or an async application, `await spider.run_async()`
+is the one to use.
+
 ### 2. Inspect Documents
 
 ```python
diff --git a/notebooks/crawl_cnn_callbacks.ipynb b/notebooks/crawl_cnn_callbacks.ipynb
index a4399a4..b3b4966 100644
--- a/notebooks/crawl_cnn_callbacks.ipynb
+++ b/notebooks/crawl_cnn_callbacks.ipynb
@@ -224,6 +224,31 @@
     "\n",
     "**For large crawls (1000+ pages):** Use streaming callbacks (`accumulate_results=False`)"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Simpler alternative: `flush_to`\n",
+    "\n",
+    "Everything above can be done with one argument. `flush_to` writes each page as\n",
+    "a JSON line the moment it is crawled (same schema as `Serializers.to_json`),\n",
+    "and nothing is retained in memory:\n",
+    "\n",
+    "```python\n",
+    "spider = Spider(\n",
+    "    start_url=\"https://cnn.com/\",\n",
+    "    max_depth=2,\n",
+    "    flush_to=\"cnn_crawl.jsonl\",   # add flush_html=True to keep page source\n",
+    ")\n",
+    "await spider.run_async()          # returns []\n",
+    "```\n",
+    "\n",
+    "Either way, streaming crawls release each `Document` after your callback runs\n",
+    "(`source` and `dom` are cleared and nothing is appended to `spider.documents`).\n",
+    "Pass `retain_documents=True` if you also want the objects in memory, and note\n",
+    "that `broken_internal_links` is only computed for retained documents."
+   ]
   }
  ],
  "metadata": {

From d14608239ec677ac6c4bf75f9d9d511e0cb11fc2 Mon Sep 17 00:00:00 2001
From: Jay Baywatch <jay.baywatch@gmail.com>
Date: Fri, 30 Oct 2026 10:07:07 -0400
Subject: [PATCH 4/5] Fix truncated HTTP bodies and silent flush failures from
 review

Two data-loss bugs and seven smaller findings from code review of the
pluggable-fetcher branch.

- HttpFetcher read the body with a single StreamReader.read(limit + 1).
  aiohttp returns whatever is buffered, so on a real network only the
  first chunk came back and the page parsed with links missing. The body
  is now read in a loop until EOF, still abandoning it the moment it
  exceeds max_response_bytes. The existing chunked test passed only
  because the local server had already written everything; the new tests
  sleep between chunks.
- Spider(flush_to=...) swallowed write errors as per-page failures, so a
  full disk let the crawl finish with rows silently missing. A failed
  write now raises FlushError (exported) and aborts the crawl once the
  current batch has settled.
- The per-domain rate limit applies to every retry attempt, not just the
  first, so a retry never beats request_delay or Crawl-delay.
- HttpFetcher rebinds to the Crawler's session on every open() unless the
  caller supplied one, so a fetcher or Spider instance can be reused
  across crawls without "Session is closed".
- Crawler.__aenter__ closes the fetcher when open() fails, so a browser
  that launched but could not create a context is not leaked.
- BrowserFetcher skips wait_for_selector / settle_ms on error responses,
  so a 404 no longer waits out the timeout and gets retried as transient.
- BrowserFetcher enforces max_response_bytes on UTF-8 bytes, not
  characters.
- final_url is included in JSONL lines and flattened DataFrame rows.
- Broken-link marking indexes redirect targets, so a link to /b resolves
  when /b was only reached via /a -> /b; a direct fetch always wins.

Docs updated (API reference, core concepts, troubleshooting, README).
15 new tests (211 total).
---
 README.md                                    |   4 +-
 docs/api-reference.md                        |  39 ++-
 docs/core-concepts.md                        |  14 +-
 docs/troubleshooting.md                      |  15 +
 linktrace/Crawler.py                         |  14 +-
 linktrace/Serializers.py                     |   2 +
 linktrace/Spider.py                          |  39 ++-
 linktrace/__init__.py                        |   3 +-
 linktrace/fetchers/browser.py                |  19 +-
 linktrace/fetchers/http.py                   |  25 +-
 tests/test_browser_fetcher.py                |  31 ++
 tests/test_rate_limiting_and_broken_links.py |   3 +-
 tests/test_review_fixes.py                   | 280 +++++++++++++++++++
 13 files changed, 460 insertions(+), 28 deletions(-)
 create mode 100644 tests/test_review_fixes.py

diff --git a/README.md b/README.md
index eaf6f74..2915147 100644
--- a/README.md
+++ b/README.md
@@ -364,7 +364,9 @@ await spider.run_async()      # returns []
 ```
 
 Streaming and flushing default to `retain_documents=False`; pass
-`retain_documents=True` to keep the `Document` list as well.
+`retain_documents=True` to keep the `Document` list as well. If a line cannot
+be written (disk full, file removed) the crawl stops with a `FlushError`
+instead of finishing with rows silently missing.
 
 ### Traversal Strategies
 
diff --git a/docs/api-reference.md b/docs/api-reference.md
index 630291c..e4e6609 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -71,7 +71,7 @@ Spider(
 | `use_sitemaps` | bool | False | Seed the queue from the site's sitemaps before link-following |
 | `fetcher` | Fetcher | None | Transport used to fetch pages (None = `HttpFetcher()`). See [Fetchers](#fetchers) |
 | `keep_hash_routes` | bool | False | Keep `#/...` and `#!...` fragments as distinct URLs (hash-router SPAs) |
-| `flush_to` | str | None | Append each crawled page as one JSON line to this file as soon as it is crawled |
+| `flush_to` | str | None | Append each crawled page as one JSON line to this file as soon as it is crawled. A failed write raises `FlushError` and aborts the crawl |
 | `flush_html` | bool | False | Include page source in flushed lines |
 | `retain_documents` | bool | None | Keep `Document` objects in memory. Defaults to False when `flush_to` is set or a callback is used without `accumulate_results`, else True |
 | `max_response_bytes` | int | 10 MiB | Largest page body a fetcher may buffer; bigger responses are dropped (not retried). `None` disables |
@@ -367,9 +367,13 @@ HttpFetcher(session: aiohttp.ClientSession = None, *, max_response_bytes=<inheri
 The default. One `GET` per URL through the Crawler's session (or the one
 supplied). Rejects bodies over the size limit before buffering them: a
 `Content-Length` above the limit is refused outright, otherwise the body is
-read up to limit + 1 bytes. Reports the served URL as `final_url` and logs
-redirect chains at DEBUG. `max_response_bytes` overrides the Crawler's limit
-for this fetcher (`None` disables). `name = "http"`.
+streamed until EOF and abandoned as soon as it passes the limit. Reports the
+served URL as `final_url` and logs redirect chains at DEBUG.
+`max_response_bytes` overrides the Crawler's limit for this fetcher (`None`
+disables). `name = "http"`.
+
+An instance can be reused across crawls: it rebinds to each Crawler's session
+on `open()` unless you passed your own `session`, which is then kept as is.
 
 ### BrowserFetcher
 
@@ -395,7 +399,7 @@ Renders pages with Playwright. Requires `pip install linktrace[browser]` and
 | Parameter | Description |
 |-----------|-------------|
 | `wait_until` | When navigation counts as done. `"networkidle"` suits SPAs; use `"domcontentloaded"` + `wait_for_selector` for long-polling sites |
-| `wait_for_selector` | CSS selector to wait for after navigation (e.g. `"a[href]"`) |
+| `wait_for_selector` | CSS selector to wait for after navigation (e.g. `"a[href]"`). Skipped on error responses (status >= 400) so a 404 does not wait out the timeout |
 | `settle_ms` | Extra fixed wait after the above |
 | `max_pages` | Max concurrent tabs, independent of the Spider's `max_concurrency` |
 | `timeout` | Navigation timeout in seconds (defaults to the Crawler's `request_timeout`) |
@@ -578,7 +582,7 @@ Document(url: str, source: str = None)
 | `internal_links` | List[HtmlLink] | Links to same domain (successful crawl) |
 | `external_links` | List[HtmlLink] | Links to other domains (successful crawl) |
 | `links` | List[HtmlLink] | All links (internal + external) |
-| `broken_internal_links` | List[BrokenLink] | Internal links whose target page returned 4xx/5xx. Filled in by the Spider after the crawl, for retained documents |
+| `broken_internal_links` | List[BrokenLink] | Internal links whose target page returned 4xx/5xx. Filled in by the Spider after the crawl, for retained documents. Links to a redirect target resolve too; a directly fetched status wins over one learned through a redirect |
 | `broken_external_links` | List[BrokenLink] | Always empty: external links are not fetched |
 
 ### Properties
@@ -709,6 +713,7 @@ serializer.to_json("output.json", include_html=False)
     "title": "Example",
     "status_code": 200,
     "domain": "example",
+    "final_url": "https://example.com/",
     "fetcher": "http",
     "response_headers": {...},
     "internal_links": [{"url": "...", "text": "..."}],
@@ -729,7 +734,7 @@ print(df[["url", "title", "link_url", "link_type"]].head())
 **Returns:** pandas DataFrame
 
 **Columns:**
-- `url`, `title`, `status_code`, `domain`, `fetcher`
+- `url`, `title`, `status_code`, `domain`, `final_url`, `fetcher`
 - `link_url`, `link_text`, `link_type` (internal/external/None)
 - `html` (if include_html=True)
 
@@ -821,6 +826,26 @@ class CrawlException(Exception):
     message: str    # Error message
 ```
 
+#### `FlushError`
+
+Raised by `Spider.run_async()` when `flush_to` is set and a page cannot be
+written (disk full, file removed, permissions). The crawl stops rather than
+finishing with silently missing lines; the original `OSError` is available as
+`__cause__`. Pages already written stay in the file, so the crawl can be
+resumed by appending to it.
+
+```python
+from linktrace import FlushError
+
+try:
+    await spider.run_async()
+except FlushError as e:
+    print(f"Output incomplete: {e}")   # e.__cause__ is the OSError
+```
+
+Fetcher-level exceptions (`FetchError`, `TransientFetchError`,
+`ResponseTooLarge`) are listed under [Fetchers](#exceptions).
+
 ---
 
 ## Logging
diff --git a/docs/core-concepts.md b/docs/core-concepts.md
index e3d96e3..16e2610 100644
--- a/docs/core-concepts.md
+++ b/docs/core-concepts.md
@@ -104,6 +104,10 @@ Default: 3 retries, backoff_factor=2. The jitter keeps a batch of concurrent
 retries against one host from landing at the same instant; set
 `backoff_jitter=False` for deterministic waits.
 
+The per-domain rate limit (`request_delay` or the robots.txt `Crawl-delay`)
+applies to every attempt, not just the first, so a retry never reaches the
+host sooner than the configured delay even when the backoff is shorter.
+
 ### Redirects
 
 aiohttp follows redirects. The Crawler records the served URL as
@@ -273,8 +277,10 @@ class Document:
 ```
 
 `broken_internal_links` is filled in once the crawl finishes, from the status
-codes of the pages actually visited. External links are never fetched, so
-`broken_external_links` stays empty.
+codes of the pages actually visited. Redirect targets count as visited, so a
+link to `/b` is marked when `/b` was only reached via `/a -> /b`; a page that
+was fetched directly always wins over a status learned through a redirect.
+External links are never fetched, so `broken_external_links` stays empty.
 
 ### HtmlLink
 
@@ -460,7 +466,9 @@ finishes. Two options change that:
 - **Flush to disk.** `flush_to="crawl.jsonl"` appends each page as one JSON
   object per line (the same schema as `Serializers.to_json`; add
   `flush_html=True` to include the source) as soon as it is crawled, and
-  likewise retains nothing.
+  likewise retains nothing. A write that fails (disk full, file removed)
+  raises `FlushError` and stops the crawl, so an incomplete file is never
+  mistaken for a finished one.
 
 Both set `retain_documents=False` implicitly; pass `retain_documents=True` to
 keep the in-memory list as well. `broken_internal_links` is only computed for
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 0841c1b..e6e34fa 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -287,6 +287,21 @@ and by default all of them are kept until the crawl finishes.
 Both streaming modes set `retain_documents=False`; `broken_internal_links` is
 only computed for retained documents.
 
+### "FlushError: Could not write ... to crawl.jsonl"
+
+**Problem:** With `flush_to` set, a page could not be appended to the output
+file (disk full, the file or directory was removed, permissions changed).
+
+**Why it stops:** continuing would finish the crawl with rows silently
+missing. The exception's `__cause__` is the original `OSError`.
+
+**Solutions:**
+
+1. Free space or fix permissions, then re-run. `flush_to` appends, so pages
+   already written are kept; use `CrawlRules` or a different `start_url` to
+   avoid re-crawling them.
+2. Write to a different volume: `flush_to="/data/crawl.jsonl"`.
+
 ---
 
 ## Caching Issues
diff --git a/linktrace/Crawler.py b/linktrace/Crawler.py
index aa1eda8..5943578 100755
--- a/linktrace/Crawler.py
+++ b/linktrace/Crawler.py
@@ -185,6 +185,12 @@ async def __aenter__(self) -> "Crawler":
                 )
             )
         except Exception:
+            # Release whatever the fetcher managed to start (e.g. a browser
+            # process whose context creation failed) before giving up.
+            try:
+                await self.fetcher.close()
+            except Exception:  # pragma: no cover - best-effort teardown
+                pass
             await self.session.close()
             self.session = None
             raise
@@ -335,15 +341,15 @@ async def crawl_document_async(self, url: str) -> Document | None:
         else:
             delay = self.request_delay
 
-        # Enforce per-domain rate limiting
-        await self._rate_limit_domain(url, delay)
-
         # Fetch with retries. Transient errors and 5xx responses are retried
-        # with exponential backoff; other failures return immediately.
+        # with exponential backoff; other failures return immediately. The
+        # per-domain rate limit applies to every attempt, not just the first,
+        # so a retry never hits the host sooner than the configured delay.
         headers = {"User-Agent": self.user_agent}
         result: FetchResult | None = None
         for attempt in range(self.max_retries):
             last_attempt = attempt >= self.max_retries - 1
+            await self._rate_limit_domain(url, delay)
             try:
                 self._logger.debug(f"Fetching {url} (attempt {attempt + 1})")
                 result = await self.fetcher.fetch(url, headers)
diff --git a/linktrace/Serializers.py b/linktrace/Serializers.py
index 2d3906e..54a26b1 100644
--- a/linktrace/Serializers.py
+++ b/linktrace/Serializers.py
@@ -45,6 +45,7 @@ def document_to_dict(doc: Document, include_html: bool = False) -> dict:
             "title": doc.title,
             "status_code": doc.status_code,
             "domain": doc.domain,
+            "final_url": doc.final_url,
             "fetcher": doc.fetcher,
             "response_headers": doc.response_headers,
             "internal_links": [
@@ -150,6 +151,7 @@ def _flatten_documents(self, include_html: bool = False) -> list[dict]:
                 "title": doc.title,
                 "status_code": doc.status_code,
                 "domain": doc.domain,
+                "final_url": doc.final_url,
                 "fetcher": doc.fetcher,
             }
 
diff --git a/linktrace/Spider.py b/linktrace/Spider.py
index effa1cb..09ed24e 100755
--- a/linktrace/Spider.py
+++ b/linktrace/Spider.py
@@ -11,6 +11,15 @@
 from linktrace.Serializers import Serializers
 
 
+class FlushError(RuntimeError):
+    """Raised when ``Spider(flush_to=...)`` cannot write a crawled page.
+
+    A failed write means the output file is incomplete, so the crawl is
+    aborted rather than continuing and finishing with silently missing rows.
+    The original ``OSError`` is available as ``__cause__``.
+    """
+
+
 class Spider:
     def __init__(
         self,
@@ -229,7 +238,13 @@ async def run_async(self) -> list[Document]:
                             self.crawl_and_collect(url, depth, crawler)
                             for url, depth in batch
                         ]
-                        await asyncio.gather(*tasks)
+                        # Let the whole batch settle, then surface a flush
+                        # failure: outstanding tasks have finished, nothing is
+                        # left half-written, and the caller sees the error.
+                        outcomes = await asyncio.gather(*tasks, return_exceptions=True)
+                        for outcome in outcomes:
+                            if isinstance(outcome, BaseException):
+                                raise outcome
 
             self._mark_broken_links()
         finally:
@@ -262,8 +277,17 @@ def _mark_broken_links(self) -> None:
         links are never fetched, so ``broken_external_links`` stays empty.
         Only retained documents are marked; with ``retain_documents=False``
         (streaming or ``flush_to`` crawls) there is nothing to mark.
+
+        Redirect targets are indexed too, so a link to ``/b`` resolves when
+        ``/b`` was only ever reached via ``/a -> /b``. A page fetched directly
+        always wins over a status learned through a redirect.
         """
-        status_by_url = {doc.url: doc.status_code for doc in self.documents}
+        status_by_url: dict[str, int] = {}
+        for doc in self.documents:
+            if doc.redirected:
+                status_by_url.setdefault(doc.final_url, doc.status_code)
+        for doc in self.documents:
+            status_by_url[doc.url] = doc.status_code
         for doc in self.documents:
             doc.broken_internal_links = [
                 BrokenLink(link.url, status_by_url[link.url])
@@ -333,7 +357,12 @@ async def crawl_and_collect(
                         self.accumulated_results.append(result)
 
                 if self._flush_file is not None:
-                    await self._flush_document(doc)
+                    try:
+                        await self._flush_document(doc)
+                    except Exception as e:
+                        raise FlushError(
+                            f"Could not write {url} to {self.flush_to}: {e}"
+                        ) from e
 
                 if not self.retain_documents:
                     # Free the heavy parts now; the Document object itself is
@@ -354,6 +383,10 @@ async def crawl_and_collect(
 
                         self.to_visit.append((link.url, current_depth + 1))
 
+        except FlushError as e:
+            # Not a per-page failure: the output file is now incomplete.
+            self._logger.error(str(e))
+            raise
         except Exception as e:
             self._logger.error(f"Failed to crawl {url}: {e}")
             if self.on_error is not None:
diff --git a/linktrace/__init__.py b/linktrace/__init__.py
index 2b878f9..fb8ec0b 100644
--- a/linktrace/__init__.py
+++ b/linktrace/__init__.py
@@ -17,7 +17,7 @@
 from linktrace.rules import CrawlRules
 from linktrace.Serializers import Serializers
 from linktrace.sitemap import SitemapParser
-from linktrace.Spider import Spider
+from linktrace.Spider import FlushError, Spider
 
 __all__ = [
     "BrokenLink",
@@ -30,6 +30,7 @@
     "Fetcher",
     "FetchError",
     "FetchResult",
+    "FlushError",
     "HtmlLink",
     "HttpFetcher",
     "ResponseCache",
diff --git a/linktrace/fetchers/browser.py b/linktrace/fetchers/browser.py
index 1e8a864..36c45a4 100644
--- a/linktrace/fetchers/browser.py
+++ b/linktrace/fetchers/browser.py
@@ -52,7 +52,8 @@ class BrowserFetcher(Fetcher):
             ``wait_for_selector`` for sites with long-polling connections that
             never go idle.
         wait_for_selector: Optional CSS selector to wait for after navigation,
-            e.g. ``"a[href]"`` or ``"#listings .card"``.
+            e.g. ``"a[href]"`` or ``"#listings .card"``. Not applied to error
+            responses (status >= 400), which never render the app.
         settle_ms: Extra fixed delay after the waits above, for apps that
             render in a second pass. Default ``0``.
         max_pages: Maximum tabs open at once (default ``4``).
@@ -191,11 +192,15 @@ async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
                     response = await page.goto(
                         url, wait_until=self.wait_until, timeout=self._timeout_ms
                     )
-                    if self.wait_for_selector:
+                    # Error pages never render the app, so waiting for its
+                    # selector would only burn the full timeout and turn a
+                    # plain 404 into a retried "transient" failure.
+                    rendered_ok = response is None or response.status < 400
+                    if self.wait_for_selector and rendered_ok:
                         await page.wait_for_selector(
                             self.wait_for_selector, timeout=self._timeout_ms
                         )
-                    if self.settle_ms:
+                    if self.settle_ms and rendered_ok:
                         await page.wait_for_timeout(self.settle_ms)
                     body = await page.content()
                 except self._timeout_error as e:
@@ -212,8 +217,12 @@ async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
             finally:
                 await page.close()
 
-        if self._max_bytes is not None and len(body) > self._max_bytes:
-            raise ResponseTooLarge(url, self._max_bytes, len(body))
+        if self._max_bytes is not None:
+            # Compare bytes, not characters: the limit is a byte budget and
+            # non-ASCII pages are larger on the wire than len(str) suggests.
+            size = len(body.encode("utf-8"))
+            if size > self._max_bytes:
+                raise ResponseTooLarge(url, self._max_bytes, size)
 
         return FetchResult(
             url=url,
diff --git a/linktrace/fetchers/http.py b/linktrace/fetchers/http.py
index cae9451..a2f57e8 100644
--- a/linktrace/fetchers/http.py
+++ b/linktrace/fetchers/http.py
@@ -46,17 +46,25 @@ def __init__(
         max_response_bytes: int | None | object = _INHERIT,
     ) -> None:
         self._session = session
+        self._user_session = session is not None
         self._max_bytes_override = max_response_bytes
         self._max_bytes: int | None = (
             None if max_response_bytes is _INHERIT else max_response_bytes  # type: ignore[assignment]
         )
 
     async def open(self, ctx: FetchContext) -> None:
-        if self._session is None:
+        # Always rebind to the Crawler's current session unless the caller
+        # supplied one: a fetcher instance may be reused across crawls, and
+        # the previous Crawler closed its session on exit.
+        if not self._user_session:
             self._session = ctx.session
         if self._max_bytes_override is _INHERIT:
             self._max_bytes = ctx.max_response_bytes
 
+    async def close(self) -> None:
+        if not self._user_session:
+            self._session = None
+
     async def fetch(self, url: str, headers: dict[str, str]) -> FetchResult:
         if self._session is None:
             raise RuntimeError(
@@ -100,9 +108,20 @@ async def _read_body(self, url: str, response: aiohttp.ClientResponse) -> str:
         if declared and declared.isdigit() and int(declared) > limit:
             raise ResponseTooLarge(url, limit, int(declared))
 
-        raw = await response.content.read(limit + 1)
-        if len(raw) > limit:
+        # StreamReader.read(n) returns whatever is buffered, up to n bytes, so
+        # a single call can hand back only the first network chunk. Keep
+        # reading until EOF or until the body provably exceeds the limit.
+        chunks: list[bytes] = []
+        received = 0
+        while received <= limit:
+            chunk = await response.content.read(limit + 1 - received)
+            if not chunk:
+                break
+            chunks.append(chunk)
+            received += len(chunk)
+        if received > limit:
             raise ResponseTooLarge(url, limit)
+        raw = b"".join(chunks)
 
         try:
             encoding = response.get_encoding()
diff --git a/tests/test_browser_fetcher.py b/tests/test_browser_fetcher.py
index 2e655f8..09d2793 100644
--- a/tests/test_browser_fetcher.py
+++ b/tests/test_browser_fetcher.py
@@ -4,6 +4,8 @@
 (``pip install 'linktrace[browser]' && playwright install chromium``).
 """
 
+import time
+
 import pytest
 
 from linktrace import BrowserFetcher, Crawler, RoutingFetcher, Spider
@@ -109,3 +111,32 @@ async def test_unopened_fetcher_raises(self):
     def test_max_pages_validated(self):
         with pytest.raises(ValueError):
             BrowserFetcher(max_pages=0)
+
+    async def test_error_page_skips_selector_wait(self, local_site):
+        """A 404 must not burn the selector timeout and become a retry."""
+        base = await local_site({"/missing": (404, "<h1>nope</h1>")})
+        fetcher = BrowserFetcher(
+            wait_until="domcontentloaded", wait_for_selector="#never-appears"
+        )
+        async with Crawler(
+            fetcher=fetcher, respect_robots_txt=False, request_timeout=5, max_retries=1
+        ) as crawler:
+            start = time.monotonic()
+            doc = await crawler.crawl_document_async(f"{base}/missing")
+            elapsed = time.monotonic() - start
+        assert doc is not None and doc.status_code == 404
+        assert elapsed < 4  # did not wait out the 5s selector timeout
+
+    async def test_size_limit_counts_utf8_bytes(self, local_site):
+        from linktrace import ResponseTooLarge
+
+        # 600 chars of a 2-byte character: under a 1000-char limit by length,
+        # over it by bytes once the HTML wrapper is added.
+        page = "<html><body><p>" + "é" * 600 + "</p></body></html>"
+        base = await local_site({"/wide": page})
+        async with Crawler(
+            fetcher=BrowserFetcher(), respect_robots_txt=False, max_response_bytes=1000
+        ) as crawler:
+            with pytest.raises(ResponseTooLarge) as exc:
+                await crawler.fetcher.fetch(f"{base}/wide", {})
+        assert exc.value.size > 1200
diff --git a/tests/test_rate_limiting_and_broken_links.py b/tests/test_rate_limiting_and_broken_links.py
index 5ac71f1..e7512ad 100644
--- a/tests/test_rate_limiting_and_broken_links.py
+++ b/tests/test_rate_limiting_and_broken_links.py
@@ -21,7 +21,8 @@ def mock_aiohttp_get(mock_get, status: int, body: str = "") -> None:
     mock_response.url = "https://example.com/"
     mock_response.history = []
     mock_response.text = AsyncMock(return_value=body)
-    mock_response.content.read = AsyncMock(return_value=body.encode())
+    # Behave like a real stream: the body, then EOF.
+    mock_response.content.read = AsyncMock(side_effect=[body.encode(), b""])
     mock_response.get_encoding = lambda: "utf-8"
 
     mock_cm = AsyncMock()
diff --git a/tests/test_review_fixes.py b/tests/test_review_fixes.py
new file mode 100644
index 0000000..d9effc6
--- /dev/null
+++ b/tests/test_review_fixes.py
@@ -0,0 +1,280 @@
+"""Regression tests for the review findings on the pluggable-fetcher branch:
+truncated streamed bodies, swallowed flush failures, rate limits on retries,
+session rebinding, partial fetcher startup, export fields and redirect-aware
+broken-link marking."""
+
+import asyncio
+import time
+
+import pytest
+from aiohttp import web
+
+from linktrace import (
+    Crawler,
+    FetchContext,
+    Fetcher,
+    FetchResult,
+    FlushError,
+    HttpFetcher,
+    Serializers,
+    Spider,
+)
+
+PAGE = '<html><head><title>P</title></head><body><a href="/x">x</a></body></html>'
+
+
+async def _serve(app: web.Application) -> tuple[web.AppRunner, str]:
+    runner = web.AppRunner(app)
+    await runner.setup()
+    site = web.TCPSite(runner, "127.0.0.1", 0)
+    await site.start()
+    return runner, f"http://127.0.0.1:{runner.addresses[0][1]}"
+
+
+class TestStreamedBodyIsReadToEof:
+    async def test_slow_chunked_body_under_limit_is_complete(self):
+        """A body arriving in several network chunks must not be truncated."""
+
+        async def handler(request: web.Request) -> web.StreamResponse:
+            resp = web.StreamResponse(headers={"Content-Type": "text/html"})
+            resp.enable_chunked_encoding()
+            await resp.prepare(request)
+            await resp.write(b"<html><body>")
+            for i in range(5):
+                await asyncio.sleep(0.02)  # force separate chunks on the wire
+                await resp.write(f'<a href="/p{i}">{i}</a>'.encode())
+            await resp.write(b"</body></html>")
+            await resp.write_eof()
+            return resp
+
+        app = web.Application()
+        app.router.add_get("/slow", handler)
+        runner, base = await _serve(app)
+        try:
+            async with Crawler(
+                respect_robots_txt=False, max_response_bytes=10_000
+            ) as crawler:
+                result = await crawler.fetcher.fetch(f"{base}/slow", {})
+                doc = await crawler.crawl_document_async(f"{base}/slow")
+        finally:
+            await runner.cleanup()
+        assert result.body.endswith("</body></html>")
+        assert doc is not None
+        assert [link.url for link in doc.internal_links] == [
+            f"{base}/p{i}" for i in range(5)
+        ]
+
+    async def test_slow_chunked_body_over_limit_still_rejected(self):
+        from linktrace import ResponseTooLarge
+
+        async def handler(request: web.Request) -> web.StreamResponse:
+            resp = web.StreamResponse(headers={"Content-Type": "text/html"})
+            resp.enable_chunked_encoding()
+            await resp.prepare(request)
+            for _ in range(10):
+                await asyncio.sleep(0.01)
+                await resp.write(b"y" * 100)
+            await resp.write_eof()
+            return resp
+
+        app = web.Application()
+        app.router.add_get("/stream", handler)
+        runner, base = await _serve(app)
+        try:
+            async with Crawler(
+                respect_robots_txt=False, max_response_bytes=500
+            ) as crawler:
+                with pytest.raises(ResponseTooLarge):
+                    await crawler.fetcher.fetch(f"{base}/stream", {})
+        finally:
+            await runner.cleanup()
+
+
+class TestFlushFailuresAbortTheCrawl:
+    async def test_write_error_raises_flush_error(self, local_site, tmp_path):
+        base = await local_site({"/": PAGE, "/x": PAGE})
+        spider = Spider(
+            f"{base}/",
+            max_depth=1,
+            flush_to=str(tmp_path / "out.jsonl"),
+            respect_robots_txt=False,
+        )
+
+        async def broken_write(_doc):
+            raise OSError("No space left on device")
+
+        spider._flush_document = broken_write  # type: ignore[method-assign]
+        with pytest.raises(FlushError, match="No space left") as exc:
+            await spider.run_async()
+        assert isinstance(exc.value.__cause__, OSError)
+        assert spider._flush_file is None  # closed by the finally block
+
+    async def test_flush_error_is_exported(self):
+        import linktrace
+
+        assert "FlushError" in linktrace.__all__
+
+
+class TestRateLimitAppliesToRetries:
+    async def test_retry_waits_for_domain_delay(self, local_site):
+        base = await local_site({"/flaky": (503, "busy")})
+        async with Crawler(
+            respect_robots_txt=False,
+            request_delay=0.3,
+            max_retries=2,
+            backoff_factor=0.01,  # type: ignore[arg-type]
+            backoff_jitter=False,
+        ) as crawler:
+            start = time.monotonic()
+            doc = await crawler.crawl_document_async(f"{base}/flaky")
+            elapsed = time.monotonic() - start
+        assert doc is not None and doc.status_code == 503
+        # Two attempts: the second must respect the 0.3s per-domain delay
+        # even though the backoff alone would only have waited 0.01s.
+        assert elapsed >= 0.3
+
+
+class TestHttpFetcherSessionRebinding:
+    async def test_fetcher_instance_survives_two_crawlers(self, local_site):
+        base = await local_site({"/": PAGE})
+        fetcher = HttpFetcher()
+        async with Crawler(fetcher=fetcher, respect_robots_txt=False) as c1:
+            assert (await c1.crawl_document_async(f"{base}/")) is not None
+        async with Crawler(fetcher=fetcher, respect_robots_txt=False) as c2:
+            doc = await c2.crawl_document_async(f"{base}/")
+        assert doc is not None and doc.title == "P"
+
+    async def test_spider_can_run_twice_with_same_fetcher(self, local_site):
+        base = await local_site({"/": PAGE, "/x": PAGE})
+        fetcher = HttpFetcher()
+        spider = Spider(
+            f"{base}/", max_depth=1, fetcher=fetcher, respect_robots_txt=False
+        )
+        first = len(await spider.run_async())
+        assert first == 2
+        # Reset crawl state and go again with the same (now re-opened) fetcher.
+        spider.visited.clear()
+        spider.in_progress.clear()
+        spider.to_visit.append((f"{base}/", 0))
+        second = await spider.run_async()
+        assert len(second) > first  # documents accumulate; no "Session is closed"
+
+    async def test_user_supplied_session_is_kept(self):
+        import aiohttp
+
+        session = aiohttp.ClientSession()
+        try:
+            fetcher = HttpFetcher(session=session)
+            async with Crawler(fetcher=fetcher, respect_robots_txt=False):
+                assert fetcher._session is session
+            assert fetcher._session is session  # close() does not drop it
+        finally:
+            await session.close()
+
+
+class TestPartialStartupIsCleanedUp:
+    async def test_fetcher_closed_when_open_fails(self):
+        class Exploding(Fetcher):
+            name = "boom"
+
+            def __init__(self) -> None:
+                self.closed = False
+
+            async def open(self, ctx: FetchContext) -> None:
+                raise RuntimeError("no browser")
+
+            async def close(self) -> None:
+                self.closed = True
+
+            async def fetch(self, url, headers):  # pragma: no cover
+                raise AssertionError
+
+        fetcher = Exploding()
+        crawler = Crawler(fetcher=fetcher, respect_robots_txt=False)
+        with pytest.raises(RuntimeError, match="no browser"):
+            async with crawler:
+                pass
+        assert fetcher.closed
+        assert crawler.session is None
+
+
+class TestExportsCarryFinalUrl:
+    def _doc(self):
+        c = Crawler()
+        doc = c.parse_document("https://x.test/a", PAGE, base_url="https://x.test/b")
+        doc.final_url = "https://x.test/b"
+        return doc
+
+    def test_document_to_dict(self):
+        data = Serializers.document_to_dict(self._doc())
+        assert data["url"] == "https://x.test/a"
+        assert data["final_url"] == "https://x.test/b"
+
+    def test_flattened_rows(self):
+        rows = Serializers([self._doc()])._flatten_documents()
+        assert rows and all(r["final_url"] == "https://x.test/b" for r in rows)
+
+
+class TestBrokenLinksViaRedirect:
+    async def test_link_to_redirect_target_is_marked_broken(self):
+        class Site(Fetcher):
+            name = "http"
+
+            async def fetch(self, url, headers):
+                if url == "https://x.test/":
+                    return FetchResult(
+                        url,
+                        200,
+                        body='<a href="/a">a</a><a href="/b">b</a>',
+                        fetcher="http",
+                    )
+                if url == "https://x.test/a":
+                    # /a redirects to /b, which is gone.
+                    return FetchResult(
+                        url, 404, fetcher="http", final_url="https://x.test/b"
+                    )
+                return FetchResult(url, 200, body="", fetcher="http")
+
+        # One URL at a time so /a's redirect marks /b visited before /b is
+        # popped; /b's status is then only known through the redirect.
+        spider = Spider(
+            "https://x.test/",
+            max_depth=1,
+            fetcher=Site(),
+            respect_robots_txt=False,
+            max_concurrency=1,
+        )
+        docs = await spider.run_async()
+        home = next(d for d in docs if d.url == "https://x.test/")
+        broken = {b.url: b.status_code for b in home.broken_internal_links}
+        assert broken == {"https://x.test/a": 404, "https://x.test/b": 404}
+
+    async def test_direct_fetch_wins_over_redirect_status(self):
+        class Site(Fetcher):
+            name = "http"
+
+            async def fetch(self, url, headers):
+                if url == "https://x.test/":
+                    return FetchResult(
+                        url,
+                        200,
+                        body='<a href="/b">b</a><a href="/a">a</a>',
+                        fetcher="http",
+                    )
+                if url == "https://x.test/a":
+                    return FetchResult(
+                        url, 500, fetcher="http", final_url="https://x.test/b"
+                    )
+                return FetchResult(url, 200, body="", fetcher="http")
+
+        spider = Spider(
+            "https://x.test/",
+            max_depth=1,
+            fetcher=Site(),
+            respect_robots_txt=False,
+            max_concurrency=1,
+        )
+        docs = await spider.run_async()
+        home = next(d for d in docs if d.url == "https://x.test/")
+        # /b was fetched directly (200) before /a's redirect reported 500 for it.
+        assert {b.url for b in home.broken_internal_links} == {"https://x.test/a"}

From f90502c2b448223f5f4c92eac9359f1ff28d4a19 Mon Sep 17 00:00:00 2001
From: Jay Baywatch <jay.baywatch@gmail.com>
Date: Fri, 30 Oct 2026 23:11:31 -0400
Subject: [PATCH 5/5] Derive the default User-Agent from the installed version

The default was hard-coded as "linktrace/0.1.0" in Spider, Crawler,
SitemapParser and FetchContext, two releases behind. A new _version module
reads the version from package metadata and exposes DEFAULT_USER_AGENT;
all four defaults use it, and linktrace.__version__ is exported.
---
 docs/api-reference.md                        | 10 +++++-----
 linktrace/Crawler.py                         |  3 ++-
 linktrace/Spider.py                          |  3 ++-
 linktrace/__init__.py                        |  2 ++
 linktrace/_version.py                        | 14 ++++++++++++++
 linktrace/fetchers/base.py                   |  4 +++-
 linktrace/sitemap.py                         |  4 +++-
 tests/test_rate_limiting_and_broken_links.py |  5 +++--
 tests/test_review_fixes.py                   | 12 ++++++++++++
 9 files changed, 46 insertions(+), 11 deletions(-)
 create mode 100644 linktrace/_version.py

diff --git a/docs/api-reference.md b/docs/api-reference.md
index e4e6609..a98187f 100644
--- a/docs/api-reference.md
+++ b/docs/api-reference.md
@@ -24,7 +24,7 @@ Spider(
     on_crawl_complete: Callable[[], None] = None,
     accumulate_results: bool = False,
     request_delay: float = 0.0,
-    user_agent: str = "linktrace/0.1.0",
+    user_agent: str = "linktrace/<version>",
     respect_robots_txt: bool = True,
     max_concurrency: int = 10,
     max_connections: int = 100,
@@ -62,7 +62,7 @@ Spider(
 | `on_crawl_complete` | Callable | None | Callback when crawl finishes. Supports async for cleanup |
 | `accumulate_results` | bool | False | If True, accumulate callback return values in results list |
 | `request_delay` | float | 0.0 | Minimum seconds between requests to same domain (0 = no forced delay) |
-| `user_agent` | str | "linktrace/0.1.0" | User-Agent header for requests (affects robots.txt rules) |
+| `user_agent` | str | "linktrace/<version>" | User-Agent header for requests (affects robots.txt rules). Default tracks the installed package version |
 | `respect_robots_txt` | bool | True | Parse and respect robots.txt Crawl-delay directives |
 | `max_concurrency` | int | 10 | Max URLs fetched concurrently per batch (must be ≥ 1) |
 | `max_connections` | int | 100 | Total size of the aiohttp connection pool |
@@ -196,7 +196,7 @@ Crawler(
     max_retries: int = 3,
     backoff_factor: int = 2,
     request_delay: float = 0.0,
-    user_agent: str = "linktrace/0.1.0",
+    user_agent: str = "linktrace/<version>",
     respect_robots_txt: bool = True,
     max_connections: int = 100,
     max_connections_per_host: int = 10,
@@ -514,7 +514,7 @@ inspection.
 ```python
 SitemapParser(
     session: aiohttp.ClientSession,
-    user_agent: str = "linktrace/0.1.0",
+    user_agent: str = "linktrace/<version>",
     logger: logging.Logger = None,
     max_sitemaps: int = 50,
     max_depth: int = 5
@@ -526,7 +526,7 @@ SitemapParser(
 | Parameter | Type | Default | Description |
 |-----------|------|---------|-------------|
 | `session` | aiohttp.ClientSession | — | Session used to fetch sitemap documents |
-| `user_agent` | str | "linktrace/0.1.0" | User-Agent sent with sitemap requests |
+| `user_agent` | str | "linktrace/<version>" | User-Agent sent with sitemap requests |
 | `logger` | logging.Logger | None | Logger to use (defaults to module logger) |
 | `max_sitemaps` | int | 50 | Hard cap on sitemap documents fetched (guards against huge indexes) |
 | `max_depth` | int | 5 | Max nesting depth for sitemap-index recursion |
diff --git a/linktrace/Crawler.py b/linktrace/Crawler.py
index 5943578..55e0555 100755
--- a/linktrace/Crawler.py
+++ b/linktrace/Crawler.py
@@ -9,6 +9,7 @@
 import lxml.html
 import tldextract
 
+from linktrace._version import DEFAULT_USER_AGENT
 from linktrace.cache import ResponseCache
 from linktrace.fetchers.base import (
     FetchContext,
@@ -109,7 +110,7 @@ def __init__(
         max_retries: int = 3,
         backoff_factor: int = 2,
         request_delay: float = 0.0,
-        user_agent: str = "linktrace/0.1.0",
+        user_agent: str = DEFAULT_USER_AGENT,
         respect_robots_txt: bool = True,
         max_connections: int = 100,
         max_connections_per_host: int = 10,
diff --git a/linktrace/Spider.py b/linktrace/Spider.py
index 09ed24e..bf64608 100755
--- a/linktrace/Spider.py
+++ b/linktrace/Spider.py
@@ -5,6 +5,7 @@
 from collections.abc import Callable
 from typing import Any, Literal
 
+from linktrace._version import DEFAULT_USER_AGENT
 from linktrace.Crawler import BrokenLink, Crawler, Document
 from linktrace.fetchers.base import Fetcher
 from linktrace.rules import CrawlRules
@@ -43,7 +44,7 @@ def __init__(
         on_crawl_complete: Callable[[], None] | None = None,
         accumulate_results: bool = False,
         request_delay: float = 0.0,
-        user_agent: str = "linktrace/0.1.0",
+        user_agent: str = DEFAULT_USER_AGENT,
         respect_robots_txt: bool = True,
         max_concurrency: int = 10,
         max_connections: int = 100,
diff --git a/linktrace/__init__.py b/linktrace/__init__.py
index fb8ec0b..9c28b9d 100644
--- a/linktrace/__init__.py
+++ b/linktrace/__init__.py
@@ -1,5 +1,6 @@
 """WebCrawler — async web crawler / spider."""
 
+from linktrace._version import __version__
 from linktrace.cache import ResponseCache
 from linktrace.Crawler import BrokenLink, Crawler, CrawlException, Document, HtmlLink
 from linktrace.fetchers import (
@@ -20,6 +21,7 @@
 from linktrace.Spider import FlushError, Spider
 
 __all__ = [
+    "__version__",
     "BrokenLink",
     "BrowserFetcher",
     "Crawler",
diff --git a/linktrace/_version.py b/linktrace/_version.py
new file mode 100644
index 0000000..c20f679
--- /dev/null
+++ b/linktrace/_version.py
@@ -0,0 +1,14 @@
+"""Package version and the default User-Agent derived from it.
+
+Kept free of package imports so Crawler, Spider and SitemapParser can use it
+for their defaults without import cycles.
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("linktrace")
+except PackageNotFoundError:  # pragma: no cover - running from an unbuilt checkout
+    __version__ = "0.0.0+unknown"
+
+DEFAULT_USER_AGENT = f"linktrace/{__version__}"
diff --git a/linktrace/fetchers/base.py b/linktrace/fetchers/base.py
index 7ada468..3f596be 100644
--- a/linktrace/fetchers/base.py
+++ b/linktrace/fetchers/base.py
@@ -24,6 +24,8 @@
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
+from linktrace._version import DEFAULT_USER_AGENT
+
 if TYPE_CHECKING:
     import aiohttp
 
@@ -74,7 +76,7 @@ class FetchContext:
 
     session: aiohttp.ClientSession
     request_timeout: float = 30
-    user_agent: str = "linktrace/0.1.0"
+    user_agent: str = DEFAULT_USER_AGENT
     ssl_verify: bool | str = True
     max_response_bytes: int | None = 10 * 1024 * 1024
 
diff --git a/linktrace/sitemap.py b/linktrace/sitemap.py
index 097aa20..1a3e341 100644
--- a/linktrace/sitemap.py
+++ b/linktrace/sitemap.py
@@ -11,6 +11,8 @@
 import aiohttp
 import lxml.etree
 
+from linktrace._version import DEFAULT_USER_AGENT
+
 
 class SitemapParser:
     """Fetches and parses XML sitemaps, returning the page URLs they list."""
@@ -18,7 +20,7 @@ class SitemapParser:
     def __init__(
         self,
         session: aiohttp.ClientSession,
-        user_agent: str = "linktrace/0.1.0",
+        user_agent: str = DEFAULT_USER_AGENT,
         logger: logging.Logger | None = None,
         max_sitemaps: int = 50,
         max_depth: int = 5,
diff --git a/tests/test_rate_limiting_and_broken_links.py b/tests/test_rate_limiting_and_broken_links.py
index e7512ad..9cb563e 100644
--- a/tests/test_rate_limiting_and_broken_links.py
+++ b/tests/test_rate_limiting_and_broken_links.py
@@ -6,6 +6,7 @@
 
 import pytest
 
+import linktrace
 from linktrace import Crawler, Document, Spider
 
 
@@ -198,7 +199,7 @@ def test_default_request_delay_is_zero(self):
     def test_default_user_agent(self):
         """Default user agent should be set."""
         crawler = Crawler()
-        assert crawler.user_agent == "linktrace/0.1.0"
+        assert crawler.user_agent == f"linktrace/{linktrace.__version__}"
 
     def test_default_respect_robots_txt_true(self):
         """Default respect_robots_txt should be True."""
@@ -209,7 +210,7 @@ def test_spider_default_parameters(self):
         """Spider should have default rate limiting parameters."""
         spider = Spider(start_url="https://example.com")
         assert spider.request_delay == 0.0
-        assert spider.user_agent == "linktrace/0.1.0"
+        assert spider.user_agent == f"linktrace/{linktrace.__version__}"
         assert spider.respect_robots_txt is True
 
     def test_spider_custom_parameters(self):
diff --git a/tests/test_review_fixes.py b/tests/test_review_fixes.py
index d9effc6..4a52179 100644
--- a/tests/test_review_fixes.py
+++ b/tests/test_review_fixes.py
@@ -278,3 +278,15 @@ async def fetch(self, url, headers):
         home = next(d for d in docs if d.url == "https://x.test/")
         # /b was fetched directly (200) before /a's redirect reported 500 for it.
         assert {b.url for b in home.broken_internal_links} == {"https://x.test/a"}
+
+
+class TestDefaultUserAgentTracksVersion:
+    def test_version_exported_and_used(self):
+        import linktrace
+        from linktrace import Crawler, SitemapParser, Spider
+
+        expected = f"linktrace/{linktrace.__version__}"
+        assert linktrace.__version__ not in ("", "0.1.0")
+        assert Crawler().user_agent == expected
+        assert Spider("https://x.test/").user_agent == expected
+        assert SitemapParser(session=None).user_agent == expected  # type: ignore[arg-type]