From a6280a00bc330439dfa71069f73eed6533f347d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 18:55:28 +0000 Subject: [PATCH 01/30] Database outputs that lose their server, and crawls whose output fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §72 asks for storage failures among the failures tested. Three held badly: - A PostgreSQL or MySQL output whose server hung up (a restart, a failover, an idle timeout) lost every item after it. The rows waiting were dropped, each write after failed, and the errors were the driver's own (OperationalError, InterfaceError (0, '')). They were not an ExportError saying how many items were not written. The output now connects again and writes the batch, or adds the column, again, once. A server that does not answer is an ExportError that says so, with how many items. The next batch connects again, so the output takes items once more when the server is back. ExportError.items carries the count (MongoDB's batches too; pymongo reconnects by itself). - An output that failed anywhere but on an item's write stopped the crawl or its shutdown: - at the flush a crawl with a disk or shared frontier makes every second; - at a checkpoint: the crawl ended with the error; - at its close: the sessions, the cache and the rest of the shutdown were skipped. Every write, flush and close now goes through Engine._to_output. The error is logged, export_errors and items_not_written count it, and the crawl goes on. crawl and goal then say how many items were not written, and exit with status 1. A flush that failed is no longer tried again with each item after it. - PostgreSQL connections (outputs, the shared frontier) give up after 10 seconds unless the URL or PGCONNECT_TIMEOUT sets connect_timeout. libpq's default waited more than a minute for a host that had vanished (measured), and the crawl waited with it. Tested against PostgreSQL 16 and MariaDB. The server ends the connection (pg_terminate_backend, KILL) between writes, before a new column and before a flush, and every item is written. Then a server that does not come back gives the ExportError with the count. The full suite passes (1,239 passed, 15 skipped). The engine, CLI and connect-timeout paths have no tests of their own yet: they are next. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FRraUvZmKWGVaNoEzqzjfH --- CHANGELOG.md | 26 +++++++ docs/cli.md | 6 +- docs/spiders.md | 7 ++ docs/storage.md | 11 ++- src/wintergrab/cli.py | 16 ++++ src/wintergrab/errors.py | 12 ++- src/wintergrab/goals/run.py | 5 ++ src/wintergrab/spider/engine.py | 27 +++++-- src/wintergrab/spider/exporters.py | 19 ++++- src/wintergrab/spider/shared.py | 5 +- src/wintergrab/storage/mongodb.py | 6 +- src/wintergrab/storage/mysql.py | 117 +++++++++++++++++++---------- src/wintergrab/storage/postgres.py | 111 +++++++++++++++++++-------- tests/test_storage.py | 70 +++++++++++++++++ 14 files changed, 350 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18733c1..ea57440 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1090,6 +1090,32 @@ an error that says so, not a setting silently ignored ### Fixes +- A PostgreSQL or MySQL output whose server hung up (a restart, a + failover, an idle timeout) lost every item after it: the rows waiting + were dropped, each write after failed, and the errors were the driver's + own (`OperationalError`, `InterfaceError (0, '')`), not an `ExportError` + saying how many items were not written. The output now connects again + and writes the batch, or adds the column, again, once; a server that does + not answer is an `ExportError` that says so and how many items. Tried with + the server ending the connection (`pg_terminate_backend`, `KILL`) between + writes, before a new column and before a flush: every item written. + +- An output that failed anywhere but on an item's write stopped the crawl or + its shutdown: at the flush a crawl with a disk or shared frontier makes + every second, at a checkpoint (the crawl ended with the error), or at its + close (the sessions, the cache and the rest of the shutdown were skipped). + Every write, flush and close of the output now goes one way: the error is + logged, `export_errors` and `items_not_written` (the items it lost, when + it says how many: `ExportError.items`) count it, and the crawl goes on. + `wintergrab crawl` and `wintergrab goal` say how many items were not + written, and exit with status 1. A flush that failed is no longer tried + again with each item after it. + +- PostgreSQL connections (outputs, and the shared frontier) give up after 10 + seconds unless the URL says `connect_timeout` or `PGCONNECT_TIMEOUT` is + set. libpq's own default waited more than a minute for a host that had + vanished (measured), and the crawl waited with it. + - A browser that crashed, or that the system killed, was never started again: every page after it failed ("Target page, context or browser has been closed"), to the end of the crawl. It is now started again for the diff --git a/docs/cli.md b/docs/cli.md index e78a367..787429a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -106,7 +106,9 @@ Items are streamed as JSON Lines to stdout unless you pass `-o FILE` (`.jsonl`, `.json`, `.csv`, `.sqlite`, `.parquet`, `.xlsx`, `.duckdb`, or a `postgresql://`, `mysql://`, `mongodb://` or `s3://` URL: see [storage.md](storage.md)). A summary goes to stderr at the end, with a short diagnosis of the most common failures (the full -[failure report](observability.md#failure-reports) with `-v`). +[failure report](observability.md#failure-reports) with `-v`). When the +output failed (a full disk, a database that stopped answering), the summary +says how many items were not written, and the exit status is 1. ```bash wintergrab crawl examples/05_quotes_spider.py -o quotes.jsonl @@ -275,6 +277,8 @@ change nothing; `--no-optimize` fetches every page the plan leads to. A request that names no site can ask a search API which sites rank for it (`--find-sites`, see [search.md](search.md#sites-for-a-goal)): they are shown, for you to pick one. See [goals.md](goals.md#records-from-the-sites-api). +When records could not be written to the output, the summary says how many, +and the exit status is 1. ## `wintergrab search`: search results, and what they say diff --git a/docs/spiders.md b/docs/spiders.md index b1a0caa..2046c25 100644 --- a/docs/spiders.md +++ b/docs/spiders.md @@ -96,6 +96,13 @@ async for item in QuotesSpider().stream(): # consume items live item (changed or not) to keep it, or `None` to drop it. This is the place for cleaning, validation or saving to a database. +An output that fails does not stop the crawl: a full disk, or a database that +stopped answering, whether on an item's write, a flush, a checkpoint or its +close. The error is logged, `stats["export_errors"]` counts it, and +`stats["items_not_written"]` counts the items it lost, when the error says +how many (a database's batch). `wintergrab crawl` then says how many items +were not written, and exits with status 1. + ## Sessions A session is a fetcher with its own cookies and settings. By default a spider diff --git a/docs/storage.md b/docs/storage.md index d4d3a27..ea1ab51 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -138,7 +138,9 @@ wintergrab crawl https://shop.example/ --auto -o "postgresql://crawler@db.exampl The table is created on first use: `items`, unless the URL says `?table=NAME` or `?table=schema.name`. The URL's other parameters -(`sslmode=require`...) are the connection's. +(`sslmode=require`...) are the connection's. Connecting gives up after 10 +seconds, unless the URL says `connect_timeout=N` or `PGCONNECT_TIMEOUT` is +set: without a limit, a host that vanished held the crawl for minutes. Each new field becomes a column, typed from its first value: `boolean`, `bigint`, `double precision`, `text`, or `jsonb` for nested values. When a @@ -158,6 +160,13 @@ table gives the field names back. first, as a file output is replaced. A resumed crawl adds to it. - **Batches**: rows are written in batches of 64 or once a second, and on checkpoints. +- **A server that hangs up** (a restart, a failover, an idle timeout) is + connected to again, and what was being written is written again, once. If + it does not answer, the items of that batch are an error that says how + many were not written, and the crawl goes on (counted in its stats: + `export_errors`, `items_not_written`). The next batch connects again, so + the output takes items again once the server is back. The same holds for + MySQL and MariaDB. wintergrab only writes to tables it created. It keeps their fields and columns in a `_wintergrab_columns` table beside them. A table of the same diff --git a/src/wintergrab/cli.py b/src/wintergrab/cli.py index d155f95..892834c 100644 --- a/src/wintergrab/cli.py +++ b/src/wintergrab/cli.py @@ -809,6 +809,14 @@ def cmd_goal(args: argparse.Namespace) -> int: if plan.goal.monitor: again = f"wintergrab goal --plan {args.save_plan or args.plan or 'PLAN.json'} --yes -o {args.output or 'OUT.jsonl'}" print(f"to watch for changes ({plan.goal.monitor}), run this again on a schedule: {again}", file=sys.stderr) + if result.counts["export_errors"]: + if args.verbose < 0: # (the summary, which says it, was not printed) + from .spider.exporters import output_failures + + print( + output_failures(result.counts["export_errors"], result.counts["not_written"], output), file=sys.stderr + ) + return 1 return 0 @@ -1738,6 +1746,14 @@ def cmd_crawl(args: argparse.Namespace) -> int: f"{stats['dead_letters']} failed request(s) recorded; retry just those with --retry-failed", file=sys.stderr, ) + if stats.get("export_errors"): + from .spider.exporters import output_failures + + print( + output_failures(stats["export_errors"], stats.get("items_not_written", 0), spider.output or "-"), + file=sys.stderr, + ) + return 1 return 0 diff --git a/src/wintergrab/errors.py b/src/wintergrab/errors.py index 1a6cbeb..78cdd95 100644 --- a/src/wintergrab/errors.py +++ b/src/wintergrab/errors.py @@ -321,7 +321,17 @@ class CheckpointError(StorageError): class ExportError(StorageError): - """Items could not be written to an output.""" + """Items could not be written to an output. + + Attributes: + items: How many items the error left unwritten, when it is known (a batch a database refused), else ``None``. + """ + + def __init__(self, message: str, *, items: int | None = None, context: Mapping[str, Any] | None = None) -> None: + super().__init__(message, context=context) + self.items = items + if items is not None: + self.context.setdefault("items", items) class BudgetExceeded(WintergrabError): diff --git a/src/wintergrab/goals/run.py b/src/wintergrab/goals/run.py index a98224e..6f422b2 100644 --- a/src/wintergrab/goals/run.py +++ b/src/wintergrab/goals/run.py @@ -41,6 +41,7 @@ from ..intel.classify import classify_page from ..request import Request from ..spider import CrawlResult, Spider +from ..spider.exporters import output_failures from ..utils import host_of from .api import MAX_API_PAGES, ApiSource, next_page, records_of, total_of from .goal import Goal @@ -284,6 +285,8 @@ def summary(self) -> str: """The records, the fields they have, and what was left out and why.""" c = self.counts lines = [f"{c['records']:,} record(s)" + (f" -> {self.output}" if self.output else "")] + if c["export_errors"]: + lines.append(output_failures(c["export_errors"], c["not_written"], self.output or "-")) if c["records"] and self.found: share = ", ".join(f"{f} {n / c['records']:.0%}" for f, n in self.found.items()) lines.append(f"fields found: {share}") @@ -391,6 +394,8 @@ def run_plan( counts["pages"] = int(crawl.stats.get("pages", 0)) counts["browser_pages"] = int(crawl.stats.get("browser_pages", 0)) counts["errors"] = int(crawl.stats.get("errors", 0)) + counts["export_errors"] = int(crawl.stats.get("export_errors", 0)) + counts["not_written"] = int(crawl.stats.get("items_not_written", 0)) counts["record_pages"] = spider.record_pages counts["incomplete"] = spider.incomplete counts["api_pages"] = spider.api_pages diff --git a/src/wintergrab/spider/engine.py b/src/wintergrab/spider/engine.py index 9b368f5..4490592 100644 --- a/src/wintergrab/spider/engine.py +++ b/src/wintergrab/spider/engine.py @@ -802,7 +802,7 @@ async def _shutdown(self) -> CrawlResult: path.with_name(path.name + suffix).unlink(missing_ok=True) await self._close_pipelines() if self.exporter is not None: - self.exporter.close() + self._to_output(self.exporter.close) restarts = sum(int(getattr(self.sessions.get(name), "restarts", 0) or 0) for name in self.sessions) if restarts: # (a browser that crashed, or was killed, and was started again) self.stats["browser_restarts"] = restarts @@ -937,10 +937,24 @@ def _ack(self, request: Request) -> None: if isinstance(self.scheduler, (DiskScheduler, SharedScheduler)): self.scheduler.ack(request) + def _to_output(self, action: Callable[[], None], *, item: bool = False) -> None: + """``action`` on the output (a write, a flush, its close): an error is counted and logged, and the crawl goes + on. ``items_not_written`` counts the items the error says it left unwritten (an item's own write: that one).""" + try: + action() + except Exception as exc: + lost = getattr(exc, "items", None) + lost = 1 if lost is None and item else lost + self.stats.inc("export_errors") + if lost: + self.stats.inc("items_not_written", lost) + output = redact_url(str(self.spider.output)) + log.error("could not write %s to %s: %s", "item" if item else "items", output, describe(exc)) + def _save_state(self, pending: list[Request]) -> None: assert self.checkpoint is not None if self.exporter is not None: - self.exporter.flush() # items on disk must match the saved queue + self._to_output(self.exporter.flush) # items on disk must match the saved queue state: dict[str, Any] = { "spider": self.spider.name, "stats": dict(self.stats), @@ -997,7 +1011,7 @@ def _periodic(self) -> None: if isinstance(self.scheduler, (DiskScheduler, SharedScheduler)) and now - self._last_commit >= 1.0: # Output first, then the queue: a crash may redo work, never lose it. if self.exporter is not None: - self.exporter.flush() + self._to_output(self.exporter.flush) self.scheduler.commit() # bounds what a crash can lose to about a second of work self._last_commit = now if self.checkpoint is not None and now - self._last_checkpoint >= spider.checkpoint_interval: @@ -1806,11 +1820,8 @@ async def _item(self, item: Any) -> Any: if self.recorder is not None: self.recorder.item(processed) if self.exporter is not None: - try: - self.exporter.write(processed) - except Exception as exc: - self.stats.inc("export_errors") - log.error("could not write item to %s: %s", redact_url(str(self.spider.output)), describe(exc)) + exporter = self.exporter + self._to_output(lambda: exporter.write(processed), item=True) if self._output_budget: exhausted = self.budget.check_output() if exhausted is not None: diff --git a/src/wintergrab/spider/exporters.py b/src/wintergrab/spider/exporters.py index 2cf786c..191170a 100644 --- a/src/wintergrab/spider/exporters.py +++ b/src/wintergrab/spider/exporters.py @@ -105,9 +105,11 @@ def _maybe_flush(self) -> None: self._unflushed += 1 now = time.monotonic() if self._unflushed >= FLUSH_EVERY or now - self._last_flush >= FLUSH_INTERVAL: - self.flush() - self._unflushed = 0 - self._last_flush = now + try: + self.flush() + finally: # (a flush that failed is not tried again with each item after it) + self._unflushed = 0 + self._last_flush = now def close(self) -> None: # pragma: no cover - interface raise NotImplementedError @@ -571,6 +573,17 @@ def open_exporter(path: str | os.PathLike[str], *, append: bool = False, unique_ return cls(target, append=append, **({"unique_key": unique_key} if cls.supports_unique_key else {})) +def output_failures(errors: int, not_written: int, output: str | os.PathLike[str]) -> str: + """The line that says a crawl's output failed (``export_errors``, ``items_not_written``): how many items it did + not take, when known.""" + from ..redact import redact_url + + where = "standard output" if os.fspath(output) == "-" else redact_url(os.fspath(output)) + if not_written: + return f"{not_written:,} item(s) not written to {where} ({errors:,} error(s), logged above)" + return f"{errors:,} error(s) writing to {where} (logged above): items may be missing from it" + + def write_items(path: str | os.PathLike[str], items: list[Any]) -> Path: """Write a list of items in one go (format chosen by extension).""" exporter = open_exporter(path) diff --git a/src/wintergrab/spider/shared.py b/src/wintergrab/spider/shared.py index 730219a..b83b039 100644 --- a/src/wintergrab/spider/shared.py +++ b/src/wintergrab/spider/shared.py @@ -123,10 +123,11 @@ def __init__( self._counted = 0.0 self._throttle: AutoThrottle | None = None self._closed = False - from ..storage.postgres import _psycopg + from ..storage.postgres import _psycopg, connect_options self._psycopg = _psycopg() self._dsn = urlunsplit(parts._replace(query=urlencode([(k, v) for k, v in query if k != "crawl"]))) + self._options = connect_options(self._dsn) self._db = self._connect() self.seen = _Seen(self) #: Whether the crawl was running already (this process joins it). @@ -135,7 +136,7 @@ def __init__( # -- the database --------------------------------------------------------------------------- # def _connect(self) -> Any: try: - db = self._psycopg.connect(self._dsn, autocommit=False) + db = self._psycopg.connect(self._dsn, autocommit=False, **self._options) # Commits need not wait for the disk: what a server crash could lose of the last moments is # redone (a request is deleted in the same transaction as the requests that came of it are queued). db.execute("SET synchronous_commit TO off") diff --git a/src/wintergrab/storage/mongodb.py b/src/wintergrab/storage/mongodb.py index a247ed5..11564de 100644 --- a/src/wintergrab/storage/mongodb.py +++ b/src/wintergrab/storage/mongodb.py @@ -254,11 +254,13 @@ def flush(self) -> None: written = owners[errors[0].get("index", 0)] raise ExportError( f"{self.collection_name}: {len(self._docs) - written} item(s) not written: " - f"{errors[0].get('errmsg', _message(exc))}" + f"{errors[0].get('errmsg', _message(exc))}", + items=len(self._docs) - written, ) from None except self._pymongo.errors.PyMongoError as exc: raise ExportError( - f"{self.collection_name}: {len(self._docs)} item(s) not written: {_message(exc)}" + f"{self.collection_name}: {len(self._docs)} item(s) not written: {_message(exc)}", + items=len(self._docs), ) from None finally: self._docs.clear() diff --git a/src/wintergrab/storage/mysql.py b/src/wintergrab/storage/mysql.py index 56a145c..9c7ae49 100644 --- a/src/wintergrab/storage/mysql.py +++ b/src/wintergrab/storage/mysql.py @@ -25,10 +25,12 @@ from __future__ import annotations +import contextlib import json +import logging import os import re -from collections.abc import Iterator +from collections.abc import Callable, Iterator from pathlib import Path from typing import Any from urllib.parse import parse_qsl, unquote, urlsplit @@ -40,6 +42,8 @@ __all__ = ["MySQLExporter", "read_mysql"] +log = logging.getLogger(__name__) + _NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$") _META = "_wintergrab_columns" _ROWID = "_wg_rowid" @@ -230,6 +234,42 @@ def _load_columns(self, rows: list[tuple[str, str, str]] | None = None) -> None: self._kinds: dict[str, str] = {col: kind for key, col, kind in rows if key} self._used = set(self._kinds) | {_ROWID} + def _rollback(self) -> None: + """Undo the transaction, and forget the columns it added. On a connection the server hung up, there is + nothing to undo: the transaction went with it.""" + if self._conn.open: + with contextlib.suppress(self._pymysql.MySQLError): + self._conn.rollback() + self._load_columns() + + def _reconnect(self) -> None: + """Connect again (the server hung up: a restart, a failover, an idle timeout), to the table as it is.""" + with contextlib.suppress(Exception): + self._conn.close() + self._conn, _ = _connect(self.url) + self._load_columns() + log.warning("%s: the database hung up; connected again", self.table_name) + + def _run(self, action: Callable[[], None], failed: str, items: int) -> None: + """Do ``action``, one transaction; when the server hung up, connect again and do it once more (``failed`` + begins the error that says it could not be done, the ``items`` it leaves unwritten).""" + for attempt in (1, 2): + try: + action() + return + except self._pymysql.MySQLError as exc: + lost = not self._conn.open + self._rollback() + if not lost or attempt == 2: + raise ExportError(f"{self.table_name}: {failed}{_message(exc)}", items=items) from None + try: + self._reconnect() + except ConfigurationError as gone: + raise ExportError( + f"{self.table_name}: {failed}the database hung up, and does not answer again ({gone})", + items=items, + ) from None + def _key_columns(self) -> list[str]: rows = self._execute( "SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = %s" @@ -304,7 +344,8 @@ def write(self, item: Any) -> None: text = dumps(row) self.bytes_written += _size(text) + 1 # type: ignore[operator] record = json.loads(text) # JSON values, as every output gets them - try: + + def columns() -> None: for key, value in record.items(): kind = kind_of(value) if kind == "null" and key not in self._columns: @@ -312,10 +353,8 @@ def write(self, item: Any) -> None: if len(key.encode()) > 2048: raise ExportError(f"{self.table_name}: a field name longer than 2,048 bytes: {key[:40]}...") self._column(key, kind) - except self._pymysql.MySQLError as exc: - self._conn.rollback() - self._load_columns() - raise ExportError(f"{self.table_name}: {_message(exc)}") from None + + self._run(columns, "", 1) self._rows.append(record) self.count += 1 self._maybe_flush() @@ -334,50 +373,50 @@ def _value(self, column: str, value: Any) -> Any: def flush(self) -> None: if not self._rows: - self._conn.commit() + if self._conn.open: + self._conn.commit() return + rows, self._rows = self._rows, [] # (written, or said not to be) + self._run(lambda: self._insert(rows), f"{len(rows)} item(s) not written: ", len(rows)) + + def _insert(self, rows: list[dict[str, Any]]) -> None: keyed = bool(self.unique_key and self._columns.get(self.unique_key)) groups: dict[tuple[str, ...], list[list[Any]]] = {} - for record in self._rows: + for record in rows: fields = [k for k in record if k in self._columns] # (a field only ever null has no column) columns = tuple(self._columns[k] for k in fields) groups.setdefault(columns, []).append([self._value(self._columns[k], record[k]) for k in fields]) - try: - with self._conn.cursor() as cursor: - for columns, values in groups.items(): - head = f"INSERT INTO {self._table} ({', '.join(map(_quote, columns))}) VALUES " - tail = "" - if keyed and columns: - if self._alias: - updates = ", ".join(f"{_quote(c)} = _wg_new.{_quote(c)}" for c in columns) - tail = f" AS _wg_new ON DUPLICATE KEY UPDATE {updates}" - else: - tail = " ON DUPLICATE KEY UPDATE " + ", ".join( - f"{_quote(c)} = VALUES({_quote(c)})" for c in columns - ) - placeholders = "(" + ", ".join(["%s"] * len(columns)) + ")" - chunk: list[str] = [] - size = 0 - for row in values: - text = cursor.mogrify(placeholders, row) # escaped as execute() escapes - if chunk and size + len(text) > _STATEMENT: - cursor.execute(head + ", ".join(chunk) + tail) - chunk, size = [], 0 - chunk.append(text) - size += len(text) + 2 - cursor.execute(head + ", ".join(chunk) + tail) - self._conn.commit() - except self._pymysql.MySQLError as exc: - self._conn.rollback() - raise ExportError(f"{self.table_name}: {len(self._rows)} item(s) not written: {_message(exc)}") from None - finally: - self._rows.clear() + with self._conn.cursor() as cursor: + for columns, values in groups.items(): + head = f"INSERT INTO {self._table} ({', '.join(map(_quote, columns))}) VALUES " + tail = "" + if keyed and columns: + if self._alias: + updates = ", ".join(f"{_quote(c)} = _wg_new.{_quote(c)}" for c in columns) + tail = f" AS _wg_new ON DUPLICATE KEY UPDATE {updates}" + else: + tail = " ON DUPLICATE KEY UPDATE " + ", ".join( + f"{_quote(c)} = VALUES({_quote(c)})" for c in columns + ) + placeholders = "(" + ", ".join(["%s"] * len(columns)) + ")" + chunk: list[str] = [] + size = 0 + for row in values: + text = cursor.mogrify(placeholders, row) # escaped as execute() escapes + if chunk and size + len(text) > _STATEMENT: + cursor.execute(head + ", ".join(chunk) + tail) + chunk, size = [], 0 + chunk.append(text) + size += len(text) + 2 + cursor.execute(head + ", ".join(chunk) + tail) + self._conn.commit() def close(self) -> None: try: self.flush() finally: - self._conn.close() + if self._conn.open: # (one the server hung up is closed already: closing it again is an error) + self._conn.close() def read_mysql(url: str) -> Iterator[dict[str, Any]]: diff --git a/src/wintergrab/storage/postgres.py b/src/wintergrab/storage/postgres.py index d637b80..7cab9e1 100644 --- a/src/wintergrab/storage/postgres.py +++ b/src/wintergrab/storage/postgres.py @@ -19,10 +19,13 @@ from __future__ import annotations +import contextlib import hashlib import json +import logging +import os import re -from collections.abc import Iterator +from collections.abc import Callable, Iterator from pathlib import Path from typing import Any from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit @@ -34,11 +37,14 @@ __all__ = ["PostgresExporter", "read_postgres"] +log = logging.getLogger(__name__) + _NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,62}$") _META = "_wintergrab_columns" _ROWID = "_wg_rowid" _TYPES = {"bool": "boolean", "int": "bigint", "float": "double precision", "str": "text", "json": "jsonb"} _OPTIONS = ("table",) # the URL's own query parameters (the others are the connection's) +_CONNECT_TIMEOUT = 10 # seconds def _psycopg() -> Any: @@ -58,11 +64,21 @@ def parse_target(url: str) -> tuple[str, list[str]]: return dsn, names +def connect_options(dsn: str) -> dict[str, Any]: + """``psycopg.connect``'s options for ``dsn``: a ``connect_timeout`` of 10 seconds, unless the URL or + ``PGCONNECT_TIMEOUT`` gives one (libpq's own waits as long as the system does, minutes, and a crawl with it).""" + if any(key == "connect_timeout" for key, _ in parse_qsl(urlsplit(dsn).query)) or os.environ.get( + "PGCONNECT_TIMEOUT" + ): + return {} + return {"connect_timeout": _CONNECT_TIMEOUT} + + def _connect(url: str) -> tuple[Any, list[str]]: psycopg = _psycopg() dsn, names = parse_target(url) try: - return psycopg.connect(dsn, autocommit=False), names + return psycopg.connect(dsn, autocommit=False, **connect_options(dsn)), names except psycopg.Error as exc: raise ConfigurationError(f"cannot connect to {redact_url(url)}: {str(exc).strip()}", key="output") from None @@ -153,9 +169,43 @@ def _load_columns(self, rows: list[tuple[str, str, str]] | None = None) -> None: self._used = set(self._types) | {_ROWID} def _rollback(self) -> None: - """Undo the transaction, and forget the columns it added (they are gone with it).""" - self._conn.rollback() + """Undo the transaction, and forget the columns it added (they are gone with it). On a connection the server + hung up, there is nothing to undo: the transaction went with it.""" + if self._alive(): + with contextlib.suppress(self._psycopg.Error): + self._conn.rollback() + self._load_columns() + + def _alive(self) -> bool: + return not (self._conn.closed or self._conn.broken) + + def _reconnect(self) -> None: + """Connect again (the server hung up: a restart, a failover, an idle timeout), to the table as it is.""" + with contextlib.suppress(Exception): + self._conn.close() + self._conn, _ = _connect(self.url) self._load_columns() + log.warning("%s: the database hung up; connected again", self.table_name) + + def _run(self, action: Callable[[], None], failed: str, items: int) -> None: + """Do ``action``, one transaction; when the server hung up, connect again and do it once more (``failed`` + begins the error that says it could not be done, the ``items`` it leaves unwritten).""" + for attempt in (1, 2): + try: + action() + return + except self._psycopg.Error as exc: + lost = not self._alive() + self._rollback() + if not lost or attempt == 2: + raise ExportError(f"{self.table_name}: {failed}{str(exc).strip()}", items=items) from None + try: + self._reconnect() + except ConfigurationError as gone: + raise ExportError( + f"{self.table_name}: {failed}the database hung up, and does not answer again ({gone})", + items=items, + ) from None def _unique_index(self) -> None: """The unique index on ``unique_key``'s column (once the column exists); others go.""" @@ -219,12 +269,12 @@ def write(self, item: Any) -> None: text = dumps(row) self.bytes_written += _size(text) + 1 # type: ignore[operator] record = json.loads(text) # JSON values, as every output gets them - try: + + def columns() -> None: for key, value in record.items(): self._column(str(key), kind_of(value)) - except self._psycopg.Error as exc: - self._rollback() - raise ExportError(f"{self.table_name}: {str(exc).strip()}") from None + + self._run(columns, "", 1) self._rows.append(record) self.count += 1 self._maybe_flush() @@ -243,35 +293,34 @@ def _value(self, column: str, value: Any) -> Any: def flush(self) -> None: if not self._rows: - self._conn.commit() + if self._alive(): + self._conn.commit() return + rows, self._rows = self._rows, [] # (written, or said not to be) + self._run(lambda: self._insert(rows), f"{len(rows)} item(s) not written: ", len(rows)) + + def _insert(self, rows: list[dict[str, Any]]) -> None: sql = self._sql key_column = self._columns.get(self.unique_key or "") if self.unique_key else None groups: dict[tuple[str, ...], list[list[Any]]] = {} - for record in self._rows: + for record in rows: columns = tuple(self._columns[str(k)] for k in record) groups.setdefault(columns, []).append([self._value(self._columns[str(k)], v) for k, v in record.items()]) - try: - with self._conn.cursor() as cursor: - for columns, values in groups.items(): - query = sql.SQL("INSERT INTO {} ({}) VALUES ({})").format( - self._table, - sql.SQL(", ").join(map(sql.Identifier, columns)), - sql.SQL(", ").join(sql.Placeholder() * len(columns)), - ) - if key_column is not None and key_column in columns: - others = [c for c in columns if c != key_column] - action = sql.SQL("DO UPDATE SET {}").format(sql.SQL(", ").join( - sql.SQL("{} = EXCLUDED.{}").format(sql.Identifier(c), sql.Identifier(c)) for c in others - )) if others else sql.SQL("DO NOTHING") # fmt: skip - query = sql.SQL("{} ON CONFLICT ({}) {}").format(query, sql.Identifier(key_column), action) - cursor.executemany(query, values) - self._conn.commit() - except self._psycopg.Error as exc: - self._rollback() - raise ExportError(f"{self.table_name}: {len(self._rows)} item(s) not written: {str(exc).strip()}") from None - finally: - self._rows.clear() + with self._conn.cursor() as cursor: + for columns, values in groups.items(): + query = sql.SQL("INSERT INTO {} ({}) VALUES ({})").format( + self._table, + sql.SQL(", ").join(map(sql.Identifier, columns)), + sql.SQL(", ").join(sql.Placeholder() * len(columns)), + ) + if key_column is not None and key_column in columns: + others = [c for c in columns if c != key_column] + action = sql.SQL("DO UPDATE SET {}").format(sql.SQL(", ").join( + sql.SQL("{} = EXCLUDED.{}").format(sql.Identifier(c), sql.Identifier(c)) for c in others + )) if others else sql.SQL("DO NOTHING") # fmt: skip + query = sql.SQL("{} ON CONFLICT ({}) {}").format(query, sql.Identifier(key_column), action) + cursor.executemany(query, values) + self._conn.commit() def close(self) -> None: try: diff --git a/tests/test_storage.py b/tests/test_storage.py index 632f687..2e37e76 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -358,6 +358,42 @@ def test_postgres(table) -> None: assert len(list(read_records(url))) == 3 +@needs_postgres +def test_postgres_connects_again_when_the_server_hangs_up(table, monkeypatch, caplog) -> None: + import psycopg + + from wintergrab.errors import ExportError + from wintergrab.storage import postgres + + url, _ = table + + def hang_up(exporter: Any) -> None: # (a restart, a failover, an idle timeout) + with psycopg.connect(str(POSTGRES), autocommit=True) as admin: + admin.execute("SELECT pg_terminate_backend(%s)", (exporter._conn.info.backend_pid,)) + + exporter = open_exporter(url, unique_key="url") + exporter.write({"url": "/1", "n": 1}) + exporter.flush() + hang_up(exporter) + exporter.write({"url": "/2", "n": 2, "price": 9.5}) # (a new column, asked for on the lost connection) + hang_up(exporter) + exporter.write({"url": "/3", "n": 3}) + exporter.close() # (the rows waiting, written on the lost connection: again, on a new one) + assert sorted((r["url"], r.get("price")) for r in read_records(url)) == [("/1", None), ("/2", 9.5), ("/3", None)] + assert caplog.text.count("the database hung up; connected again") == 2 + + exporter = open_exporter(url, unique_key="url", append=True) + exporter.write({"url": "/4", "n": 4}) + hang_up(exporter) + + def gone(target: str) -> Any: + raise ConfigurationError("cannot connect: Connection refused") + + monkeypatch.setattr(postgres, "_connect", gone) # (and it does not come back) + with pytest.raises(ExportError, match="1 item\\(s\\) not written: the database hung up, and does not answer again"): + exporter.close() + + @needs_postgres def test_postgres_refuses_what_it_did_not_create(table) -> None: import psycopg @@ -598,6 +634,40 @@ def test_mysql_a_key_whose_column_widens(mysql_table) -> None: assert [(r["sku"], r["n"]) for r in read_records(url)] == [("1", 1), ("2", 22), ("A-3", 3)] # still upserted +@needs_mysql +def test_mysql_connects_again_when_the_server_hangs_up(mysql_table, monkeypatch, caplog) -> None: + from wintergrab.errors import ExportError + from wintergrab.storage import mysql + + url, _ = mysql_table + + def hang_up(exporter: Any) -> None: # (a restart, a failover, wait_timeout) + with _mysql() as admin, admin.cursor() as cursor: + cursor.execute(f"KILL {exporter._conn.thread_id()}") + + exporter = open_exporter(url, unique_key="url") + exporter.write({"url": "/1", "n": 1}) + exporter.flush() + hang_up(exporter) + exporter.write({"url": "/2", "n": 2, "price": 9.5}) # (a new column, asked for on the lost connection) + hang_up(exporter) + exporter.write({"url": "/3", "n": 3}) + exporter.close() # (the rows waiting, written on the lost connection: again, on a new one) + assert sorted((r["url"], r.get("price")) for r in read_records(url)) == [("/1", None), ("/2", 9.5), ("/3", None)] + assert caplog.text.count("the database hung up; connected again") == 2 + + exporter = open_exporter(url, unique_key="url", append=True) + exporter.write({"url": "/4", "n": 4}) + hang_up(exporter) + + def gone(target: str) -> Any: + raise ConfigurationError("cannot connect: Connection refused") + + monkeypatch.setattr(mysql, "_connect", gone) # (and it does not come back) + with pytest.raises(ExportError, match="1 item\\(s\\) not written: the database hung up, and does not answer again"): + exporter.close() + + @needs_mysql def test_mysql_refuses_what_it_did_not_create(mysql_table) -> None: url, name = mysql_table From e91d11b021f21ad3032b50dc38a4f57a959cebdb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 19:48:21 +0000 Subject: [PATCH 02/30] Tests for a crawl whose output fails, and the output closed before the state The last commit made every write, flush and close of the output one way (logged, counted, the crawl goes on) but tested only the database outputs. Now tested: an output that refuses writes, whose flush fails at the checkpoint (as a database batch does, saying how many items it lost) and whose close fails, in a crawl with a disk frontier that pauses and is resumed. The crawl saves its state and shuts down, the resumed crawl finishes, and nothing is lost but what the output refused, nothing twice. export_errors and items_not_written count over both runs, and the log says what could not be written where. `wintergrab crawl` and `wintergrab goal` say how many items were not written and exit with 1 (goal: with -q too, when the summary is not printed). A flush that failed is not tried again with each item after it. The failure line, ExportError.items (pickled too) and connect_options (with and without connect_timeout, from the URL and from PGCONNECT_TIMEOUT; what psycopg is asked, for outputs and the shared frontier) have tests of their own. The test found one thing: a close that failed after a pause was not in the stats the resumed crawl carried on, nor in the summary, because the output was closed after the state and the summary were saved. The output is now closed first: what it took is on disk before the queue's state says so, and what it did not take is in the counts the state and the summary keep. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 4 +- src/wintergrab/spider/engine.py | 7 +- tests/test_errors.py | 4 + tests/test_storage.py | 237 +++++++++++++++++++++++++++++++- 4 files changed, 246 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea57440..68e3f39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1109,7 +1109,9 @@ an error that says so, not a setting silently ignored it says how many: `ExportError.items`) count it, and the crawl goes on. `wintergrab crawl` and `wintergrab goal` say how many items were not written, and exit with status 1. A flush that failed is no longer tried - again with each item after it. + again with each item after it. The output is closed before the crawl's + state and summary are saved, so a close that fails is in their counts too + (before, a paused crawl resumed without it). - PostgreSQL connections (outputs, and the shared frontier) give up after 10 seconds unless the URL says `connect_timeout` or `PGCONNECT_TIMEOUT` is diff --git a/src/wintergrab/spider/engine.py b/src/wintergrab/spider/engine.py index 4490592..caba21c 100644 --- a/src/wintergrab/spider/engine.py +++ b/src/wintergrab/spider/engine.py @@ -765,6 +765,11 @@ async def _shutdown(self) -> CrawlResult: len(self.scheduler) + len(self._delayed) + len(self._inflight) if self._persistent else len(pending) ) keep = status in ("paused", "limit") or self._fatal is not None + # The output first: closed, so what it took is on disk before the queue's state says so, and what it did + # not take (a flush or the close that failed) is in the stats the state and the summary keep. + if self.exporter is not None: + exporter, self.exporter = self.exporter, None + self._to_output(exporter.close) if self.checkpoint is not None and self._state_ready: try: if keep and pending_count: @@ -801,8 +806,6 @@ async def _shutdown(self) -> CrawlResult: for suffix in ("", "-wal", "-shm"): path.with_name(path.name + suffix).unlink(missing_ok=True) await self._close_pipelines() - if self.exporter is not None: - self._to_output(self.exporter.close) restarts = sum(int(getattr(self.sessions.get(name), "restarts", 0) or 0) for name in self.sessions) if restarts: # (a browser that crashed, or was killed, and was started again) self.stats["browser_restarts"] = restarts diff --git a/tests/test_errors.py b/tests/test_errors.py index c1b4f18..24b446b 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -64,6 +64,9 @@ def test_context_and_with_context() -> None: assert cfg.key == "crawl.concurrency" and cfg.context["key"] == "crawl.concurrency" status = errors.HTTPStatusError(Response("http://x/", status=503)) assert status.context == {"url": "http://x/", "status": 503} + lost = errors.ExportError("items: 64 item(s) not written: the database hung up", items=64) + assert lost.items == 64 and lost.context == {"items": 64} # (a crawl counts them: items_not_written) + assert errors.ExportError("could not write items.parquet").items is None # how many is not always known @pytest.mark.parametrize( @@ -78,6 +81,7 @@ def test_context_and_with_context() -> None: errors.BrowserFetchError("http://x/", "crashed"), errors.CacheMiss("http://x/"), errors.ConfigurationError("bad", key="k"), + errors.ExportError("shop: 2 item(s) not written", items=2), errors.ValidationError("invalid", issues=["a"]), errors.BudgetExceeded("max_requests"), errors.HTTPStatusError(Response("http://x/", status=500), "server error"), diff --git a/tests/test_storage.py b/tests/test_storage.py index 2e37e76..aa77363 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -2,17 +2,29 @@ from __future__ import annotations +import errno +import json +import logging import os import sys import uuid from pathlib import Path -from typing import Any +from types import SimpleNamespace +from typing import Any, ClassVar import pytest from wintergrab.data.io import read_records, register_reader -from wintergrab.errors import ConfigurationError -from wintergrab.spider.exporters import Exporter, open_exporter, register_exporter +from wintergrab.errors import ConfigurationError, ExportError +from wintergrab.spider.exporters import ( + EXPORTERS, + FLUSH_EVERY, + Exporter, + JsonLinesExporter, + open_exporter, + output_failures, + register_exporter, +) ITEMS: list[Any] = [ {"url": "https://s.example/1", "price": 10, "tags": ["a", "b"], "ok": True, "code": 7}, @@ -311,6 +323,189 @@ def test_registering_outputs_and_readers(tmp_path) -> None: register_exporter("not a key", _Memory) +# -- An output that fails: the crawl goes on, and says what it did not write --------------------------- # +class _Errors(logging.Handler): + """The errors wintergrab logs while the handler is on its logger (``caplog`` misses them once the CLI has + configured the logger, which stops it propagating).""" + + def __init__(self) -> None: + super().__init__(logging.ERROR) + self.messages: list[str] = [] + self.logger = logging.getLogger("wintergrab") + self.logger.addHandler(self) + + def emit(self, record: logging.LogRecord) -> None: + self.messages.append(record.getMessage()) + + def close(self) -> None: + self.logger.removeHandler(self) + super().close() + + +class FullDisk(JsonLinesExporter): + """An output that takes nothing: every write is a full disk (registered as ``.full`` by the tests).""" + + def write(self, item: Any) -> None: + raise OSError(errno.ENOSPC, "No space left on device") + + +class Unreliable(JsonLinesExporter): + """A JSON Lines output that fails now and then (registered as ``.failing``): the write of an item whose name + ends in 3 (a full disk), its first flush (as a database that hung up does, a batch of 3 items with it) and + its close. ``opened`` keeps the instances for the tests to look at.""" + + opened: ClassVar[list[Unreliable]] = [] + + def __init__(self, path: Path, *, append: bool) -> None: + super().__init__(path, append=append) + self.flushes = 0 + self.closed = False + Unreliable.opened.append(self) + + def write(self, item: Any) -> None: + if str(item.get("name", "")).endswith("3"): + raise OSError(errno.ENOSPC, "No space left on device") + super().write(item) + + def flush(self) -> None: + self.flushes += 1 + if self.flushes == 1: + raise ExportError("items: 3 item(s) not written: the database hung up, and does not answer again", items=3) + super().flush() + + def close(self) -> None: + self.closed = True + super().close() + raise OSError(errno.EIO, "Input/output error") + + +def test_an_output_that_fails_does_not_stop_the_crawl(site, tmp_path, monkeypatch) -> None: + """A write, a flush (at a checkpoint) or a close that fails is logged and counted, and the crawl goes on: with + its shutdown (the state saved, the frontier closed), and when it is resumed.""" + from wintergrab import Spider + + class Catalog(Spider): # 5 listing pages x 4 products; pauses itself after ``pause_after`` items + log_level = None + obey_robots_txt = False + concurrency = 2 + pause_after: int | None = None + + def parse(self, response: Any) -> Any: + for link in response.css(".product .name a"): + yield response.follow(link, callback=self.parse_product) + yield response.follow_next() + + def parse_product(self, response: Any) -> Any: + if self.pause_after is not None and self.stats.get("items", 0) + 1 >= self.pause_after: + self.pause() + yield {"name": response.css("h1::text").get()} + + monkeypatch.setitem(EXPORTERS, ".failing", Unreliable) + monkeypatch.setattr(Unreliable, "opened", []) + out, crawl_dir = tmp_path / "items.failing", tmp_path / "crawl" + settings: dict[str, Any] = { + "start_urls": [site.url + "/products/page/1"], + "frontier": "disk", + "crawl_dir": str(crawl_dir), + "output": str(out), + } + errors = _Errors() + first = Catalog(pause_after=8, **settings).run() + # The checkpoint at the start flushed the output, and the flush failed; the close failed. Neither stopped the + # crawl: it paused when told to, saved its state, and shut down. + assert first.status == "paused" and (crawl_dir / "state.pickle").exists() and first.stats["status"] == "paused" + output = Unreliable.opened[0] + assert output.flushes == 1 and output.closed # the checkpoint at the start; the pause closed it, then saved + assert first.stats["export_errors"] >= 2 and first.stats["items_not_written"] >= 3 + assert first.stats["items"] > len(out.read_text(encoding="utf-8").splitlines()) # (the flush: 3 items said lost) + second = Catalog(**settings).run() + errors.close() + assert second.status == "finished" and second.stats["status"] == "finished" and second.stats["runs"] == 2 + assert not (crawl_dir / "state.pickle").exists() and not (crawl_dir / "frontier.sqlite3").exists() + assert Unreliable.opened[1].append and Unreliable.opened[1].closed # continued, and closed (which failed too) + rows = [json.loads(line) for line in out.read_text(encoding="utf-8").splitlines()] + expected = sorted(f"Product {i}" for i in range(1, 21) if i not in (3, 13)) # what the output refused is gone + assert sorted(r["name"] for r in rows) == expected # nothing else lost, nothing twice + assert second.stats["items"] == 20 # (cumulative over the two runs, as the errors are) + # (the first run's close failed after its state was saved: the output is closed before, so the count has it) + assert second.stats["export_errors"] == 2 + 2 + 2 # two writes, a flush and a close per run + assert second.stats["items_not_written"] == 2 + 3 + 3 # the two items refused; what each failed flush said + messages = errors.messages + assert sum(f"could not write item to {out}: " in m and "No space left on device" in m for m in messages) == 2 + assert sum(f"could not write items to {out}: " in m and "the database hung up" in m for m in messages) == 2 + assert sum(f"could not write items to {out}: " in m and "Input/output error" in m for m in messages) == 2 + summary = json.loads((crawl_dir / "summary.json").read_text(encoding="utf-8")) + assert summary["status"] == "finished" and summary["stats"]["export_errors"] == 6 # (the close's error too) + + +def test_crawl_says_what_its_output_did_not_take(site, tmp_path, monkeypatch, capsys) -> None: + from wintergrab.cli import main + + monkeypatch.setitem(EXPORTERS, ".full", FullDisk) + out = tmp_path / "items.full" + errors = _Errors() + code = main(["-q", "crawl", site.url + "/products/page/1", "--follow", "a.next", "--follow", ".product .name", + "--each", "h1", "--field", "name=::text", "-o", str(out), "--no-progress"]) # fmt: skip + errors.close() + assert code == 1 + assert f"20 item(s) not written to {out} (20 error(s), logged above)" in capsys.readouterr().err + assert ( + sum(f"could not write item to {out}: " in m and "No space left on device" in m for m in errors.messages) == 20 + ) + + +def test_goal_says_what_its_output_did_not_take(site, tmp_path, monkeypatch, capsys) -> None: + from wintergrab.cli import main + from wintergrab.goals import parse_goal, plan_goal + + monkeypatch.setitem(EXPORTERS, ".full", FullDisk) + plan = tmp_path / "books.plan.json" + goal = parse_goal("books rated 4 stars or more with title and price", sites=[site.url + "/books/"]) + plan_goal(goal, sample=15, log_level=None).save(plan) + out = tmp_path / "books.full" + assert main(["goal", "--plan", str(plan), "--yes", "-o", str(out)]) == 1 + err = capsys.readouterr().err + assert "4 record(s)" in err and f"4 item(s) not written to {out} (4 error(s), logged above)" in err + assert main(["-q", "goal", "--plan", str(plan), "--yes", "-o", str(out)]) == 1 # no summary: the line alone + err = capsys.readouterr().err + assert "record(s)" not in err and f"4 item(s) not written to {out} (4 error(s), logged above)" in err + + +def test_the_line_that_says_what_an_output_did_not_take() -> None: + assert output_failures(3, 2, "items.jsonl") == "2 item(s) not written to items.jsonl (3 error(s), logged above)" + assert output_failures(1, 0, "-") == ( + "1 error(s) writing to standard output (logged above): items may be missing from it" + ) # (an error that did not say how many items it lost) + assert output_failures(2, 1200, "postgresql://crawler:hunter2@db.example/shop") == ( + "1,200 item(s) not written to postgresql://***@db.example/shop (2 error(s), logged above)" + ) + + +def test_a_flush_that_failed_is_not_tried_again_with_each_item(tmp_path) -> None: + class Stuck(JsonLinesExporter): + def __init__(self, path: Path, *, append: bool) -> None: + super().__init__(path, append=append) + self.flushes = 0 + + def flush(self) -> None: + self.flushes += 1 + raise OSError(errno.EIO, "Input/output error") + + exporter = Stuck(tmp_path / "items.jsonl", append=False) + for i in range(FLUSH_EVERY - 1): + exporter.write({"i": i}) + with pytest.raises(OSError, match="Input/output error"): + exporter.write({"i": FLUSH_EVERY - 1}) # the batch is full: flushed, which fails + assert exporter.flushes == 1 + for i in range(FLUSH_EVERY - 1): + exporter.write({"i": i}) # the next batch: nothing tried until it is full + assert exporter.flushes == 1 + with pytest.raises(OSError, match="Input/output error"): + exporter.write({"i": FLUSH_EVERY - 1}) + assert exporter.flushes == 2 + exporter._fh.close() + + # -- PostgreSQL: WINTERGRAB_TEST_POSTGRES=postgresql://user@host:port/db names a server to test against -- # POSTGRES = os.environ.get("WINTERGRAB_TEST_POSTGRES") needs_postgres = pytest.mark.skipif(not POSTGRES, reason="set WINTERGRAB_TEST_POSTGRES to a PostgreSQL URL") @@ -415,6 +610,42 @@ def test_postgres_errors_keep_passwords_out() -> None: open_exporter("postgresql://crawler@127.0.0.1:1/shop?table=items;drop") +def test_postgres_connections_give_up_after_ten_seconds(monkeypatch) -> None: + """libpq waits minutes for a host that vanished: connections get connect_timeout=10, unless the URL or + PGCONNECT_TIMEOUT sets one.""" + from wintergrab.spider.shared import SharedScheduler + from wintergrab.storage import postgres + + monkeypatch.delenv("PGCONNECT_TIMEOUT", raising=False) + assert postgres.connect_options("postgresql://crawler@db.example/shop") == {"connect_timeout": 10} + assert postgres.connect_options("postgresql://crawler@db.example/shop?sslmode=require&connect_timeout=3") == {} + monkeypatch.setenv("PGCONNECT_TIMEOUT", "5") + assert postgres.connect_options("postgresql://crawler@db.example/shop") == {} + monkeypatch.delenv("PGCONNECT_TIMEOUT") + + connections: list[tuple[str, dict[str, Any]]] = [] # what psycopg is asked (a fake: no server, no library) + + class Error(Exception): + pass + + def connect(dsn: str, **options: Any) -> Any: + connections.append((dsn, options)) + raise Error("connection refused") + + monkeypatch.setattr(postgres, "_psycopg", lambda: SimpleNamespace(Error=Error, connect=connect)) + for query in ("table=items", "connect_timeout=3"): + with pytest.raises(ConfigurationError, match="cannot connect to") as error: + open_exporter("postgresql://crawler@db.example/shop?" + query) + assert "postgresql://***@db.example/shop" in str(error.value) + with pytest.raises(ConfigurationError, match="frontier: cannot connect to"): + SharedScheduler("postgresql://crawler@db.example/shop?crawl=books", None) + assert connections == [ + ("postgresql://crawler@db.example/shop", {"autocommit": False, "connect_timeout": 10}), + ("postgresql://crawler@db.example/shop?connect_timeout=3", {"autocommit": False}), + ("postgresql://crawler@db.example/shop", {"autocommit": False, "connect_timeout": 10}), # the shared frontier + ] + + def test_output_urls_keep_their_password_out(site, tmp_path, capsys) -> None: from wintergrab import Spider from wintergrab.cli import main From 70de0ee5abd4db5c5208b5ae95833c309e5581ac Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 19:48:21 +0000 Subject: [PATCH 03/30] README and docs say what the browser is; live tests that meet a bot check Two README claims were false since the responsible-access work: that a site with bot checks is one the API "scales up" to, and that the browser "hides common automation tells and waits out 'checking your browser' interstitials". The browser is Playwright's Chromium as it is: nothing hides that it is automated, and a check that clears by itself is waited for. The fetching docs' table said the browser was for "stubborn bot checks"; the benchmarks README linked to the anti-blocking page that the responsible-access page replaced. pypi.org's CDN sends some networks its JavaScript bot check instead of the page. test_pypi_project_page skipped on it (checking that it was the check that was flagged); test_pypi_feed_and_cache failed instead, on the cache not storing a page that says no-store. Both now skip the same way, through one check, and the cache test asserts that the check was not stored. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- README.md | 9 +++++---- benchmarks/README.md | 2 +- docs/fetching.md | 2 +- tests/test_live.py | 19 ++++++++++++++----- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index cbafccc..2892762 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,15 @@ for quote in page.css(".quote"): ``` wintergrab is a Python toolkit for grabbing data from websites. Simple things -are one line. When a site is harder (JavaScript, bot checks, rate limits, -thousands of pages), the same API scales up. +are one line. When a site is harder (JavaScript, rate limits, thousands of +pages), the same API scales up. - **Fetch like a real browser.** HTTP requests carry Chrome/Firefox/Safari TLS and HTTP/2 fingerprints (via [curl_cffi](https://github.com/lexiforest/curl_cffi)). A headless Chromium (via [Playwright](https://playwright.dev/python/)) is - one flag away for JavaScript pages. It hides common automation tells and - waits out "checking your browser" interstitials. + one flag away for JavaScript pages. It is Chromium as it is: nothing hides + that it is automated, and a "checking your browser" page that clears by + itself is waited for. - **Parse with CSS or XPath.** Scrapy-style `::text` / `::attr(href)`, extraction schemas, search by text, "find similar elements", and HTML → Markdown/text conversion. diff --git a/benchmarks/README.md b/benchmarks/README.md index 15fee8c..6e4cda5 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -217,7 +217,7 @@ Done for 0.2, guided by the "before" profile. Each shortcut is tested to return Tried and rejected: - **Large LRU caches (65,536 entries) on the URL functions.** They gave a little more throughput but pushed the disk frontier's flat-memory test over its 16 MB budget. The regex fast paths get the win and hold no memory. - **One regex for the block-page markers.** An alternation was 5× slower than the 14 substring scans, and a case-insensitive one was 35× slower. The scans already run at memory speed (≈6 GB/s). -- **Scanning only the `` and the first 2 KB for the phrase markers.** It is faster, but it would miss challenge pages whose text sits further down. Detection wins here. A spider that only crawls sites known never to serve challenge pages can override `is_blocked` (see [anti-blocking](../docs/anti-blocking.md#6-detecting-blocks)). +- **Scanning only the `<title>` and the first 2 KB for the phrase markers.** It is faster, but it would miss challenge pages whose text sits further down. Detection wins here. A spider that only crawls sites known never to serve challenge pages can override `is_blocked` (see [responsible access](../docs/responsible-access.md#blocked-pages)). Remaining ideas, ranked by expected gain: 1. **Stop paying for curl_cffi's `requests` layer on the crawl path** (`fetchers/http.py`). Drive `curl_cffi.AsyncCurl` with a pool of `Curl` handles directly: diff --git a/docs/fetching.md b/docs/fetching.md index 4e46b33..0cdae05 100644 --- a/docs/fetching.md +++ b/docs/fetching.md @@ -5,7 +5,7 @@ wintergrab has two kinds of fetchers, each in a sync and an async flavour: | | Sync | Async | Use for | |---|---|---|---| | HTTP | `Fetcher` / `wg.get()` | `AsyncFetcher` / `wg.aget()` | Most pages. Fast, looks like a real browser. | -| Browser | `BrowserFetcher` / `wg.render()` | `AsyncBrowserFetcher` / `wg.arender()` | JavaScript pages, clicking/scrolling, stubborn bot checks. | +| Browser | `BrowserFetcher` / `wg.render()` | `AsyncBrowserFetcher` / `wg.arender()` | JavaScript pages, clicking and scrolling. | All of them return the same [`Response`](#the-response-object). diff --git a/tests/test_live.py b/tests/test_live.py index e738e0c..2dfc990 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -95,14 +95,20 @@ def test_browser_rendering() -> None: assert len(texts) == 10 and all(texts) +def bot_check(page: wg.Response) -> bool: + """Whether pypi.org's CDN sent its JavaScript bot check instead of the page (it does, to some networks). + Then it must really be that check that was flagged, not the page misjudged.""" + if not looks_blocked(page): + return False + title = page.css("title::text").get("") + assert "challenge" in title.lower(), f"flagged as blocked but not a challenge page: {title!r}" + return True + + def test_pypi_project_page() -> None: page = wg.get("https://pypi.org/project/lxml/") assert page.status == 200 - title = page.css("title::text").get("") - if looks_blocked(page): - # pypi.org's CDN sends some networks a JavaScript bot check instead of - # the page. Then it must really be that check, not a misdetected page. - assert "challenge" in title.lower(), f"flagged as blocked but not a challenge page: {title!r}" + if bot_check(page): pytest.skip("pypi.org served its bot check to this network (detected correctly)") assert page.css("h1.project-header__name::text").get("").strip().startswith("lxml") assert page.structured_data()["opengraph"] @@ -112,6 +118,9 @@ def test_pypi_feed_and_cache(tmp_path) -> None: assert len(wg.sitemap("https://pypi.org/rss/updates.xml", max_sitemaps=1)) > 0 with wg.Fetcher(cache=str(tmp_path)) as http: first = http.get("https://pypi.org/project/cssselect/") + if bot_check(first): + assert first.cache_status is None # (the check says no-store, and is not a page to keep) + pytest.skip("pypi.org served its bot check to this network (detected correctly)") second = http.get("https://pypi.org/project/cssselect/") assert first.cache_status == "stored" assert second.cache_status in ("hit", "revalidated") From 6029f10ae7eed00af7bb50623cff667becca2c14 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 26 Sep 2026 20:02:27 +0000 Subject: [PATCH 04/30] Goal runs that are the whole loop: provenance, healing, questions, fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §75 asks for one path from a goal through extraction, provenance, storage, change, self-heal-or-flag, and around again. It existed only as `crawl --extract --provenance --heal`, set up by hand; goal runs attached no provenance and healed nothing. `plan.run(output, provenance=True, heal="DIR")`, and `wintergrab goal --provenance --heal DIR [--review FILE]`: - every record says where each value came from (`_provenance`); a record from the site's API names the call, its page and the field of the answer each value was read from; - the records are read by a self-healing extractor kept in DIR, whose first version is the plan's schema: selectors (a generated scraper's, a schema of your own) are repaired when the site changes, what it cannot decide and the values it found with little confidence are questions in DIR/review.jsonl, and the first complete record of each site is kept as a regression fixture, which every repair is tested against and `wintergrab heal DIR --check` runs at any time; - the summary says the version, the repairs, the fixtures and the questions waiting (`result.counts["repairs"]`, `["questions"]`, `["fixtures"]`, `result.review`). Tested on the test site: the products section learns the baseline and keeps the fixture, the books section ("the redesign") gets the price selector repaired, the records carry their evidence, the command says so; API records carry their call. `review` without `heal` is an error, in goal runs and on the command line, not an option ignored. `--heal DIR` without `--review` now asks in DIR/review.jsonl (get, crawl, goal, heal); before, questions were not asked at all without a queue. `add_fixture` keeps typed values as JSON holds them (a Money crashed it). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 18 ++++- README.md | 3 + docs/api.md | 7 +- docs/cli.md | 28 ++++--- docs/goals.md | 40 ++++++++++ docs/healing.md | 8 +- src/wintergrab/cli.py | 41 ++++++++-- src/wintergrab/extraction/healing.py | 7 +- src/wintergrab/goals/run.py | 114 ++++++++++++++++++++++++++- tests/test_goal_api.py | 12 +++ tests/test_goals.py | 51 ++++++++++++ tests/test_healing.py | 32 ++++++++ 12 files changed, 335 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68e3f39..3002bfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -283,6 +283,18 @@ an error that says so, not a setting silently ignored - `plan.run(output)` collects the records with one spider: extraction, conditions and de-duplication as a data pipeline, adaptive fetching when some pages need JavaScript, the goal's limit, history when watching. +- **The whole loop in one run**: `plan.run(output, provenance=True, + heal="DIR")` (`wintergrab goal --provenance --heal DIR [--review FILE]`). + Every record says where each value came from (`_provenance`; a record + from the site's API names the call, its page and the field each value was + read from). The records are read by a self-healing extractor kept in DIR: + the plan's schema is its first version, selectors are repaired when the + site changes, what it cannot decide and low-confidence values are + questions in `DIR/review.jsonl`, and the first complete record of each + site is kept as a regression fixture (`wintergrab heal DIR --check`). The + summary says the version, the repairs, the fixtures and the questions + waiting; `result.counts["repairs"]`, `["questions"]`, `["fixtures"]`. + `review` without `heal` is a `ConfigurationError`. - `wintergrab goal "..."` shows how the request was understood and the plan, asks before big crawls (`--yes`), and writes the records; `--plan-only`, `--save-plan`, `--plan`, `--explain`, `--json`. @@ -337,7 +349,11 @@ an error that says so, not a setting silently ignored FIELD` (why a field is what it is on a page), `wintergrab heal DIR` (versions, health, `--log`, `--diff`, `--rollback`, `--activate`, `--import`, `--check`), and `wintergrab review FILE` (`--accept`, `--choice`, - `--reject`, `--correct`, `--note`). + `--reject`, `--correct`, `--note`). `--heal DIR` without `--review` keeps + the questions in `DIR/review.jsonl` (before, they were not asked); + `--review` without `--heal` is an error, not an option ignored. + `ExtractorVersions.add_fixture` keeps typed values (a price) as JSON + holds them. ### Crawl optimization (`wintergrab.spider.optimizer`) diff --git a/README.md b/README.md index 2892762..e40cc11 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ pages), the same API scales up. to a browser, the others stay on fast HTTP; when the pages call a JSON API that holds the records, it is read instead, page by page. No site in mind? `--find-sites` asks a search API (with your key) which sites rank for it. + Run it again and again with `--provenance --heal DIR` and it is the whole + loop: where each value came from, selectors repaired when the site + changes, questions for you, a regression fixture per site. - **Click to build.** `wintergrab build URL -o FILE` shows the page without its scripts. Click a field, a repeated card, a table or the next-page link, and get a schema you can read, edit and test on the page. It diff --git a/docs/api.md b/docs/api.md index b895b61..687353f 100644 --- a/docs/api.md +++ b/docs/api.md @@ -787,10 +787,11 @@ checks that it is up to date. - `run(self, output: str | None = None, **options: Any) -> GoalResult`: Collect the records (see :func:`~wintergrab.goals.run.run_plan`). - `save(self, path: str | Path)`: Write the plan as JSON (edit it, and run it with :meth:`load` and :meth:`run`). - `to_dict(self, *, embed_schema: bool = False) -> dict[str, Any]`: The plan as JSON holds it; ``embed_schema``: a schema file's content rather than its name. -- **`GoalResult(plan: GoalPlan, crawl: CrawlResult | None = None, records: list[dict[str, Any]] = ..., output: str | None = None, counts: Counter[str] = ..., found: Counter[str] = ..., pages: list[Response] = ..., notes: list[str] = ...)`** (class). What running a plan gave: the records (when kept), the crawl's result, and counts. +- **`GoalResult(plan: GoalPlan, crawl: CrawlResult | None = None, records: list[dict[str, Any]] = ..., output: str | None = None, counts: Counter[str] = ..., found: Counter[str] = ..., pages: list[Response] = ..., notes: list[str] = ..., extractor: str | None = None, extractor_version: int | None = None, review: str | None = None)`** (class). What running a plan gave: the records (when kept), the crawl's result, and counts. - `summary(self) -> str`: The records, the fields they have, and what was left out and why. -- **`GoalSpider(goal: Goal, plans: list[SitePlan], *, schema: Schema | None = None, keep_pages: bool = False, use_api: bool = True, **settings: Any)`** (class). Collects a goal's records, following a plan per site (see the module docs). +- **`GoalSpider(goal: Goal, plans: list[SitePlan], *, schema: Schema | None = None, keep_pages: bool = False, use_api: bool = True, provenance: bool = False, heal: str | os.PathLike[str] | None = None, review: ReviewQueue | str | os.PathLike[str] | None = None, **settings: Any)`** (class). Collects a goal's records, following a plan per site (see the module docs). - `api_failed(self, request: Request, error: BaseException) -> Any`: An API request that failed: a refusal is reported; another failure on the first page leaves the site to its pages. + - `on_close(self, result: CrawlResult)`: A self-healing extractor keeps what it learned for the next run. - `parse(self, response: Response) -> Any`: A page of a ``follow`` plan: its record if it has one, and the links that lead to more. - `parse_api(self, response: Response) -> Any`: A page of a site's API: its records, and the next page. - `parse_record(self, response: Response) -> Any`: A page that holds a record: extract it. @@ -806,7 +807,7 @@ checks that it is up to date. - **`parse_goal(text: str, *, sites: list[str] | None = None, parser: Callable[[str], Mapping[str, Any]] | None = None, now: datetime | None = None) -> Goal`**. A :class:`Goal` from a request in plain words (see the module docs). - **`path_pattern(urls: list[str]) -> str`**. A path pattern covering ``urls``: segments they share stay, the others become ``*``. - **`plan_goal(goal: Goal, *, sample: int = 30, obey_robots: bool = True, browser: bool = False, timeout: float = 20.0, surveys: dict[str, SiteSurvey] | None = None, log_level: str | None = 'WARNING', settings: Mapping[str, Any] | None = None, api: bool = True, probe: int = 3) -> GoalPlan`**. Plan ``goal`` for each of its sites (see the module docs). -- **`run_plan(plan: GoalPlan, output: str | None = None, *, max_pages: int | None = None, keep_items: bool | None = None, keep_pages: bool = False, use_api: bool = True, log_level: str | None = 'INFO', progress: bool | None = None, **settings: Any) -> GoalResult`**. Collect ``plan``'s records into ``output`` (``.jsonl``, ``.csv``, ``.json``...; see the module docs). +- **`run_plan(plan: GoalPlan, output: str | None = None, *, max_pages: int | None = None, keep_items: bool | None = None, keep_pages: bool = False, use_api: bool = True, log_level: str | None = 'INFO', progress: bool | None = None, provenance: bool = False, heal: str | os.PathLike[str] | None = None, review: ReviewQueue | str | os.PathLike[str] | None = None, **settings: Any) -> GoalResult`**. Collect ``plan``'s records into ``output`` (``.jsonl``, ``.csv``, ``.json``...; see the module docs). ## `wintergrab.goals.api`: Records from the API a site's pages call diff --git a/docs/cli.md b/docs/cli.md index 787429a..680475b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -75,7 +75,7 @@ Options: | `--each SEL`, `--field NAME=SEL` | Build records (repeatable). | | `--extract SCHEMA`, `--all`, `--container SEL`, `--explain`, `--provenance` | Typed extraction with a [data schema](extraction.md). | | `--why FIELD` | (`--extract`) Why FIELD is what it is on the page, or empty: what each strategy saw, and the likely causes, each saying how sure it is ([extraction](extraction.md#why-is-this-field-empty)). | -| `--heal DIR`, `--review FILE` | (`--extract`) A [self-healing extractor](healing.md) kept in DIR, and a review queue for what needs a person. | +| `--heal DIR`, `--review FILE` | (`--extract`) A [self-healing extractor](healing.md) kept in DIR; what needs a person waits in `DIR/review.jsonl`, or in FILE. | | `-f/--format`, `-o/--output FILE` | Output format and destination. | | `--main-content` | Markdown/text of the main content only. | | `--adaptive` | Use [adaptive selectors](adaptive-selectors.md). | @@ -146,7 +146,7 @@ Options for any crawl: | `--block-trackers` | `resource_filter = True` | | `--extract SCHEMA` | (URL mode) extract typed records with [the extractor](extraction.md) (`--all`, `--container`, `--provenance`) | | `--model PROVIDER:NAME`, `--model-url URL` | (with `--extract`) ask a [language model](models.md) for the fields the page's own data does not give: `openai:NAME`, `anthropic:NAME`, `ollama:NAME` | -| `--heal DIR`, `--review FILE` | (with `--extract`) a [self-healing extractor](healing.md): versions in DIR, selectors repaired when the site changes, questions in FILE | +| `--heal DIR`, `--review FILE` | (with `--extract`) a [self-healing extractor](healing.md): versions in DIR, selectors repaired when the site changes, questions in `DIR/review.jsonl` (or FILE) | | `--quality FILE` | measure the items' [quality](data.md#quality) and compare it with the last run's report, kept in FILE (events `quality_degraded`, `schema_changed`) | | `--pipeline FILE` | appends a [data pipeline](data.md#pipelines-as-configuration) to `pipelines` (`--allow-imports` if it names Python functions) | | `-s/--set NAME=VALUE` | any attribute; values are parsed as JSON when possible (`-s retries=5`, `-s 'allowed_statuses=[404]'`) | @@ -259,7 +259,8 @@ wintergrab goal "REQUEST" [--site URL] [--sample N] [--plan-only] [--save-plan F [-y] [--confirm-over N] [--max-pages N] [--browser] [--no-api] [--no-optimize] [--find-sites [--provider P] [--endpoint URL]] [--record] [--quality FILE] [--model PROVIDER:NAME [--model-url URL]] [-o FILE] -wintergrab goal --plan FILE [-y] [--no-api] [-o FILE] + [--provenance] [--heal DIR [--review FILE]] +wintergrab goal --plan FILE [-y] [--no-api] [-o FILE] [--provenance] [--heal DIR] ``` Reads the request ("Find all laptops under $1000 on shop.example with name, @@ -280,6 +281,13 @@ shown, for you to pick one. See [goals.md](goals.md#records-from-the-sites-api). When records could not be written to the output, the summary says how many, and the exit status is 1. +`--provenance` adds where each value came from to every record, and +`--heal DIR` reads the records with a [self-healing extractor](healing.md) +kept in DIR: selectors repaired when the site changes, questions for you in +`DIR/review.jsonl` (`--review FILE` puts them elsewhere), and the first +complete record of each site kept as a regression fixture. Together they +make one run [the whole loop](goals.md#the-whole-loop). + ## `wintergrab search`: search results, and what they say ```bash @@ -341,9 +349,10 @@ wintergrab heal DIR --rollback | --activate N | --import SCHEMA [--note TEXT] wintergrab heal DIR --check # the regression fixtures against the active version ``` -DIR is the directory of `get`/`crawl --extract SCHEMA --heal DIR`. `--check` -exits with 1 when the active version reads a confirmed page differently. -See [healing.md](healing.md). +DIR is the directory of `get`/`crawl --extract SCHEMA --heal DIR` or +`goal --heal DIR`; its review queue is `DIR/review.jsonl` unless `--review` +names another file. `--check` exits with 1 when the active version reads a +confirmed page differently. See [healing.md](healing.md). ## `wintergrab review`: decide what an extractor was unsure of @@ -352,9 +361,10 @@ wintergrab review FILE [--all] [--json] wintergrab review FILE --accept ID [--choice B] | --reject ID | --correct ID VALUE [--note TEXT] ``` -Lists the pending items of a review queue (`--review FILE`): low-confidence -values with their candidates, selector repairs waiting for a person, and -fields whose selectors broke. Decisions are kept in the file. The extractor +Lists the pending items of a review queue (`DIR/review.jsonl` of a `--heal +DIR` extractor, or the `--review FILE` given): low-confidence values with +their candidates, selector repairs waiting for a person, and fields whose +selectors broke. Decisions are kept in the file. The extractor applies them the next time it runs: repairs become versions, and confirmed values become regression fixtures. diff --git a/docs/goals.md b/docs/goals.md index b0aaf0d..9be1b31 100644 --- a/docs/goals.md +++ b/docs/goals.md @@ -215,6 +215,46 @@ When the API does not work out: `--no-api` reads the pages instead (`plan_goal(goal, api=False)`, `plan.run(use_api=False)`, `wg.plan(..., api=False)`). +## The whole loop + +A goal is rarely collected once. Run with `--provenance --heal DIR`, a plan +is the whole loop in one command, repeated on a schedule or by hand: + +```bash +wintergrab goal --plan shop.plan.json --yes -o shop.jsonl --provenance --heal shop.extractor +``` + +- **Provenance.** Every record carries `_provenance`: for a record read + from a page, the page, when it was fetched, the extractor, and per field + the method, the confidence and the evidence + ([extraction](extraction.md#provenance)); for a record from the site's + API, the call, its page, and the field of the answer each value was read + from. +- **Healing.** The records are read by a + [self-healing extractor](healing.md) kept in `DIR`: the plan's schema is + its first version, and the fields read with selectors (a + [generated scraper](generate.md)'s, or a schema of your own; the goal's + own fields need none) are watched. When the site changes and a field's + selectors stop matching, a replacement is found, tested and applied, or + put to you. +- **Questions.** What the extractor cannot decide, and values it found + with little confidence, wait in `DIR/review.jsonl` (`--review FILE` puts + them elsewhere): `wintergrab review DIR/review.jsonl` lists them, your + decisions are applied the next time the plan runs. The summary says how + many questions wait. +- **Fixtures.** The first complete record of each site is kept in `DIR` as + a regression fixture: the page and the values read from it. Every repair + is tested against the fixtures before it is applied, and `wintergrab heal + DIR --check` runs them against the active version at any time. Values + you confirm in the review queue become fixtures too. + +The summary of such a run ends with the extractor's version, the repairs +made, the fixtures kept and the questions waiting. In code: +`plan.run(output, provenance=True, heal="shop.extractor")`, then +`result.counts["repairs"]`, `["questions"]`, `["fixtures"]`, and +`result.review` (the queue's file). Add `--record` to keep the pages, and a +goal that "tracks changes" keeps the pages' [history](history.md) as well. + ## In code `WinterGrab` is one entry point for goals, pages and sites, with settings diff --git a/docs/healing.md b/docs/healing.md index b28948b..33af293 100644 --- a/docs/healing.md +++ b/docs/healing.md @@ -241,7 +241,10 @@ and returns a `RepairResult` with every candidate and its test. In a [spider](spiders.md), use it where you would use an `Extractor`, and close it at the end (`close()` keeps what it learned). `wintergrab crawl ---heal` does that. +--heal` does that, and so does `wintergrab goal --heal DIR`, which also +keeps the first complete record of each site as a fixture +([the whole loop](goals.md#the-whole-loop)). On the command line, `--heal +DIR` without `--review` puts the questions in `DIR/review.jsonl`. ## Cost @@ -259,7 +262,8 @@ every 40 pages. The kept pages are held in memory. | `versions.json` | which version is active; each version's reason, author (`initial`, `auto`, `human`), status (`active`, `retired`, `candidate`, `accepted`, `rejected`, `superseded`, `rolled back`), parent, and the test its repair passed | | `repairs.jsonl` | every attempt (`applied`, `queued`, `waiting`, `flagged`), confirmation, rollback, fixture and decision applied | | `state.json` | baselines, remembered elements, the last run's health, repairs on probation | -| `fixtures/` | pages with confirmed values | +| `fixtures/` | pages with confirmed values: those a person chose in the review queue, and (`by`: `goal`) the first complete record of each site a goal run read | +| `review.jsonl` | the questions for a person, and the decisions, unless `--review FILE` puts them elsewhere | `wintergrab heal DIR --log` prints the log, `--rollback` goes back to the version the active one came from, `--activate N` makes a version active, diff --git a/src/wintergrab/cli.py b/src/wintergrab/cli.py index 892834c..2deecba 100644 --- a/src/wintergrab/cli.py +++ b/src/wintergrab/cli.py @@ -106,6 +106,9 @@ wintergrab goal "jobs posted in the last 30 days with title, company and salary" --site jobs.example wintergrab goal "articles from news.example published in 2025" --plan-only --save-plan news.plan.json wintergrab goal --plan news.plan.json --yes -o articles.jsonl # run a saved (maybe edited) plan + wintergrab goal --plan shop.plan.json --yes -o shop.jsonl --provenance --heal shop.extractor + # the whole loop, run again and again: where each value came from, selectors repaired when the + # site changes, questions for you in shop.extractor/review.jsonl, a regression fixture per site The request is read by rules (entities, fields, conditions such as "under $1000", "rated 4 or more", "in the last 30 days", "in stock"); the plan shows how it was @@ -608,7 +611,8 @@ def _records(self, response: Response) -> Any: if self.heal: from .extraction.healing import HealingExtractor - extractor = HealingExtractor(self.heal, self.extract, review=self.review, provenance=self.provenance, + review = self.review or Path(self.heal) / "review.jsonl" + extractor = HealingExtractor(self.heal, self.extract, review=review, provenance=self.provenance, model=model) # fmt: skip else: from .extraction import Extractor @@ -790,11 +794,17 @@ def cmd_goal(args: argparse.Namespace) -> int: print(f"the plan makes about {estimate.requests:,} requests: add --yes to run it", file=sys.stderr) return 0 output = args.output or "-" + if args.review and not args.heal: + print("error: --review needs --heal DIR (the review queue is a self-healing extractor's)", file=sys.stderr) + return 2 result = plan.run( output, max_pages=args.max_pages, keep_items=output == "-", use_api=not args.no_api, + provenance=args.provenance, + heal=args.heal, + review=args.review, log_level="DEBUG" if args.verbose > 0 else ("WARNING" if args.verbose < 0 else "INFO"), progress=False if output == "-" else None, optimize=not args.no_optimize, @@ -1446,7 +1456,7 @@ def cmd_heal(args: argparse.Namespace) -> int: from .extraction.healing import HealingExtractor try: - extractor = HealingExtractor(args.directory, review=args.review) + extractor = HealingExtractor(args.directory, review=args.review or Path(args.directory) / "review.jsonl") versions = extractor.versions if args.rollback: version = versions.rollback(reason=args.note or "rolled back by hand", by="human") @@ -1549,7 +1559,7 @@ def _crawl_settings(args: argparse.Namespace) -> tuple[type[Spider], dict[str, A container=args.container, provenance=args.provenance, heal=args.heal, - review=args.review, + review=_review_file(args), model=args.model, model_url=args.model_url, ) @@ -2580,7 +2590,9 @@ def _add_typed_extract_options(p: Any, *, schema_flag: bool = True) -> None: metavar="DIR", help="(--extract) keep versions of the extractor in DIR and repair its selectors when the site changes", ) - p.add_argument("--review", metavar="FILE", help="(--heal) queue what needs a person in FILE (wintergrab review)") + p.add_argument( + "--review", metavar="FILE", help="(--heal) queue what needs a person in FILE instead of DIR/review.jsonl" + ) p.add_argument( "--model", metavar="PROVIDER:NAME", @@ -2600,14 +2612,23 @@ def _model(args: argparse.Namespace) -> Any: return load_model(spec, base_url=getattr(args, "model_url", None)) +def _review_file(args: argparse.Namespace) -> str | None: + """``--review FILE``, which needs ``--heal DIR`` (whose ``review.jsonl`` it is by default).""" + review, heal = getattr(args, "review", None), getattr(args, "heal", None) + if review and not heal: + raise SystemExit("error: --review needs --heal DIR (the review queue is a self-healing extractor's)") + return review + + def _extractor(args: argparse.Namespace) -> Any: + review = _review_file(args) if not getattr(args, "extract", None) and not getattr(args, "heal", None): return None if getattr(args, "heal", None): from .extraction.healing import HealingExtractor - return HealingExtractor(args.heal, args.extract, review=args.review, provenance=args.provenance, - model=_model(args)) # fmt: skip + return HealingExtractor(args.heal, args.extract, review=review or Path(args.heal) / "review.jsonl", + provenance=args.provenance, model=_model(args)) # fmt: skip from .extraction import Extractor return Extractor( @@ -3157,6 +3178,14 @@ def build_parser() -> argparse.ArgumentParser: "(by default the plan collects from it)", ) gp.add_argument("--timeout", type=float, default=20, metavar="SEC", help="per request (default 20)") + gp.add_argument("--provenance", action="store_true", help="add where each value came from to the records") + gp.add_argument( + "--heal", + metavar="DIR", + help="read the records with a self-healing extractor kept in DIR: selectors repaired when the site " + "changes, questions for you in DIR/review.jsonl (wintergrab review), a regression fixture per site", + ) + gp.add_argument("--review", metavar="FILE", help="(--heal) the review queue's file, instead of DIR/review.jsonl") gp.add_argument("-o", "--output", metavar="FILE", help="save the records (.jsonl, .csv, .json); default stdout") gp.add_argument("--json", action="store_true", help="print the plan as JSON (and collect nothing)") gp.set_defaults(func=cmd_goal) diff --git a/src/wintergrab/extraction/healing.py b/src/wintergrab/extraction/healing.py index d1b2940..4818033 100644 --- a/src/wintergrab/extraction/healing.py +++ b/src/wintergrab/extraction/healing.py @@ -280,10 +280,11 @@ def add_fixture(self, url: str, html: bytes | str, expected: Mapping[str, Any], folder.mkdir(exist_ok=True) raw = html.encode("utf-8") if isinstance(html, str) else html number = len(list(folder.glob("*.json.gz"))) + 1 - payload = {"url": url, "expected": dict(expected), "by": by, "html": raw.decode("utf-8", "replace")} + values = {str(k): _plain(v) for k, v in expected.items()} # (typed values, a Money, as JSON holds them) + payload = {"url": url, "expected": values, "by": by, "html": raw.decode("utf-8", "replace")} (folder / f"{number:04d}.json.gz").write_bytes(gzip.compress(json.dumps(payload).encode("utf-8"))) - self.log({"event": "fixture", "url": url, "fields": sorted(expected), "by": by}) - return Fixture(url, dict(expected), raw, by) + self.log({"event": "fixture", "url": url, "fields": sorted(values), "by": by}) + return Fixture(url, values, raw, by) def fixtures(self) -> list[Fixture]: folder = self.directory / "fixtures" diff --git a/src/wintergrab/goals/run.py b/src/wintergrab/goals/run.py index 6f422b2..a64f94d 100644 --- a/src/wintergrab/goals/run.py +++ b/src/wintergrab/goals/run.py @@ -4,6 +4,7 @@ result = plan.run("laptops.jsonl") # or run_plan(plan, "laptops.jsonl") print(result.summary()) + result = plan.run("laptops.jsonl", provenance=True, heal="laptops.extractor") # the whole loop Each site's :class:`GoalSpider` follows its :class:`~wintergrab.goals.plan.SitePlan`: the API the site's pages call when the plan found one (:mod:`wintergrab.goals.api`), @@ -19,22 +20,34 @@ pages, and the crawl stops at the goal's limit. The crawl learns as it goes (:mod:`wintergrab.spider.optimizer`): URL patterns that give nothing are skipped, and query parameters that change nothing are dropped. + +With ``provenance=True`` every record says where each value came from. With ``heal=DIR`` the records are +read by a self-healing extractor kept in that directory (:mod:`wintergrab.extraction.healing`): the +schema's selectors are repaired when the site changes, what it cannot decide and the values it found with +little confidence are questions for a person in its review queue, and the first complete record of each +site is kept as a regression fixture, so the whole loop of a goal (collect, keep provenance, notice a +change, repair or ask, test) is one run, repeated. """ from __future__ import annotations import json +import os import re +import time from collections import Counter from collections.abc import Iterable, Iterator from dataclasses import dataclass, field +from datetime import datetime, timezone from pathlib import Path from typing import TYPE_CHECKING, Any, cast from ..data.pipeline import Deduplicate, Filter, Pipeline from ..data.schema import Schema -from ..errors import RobotsPolicyError, describe +from ..errors import ConfigurationError, RobotsPolicyError, describe from ..extraction import Extractor +from ..extraction.healing import HealingExtractor +from ..extraction.review import ReviewQueue from ..fetchers.resources import registrable_domain from ..fetchers.response import Response from ..fetchers.strategy import FetchStrategy @@ -73,6 +86,15 @@ class GoalSpider(Spider): schema: What records are read with (default: the goal's fields). keep_pages: Keep the record pages fetched (:attr:`pages`). use_api: Ask the API a plan found (``False``: read the pages). + provenance: Records say where each value came from (``_provenance``: for a page's record, the + extractor's evidence per field; for an API's, the call, its page and the field each value was + read from). + heal: A directory: records are read by a :class:`~wintergrab.extraction.healing.HealingExtractor` + kept there. It repairs the schema's selectors when the site changes, puts what it cannot decide + and the values it found with little confidence to a person (``review``), and the first complete + record of each site is kept there as a regression fixture (``wintergrab heal DIR --check``). + review: With ``heal``: the review queue (a :class:`~wintergrab.extraction.review.ReviewQueue` or + its file; by default ``review.jsonl`` in the extractor's directory). settings: More :class:`~wintergrab.Spider` settings. """ @@ -87,6 +109,9 @@ def __init__( schema: Schema | None = None, keep_pages: bool = False, use_api: bool = True, + provenance: bool = False, + heal: str | os.PathLike[str] | None = None, + review: ReviewQueue | str | os.PathLike[str] | None = None, **settings: Any, ) -> None: self.goal = goal @@ -104,7 +129,24 @@ def __init__( # and the hosts of the APIs, which may be elsewhere (api.shop-cdn.example) settings.setdefault("allowed_domains", sorted({*self.plans, *(host_of(a.url) for a in self.apis.values())})) super().__init__(**settings) - self.extractor = Extractor(schema if schema is not None else goal.schema()) + self.provenance = provenance + read_with = schema if schema is not None else goal.schema() + #: The self-healing extractor's directory, with ``heal``. + self.heal = Path(heal) if heal is not None else None + self.extractor: Extractor | HealingExtractor + if self.heal is not None: + queue = review if review is not None else self.heal / "review.jsonl" + self.extractor = HealingExtractor(self.heal, read_with, review=queue, provenance=provenance) + elif review is not None: + raise ConfigurationError("review needs heal: the review queue is a self-healing extractor's", key="review") + else: + self.extractor = Extractor(read_with, provenance=provenance) + #: Regression fixtures kept in this run (one per site, with ``heal``). + self.fixtures_kept = 0 + self._fixture_sites: set[str] = set() + if isinstance(self.extractor, HealingExtractor): + fixtures = self.extractor.versions.fixtures() + self._fixture_sites = {registrable_domain(host_of(f.url)) for f in fixtures if f.by == "goal"} self.identity = next((f for f in goal.fields if f in ("name", "title")), goal.fields[0]) #: Record pages where the record's name (or title) was not found. self.incomplete = 0 @@ -123,6 +165,16 @@ def __init__( self.api_notes: list[str] = [] self._api_asked: set[str] = set() + @property + def healer(self) -> HealingExtractor | None: + """The self-healing extractor (with ``heal``).""" + return self.extractor if isinstance(self.extractor, HealingExtractor) else None + + def on_close(self, result: CrawlResult) -> None: + """A self-healing extractor keeps what it learned for the next run.""" + if isinstance(self.extractor, HealingExtractor): + self.extractor.close() + # -- the APIs ------------------------------------------------------------------------------ # def start_requests(self) -> Iterator[Request | str]: """The plans' sitemaps and start pages, and the first page of each API (whose sites' pages wait).""" @@ -171,6 +223,8 @@ def parse_api(self, response: Response) -> Any: self.api_incomplete += 1 continue self.api_records += 1 + if self.provenance: + record["_provenance"] = self._api_provenance(source, asked, number) yield record following = next_page(source, asked, body, answer, len(records)) if following is None: @@ -184,6 +238,13 @@ def parse_api(self, response: Response) -> Any: return yield self._api_request(site, url, next_body, number + 1) + def _api_provenance(self, source: ApiSource, url: str, page: int) -> dict[str, Any]: + """Where an API record's values came from: the call, its page, and the field each value was read from + (as a page's record says the page and the evidence per field).""" + stamp = datetime.now(timezone.utc).isoformat(timespec="seconds") # (the answer was just received) + api = {"method": source.method, "page": page, "records": source.path, "fields": dict(source.fields)} + return {"url": url, "fetched_at": stamp, "extractor": self.extractor.name, "api": api} + def api_failed(self, request: Request, error: BaseException) -> Any: """An API request that failed: a refusal is reported; another failure on the first page leaves the site to its pages.""" @@ -244,6 +305,14 @@ def parse_record(self, response: Response) -> Any: return if not data.get("url"): data["url"] = response.url + healer = self.healer + if healer is not None: + site = registrable_domain(host_of(response.url)) + if site not in self._fixture_sites: # a site's first complete record: what later versions must read + values = {k: v for k, v in record.data.items() if not k.startswith("_") and v not in (None, "", [], {})} + healer.versions.add_fixture(response.url, response.body, values, by="goal") + self._fixture_sites.add(site) + self.fixtures_kept += 1 yield data def parse(self, response: Response) -> Any: @@ -280,6 +349,10 @@ class GoalResult: pages: list[Response] = field(default_factory=list) #: What happened to an API that failed or stopped early. notes: list[str] = field(default_factory=list) + #: With ``heal``: the self-healing extractor's directory, its active version, and its review queue's file. + extractor: str | None = None + extractor_version: int | None = None + review: str | None = None def summary(self) -> str: """The records, the fields they have, and what was left out and why.""" @@ -309,6 +382,15 @@ def summary(self) -> str: + (f", {c['browser_pages']:,} in a browser" if c["browser_pages"] else "") + f", {c['errors']:,} error(s)" ) + if self.extractor: + healing = f"self-healing extractor {self.extractor}: version {self.extractor_version}" + if c["repairs"]: + healing += f", {c['repairs']:,} repair(s) this run" + if c["fixtures"]: + healing += f", {c['fixtures']:,} regression fixture(s) kept" + lines.append(healing) + if c["questions"]: + lines.append(f"{c['questions']:,} question(s) waiting for you: wintergrab review {self.review}") lines.extend(f"note: {note}" for note in self.notes) return "\n".join(lines) @@ -323,6 +405,9 @@ def run_plan( use_api: bool = True, log_level: str | None = "INFO", progress: bool | None = None, + provenance: bool = False, + heal: str | os.PathLike[str] | None = None, + review: ReviewQueue | str | os.PathLike[str] | None = None, **settings: Any, ) -> GoalResult: """Collect ``plan``'s records into ``output`` (``.jsonl``, ``.csv``, ``.json``...; see the module docs). @@ -334,6 +419,11 @@ def run_plan( keep_items: Keep the records in memory (``result.records``); by default when there is no ``output``. keep_pages: Keep the record pages fetched (``result.pages``). use_api: Collect from the API the plan found, where it found one (``False``: read the pages). + provenance: Records say where each value came from (``_provenance``). + heal: A directory for a self-healing extractor (see :class:`GoalSpider`): selectors repaired when + the site changes, questions for a person in its review queue, a regression fixture per site. + ``result.summary()`` says what it did; ``result.counts``: ``repairs``, ``questions``, ``fixtures``. + review: With ``heal``: the review queue's file (by default ``review.jsonl`` in the directory). settings: More :class:`~wintergrab.Spider` settings (``concurrency``, ``cache``, ``obey_robots_txt``...); ``optimize=False`` fetches every page the plan leads to (see :mod:`wintergrab.spider.optimizer`). """ @@ -371,12 +461,16 @@ def run_plan( options.setdefault("history", str(Path(".wintergrab") / f"{goal.kind.name}.history")) if goal.limit: options.setdefault("max_items", goal.limit) + started = time.time() spider = GoalSpider( goal, sites, schema=plan.extraction_schema() if plan.schema is not None else None, keep_pages=keep_pages, use_api=use_api, + provenance=provenance, + heal=heal, + review=review, output=output, keep_items=keep, max_pages=max_pages, @@ -401,6 +495,22 @@ def run_plan( counts["api_pages"] = spider.api_pages counts["api_records"] = spider.api_records counts["api_incomplete"] = spider.api_incomplete + healer = spider.healer + if healer is not None: + directory = str(healer.versions.directory) + result.extractor, result.extractor_version = directory, healer.versions.active_number + history = healer.versions.history() + counts["repairs"] = sum( + 1 + for e in history + if e.get("event") == "repair" and e.get("outcome") == "applied" and e.get("at", 0) >= started + ) + counts["fixtures"] = spider.fixtures_kept + if healer.review is not None: + result.review = str(healer.review.path) + counts["questions"] = sum( + 1 for item in healer.review.pending() if item.details.get("extractor") == directory + ) result.notes = list(spider.api_notes) cut_short = crawl.status != "finished" or bool(goal.limit and counts["records"] >= goal.limit) for site, total in spider.api_total.items(): diff --git a/tests/test_goal_api.py b/tests/test_goal_api.py index a774c63..4ed5600 100644 --- a/tests/test_goal_api.py +++ b/tests/test_goal_api.py @@ -286,6 +286,18 @@ def test_a_recorded_run_replays_its_api(shop, modes, tmp_path) -> None: assert again.same and Shop.hits["api"] == 3 +def test_api_records_say_where_they_came_from(shop, modes) -> None: + result = _api_plan(shop).run(log_level=None, provenance=True) + first = next(r for r in result.records if r["name"] == "Laptop 1") + where = first["_provenance"] + assert where["url"] == f"{shop}/api/laptops?page=1&per_page={PER_PAGE}" and "fetched_at" in where + assert where["extractor"] == "product@1" and where["api"]["page"] == 1 and where["api"]["records"] == "items[]" + assert where["api"]["fields"]["price"] == "price" and where["api"]["method"] == "GET" + last = next(r for r in result.records if r["name"] == f"Laptop {TOTAL}") + assert last["_provenance"]["api"]["page"] == 3 and last["_provenance"]["url"].endswith("page=3&per_page=4") + assert all("_provenance" not in r for r in _api_plan(shop).run(log_level=None).records) # (only when asked) + + def test_the_pages_when_asked(shop, modes, tmp_path) -> None: result = _api_plan(shop).run(log_level=None, use_api=False) assert len(result.records) == TOTAL and Shop.hits["api"] == 0 and result.notes == [] diff --git a/tests/test_goals.py b/tests/test_goals.py index 2caada9..97fd359 100644 --- a/tests/test_goals.py +++ b/tests/test_goals.py @@ -220,6 +220,57 @@ def test_plans_run_and_replay(site, tmp_path) -> None: assert limited.run(log_level=None).counts["records"] == 2 +def test_the_whole_loop_in_one_run(site, tmp_path, capsys) -> None: + """provenance=True, heal=DIR: records say where each value came from, the plan's schema is read by a + self-healing extractor kept in DIR (a fixture per site, questions in DIR/review.jsonl), and a redesign is + repaired the next time the plan runs.""" + from wintergrab.cli import main + from wintergrab.extraction.healing import ExtractorVersions + from wintergrab.extraction.review import ReviewQueue + + schema = tmp_path / "product.schema.json" + fields = {"name": {"type": "string", "selectors": ["h1"]}, "price": {"type": "money", "selectors": ["p.price"]}} + schema.write_text(json.dumps({"name": "product", "fields": {**fields, "url": "url"}}), encoding="utf-8") + heal = tmp_path / "ext" + products = plan_goal(read("products with name and price", sites=[site.url + "/products/page/1"]), sample=15) + products.schema = str(schema) + out = tmp_path / "products.jsonl" + result = products.run(str(out), log_level=None, provenance=True, heal=str(heal)) + rows = [json.loads(line) for line in out.read_text(encoding="utf-8").splitlines()] + assert result.counts["records"] == len(rows) == 20 + where = rows[0]["_provenance"] + assert where["url"] == rows[0]["url"] and where["extractor"] == "product@1" and "fetched_at" in where + assert where["fields"]["price"]["method"] == "selector" # (the evidence per field, as the extractor keeps it) + versions = ExtractorVersions(heal) + assert versions.active_number == 1 and versions.load_state()["fields"]["price"]["baseline"] == 1.0 + fixtures = versions.fixtures() # the site's first complete record, as later versions must read it + assert len(fixtures) == 1 and fixtures[0].by == "goal" and "/product/" in fixtures[0].url + assert fixtures[0].expected["name"].startswith("Product") and fixtures[0].expected["price"]["currency"] == "USD" + assert versions.check_fixtures() == [] + assert (result.extractor, result.extractor_version, result.review) == (str(heal), 1, str(heal / "review.jsonl")) + assert result.counts["fixtures"] == 1 and result.counts["repairs"] == 0 and result.counts["questions"] == 0 + assert f"self-healing extractor {heal}: version 1, 1 regression fixture(s) kept" in result.summary() + + # "the redesign": the books section, whose prices are p.price_color; the plan runs again, on the same extractor + books = plan_goal(read("books with title and price", sites=[site.url + "/books/"]), sample=15) + books.schema = str(schema) + books.save(tmp_path / "books.plan.json") + again = books.run(str(tmp_path / "books.jsonl"), log_level=None, provenance=True, heal=str(heal)) + assert again.counts["records"] > 10 and again.counts["repairs"] == 1 and again.counts["fixtures"] == 0 + assert ExtractorVersions(heal).active.reason == "repair of price: p.price -> .price_color (anchored)" + assert ", 1 repair(s) this run" in again.summary() and "question(s)" not in again.summary() + assert ReviewQueue(heal / "review.jsonl").pending() == [] # (nothing it could not decide) + # the command: the same, and the summary says what the extractor did + assert main(["goal", "--plan", str(tmp_path / "books.plan.json"), "--yes", "-o", str(tmp_path / "b.jsonl"), + "--provenance", "--heal", str(heal)]) == 0 # fmt: skip + err = capsys.readouterr().err + assert f"self-healing extractor {heal}: version 2" in err + assert main(["goal", "--plan", str(tmp_path / "books.plan.json"), "--yes", "--review", "r.jsonl"]) == 2 + assert "--review needs --heal" in capsys.readouterr().err + with pytest.raises(ConfigurationError, match="review needs heal"): + books.run(log_level=None, review=str(tmp_path / "r.jsonl")) + + def test_goal_command(site, tmp_path, capsys) -> None: from wintergrab.cli import main diff --git a/tests/test_healing.py b/tests/test_healing.py index ef37583..1c31f7d 100644 --- a/tests/test_healing.py +++ b/tests/test_healing.py @@ -277,6 +277,38 @@ def test_crawl_heals(site, tmp_path) -> None: assert all(r["price"]["currency"] == "GBP" for r in rows if "/catalogue/book-" in r["url"]) +def test_the_questions_go_in_the_extractors_directory_unless_told_otherwise(site, tmp_path, capsys) -> None: + from argparse import Namespace + + from wintergrab.cli import _extractor, main + + schema = tmp_path / "product.schema.json" + schema.write_text(json.dumps({"name": "product", "fields": {"name": {"type": "string", "selectors": ["h1"]}}})) + args = Namespace(extract=str(schema), heal=str(tmp_path / "ext"), review=None, provenance=False, model=None) + assert _extractor(args).review.path == tmp_path / "ext" / "review.jsonl" + args.review = str(tmp_path / "elsewhere.jsonl") + assert _extractor(args).review.path == tmp_path / "elsewhere.jsonl" + args.heal = None + with pytest.raises(SystemExit, match="--review needs --heal"): + _extractor(args) + with pytest.raises(SystemExit, match="--review needs --heal"): + main(["crawl", site.url + "/product/1", "--extract", str(schema), "--review", str(tmp_path / "r.jsonl")]) + assert main(["get", site.url + "/product/1", "--extract", str(schema), "--heal", str(tmp_path / "g")]) == 0 + capsys.readouterr() + assert not (tmp_path / "g" / "review.jsonl").exists() # (no question: no file) + assert main(["heal", str(tmp_path / "g")]) == 0 # (reads the same queue) + + +def test_fixtures_hold_typed_values_as_json_does(tmp_path) -> None: + from wintergrab.data.normalize import Money + + schema = {"name": "product", "fields": {"price": {"type": "money", "selectors": ["p.price"]}}} + versions = ExtractorVersions(tmp_path / "ext", schema) + kept = versions.add_fixture("https://s.example/1", "<p class='price'>$9.50</p>", {"price": Money(9.5, "USD")}) + assert kept.expected == {"price": "9.5 USD"} and versions.fixtures()[0].expected == kept.expected + assert versions.check_fixtures() == [] # (read back as the field's type: the same price) + + def test_rolling_back_one_repair_keeps_the_others(tmp_path) -> None: schema = {"name": "product", "fields": {"name": {"type": "string", "selectors": ["h1.title"]}, "price": {"type": "money", "selectors": [".price"]}}} # fmt: skip From 5c2c1470bd8fa6bb42fd5d595688b04cf424c32b Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 26 Sep 2026 20:20:59 +0000 Subject: [PATCH 05/30] Provenance you can trace: the run, the output, and what the pipeline did MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §24 asks that every value can be traced. Records collected with --provenance carried the page, the fetch time, the extractor and the evidence per field, but not the run they came from, nor what happened to a value after extraction; and nothing read that back for one record. A crawl now stamps each record that carries _provenance with `run` (its id in the run registry) and `output` (where the record went). A data pipeline records, under each field's `transforms`, what every stage did to it: the value before it changed it, the old name of a renamed field (whose evidence follows it), or that the stage added or dropped it. Records without provenance pay nothing: a stage is traced only when the record has a _provenance dict. `wintergrab data trace INPUT [FIELD...] --where FIELD=VALUE` (and `describe_provenance(record)`, `find_records(path, where)` from wintergrab.data) tells the story of a value: the page or API call, when, the extractor, the run, the output, how it was read, what else was found, and how it was changed. The exit status is 1 when no record matches or none has provenance. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 10 +++ docs/api.md | 2 + docs/cli.md | 9 ++- docs/data.md | 27 ++++++++ docs/extraction.md | 10 +++ src/wintergrab/cli.py | 38 +++++++++++ src/wintergrab/data/__init__.py | 3 + src/wintergrab/data/pipeline.py | 44 ++++++++++++ src/wintergrab/data/trace.py | 115 ++++++++++++++++++++++++++++++++ src/wintergrab/spider/engine.py | 9 +++ tests/test_data_pipeline.py | 71 ++++++++++++++++++++ 11 files changed, 336 insertions(+), 2 deletions(-) create mode 100644 src/wintergrab/data/trace.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3002bfe..e2ce817 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -141,6 +141,16 @@ an error that says so, not a setting silently ignored pipeline about 4,200 records/s, near-duplicate checks 158 µs per record, expressions 1 µs. +- **Provenance you can trace.** A crawl stamps every record that carries + `_provenance` with `run` (its id in the run registry) and `output`; a + pipeline records under each field's `transforms` what every stage did to + it (the value before, the old name of a renamed field, whose evidence + follows it, added, dropped); records without provenance pay nothing. + `wintergrab data trace INPUT [FIELD...] --where FIELD=VALUE` (or + `describe_provenance(record)` and `find_records(path, where)` from + `wintergrab.data`) tells where a value came from: the page or API call, + when, the extractor, the run, the output, how it was read, what else was + found, and how it was changed. ### Extraction engine (`wintergrab.extraction`) - `Extractor(schema).extract(page)` finds every field of a data schema diff --git a/docs/api.md b/docs/api.md index 687353f..e9b2b12 100644 --- a/docs/api.md +++ b/docs/api.md @@ -692,9 +692,11 @@ checks that it is up to date. - `to_config(self) -> dict[str, Any]`: The stage's configuration-file form: ``{kind: options}``. - **`compile_expression(source: str) -> Expression`**. :class:`Expression` with a cache (the same source compiles once). - **`content_hash(value: Any) -> str`**. A stable hex digest of ``value``'s normalized text (dicts and lists: of their sorted items). +- **`describe_provenance(record: Mapping[str, Any], fields: Sequence[str] | None = None) -> str`**. Where the values of ``fields`` (every field by default) came from, as lines: the page or API call, the fetch time, the extractor, the run and the output; then per field how it was read, what else was found, and what the pipeline did to it. - **`diff_records(old: Iterable[Mapping[str, Any]], new: Iterable[Mapping[str, Any]], key: str | Sequence[str] | None = None, *, ignore: Iterable[str] = (), private: bool = False) -> DatasetDiff`**. The differences between two datasets. - **`distance_km(a: Any, b: Any) -> float | None`**. The great-circle distance between two points, in kilometres (to 0.1 km); ``None`` when either has no coordinates. - **`explain_inference(records: Iterable[Mapping[str, Any]], **options: Any) -> list[TypeGuess]`**. Why each field got its type (share of values the type could read, presence...). +- **`find_records(path: str | Path, where: Mapping[str, Any] | None = None, *, limit: int | None = None) -> Iterator[dict[str, Any]]`**. The records of ``path`` (see :func:`~wintergrab.data.io.read_records`) whose fields have the values ``where`` gives, compared as text (``{"url": "https://shop.example/p/1"}``); all of them without ``where``. - **`get_path(record: Any, path: str, default: Any = None) -> Any`**. The value at a dotted ``path`` (``"offers.0.price"``); a key with that exact name wins. - **`group_records(records: Iterable[Mapping[str, Any]], by: str | Callable[[Mapping[str, Any]], Any], *, stats: Sequence[str] = (), country: str | None = None, fields: Mapping[str, str | Sequence[str]] | None = None, places: Sequence[Place] | None = None) -> list[Group]`**. Group ``records`` by a place part (``"country"``, ``"region"``, ``"city"``, ``"postal_code"``, ``"remote"``: read with :func:`places_of`, so ``"Germany"``, ``"DE"`` and ``"Deutschland"`` are one group), by another field (a dotted path), or by a function of the record. - **`hamming(a: int, b: int) -> int`**. Number of differing bits. diff --git a/docs/cli.md b/docs/cli.md index 680475b..222b7c9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -192,6 +192,7 @@ wintergrab data graph [KIND=]INPUT... [--kind KIND] [--relation FIELD=RELATION:K wintergrab data places INPUT [--country C] [--in PLACE] [--near LAT,LON --within KM] [--remote] [--by PART [--stats FIELD] [--top N] [--json]] [--add PARTS] [--field PART=NAME] [-o OUT] wintergrab data diff OLD NEW [--key FIELD] [--ignore FIELD] [-o CHANGES] [--json] [--exit-code] +wintergrab data trace INPUT [FIELD...] [--where FIELD=VALUE] [--limit N] [--json] # where a value came from ``` Inputs are JSON Lines, JSON or CSV files (`-` reads JSON Lines from stdin); @@ -209,8 +210,12 @@ versions (`DIR@v2`, `DIR@previous`, `DIR@latest`), prints the counts and the fields that changed, writes the details with `-o`, and with `--exit-code` exits with status 1 when they differ. `places` reads where each record is, normalized, keeps those in a country, region or city or near a point, and -with `--by` prints them grouped by place. See [data.md](data.md), -[entities.md](entities.md) and [places.md](places.md). +with `--by` prints them grouped by place. `trace` tells where the values of +a record collected with `--provenance` came from: the page or API call, the +run and the output, how each field was read, and what the pipeline did to +it; the exit status is 1 when no record matches or none has provenance. +See [data.md](data.md), [entities.md](entities.md) and +[places.md](places.md). ## `wintergrab extract`: records from a site diff --git a/docs/data.md b/docs/data.md index 2200d7e..f1148a6 100644 --- a/docs/data.md +++ b/docs/data.md @@ -316,6 +316,32 @@ sets the value to `None` and counts an error; `on_error="keep"` leaves the field as it was, `"drop"` drops the record, `"raise"` raises. Add operations with `register_operation(name, build)`. +### Where a value came from + +A record that carries `_provenance` (collected with `--provenance`, see +[extraction](extraction.md#provenance)) keeps what the pipeline did to it: +under `_provenance.fields.NAME.transforms`, each stage that changed the +value (with the value before), renamed the field (with its old name; the +field's evidence follows it), added it, or dropped it. Records without +provenance pay nothing. `wintergrab data trace` tells the story of a value: + +```bash +wintergrab data trace items.jsonl price --where url=https://shop.example/p/1 +``` + +```text +from: https://shop.example/p/1, fetched 2026-09-26T19:04:11+00:00, extractor product@2, run run-7, output items.jsonl +price: 1299 + read by selector from selector:.price ('$1,299'), confidence 0.90, agreeing with json-ld + also found: 1399 (dom, 0.41) + rename (names): was named cost + transform: was '$1,299' +``` + +In code: `find_records(path, {"url": ...})` and +`describe_provenance(record, ["price"])` from `wintergrab.data`. `--json` +prints the records with their provenance instead. + ### Pipelines in crawls A pipeline is an item pipeline, so it plugs into any spider: @@ -512,6 +538,7 @@ wintergrab data graph job=jobs.jsonl company=companies.jsonl -o graph.graphml wintergrab data places jobs.jsonl --country US --by region --stats salary wintergrab data diff prices/@previous prices/@latest -o changes.jsonl wintergrab data diff yesterday.jsonl today.jsonl --key sku --exit-code +wintergrab data trace items.jsonl price --where url=https://shop.example/p/1 wintergrab crawl https://shop.example --field ... --pipeline pipeline.yaml -o items.jsonl ``` diff --git a/docs/extraction.md b/docs/extraction.md index b4498bc..01d3f1a 100644 --- a/docs/extraction.md +++ b/docs/extraction.md @@ -162,6 +162,16 @@ the schema's name and version), each field's method, source, raw value, confidence, agreement, alternatives, notes and validation, and the record's validation issues. [Quality monitoring](data.md#quality) reads `_confidence`. +A crawl (`crawl`/`goal --provenance`) adds `run`, the run's id in the +[run registry](runs.md), and `output`, where the record went; a record from +a site's API names the call, its page and the field of the answer each +value was read from. A [pipeline](data.md#pipelines) adds `transforms` to +each field it touched: the stage, the value before, the name a renamed +field had, or that it was added or dropped. `wintergrab data trace +items.jsonl price --where url=URL` (or `describe_provenance(record)`) reads +it all back as lines: where the record came from, and how each value was +read and changed ([data.md](data.md#where-a-value-came-from)). + ## Why is this field empty? `extractor.why("price", page)` (`wintergrab get URL --extract SCHEMA --why diff --git a/src/wintergrab/cli.py b/src/wintergrab/cli.py index 2deecba..43ffdc1 100644 --- a/src/wintergrab/cli.py +++ b/src/wintergrab/cli.py @@ -2526,6 +2526,30 @@ def cmd_data_quality(args: argparse.Namespace) -> int: return 1 if any(issue.severity == "error" for issue in comparison) else 0 +def cmd_data_trace(args: argparse.Namespace) -> int: + from .data.trace import describe_provenance, find_records + + where = _parse_pairs(args.where, "=", "--where") + shown = traced = 0 + try: + for record in find_records(args.input, where, limit=args.limit): + shown += 1 + if args.json: + print(json.dumps(record, ensure_ascii=False, default=str)) + else: + if shown > 1: + print() + print(describe_provenance(record, args.field)) + traced += isinstance(record.get("_provenance"), dict) + except WintergrabError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if not shown: + print("no record matches" if where else "no records", file=sys.stderr) + return 1 + return 0 if traced else 1 + + # --------------------------------------------------------------------------- # # argument parsing # --------------------------------------------------------------------------- # @@ -2953,6 +2977,20 @@ def build_parser() -> argparse.ArgumentParser: qual.add_argument("--json", action="store_true", help="print the report as JSON") qual.add_argument("--limit", type=int, metavar="N", help="only the first N records") qual.set_defaults(func=cmd_data_quality) + trace = actions.add_parser( + "trace", + help="where a record's values came from (records collected with --provenance)", + description="For records collected with --provenance: the page or API call, when, which extractor, the " + "run and the output; per field how it was read, what else was found, and what the pipeline did to it.", + ) + trace.add_argument("input", metavar="INPUT") + trace.add_argument("field", nargs="*", metavar="FIELD", help="only these fields (default: every field)") + trace.add_argument( + "--where", action="append", metavar="FIELD=VALUE", help="the record(s) with this value (repeatable: all)" + ) + trace.add_argument("--limit", type=int, default=5, metavar="N", help="records to show (default 5)") + trace.add_argument("--json", action="store_true", help="print the record(s), provenance included, as JSON") + trace.set_defaults(func=cmd_data_trace) ent = actions.add_parser("entities", help="find which names are the same company, brand, product, person or place") ent.add_argument("input", metavar="INPUT") ent.add_argument("--field", required=True, metavar="FIELD", help="the field holding the names (dotted paths work)") diff --git a/src/wintergrab/data/__init__.py b/src/wintergrab/data/__init__.py index 97e5544..4d94aff 100644 --- a/src/wintergrab/data/__init__.py +++ b/src/wintergrab/data/__init__.py @@ -68,6 +68,7 @@ simhash, simhash_similarity, ) +from .trace import describe_provenance, find_records from .validate import Rule, is_valid, validate_record from .versions import DatasetDiff, DatasetVersions, diff_records @@ -120,9 +121,11 @@ "Validate", "compile_expression", "content_hash", + "describe_provenance", "diff_records", "distance_km", "explain_inference", + "find_records", "get_path", "group_records", "hamming", diff --git a/src/wintergrab/data/pipeline.py b/src/wintergrab/data/pipeline.py index 08ba52d..c72c596 100644 --- a/src/wintergrab/data/pipeline.py +++ b/src/wintergrab/data/pipeline.py @@ -161,6 +161,44 @@ class RecordContext: reason: str = "" +def _traced(before: Mapping[str, Any], after: dict[str, Any], stage: Stage) -> None: + """What ``stage`` did to a record that carries provenance, under ``_provenance["fields"][name]["transforms"]``: + the value before it changed it, the name a renamed field had (its provenance follows it), or that the stage + added or dropped the field.""" + where = after.get("_provenance") + if not isinstance(where, dict) or stage.kind == "pipeline": # (a nested pipeline's stages say it themselves) + return + fields = where.setdefault("fields", {}) + if not isinstance(fields, dict): + return + step = {"stage": stage.name, "kind": stage.kind} + gone = [k for k in before if k not in after and not k.startswith("_")] + for name, value in after.items(): + if name.startswith("_"): + continue + if name in before: + old = before[name] + if old is value or old == value: + continue + note = {**step, "before": old} + else: + source = next((k for k in gone if before[k] == value), None) + if source is None: + note = {**step, "added": True} + else: + gone.remove(source) + note = {**step, "from": source} + if source in fields and name not in fields: + fields[name] = fields.pop(source) + evidence = fields.setdefault(name, {}) + if isinstance(evidence, dict): + evidence.setdefault("transforms", []).append(note) + for name in gone: + evidence = fields.setdefault(name, {}) + if isinstance(evidence, dict): + evidence.setdefault("transforms", []).append({**step, "dropped": True}) + + def _item_result(record: dict[str, Any] | None, ctx: RecordContext) -> dict[str, Any]: if record is None: from ..spider.middleware import DropItem @@ -2385,10 +2423,13 @@ def process(self, record: dict[str, Any], ctx: RecordContext | None = None) -> d ctx = ctx if ctx is not None else RecordContext() current = record for stage in self.stages: + before = dict(current) if "_provenance" in current else None # (a record with provenance: traced) result = stage.process(current, ctx) if result is None: return None current = result + if before is not None: + _traced(before, current, stage) return current async def aprocess(self, record: dict[str, Any], ctx: RecordContext | None = None) -> dict[str, Any] | None: @@ -2396,10 +2437,13 @@ async def aprocess(self, record: dict[str, Any], ctx: RecordContext | None = Non ctx = ctx if ctx is not None else RecordContext() current = record for stage in self.stages: + before = dict(current) if "_provenance" in current else None result = await stage.aprocess(current, ctx) if stage.is_async else stage.process(current, ctx) if result is None: return None current = result + if before is not None: + _traced(before, current, stage) return current def __call__(self, record: Mapping[str, Any]) -> dict[str, Any] | None: diff --git a/src/wintergrab/data/trace.py b/src/wintergrab/data/trace.py new file mode 100644 index 0000000..e259498 --- /dev/null +++ b/src/wintergrab/data/trace.py @@ -0,0 +1,115 @@ +"""Where a record's values came from: the page or the API call, the extractor's evidence per field, the crawl's +run and output, and what the pipeline did to each value. + +Records collected with ``provenance=True`` (``crawl``/``goal --provenance``) carry ``_provenance`` +(:meth:`~wintergrab.extraction.ExtractedRecord.provenance`): the page URL, the fetch time, the extractor, per +field the method, source, raw value, confidence, agreement and alternatives; a record from a site's API names +the call, its page and the field of the answer each value was read from. A crawl adds ``run`` (its id in the +run registry) and ``output`` (where the record went). A :class:`~wintergrab.data.Pipeline` adds, per field, +``transforms``: each stage that changed the value (with the value before), renamed the field (with its old +name), added it or dropped it. + +:: + + for record in find_records("items.jsonl", {"url": "https://shop.example/p/1"}): + print(describe_provenance(record, ["price"])) + +On the command line: ``wintergrab data trace items.jsonl price --where url=https://shop.example/p/1``. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Mapping, Sequence +from pathlib import Path +from typing import Any + +from .io import read_records + +__all__ = ["describe_provenance", "find_records"] + + +def find_records( + path: str | Path, where: Mapping[str, Any] | None = None, *, limit: int | None = None +) -> Iterator[dict[str, Any]]: + """The records of ``path`` (see :func:`~wintergrab.data.io.read_records`) whose fields have the values + ``where`` gives, compared as text (``{"url": "https://shop.example/p/1"}``); all of them without ``where``.""" + wanted = {k: str(v) for k, v in (where or {}).items()} + found = 0 + for record in read_records(path): + if all(str(record.get(k)) == v for k, v in wanted.items()): + yield record + found += 1 + if limit is not None and found >= limit: + return + + +def describe_provenance(record: Mapping[str, Any], fields: Sequence[str] | None = None) -> str: + """Where the values of ``fields`` (every field by default) came from, as lines: the page or API call, the + fetch time, the extractor, the run and the output; then per field how it was read, what else was found, + and what the pipeline did to it. A record without ``_provenance`` says so.""" + where = record.get("_provenance") + if not isinstance(where, Mapping): + return "no provenance: the records were collected without it (crawl/goal --provenance, provenance=True)" + api = where.get("api") if isinstance(where.get("api"), Mapping) else None + origin = str(where.get("url") or "?") + if api is not None: + origin = f"{api.get('method', 'GET')} {origin} (page {api.get('page', '?')} of the site's API)" + facts = [origin] + for key, label in (("fetched_at", "fetched"), ("extractor", "extractor"), ("run", "run"), ("output", "output")): + if where.get(key): + facts.append(f"{label} {where[key]}") + lines = ["from: " + ", ".join(facts)] + evidence: Mapping[str, Any] = where["fields"] if isinstance(where.get("fields"), Mapping) else {} + names = list(fields) if fields else [k for k in record if not k.startswith("_")] + for name in names: + lines.append(f"{name}: {_show(record[name])}" if name in record else f"{name}: (not in the record)") + if api is not None and isinstance(api.get("fields"), Mapping) and name in api["fields"]: + lines.append(f" read from the answer's {api['fields'][name]}") + fv = evidence.get(name) + if not isinstance(fv, Mapping): + if api is None: + lines.append(" not read by the extractor") + continue + if fv.get("method"): + how = f"read by {fv['method']}" + (f" from {fv['source']}" if fv.get("source") else "") + if fv.get("raw") is not None: + how += f" ({_show(fv['raw'])})" + if fv.get("confidence") is not None: + how += f", confidence {float(fv['confidence']):.2f}" + if fv.get("agreed"): + how += ", agreeing with " + ", ".join(str(m) for m in fv["agreed"]) + lines.append(" " + how) + alternatives = fv.get("alternatives") or () + if alternatives: + others = "; ".join( + f"{_show(a.get('value'))} ({'/'.join(a.get('methods') or ()) or '?'}, {float(a.get('confidence') or 0):.2f})" + for a in alternatives[:3] + if isinstance(a, Mapping) + ) + lines.append(f" also found: {others}") + if fv.get("notes"): + lines.append(" notes: " + ", ".join(str(n) for n in fv["notes"])) + if fv.get("validation") not in (None, "ok"): + lines.append(f" validation: {fv['validation']}") + for step in fv.get("transforms") or (): + if isinstance(step, Mapping): + lines.append(" " + _transform(step)) + return "\n".join(lines) + + +def _transform(step: Mapping[str, Any]) -> str: + kind, stage = str(step.get("kind", "stage")), str(step.get("stage", "")) + what = kind if not stage or stage == kind else f"{kind} ({stage})" + if "from" in step: + return f"{what}: was named {step['from']}" + if step.get("dropped"): + return f"{what}: dropped" + if step.get("added"): + return f"{what}: added" + return f"{what}: was {_show(step.get('before'))}" + + +def _show(value: Any) -> str: + if isinstance(value, str): + return repr(value) if len(value) <= 60 else repr(value[:57] + "...") + return repr(value) diff --git a/src/wintergrab/spider/engine.py b/src/wintergrab/spider/engine.py index caba21c..b71338a 100644 --- a/src/wintergrab/spider/engine.py +++ b/src/wintergrab/spider/engine.py @@ -1792,6 +1792,13 @@ def _item_dropped(self, pipeline: str, reason: str) -> None: self.stats.inc(f"items_dropped/{pipeline}") self.events.emit("item_dropped", pipeline=pipeline, reason=reason) + def _stamp(self, where: dict[str, Any]) -> None: + """A record's provenance says the run it was collected in and the output it went to.""" + if self.recorder is not None and self.recorder.run is not None: + where.setdefault("run", self.recorder.run.id) + if self.spider.output: + where.setdefault("output", redact_url(str(self.spider.output))) + def _is_duplicate_item(self, item: Any) -> bool: data = to_dict(item) if not isinstance(data, dict) or data.get(self.spider.unique_key) is None: @@ -1816,6 +1823,8 @@ async def _item(self, item: Any) -> Any: processed = await self._run_pipelines(processed) if processed is None: return None + if isinstance(processed, dict) and isinstance(processed.get("_provenance"), dict): + self._stamp(processed["_provenance"]) if spider.unique_key and self._is_duplicate_item(processed): self.stats.inc("items_duplicate") return None diff --git a/tests/test_data_pipeline.py b/tests/test_data_pipeline.py index 350324a..6bcfbf9 100644 --- a/tests/test_data_pipeline.py +++ b/tests/test_data_pipeline.py @@ -688,3 +688,74 @@ def test_a_schema_inferred_from_records() -> None: schema = Schema.infer([{"price": 10.5, "name": "A"}, {"price": 12, "name": "B"}], name="products") assert schema.name == "products" and {f.name for f in schema.fields} >= {"price", "name"} + + +# --------------------------------------------------------------------------- # +# where a value came from +# --------------------------------------------------------------------------- # +def test_a_record_with_provenance_says_what_the_pipeline_did_to_it() -> None: + evidence = {"cost": {"method": "selector", "source": "selector:.price", "confidence": 0.9}} + record = { + "cost": "$1,299", + "title": "Laptop X ", + "note": "old", + "_provenance": {"url": "https://s.example/1", "fields": evidence}, + } + pipeline = Pipeline( + [ + Rename({"cost": "price", "title": "name"}, name="names"), + Transform("price", ["strip", {"regex": "[0-9.,]+"}, "number"]), + Transform("name", ["strip"]), + Exclude(["note"]), + ] + ) + out = pipeline(record) + assert (out["price"], out["name"]) == (1299, "Laptop X") and "note" not in out + fields = out["_provenance"]["fields"] + assert "cost" not in fields and fields["price"]["method"] == "selector" # the evidence follows the new name + assert fields["price"]["transforms"] == [ + {"stage": "names", "kind": "rename", "from": "cost"}, + {"stage": "transform", "kind": "transform", "before": "$1,299"}, + ] + assert fields["name"]["transforms"] == [ + {"stage": "names", "kind": "rename", "from": "title"}, + {"stage": "transform#2", "kind": "transform", "before": "Laptop X "}, # (a second stage of a kind is numbered) + ] + assert fields["note"]["transforms"] == [{"stage": "exclude", "kind": "exclude", "dropped": True}] + assert pipeline({"cost": "$5", "title": "Y", "note": "n"}) == {"price": 5, "name": "Y"} # no provenance: as before + + from wintergrab.data import describe_provenance + + assert describe_provenance(out, ["price"]).splitlines() == [ + "from: https://s.example/1", + "price: 1299", + " read by selector from selector:.price, confidence 0.90", + " rename (names): was named cost", + " transform: was '$1,299'", + ] + assert describe_provenance({"price": 5}).startswith("no provenance") + + +def test_data_trace_says_where_a_value_came_from(site, tmp_path, capsys) -> None: + schema = tmp_path / "product.json" + fields = {"name": {"type": "string", "required": True}, "price": "money", "currency": "currency"} + schema.write_text(json.dumps({"name": "product", "fields": fields}), encoding="utf-8") + out, workspace = tmp_path / "items.jsonl", tmp_path / "ws" + assert main(["-q", "crawl", site.url + "/product/1", "--extract", str(schema), "--provenance", "--max-pages", + "1", "--workspace", str(workspace), "-o", str(out), "--no-progress"]) == 0 # fmt: skip + row = json.loads(out.read_text(encoding="utf-8").splitlines()[0]) + assert row["_provenance"]["run"] == "run-1" and row["_provenance"]["output"] == str(out) # (the crawl's stamp) + capsys.readouterr() + assert main(["data", "trace", str(out), "price", "--where", f"url={site.url}/product/1"]) == 0 + told = capsys.readouterr().out.splitlines() + fetched = row["_provenance"]["fetched_at"] + assert told[0] == f"from: {site.url}/product/1, fetched {fetched}, extractor product@1, run run-1, output {out}" + assert told[1] == "price: 6.25" and told[2].startswith(" read by ") and "confidence 0." in told[2] + assert main(["data", "trace", str(out), "--where", "url=nope"]) == 1 + assert "no record matches" in capsys.readouterr().err + plain = tmp_path / "plain.jsonl" + plain.write_text('{"url": "u", "price": 5}\n', encoding="utf-8") + assert main(["data", "trace", str(plain)]) == 1 # (nothing to trace: collected without provenance) + assert capsys.readouterr().out.startswith("no provenance") + assert main(["data", "trace", str(out), "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["_provenance"]["run"] == "run-1" From 8b46a8ca03f549d1a80299206913b2fc6c43ef04 Mon Sep 17 00:00:00 2001 From: Claude <noreply@anthropic.com> Date: Sat, 26 Sep 2026 20:21:41 +0000 Subject: [PATCH 06/30] RDFa: the structured-data format the brief named that was missing structured_data()["rdfa"] reads RDFa (Lite, and the value attributes of RDFa 1.1: vocab, prefix, typeof, property, about, resource, content) into nested dicts of microdata's shape: an element with typeof is an item, its property descendants are its properties, and a property element with a typeof of its own is a nested item. Terms are expanded with the vocabulary in effect and the prefixes declared (the common ones are known); a schema.org property keeps its short name whatever prefix wrote it, one of another vocabulary its CURIE (dc:creator), as microdata keeps its itemprop. The extractor reads it like JSON-LD and microdata (method rdfa, prior 0.93, sources=["rdfa:Product.offers.price"]), page classification and the history's types count its types (the history's structured digest is unchanged for pages without RDFa, so existing histories see no spurious change), and `wintergrab get --sources` lists it when a page has some. The kinds and their keys are one table (STRUCTURED_KINDS) rather than a tuple repeated in six places. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 10 ++ README.md | 2 +- docs/api.md | 12 +- docs/architecture.md | 2 +- docs/extraction.md | 2 +- docs/history.md | 2 +- docs/intelligence.md | 2 +- docs/power-features.md | 1 + docs/sources.md | 4 +- src/wintergrab/extraction/engine.py | 5 +- src/wintergrab/extraction/explain.py | 2 +- src/wintergrab/extraction/page.py | 17 ++- src/wintergrab/extraction/strategies.py | 11 +- src/wintergrab/fetchers/response.py | 2 +- src/wintergrab/history/snapshot.py | 12 +- src/wintergrab/intel/classify.py | 4 +- src/wintergrab/intel/sources.py | 7 ++ src/wintergrab/parser/selector.py | 2 +- src/wintergrab/parser/structured.py | 151 +++++++++++++++++++++++- tests/test_extraction.py | 11 ++ tests/test_structured.py | 70 ++++++++++- 21 files changed, 289 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2ce817..c063414 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -153,6 +153,16 @@ an error that says so, not a setting silently ignored found, and how it was changed. ### Extraction engine (`wintergrab.extraction`) +- **RDFa.** `structured_data()["rdfa"]` reads RDFa (Lite, and the value + attributes of RDFa 1.1: `vocab`, `prefix`, `typeof`, `property`, `about`, + `resource`, `content`) into nested dicts of microdata's shape, terms + expanded with the vocabulary in effect and the prefixes declared (the + common ones are known); a property of another vocabulary keeps its CURIE + (`dc:creator`). The extractor reads it like JSON-LD and microdata + (method `rdfa`, `sources=["rdfa:Product.offers.price"]`), page + classification and the history's `types` count its types, and + `wintergrab get --sources` lists it. + - `Extractor(schema).extract(page)` finds every field of a data schema without selectors: schema.org JSON-LD and microdata, OpenGraph/Twitter/meta tags, your selectors, embedded app state, labelled values ("Weight: 1.2 kg", diff --git a/README.md b/README.md index e40cc11..90b4592 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ pages), the same API scales up. sessions (HTTP + browser, several accounts…), proxy rotation with health checks, **AutoThrottle** that backs off when a site pushes back, robots.txt support, and **pause/resume** (Ctrl+C, then run again). -- **Scrape without selectors.** Pull JSON-LD/microdata/OpenGraph, the JSON +- **Scrape without selectors.** Pull JSON-LD/microdata/RDFa/OpenGraph, the JSON state that React/Next/Vue apps embed in their HTML, and every table. `auto_extract()` finds a page's product grid or result list and names the fields. `learn({"title": "…", "price": "…"})` writes the selectors for you diff --git a/docs/api.md b/docs/api.md index e9b2b12..73bfa2c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -173,7 +173,7 @@ checks that it is up to date. - `re_first(self, pattern: str | Pattern[str], default: str | None = None, flags: int = 0) -> str | None` - `save(self, path: str | Path) -> Path`: Write the raw body to a file and return its path. - `select(self, query: str, **kwargs: Any) -> SelectorList`: CSS or XPath, guessed from the query. - - `structured_data(self) -> dict[str, Any]`: JSON-LD, microdata, OpenGraph, Twitter cards and meta tags of the page. + - `structured_data(self) -> dict[str, Any]`: JSON-LD, microdata, RDFa, OpenGraph, Twitter cards and meta tags of the page. - `tables(self) -> list[dict[str, Any]]`: Every HTML table as records. - `urljoin(self, url: str) -> str`: Resolve a relative URL against this page. - `xpath(self, query: str, **kwargs: Any) -> SelectorList`: XPath query on the page. @@ -209,7 +209,7 @@ checks that it is up to date. - `re_first(self, pattern: str | Pattern[str], default: str | None = None, flags: int = 0) -> str | None` - `remove_namespaces(self)`: Strip XML namespaces so ``//loc`` works on sitemaps and feeds. - `select(self, query: str, **kwargs: Any) -> SelectorList`: CSS or XPath, guessed from the query (XPath starts with ``/``, ``./`` or ``(``). - - `structured_data(self) -> dict[str, Any]`: Machine-readable data the page publishes: JSON-LD, microdata, OpenGraph, Twitter cards, meta tags. + - `structured_data(self) -> dict[str, Any]`: Machine-readable data the page publishes: JSON-LD, microdata, RDFa, OpenGraph, Twitter cards, meta tags. - `tables(self) -> list[dict[str, Any]]`: Every ``<table>`` as ``{"headers", "rows": [{header: value}], "caption"}`` (colspan/rowspan handled). - `urljoin(self, url: str) -> str`: Resolve a (possibly relative) URL against the page URL / ``<base>``. - `xpath(self, query: str, *, namespaces: Mapping[str, str] | None = None, adaptive: bool = False, auto_save: bool = False, identifier: str | None = None, min_score: float = 0.55, **variables: Any) -> SelectorList`: Select with XPath 1.0. @@ -414,7 +414,7 @@ checks that it is up to date. - **`ModelRequest(fields: list[ModelField], text: str, url: str | None = None, schema_name: str = 'record', known: dict[str, Any] = ..., images: list[Image] = ...)`** (class). What an extraction model gets: the fields wanted and the page's content (Markdown), and a screenshot of the page when the extractor was asked to send one (``Extractor(vision=True)``). - `prompt(self) -> str`: A ready-made instruction for chat models (use it or build your own from the attributes). - **`PageContext(source: Any, *, url: str | None = None, fetched_at: float | None = None, scope: Selector | None = None, parent: PageContext | None = None)`** (class). A page (a :class:`~wintergrab.Response`, a :class:`~wintergrab.Selector` or HTML) ready for extraction. - - `iter_nodes(self, kind: str) -> Iterator[tuple[str, dict[str, Any]]]`: ``(path, node)`` for every typed object in the page's JSON-LD (``kind="json-ld"``) or microdata. + - `iter_nodes(self, kind: str) -> Iterator[tuple[str, dict[str, Any]]]`: ``(path, node)`` for every typed object in the page's JSON-LD (``kind="json-ld"``), microdata or RDFa. - `nodes(self, kind: str) -> list[tuple[str, dict[str, Any]]]`: :meth:`iter_nodes` as a list, computed once per page. - `scoped(self, element: Selector) -> PageContext`: The same page, looking only at ``element`` (one record of a listing). - **`Patterns()`** (class). Regular expressions over the visible text, for values with a recognisable shape. @@ -435,7 +435,7 @@ checks that it is up to date. - `candidates(self, page: PageContext, f: SchemaField, schema: Schema) -> list[Candidate]` - **`Strategy()`** (class). Base class: ``candidates(page, field, schema)`` returns what this strategy finds for one field. - `candidates(self, page: PageContext, f: SchemaField, schema: Schema) -> list[Candidate]` -- **`StructuredData(node: tuple[str, dict[str, Any]] | None = None, kind: str | None = None)`** (class). JSON-LD and microdata (schema.org): what the site publishes for machines. +- **`StructuredData(node: tuple[str, dict[str, Any]] | None = None, kind: str | None = None)`** (class). JSON-LD, microdata and RDFa (schema.org): what the site publishes for machines. - `candidates(self, page: PageContext, f: SchemaField, schema: Schema) -> list[Candidate]` - **`VisualLayout()`** (class). Values beside, under or over a label named like the field, where the page draws them (a page fetched in a browser with ``layout=True``; see the module docs). - `candidates(self, page: PageContext, f: SchemaField, schema: Schema) -> list[Candidate]` @@ -448,7 +448,7 @@ checks that it is up to date. - **`layout_pairs(layout: Layout) -> list[LabelledPair]`**. The labelled values drawn on the page (see the module docs). - **`layout_tables(layout: Layout, *, min_rows: int = 3, min_columns: int = 2) -> list[VisualTable]`**. The tables drawn on the page (see the module docs), in the order they are drawn. - **`register_strategy(strategy: type[Strategy], *, before: str | None = None)`**. Add a strategy extractors use by default: last, or before the one whose ``method`` is ``before``. -- **`schema_types(node: Any) -> list[str]`**. The schema.org type names of a JSON-LD/microdata node (``"https://schema.org/Product"`` -> ``"Product"``). +- **`schema_types(node: Any) -> list[str]`**. The schema.org type names of a JSON-LD, microdata or RDFa node (``"https://schema.org/Product"`` -> ``"Product"``). - **`value_key(value: Any) -> Any`**. What two values must share to count as the same (``$299.99`` = ``299.99``; case and spacing ignored). ## `wintergrab.extraction.templates`: Ready-made schemas @@ -839,7 +839,7 @@ checks that it is up to date. - `describe(self, fields: int = 6) -> str` - `schema(self, name: str = 'records') -> Schema`: A data schema for these records (:func:`~wintergrab.data.inference.infer_schema`): a start to review. - `to_dict(self) -> dict[str, Any]` -- **`DataSources(url: str, html: list[HtmlRecords] = ..., tables: list[dict[str, Any]] = ..., json_ld: dict[str, int] = ..., microdata: dict[str, int] = ..., meta: list[str] = ..., embedded: dict[str, list[Collection]] = ..., api: list[ApiCall] = ..., recorded: bool = False, endpoints: list[tuple[str | None, str]] = ..., document: list[Collection] | None = None, pagination: Pagination | None = None, _json_ld_collections: list[tuple[str, Collection]] = ...)`** (class). Where a page's data is (see the module docs). +- **`DataSources(url: str, html: list[HtmlRecords] = ..., tables: list[dict[str, Any]] = ..., json_ld: dict[str, int] = ..., microdata: dict[str, int] = ..., rdfa: dict[str, int] = ..., meta: list[str] = ..., embedded: dict[str, list[Collection]] = ..., api: list[ApiCall] = ..., recorded: bool = False, endpoints: list[tuple[str | None, str]] = ..., document: list[Collection] | None = None, pagination: Pagination | None = None, _json_ld_collections: list[tuple[str, Collection]] = ...)`** (class). Where a page's data is (see the module docs). - `describe(self) -> str` - `richest(self) -> Source | None`: The place holding the most values (records x fields): often the one to read. - `sources(self) -> list[Source]`: Every place holding at least two records, those holding the most values (records x fields) first. diff --git a/docs/architecture.md b/docs/architecture.md index 74f9556..631305f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,7 +9,7 @@ where something happens. Users start with [getting started](getting-started.md). fingerprint. A browser renders only the pages that need one. A page that is blocked or behind a bot check is never a reason to switch to a browser. - **Evidence before guesses.** Extraction reads what a page publishes for - machines first (JSON-LD, microdata, meta tags), then your selectors, then + machines first (JSON-LD, microdata, RDFa, meta tags), then your selectors, then labels, layout and text patterns. Every value keeps where it came from and how sure wintergrab is. A language model, when you name one, only fills gaps, and its answers are checked against the page. diff --git a/docs/extraction.md b/docs/extraction.md index 01d3f1a..dbb794e 100644 --- a/docs/extraction.md +++ b/docs/extraction.md @@ -91,7 +91,7 @@ reliable ones come first: | Method | Looks at | |---|---| -| `json-ld`, `microdata` | schema.org data (`Product.offers.price`, `aggregateRating.ratingValue`...), picked by the schema's name (`product` -> `Product`) | +| `json-ld`, `microdata`, `rdfa` | schema.org data (`Product.offers.price`, `aggregateRating.ratingValue`...), picked by the schema's name (`product` -> `Product`) | | `opengraph`, `twitter`, `meta` | `og:title`, `product:price:amount`, `<meta name=description>`, the `<title>` without the site name | | `selector` | the field's own `selectors` (CSS or XPath, `::text`, `::attr()`) | | `embedded-json` | state embedded by JavaScript apps (`__NEXT_DATA__`, `window.__STATE__`), by key name | diff --git a/docs/history.md b/docs/history.md index 915b24d..e032ef8 100644 --- a/docs/history.md +++ b/docs/history.md @@ -36,7 +36,7 @@ A `PageSnapshot` keeps hashes rather than the page, about 1.3 KB per page: | `text` | The visible text changed (spacing aside). `text_similarity` tells by how much. | | `title`, `description` | The title or the meta description changed (before and after are kept). | | `meta` | Other `<meta>` tags changed. Tokens, nonces and timestamps are left out. | -| `structured`, `types` | The JSON-LD or microdata changed, or its schema.org types did (`schema`). | +| `structured`, `types` | The JSON-LD, microdata or RDFa changed, or its schema.org types did (`schema`). | | `price`, `currency`, `availability` | The product's price or availability changed, read from schema.org or OpenGraph data, with no extraction schema needed. | | `layout` | The tag structure changed: a SimHash of the page's tag paths moved by 8 bits or more (`layout_similarity`). | | `images` | The image URLs changed. Cache-busting queries (`?v=2`) are ignored. | diff --git a/docs/intelligence.md b/docs/intelligence.md index bab7765..4c71dd8 100644 --- a/docs/intelligence.md +++ b/docs/intelligence.md @@ -31,7 +31,7 @@ It adds up evidence. Each rule that applies gives its type a weight: | Evidence | Examples | Weight | |---|---|---:| -| schema.org types in JSON-LD or microdata | `Product`, `NewsArticle`, `JobPosting`, `SearchResultsPage` (`Organization`: 1) | 1-6 | +| schema.org types in JSON-LD, microdata or RDFa | `Product`, `NewsArticle`, `JobPosting`, `SearchResultsPage` (`Organization`: 1) | 1-6 | | `og:type` | `product`, `article`, `profile` | 3 | | a 404 status, "Page not found" in the title | | 8, 5 | | a password field in a small form | | 6 | diff --git a/docs/power-features.md b/docs/power-features.md index e33502c..364196c 100644 --- a/docs/power-features.md +++ b/docs/power-features.md @@ -31,6 +31,7 @@ page = wg.get("https://shop.example/product/42") data = page.structured_data() data["json_ld"] # [{"@type": "Product", "name": ..., "offers": {"price": ...}}] data["microdata"] # itemscope/itemprop trees as dicts +data["rdfa"] # vocab/typeof/property trees, the same shape data["opengraph"] # {"title": ..., "image": [...], "product:price:amount": ...} data["twitter"], data["meta"] # twitter cards; title, description, canonical, language, feeds... ``` diff --git a/docs/sources.md b/docs/sources.md index e7436d2..2286d6c 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -37,8 +37,8 @@ https://shop.example/catalog - **HTML**: repeated elements with the same fields (cards, rows), as [`auto_extract`](power-features.md) finds them. -- **tables**, **JSON-LD**, **microdata**, **meta**: what `--tables` and - `--structured` read, counted. +- **tables**, **JSON-LD**, **microdata**, **RDFa**, **meta**: what `--tables` + and `--structured` read, counted (RDFa is listed when the page has some). - **embedded JSON**: the lists of records in each blob `--json-data` reads. - **API calls**: with `--browser`, the calls the page made as it rendered, and the lists of records in their answers. Calls that differ only in diff --git a/src/wintergrab/extraction/engine.py b/src/wintergrab/extraction/engine.py index a2f551f..4f48619 100644 --- a/src/wintergrab/extraction/engine.py +++ b/src/wintergrab/extraction/engine.py @@ -45,7 +45,7 @@ from ..errors import ConfigurationError from ..parser import Selector from .model import Image, ModelField, ModelRequest, call_model, grounding, model_name, parse_answer -from .page import PageContext, schema_types +from .page import STRUCTURED_KINDS, PageContext, schema_types from .schemaorg import target_types from .strategies import STRATEGIES, Candidate, DomHeuristics, RecordFields, Strategy, StructuredData, field_kind @@ -60,6 +60,7 @@ DEFAULT_PRIORS: dict[str, float] = { "json-ld": 0.95, # published by the site for machines (search engines check it) "microdata": 0.93, + "rdfa": 0.93, "selector": 0.92, # written by you for this site "opengraph": 0.88, # published for link previews: titles and images are reliable, prices less often present "twitter": 0.85, @@ -620,7 +621,7 @@ def extract_all( container = container or self.schema.container wanted = target_types(self.schema.name) if wanted and container is None: - for kind in ("json-ld", "microdata"): + for kind in STRUCTURED_KINDS: nodes = [(p, n) for p, n in ctx.nodes(kind) if wanted & set(schema_types(n))] if len(nodes) >= min_records: records = [] diff --git a/src/wintergrab/extraction/explain.py b/src/wintergrab/extraction/explain.py index 929a852..b553669 100644 --- a/src/wintergrab/extraction/explain.py +++ b/src/wintergrab/extraction/explain.py @@ -42,7 +42,7 @@ __all__ = ["FieldDiagnosis", "diagnose"] _SURE_TYPE = 0.5 # page classifications at least this sure are named as causes -_NAMES = {"json-ld": "JSON-LD", "microdata": "microdata", "opengraph": "OpenGraph", "meta": "meta tags", +_NAMES = {"json-ld": "JSON-LD", "microdata": "microdata", "rdfa": "RDFa", "opengraph": "OpenGraph", "meta": "meta tags", "twitter": "Twitter cards", "selector": "the field's selectors", "embedded-json": "embedded JSON", "label": "labels", "records": "the page's records", "dom": "the page's layout", "pattern": "text patterns", "model": "the model"} # fmt: skip diff --git a/src/wintergrab/extraction/page.py b/src/wintergrab/extraction/page.py index 92b3638..5c073e1 100644 --- a/src/wintergrab/extraction/page.py +++ b/src/wintergrab/extraction/page.py @@ -17,8 +17,14 @@ _MAX_MODEL_TEXT = 24_000 # characters of page text handed to an extraction model +#: The kinds of structured data that hold typed objects, and their key in :func:`~wintergrab.parser.structured_data`. +STRUCTURED_KEYS = {"json-ld": "json_ld", "microdata": "microdata", "rdfa": "rdfa"} +STRUCTURED_KINDS = tuple(STRUCTURED_KEYS) + + def schema_types(node: Any) -> list[str]: - """The schema.org type names of a JSON-LD/microdata node (``"https://schema.org/Product"`` -> ``"Product"``).""" + """The schema.org type names of a JSON-LD, microdata or RDFa node (``"https://schema.org/Product"`` -> + ``"Product"``).""" if not isinstance(node, dict): return [] declared = node.get("@type") @@ -94,13 +100,14 @@ def layout(self) -> Layout | None: # -- structured data ---------------------------------------------------- # @cached_property def structured(self) -> dict[str, Any]: - """JSON-LD, microdata, OpenGraph, Twitter and meta tags (see :func:`~wintergrab.parser.structured_data`).""" + """JSON-LD, microdata, RDFa, OpenGraph, Twitter and meta tags (see + :func:`~wintergrab.parser.structured_data`).""" if self.parent is not None: return self.parent.structured try: return self.selector.structured_data() except Exception: # pragma: no cover - defensive: broken markup must not stop extraction - return {"json_ld": [], "microdata": [], "opengraph": {}, "twitter": {}, "meta": {}} + return {"json_ld": [], "microdata": [], "rdfa": [], "opengraph": {}, "twitter": {}, "meta": {}} def nodes(self, kind: str) -> list[tuple[str, dict[str, Any]]]: """:meth:`iter_nodes` as a list, computed once per page.""" @@ -110,12 +117,12 @@ def nodes(self, kind: str) -> list[tuple[str, dict[str, Any]]]: return list(cache[kind]) def iter_nodes(self, kind: str) -> Iterator[tuple[str, dict[str, Any]]]: - """``(path, node)`` for every typed object in the page's JSON-LD (``kind="json-ld"``) or microdata. + """``(path, node)`` for every typed object in the page's JSON-LD (``kind="json-ld"``), microdata or RDFa. Nested objects that carry their own ``@type`` (a ``WebPage``'s ``mainEntity``, an ``ItemList``'s items) are included with their path. """ - roots = self.structured.get("json_ld" if kind == "json-ld" else "microdata") or [] + roots = self.structured.get(STRUCTURED_KEYS.get(kind, kind)) or [] stack: list[tuple[str, Any]] = [(f"[{i}]", node) for i, node in reversed(list(enumerate(roots)))] while stack: path, node = stack.pop() diff --git a/src/wintergrab/extraction/strategies.py b/src/wintergrab/extraction/strategies.py index 1b33a5d..f7aae2e 100644 --- a/src/wintergrab/extraction/strategies.py +++ b/src/wintergrab/extraction/strategies.py @@ -26,7 +26,7 @@ from ..data.normalize import coordinates_in_url, iter_numbers, normalize_availability, normalize_phone, parse_rating from ..parser import Selector from ..parser.text import tag_name, text_content -from .page import PageContext, schema_types +from .page import STRUCTURED_KINDS, PageContext, schema_types from .schemaorg import FIELD_PATHS, META_KEYS, camel, candidate_names, field_key, read_path, target_types if TYPE_CHECKING: @@ -220,17 +220,18 @@ def _paths_for(name: str, aliases: tuple[str, ...]) -> tuple[str, ...]: class StructuredData(Strategy): - """JSON-LD and microdata (schema.org): what the site publishes for machines. + """JSON-LD, microdata and RDFa (schema.org): what the site publishes for machines. Records come from the objects whose ``@type`` fits the schema's name (``product`` -> ``Product``...). A field is read from the paths schema.org uses for it (``price`` -> ``offers.price``...), from its own name in - camelCase, or from explicit ``sources`` such as ``"jsonld:Product.offers.price"``. + camelCase, or from explicit ``sources`` such as ``"jsonld:Product.offers.price"`` + (``microdata:``, ``rdfa:`` likewise). """ method = "json-ld" page_level = True - kinds = ("json-ld", "microdata") + kinds = STRUCTURED_KINDS def __init__(self, node: tuple[str, dict[str, Any]] | None = None, kind: str | None = None) -> None: #: Restrict to one object (a record of a listing made of several JSON-LD objects). @@ -240,7 +241,7 @@ def __init__(self, node: tuple[str, dict[str, Any]] | None = None, kind: str | N @staticmethod def _explicit(f: SchemaField, kind: str) -> list[tuple[str | None, str]]: """``(type or None, path)`` from ``sources`` like ``jsonld:Product.offers.price`` / ``microdata:offers.price``.""" - prefixes = ("jsonld:", "json-ld:") if kind == "json-ld" else ("microdata:",) + prefixes = ("jsonld:", "json-ld:") if kind == "json-ld" else (f"{kind}:",) out: list[tuple[str | None, str]] = [] for source in f.sources: for prefix in prefixes: diff --git a/src/wintergrab/fetchers/response.py b/src/wintergrab/fetchers/response.py index 3535a1c..089117d 100644 --- a/src/wintergrab/fetchers/response.py +++ b/src/wintergrab/fetchers/response.py @@ -359,7 +359,7 @@ def markdown(self, *, main_content: bool = False) -> str: # zero-selector extraction (see Selector) # ------------------------------------------------------------------ # def structured_data(self) -> dict[str, Any]: - """JSON-LD, microdata, OpenGraph, Twitter cards and meta tags of the page.""" + """JSON-LD, microdata, RDFa, OpenGraph, Twitter cards and meta tags of the page.""" return self.selector.structured_data() def embedded_json(self) -> dict[str, Any]: diff --git a/src/wintergrab/history/snapshot.py b/src/wintergrab/history/snapshot.py index a89dde4..7b6f052 100644 --- a/src/wintergrab/history/snapshot.py +++ b/src/wintergrab/history/snapshot.py @@ -20,7 +20,7 @@ from ..data.normalize import normalize_availability, parse_money from ..data.similarity import hamming, simhash, simhash_similarity -from ..extraction.page import PageContext, schema_types +from ..extraction.page import STRUCTURED_KEYS, STRUCTURED_KINDS, PageContext, schema_types from ..extraction.schemaorg import read_path from ..parser.text import tag_name @@ -151,7 +151,7 @@ def _items_digest(items: Iterable[Any]) -> tuple[str | None, int]: def _offer(ctx: PageContext) -> tuple[float | None, str | None, str | None]: """The price, currency and availability in the page's product data.""" price = currency = availability = None - nodes = [node for kind in ("json-ld", "microdata") for _, node in ctx.nodes(kind)] + nodes = [node for kind in STRUCTURED_KINDS for _, node in ctx.nodes(kind)] products = [n for n in nodes if {"Product", "ProductGroup", "IndividualProduct", "Offer"} & set(schema_types(n))] for node in products: for path in _PRICE_PATHS: @@ -217,12 +217,10 @@ def snapshot_page( if response is not None: snap.etag = response.headers.get("etag") snap.last_modified = response.headers.get("last-modified") - structured = [ctx.structured.get("json_ld") or [], ctx.structured.get("microdata") or []] + structured = [ctx.structured.get(key) or [] for key in STRUCTURED_KEYS.values()] if any(structured): - snap.structured = _digest(structured) - snap.types = sorted( - {t for kind in ("json-ld", "microdata") for _, n in ctx.nodes(kind) for t in schema_types(n)} - ) + snap.structured = _digest(structured[:2] if not structured[2] else structured) # (as before, without RDFa) + snap.types = sorted({t for kind in STRUCTURED_KINDS for _, n in ctx.nodes(kind) for t in schema_types(n)}) snap.price, snap.currency, snap.availability = _offer(ctx) if items is not None: snap.items, snap.item_count = _items_digest(items) diff --git a/src/wintergrab/intel/classify.py b/src/wintergrab/intel/classify.py index 5fcfc48..4d1cb44 100644 --- a/src/wintergrab/intel/classify.py +++ b/src/wintergrab/intel/classify.py @@ -30,7 +30,7 @@ from typing import Any from urllib.parse import parse_qs, urlsplit -from ..extraction.page import PageContext, schema_types +from ..extraction.page import STRUCTURED_KINDS, PageContext, schema_types from ..parser.text import tag_name __all__ = ["PAGE_TYPES", "PageClassifier", "PageFeatures", "PageType", "classify_page", "classify_url"] @@ -198,7 +198,7 @@ def __init__(self, page: PageContext, status: int | None = None) -> None: @cached_property def schema_types(self) -> list[str]: types = [] - for kind in ("json-ld", "microdata"): + for kind in STRUCTURED_KINDS: for _path, node in self.page.nodes(kind): types.extend(schema_types(node)) return types diff --git a/src/wintergrab/intel/sources.py b/src/wintergrab/intel/sources.py index 3b25e8e..1a66265 100644 --- a/src/wintergrab/intel/sources.py +++ b/src/wintergrab/intel/sources.py @@ -679,6 +679,7 @@ class DataSources: tables: Its ``<table>`` elements: rows and column names. json_ld: JSON-LD objects by ``@type``, and how many. microdata: Microdata items by type, and how many. + rdfa: RDFa items by type, and how many. meta: The OpenGraph, Twitter and meta fields it declares. embedded: The JSON an app embeds, by name, with the records in each. api: The API calls it made as it rendered (a browser fetch with ``capture=True``). @@ -693,6 +694,7 @@ class DataSources: tables: list[dict[str, Any]] = field(default_factory=list) json_ld: dict[str, int] = field(default_factory=dict) microdata: dict[str, int] = field(default_factory=dict) + rdfa: dict[str, int] = field(default_factory=dict) meta: list[str] = field(default_factory=list) embedded: dict[str, list[Collection]] = field(default_factory=dict) api: list[ApiCall] = field(default_factory=list) @@ -745,6 +747,8 @@ def describe(self) -> str: lines.append(_line("tables", "; ".join(tables[:3]) or "none")) lines.append(_line("JSON-LD", _counts(self.json_ld))) lines.append(_line("microdata", _counts(self.microdata))) + if self.rdfa: + lines.append(_line("RDFa", _counts(self.rdfa))) lines.append(_line("meta", _names(self.meta))) embedded = [ f"{name}: " + (collections[0].describe() if collections else "no list of records") @@ -793,6 +797,7 @@ def to_dict(self) -> dict[str, Any]: "tables": self.tables, "json_ld": self.json_ld, "microdata": self.microdata, + "rdfa": self.rdfa, "meta": self.meta, "embedded": {name: [c.to_dict() for c in cs] for name, cs in self.embedded.items()}, "api": [call.to_dict() for call in self.api] if self.recorded else None, @@ -860,6 +865,7 @@ def data_sources(response: Any, *, recorded: bool | None = None) -> DataSources: if len(items) >= 2: json_ld_collections.append((kind, _collection("[]", items))) microdata = Counter(_type_name((item.get("@type") or "").split()) for item in structured.get("microdata") or []) + rdfa = Counter(_type_name(item.get("@type")) for item in structured.get("rdfa") or []) # (several: a list) meta = [f"og:{k}" for k in structured.get("opengraph") or {}] meta += [f"twitter:{k}" for k in structured.get("twitter") or {}] meta += [str(k) for k in structured.get("meta") or {}] @@ -879,6 +885,7 @@ def data_sources(response: Any, *, recorded: bool | None = None) -> DataSources: tables=tables, json_ld=dict(json_ld), microdata=dict(microdata), + rdfa=dict(rdfa), meta=meta, embedded=embedded, api=calls, diff --git a/src/wintergrab/parser/selector.py b/src/wintergrab/parser/selector.py index 51cf3ad..cbdfd9b 100644 --- a/src/wintergrab/parser/selector.py +++ b/src/wintergrab/parser/selector.py @@ -674,7 +674,7 @@ def extract_all(self, query: str, schema: Mapping[str, Any], **select_kwargs: An # zero-selector extraction # ------------------------------------------------------------------ # def structured_data(self) -> dict[str, Any]: - """Machine-readable data the page publishes: JSON-LD, microdata, OpenGraph, Twitter cards, meta tags. + """Machine-readable data the page publishes: JSON-LD, microdata, RDFa, OpenGraph, Twitter cards, meta tags. Product pages, articles, recipes and events very often carry clean structured data - no selectors needed. diff --git a/src/wintergrab/parser/structured.py b/src/wintergrab/parser/structured.py index 08fbd4f..557699b 100644 --- a/src/wintergrab/parser/structured.py +++ b/src/wintergrab/parser/structured.py @@ -1,4 +1,4 @@ -"""Machine-readable page data: JSON-LD, microdata, meta tags, SPA state, tables and pagination.""" +"""Machine-readable page data: JSON-LD, microdata, RDFa, meta tags, SPA state, tables and pagination.""" from __future__ import annotations @@ -181,13 +181,15 @@ def _load_json(raw: str) -> Any: def structured_data(root: etree._Element, base_url: str | None = None) -> dict[str, Any]: """Everything a page declares about itself in machine-readable form. - Returns ``{"json_ld": [...], "microdata": [...], "opengraph": {...}, "twitter": {...}, "meta": {...}}``. - ``base_url`` makes link-like values absolute (pass the page URL, or the ``<base href>`` one). + Returns ``{"json_ld": [...], "microdata": [...], "rdfa": [...], "opengraph": {...}, "twitter": {...}, + "meta": {...}}``. Microdata and RDFa items are nested dicts of the same shape (``@type``, ``@id``, the + properties). ``base_url`` makes link-like values absolute (pass the page URL, or the ``<base href>`` one). """ opengraph, twitter, meta = _meta_tags(root, base_url) return { "json_ld": _json_ld(root), "microdata": _microdata(root, base_url), + "rdfa": _rdfa(root, base_url), "opengraph": opengraph, "twitter": twitter, "meta": meta, @@ -283,6 +285,149 @@ def _microdata_value(el: etree._Element, base_url: str | None) -> str: return text_content(el) +# RDFa (Lite, and the value attributes of RDFa 1.1): ``vocab``, ``prefix``, ``typeof``, ``property``, ``about``, +# ``resource``, ``content``, ``datatype``. Items are read like microdata's: an element with ``typeof`` is an item, +# its ``property`` descendants are its properties, and a ``property`` element with ``typeof`` of its own is a nested +# item. Terms are expanded with the vocabulary in effect and the prefixes declared (the common ones are known); +# a property of the vocabulary keeps its short name (``name``), one of another vocabulary its CURIE +# (``dc:creator``), as microdata keeps its ``itemprop``. +_RDFA_PREFIXES = { + "schema": "https://schema.org/", + "og": "http://ogp.me/ns#", + "article": "http://ogp.me/ns/article#", + "book": "http://ogp.me/ns/book#", + "profile": "http://ogp.me/ns/profile#", + "product": "http://ogp.me/ns/product#", + "fb": "http://ogp.me/ns/fb#", + "dc": "http://purl.org/dc/terms/", + "dcterms": "http://purl.org/dc/terms/", + "dc11": "http://purl.org/dc/elements/1.1/", + "foaf": "http://xmlns.com/foaf/0.1/", + "gr": "http://purl.org/goodrelations/v1#", + "v": "http://rdf.data-vocabulary.org/#", + "vcard": "http://www.w3.org/2006/vcard/ns#", + "sioc": "http://rdfs.org/sioc/ns#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "cc": "http://creativecommons.org/ns#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "owl": "http://www.w3.org/2002/07/owl#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "prov": "http://www.w3.org/ns/prov#", + "dcat": "http://www.w3.org/ns/dcat#", +} +_RDFA_URL_ATTRS = {**_MICRODATA_URL_ATTRS} # (the same elements carry a resource in an attribute) +_SCHEMA_ORG = ("https://schema.org/", "http://schema.org/") + + +class _RdfaEnv: + """The vocabulary and prefixes in effect on an element (``vocab`` and ``prefix`` are inherited).""" + + __slots__ = ("prefixes", "vocab") + + def __init__(self, vocab: str | None, prefixes: dict[str, str]) -> None: + self.vocab = vocab + self.prefixes = prefixes + + def under(self, el: etree._Element) -> _RdfaEnv: + """The environment inside ``el``: its own ``vocab`` and ``prefix`` declarations over the inherited ones.""" + vocab, prefixes = self.vocab, self.prefixes + declared = el.get("vocab") + if declared is not None: + vocab = declared.strip() or None # (vocab="" ends the inherited vocabulary) + mapping = el.get("prefix") + if mapping: + tokens = mapping.split() + found = { + tokens[i].rstrip(":"): tokens[i + 1] + for i in range(0, len(tokens) - 1, 2) + if tokens[i].endswith(":") and tokens[i + 1] + } + if found: + prefixes = {**prefixes, **found} + return self if vocab == self.vocab and prefixes is self.prefixes else _RdfaEnv(vocab, prefixes) + + def expand(self, term: str) -> str: + """A term or CURIE as a full IRI (``Product`` -> ``https://schema.org/Product`` under that vocabulary; + ``schema:name`` -> ``https://schema.org/name``); an IRI, or a term without a vocabulary, as it is.""" + if "://" in term or term.startswith(("#", "/", "_:")): + return term + prefix, colon, local = term.partition(":") + if colon and prefix in self.prefixes: + return self.prefixes[prefix] + local + if colon and prefix in ("http", "https", "urn", "mailto", "tel"): + return term + return f"{self.vocab}{term}" if self.vocab and not colon else term + + def property_name(self, term: str) -> str: + """The name a property is kept under: its short name under the vocabulary in effect or schema.org, + else the CURIE or IRI as written.""" + iri = self.expand(term) + for namespace in ((self.vocab,) if self.vocab else ()) + _SCHEMA_ORG: + if namespace and iri.startswith(namespace) and len(iri) > len(namespace): + return iri[len(namespace) :] + return term + + +def _rdfa_env_of(el: etree._Element) -> _RdfaEnv: + """The environment on ``el``, from the declarations of its ancestors and its own.""" + env = _RdfaEnv(None, _RDFA_PREFIXES) + for ancestor in reversed(list(el.iterancestors())): + env = env.under(ancestor) + return env.under(el) + + +def _rdfa(root: etree._Element, base_url: str | None) -> list[dict[str, Any]]: + """Top-level RDFa items (``typeof`` without ``property``) as nested dicts, like microdata's.""" + scopes = root.xpath("descendant-or-self::*[@typeof and not(@property)]") + return [_rdfa_item(scope, _rdfa_env_of(scope), base_url, 0) for scope in scopes] + + +def _rdfa_item(scope: etree._Element, env: _RdfaEnv, base_url: str | None, depth: int) -> dict[str, Any]: + item: dict[str, Any] = {} + types = [env.expand(t) for t in (scope.get("typeof") or "").split()] + if types: + item["@type"] = types[0] if len(types) == 1 else types + subject = scope.get("about") if scope.get("about") is not None else scope.get("resource") + if subject is not None and subject.strip(): + item["@id"] = _absolute(subject.strip(), base_url) + stack = [(c, env) for c in reversed(scope) if isinstance(c.tag, str)] + while stack: + el, inherited = stack.pop() + inside = inherited.under(el) + is_scope = el.get("typeof") is not None + names = (el.get("property") or "").split() + if names: + if is_scope and depth < _MAX_ITEM_DEPTH: + value: Any = _rdfa_item(el, inside, base_url, depth + 1) + else: + value = _rdfa_value(el, base_url) + for name in names: + _add(item, inside.property_name(name), value) + if not is_scope: # a nested item's properties belong to it, not to us + stack.extend((c, inside) for c in reversed(el) if isinstance(c.tag, str)) + return item + + +def _rdfa_value(el: etree._Element, base_url: str | None) -> str: + content = el.get("content") + if content is not None: + return content.strip() + resource = el.get("resource") + if resource is not None and resource.strip(): + return _absolute(resource.strip(), base_url) + name = tag_name(el) + if name in _RDFA_URL_ATTRS: + url = el.get(_RDFA_URL_ATTRS[name]) + if url is not None: + return _absolute(url, base_url) + elif name in _MICRODATA_ATTRS: + value = el.get(_MICRODATA_ATTRS[name]) + if value is not None: + return value.strip() + return text_content(el) + + _OG_PREFIXES = ("og:", "article:", "product:", "book:", "profile:", "music:", "video:", "fb:", "place:") _OG_URL_KEYS = frozenset( { diff --git a/tests/test_extraction.py b/tests/test_extraction.py index 7818293..590b714 100644 --- a/tests/test_extraction.py +++ b/tests/test_extraction.py @@ -109,6 +109,17 @@ def test_provenance_and_to_dict(extractor: Extractor) -> None: assert record["price"] == 299.99 and record.get("nope", 1) == 1 +def test_rdfa() -> None: + page = """<div vocab="https://schema.org/" typeof="Product"><h1 property="name">Phone R</h1> + <div property="offers" typeof="Offer"><span property="price" content="149">$149</span> + <meta property="priceCurrency" content="USD"></div></div>""" + record = Extractor(PRODUCT).extract(page) + assert (record.data["name"], record.data["price"]) == ("Phone R", 149) + assert record.fields["price"].method == "rdfa" and record.fields["price"].source == "rdfa:Product.offers.price" + explicit = {"name": "product", "fields": {"amount": {"type": "number", "sources": ["rdfa:Product.offers.price"]}}} + assert Extractor(explicit).extract(page).data["amount"] == 149 + + def test_microdata_and_opengraph() -> None: microdata = """<div itemscope itemtype="https://schema.org/Product"><h1 itemprop="name">Phone Y</h1> <div itemprop="offers" itemscope itemtype="https://schema.org/Offer"><span itemprop="price" content="199.00">$199</span> diff --git a/tests/test_structured.py b/tests/test_structured.py index 477bf41..a46eccc 100644 --- a/tests/test_structured.py +++ b/tests/test_structured.py @@ -135,7 +135,8 @@ def product_data() -> dict[str, Any]: def test_structured_data_shape(product_data: dict[str, Any]) -> None: - assert list(product_data) == ["json_ld", "microdata", "opengraph", "twitter", "meta"] + assert list(product_data) == ["json_ld", "microdata", "rdfa", "opengraph", "twitter", "meta"] + assert product_data["rdfa"] == [] # (the page has none) def test_json_ld_graph_list_and_lenient_parsing(product_data: dict[str, Any]) -> None: @@ -220,6 +221,71 @@ def test_microdata_value_fallbacks() -> None: } +RDFA_PAGE = """<!DOCTYPE html> +<html vocab="https://schema.org/" prefix="dc: http://purl.org/dc/terms/ ex: https://example.org/ns#"> +<body> + <div typeof="Product" about="/products/widget#it"> + <h1 property="name alternateName">Acme Widget</h1> + <img property="image" src="/img/widget.jpg" alt=""> + <a property="url" href="/products/widget?color=red">buy</a> + <span property="dc:creator">Acme Ltd</span> + <span property="ex:tag">tools</span><span property="ex:tag">gadgets</span> + <meta property="sku" content="W-1"> + <div property="offers" typeof="Offer" resource="#offer"> + <span property="price" content="19.99">€19.99</span> + <meta property="priceCurrency" content="EUR"> + <link property="availability" href="https://schema.org/InStock"> + <time property="priceValidUntil" datetime="2026-12-31">end of the year</time> + </div> + <div property="review" typeof="Review"><span property="author">Bob</span></div> + <div property="review" typeof="Review"><span property="author">Eve</span></div> + <div typeof="Thing"><span property="name">Independent thing</span></div> + </div> + <section vocab="http://purl.org/dc/terms/" typeof="schema:Article"> + <span property="title">A note</span> + <span property="schema:headline">Headline</span> + </section> + <p typeof="">no type</p> +</body></html>""" + + +def test_rdfa_items(product_data: dict[str, Any]) -> None: + product, thing, article, blank = structured_data(root_of(RDFA_PAGE), URL)["rdfa"] + assert product["@type"] == "https://schema.org/Product" and product["@id"] == "https://shop.test/products/widget#it" + assert product["name"] == product["alternateName"] == "Acme Widget" # (two property names on one element) + assert product["image"] == "https://shop.test/img/widget.jpg" + assert product["url"] == "https://shop.test/products/widget?color=red" + assert product["dc:creator"] == "Acme Ltd" # another vocabulary: its CURIE, as written + assert product["ex:tag"] == ["tools", "gadgets"] + assert product["sku"] == "W-1" + assert product["offers"] == { + "@type": "https://schema.org/Offer", + "@id": "https://shop.test/products/widget#offer", + "price": "19.99", # (content= over the text) + "priceCurrency": "EUR", + "availability": "https://schema.org/InStock", + "priceValidUntil": "2026-12-31", + } + assert product["review"] == [ + {"@type": "https://schema.org/Review", "author": "Bob"}, + {"@type": "https://schema.org/Review", "author": "Eve"}, + ] + assert "author" not in product and "price" not in product # nested items keep their properties + assert thing == {"@type": "https://schema.org/Thing", "name": "Independent thing"} # (not a property: its own) + # a vocabulary of its own; a schema.org type and property by CURIE (schema.org's always get short names) + assert article == {"@type": "https://schema.org/Article", "title": "A note", "headline": "Headline"} + assert blank == {} # typeof="": an item of no type + + +def test_rdfa_terms_without_a_vocabulary_and_the_known_prefixes() -> None: + html = """<div typeof="Product"><span property="name">Bare</span> + <span property="gr:hasCurrency">EUR</span></div> + <div typeof="schema:Person" prefix="s: https://schema.org/"><span property="s:name">Ann</span></div>""" + bare, person = structured_data(root_of(html), "https://x.test/")["rdfa"] + assert bare == {"@type": "Product", "name": "Bare", "gr:hasCurrency": "EUR"} # (no vocab: terms as written) + assert person == {"@type": "https://schema.org/Person", "name": "Ann"} # (schema.org by any prefix: short names) + + def test_opengraph_and_twitter(product_data: dict[str, Any]) -> None: assert product_data["opengraph"] == { "type": "product", @@ -254,7 +320,7 @@ def test_meta(product_data: dict[str, Any]) -> None: def test_structured_data_on_a_bare_page() -> None: data = structured_data(root_of("<p>nothing to see</p>")) - assert data == {"json_ld": [], "microdata": [], "opengraph": {}, "twitter": {}, "meta": {}} + assert data == {"json_ld": [], "microdata": [], "rdfa": [], "opengraph": {}, "twitter": {}, "meta": {}} only_touch = structured_data(root_of('<link rel="apple-touch-icon" href="/t.png"><title>')) assert only_touch["meta"] == {"favicon": "/t.png"} # no base URL: left relative From 1a76221f4388f1192728def18d75c5c9261d55af Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 20:24:42 +0000 Subject: [PATCH 07/30] The same page under other URLs, once: content digests and rel=canonical MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §25 asks that a crawl skip pages whose content it has seen, and §7 that rel=canonical de-duplicate the frontier. URL de-duplication caught neither the same page served under several URLs, nor a variant that names its canonical page. Two spider settings, off by default: - skip_duplicate_pages: True (or "exact") skips a fetched page whose bytes were processed already; "near" also skips one whose visible text nearly was (a SimHash within 3 bits of an earlier page's, the exact pigeonhole index the record de-duplicator uses). The page is fetched, no callback runs for it, and stats["duplicate_pages"] counts it. The digests live in memory, about 100 bytes a page; "near" reads each page's text. - canonical_dedupe: a page that names a canonical URL counts as that page. When the canonical URL was seen already (fetched, or queued) the page is skipped (stats["canonical_skipped"]); otherwise it stands for it, and the canonical URL is marked seen so it is not fetched. The URL normalizer applies to the canonical URL as to every other. wintergrab.parser.structured.canonical_url reads the link. The record de-duplicator (and the dedupe stage) take near="simhash" with a distance in bits, beside MinHash. Tested on the test site's pages served under three URLs, nearly the same under three, and a product under a URL that names the product page as canonical. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 11 +++++++ docs/api.md | 4 +-- docs/data.md | 5 ++- docs/spiders.md | 18 +++++++++++ src/wintergrab/data/dedupe.py | 27 ++++++++++++++--- src/wintergrab/data/pipeline.py | 15 ++++++--- src/wintergrab/parser/structured.py | 20 +++++++++++- src/wintergrab/spider/engine.py | 47 +++++++++++++++++++++++++++++ src/wintergrab/spider/spider.py | 8 +++++ tests/test_data_pipeline.py | 19 ++++++++++++ tests/test_spider.py | 39 ++++++++++++++++++++++++ tests/testsite.py | 18 +++++++++++ 12 files changed, 218 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c063414..fb139bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,14 @@ an error that says so, not a setting silently ignored `Response.blocked_resources` reports what was blocked. - Spider settings can now hold callable values (a URL normalizer, a priority function); only methods are rejected as overrides. +- **The same page under other URLs, once.** `skip_duplicate_pages = True` + skips a fetched page whose bytes were processed already, `"near"` one + whose visible text nearly was (a SimHash within 3 bits); + `stats["duplicate_pages"]`. `canonical_dedupe = True` makes a page that + names a canonical URL count as that page: skipped when the canonical URL + was seen, else standing for it, so the canonical URL is not fetched; + `stats["canonical_skipped"]`. `wintergrab.parser.structured.canonical_url` + reads the link. ### Extension points, budgets and observability @@ -141,6 +149,8 @@ an error that says so, not a setting silently ignored pipeline about 4,200 records/s, near-duplicate checks 158 µs per record, expressions 1 µs. +- `Deduplicator(near="simhash", distance=3)` (the `dedupe` stage too) finds + near-duplicate records by SimHash as well as by MinHash. - **Provenance you can trace.** A crawl stamps every record that carries `_provenance` with `run` (its id in the run registry) and `output`; a pipeline records under each field's `transforms` what every stage did to @@ -151,6 +161,7 @@ an error that says so, not a setting silently ignored `wintergrab.data`) tells where a value came from: the page or API call, when, the extractor, the run, the output, how it was read, what else was found, and how it was changed. + ### Extraction engine (`wintergrab.extraction`) - **RDFa.** `structured_data()["rdfa"]` reads RDFa (Lite, and the value diff --git a/docs/api.md b/docs/api.md index 73bfa2c..c446fd5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -523,12 +523,12 @@ checks that it is up to date. - `diff(self, a: int | str = 'previous', b: int | str = 'latest', **options: Any) -> DatasetDiff`: The differences between two versions (by default the last two). - `get(self, ref: int | str) -> Version`: A version by number (``3``), name (``"v3"``), ``"latest"`` or ``"previous"``. - `load(self, ref: int | str = 'latest') -> list[dict[str, Any]]`: The records of a version. -- **`Deduplicate(key: str | Sequence[str] | None = None, *, fields: Sequence[str] | None = None, near: bool = False, text_fields: Sequence[str] | None = None, similarity: float = 0.8, mark: bool = False, name: str | None = None)`** (class). Drop (or mark) duplicate records; see :class:`~wintergrab.data.dedupe.Deduplicator`. +- **`Deduplicate(key: str | Sequence[str] | None = None, *, fields: Sequence[str] | None = None, near: bool | str = False, text_fields: Sequence[str] | None = None, similarity: float = 0.8, distance: int = 3, mark: bool = False, name: str | None = None)`** (class). Drop (or mark) duplicate records; see :class:`~wintergrab.data.dedupe.Deduplicator`. - `apply(self, record: dict[str, Any], ctx: RecordContext) -> dict[str, Any] | None`: Return the record (changed in place, or a new dict), or ``None`` to drop it (set ``ctx.reason``). - `details(self) -> str`: A short note for :meth:`Pipeline.describe`. - `classmethod from_config(cls, options: Any, loader: ConfigLoader) -> Deduplicate` - `to_config(self) -> dict[str, Any]`: The stage's configuration-file form: ``{kind: options}``. -- **`Deduplicator(key: str | Sequence[str] | None = None, *, fields: Sequence[str] | None = None, near: bool = False, text_fields: Sequence[str] | None = None, similarity: float = 0.8, mark: bool = False)`** (class). Drop (or mark) duplicate records. +- **`Deduplicator(key: str | Sequence[str] | None = None, *, fields: Sequence[str] | None = None, near: bool | str = False, text_fields: Sequence[str] | None = None, similarity: float = 0.8, distance: int = 3, mark: bool = False)`** (class). Drop (or mark) duplicate records. - `check(self, record: Mapping[str, Any]) -> tuple[str, int] | None`: ``(kind, index of the first record it duplicates)`` or ``None``; remembers new records. - `process_item(self, item: Any, spider: Any = None) -> Any` - `run(self, records: Iterable[Mapping[str, Any]]) -> list[Any]`: De-duplicate a list (the first occurrence is kept). diff --git a/docs/data.md b/docs/data.md index f1148a6..2c73b7c 100644 --- a/docs/data.md +++ b/docs/data.md @@ -395,7 +395,10 @@ always safe. 2. **Same content**: without a key, records whose fields are equal after normalizing case, spacing and punctuation. 3. **Near duplicates** (`near=True`): records whose text shares at least - `similarity` (default 0.8) of its word 3-grams with an earlier record. + `similarity` (default 0.8) of its word 3-grams with an earlier record; + or, with `near="simhash"`, whose SimHash fingerprint is within + `distance` (3) bits of an earlier record's: cheaper on long texts, and + exact within that distance. Similarity is the Jaccard index of 3-word shingles. A single changed word removes three shingles, so 0.8 catches texts with up to about one word in 30 diff --git a/docs/spiders.md b/docs/spiders.md index 2046c25..2b94c4e 100644 --- a/docs/spiders.md +++ b/docs/spiders.md @@ -506,6 +506,22 @@ class Shop(Spider): `wintergrab.url_template(url)` turns a URL into its route pattern (`/product/123` -> `/product/{int}`), handy for grouping pages by template. +Two more settings catch the same page under different URLs once it is +fetched, when the URL alone cannot tell: + +- `skip_duplicate_pages = True` skips a page whose bytes were processed + already; `"near"` also skips one whose visible text is nearly the same + (a SimHash within 3 bits of an earlier page's, as + [near-duplicate records](data.md#duplicates) are found). The page is + fetched (it counts in `pages`) but no callback runs for it, and + `stats["duplicate_pages"]` counts it. The digests live in memory, about + 100 bytes a page; `"near"` reads each page's text, about a millisecond. +- `canonical_dedupe = True` makes a page that names a canonical URL + (``) count as that page: when the canonical URL was + seen already (fetched, or queued), the page is skipped and + `stats["canonical_skipped"]` counts it; otherwise the page stands for it, + and the canonical URL is not fetched. + ## robots.txt `obey_robots_txt = True` (the default) fetches each site's robots.txt once @@ -564,6 +580,8 @@ wintergrab crawl my_spider.py -o items.jsonl --crawl-dir .crawl/mine -s max_page | `obey_robots_txt` | `True` | Respect robots.txt. | | `robots_user_agent` | `"*"` | User agent used to match robots.txt rules. | | `dedupe` | `True` | Filter already-seen URLs. | +| `skip_duplicate_pages` | `False` | Skip pages whose content was processed already: `True` the same bytes, `"near"` nearly the same text ([which URLs get crawled](#which-urls-get-crawled)). | +| `canonical_dedupe` | `False` | A page that names a canonical URL counts as that page: skipped when it was seen, else it stands for it. | | `url_normalizer` | `None` | Rewrite queued URLs to one canonical spelling (`True`, a dict of options, or a function). | | `url_rules` | `None` | Filter discovered links: patterns, domains, extensions, crawler traps (`True`, a dict, or `URLRules`). | | `network_policy` | `None` | Where requests may go: `"public"`, `"private"`, a dict, or a `NetworkPolicy`. `None` = anywhere. | diff --git a/src/wintergrab/data/dedupe.py b/src/wintergrab/data/dedupe.py index 753a429..5c687fc 100644 --- a/src/wintergrab/data/dedupe.py +++ b/src/wintergrab/data/dedupe.py @@ -23,7 +23,7 @@ from typing import Any from ..urls import normalize_url -from .similarity import MinHashLSH, content_hash, minhash, normalize_for_hash +from .similarity import MinHashLSH, SimHashIndex, content_hash, minhash, normalize_for_hash, simhash __all__ = ["Deduplicator", "normalize_key"] @@ -47,8 +47,11 @@ class Deduplicator: Args: key: Fields forming the record's identity. Records missing any of them are not key-checked. fields: Fields compared for content duplicates (default: all non-metadata fields). - near: Also catch near duplicates, comparing ``text_fields`` (default ``fields``). + near: Also catch near duplicates, comparing ``text_fields`` (default ``fields``): ``True`` (or + ``"minhash"``) by the Jaccard similarity of their word 3-grams, ``"simhash"`` by the Hamming + distance of their SimHash fingerprints (cheaper on long texts, and exact within ``distance``). similarity: The Jaccard similarity (0-1) from which texts count as near duplicates. + distance: With ``near="simhash"``: the bits (of 64) within which fingerprints count as the same. mark: Keep duplicates and set ``_duplicate_of`` (and ``_duplicate_kind``) instead of dropping them. """ @@ -57,11 +60,14 @@ def __init__( key: str | Sequence[str] | None = None, *, fields: Sequence[str] | None = None, - near: bool = False, + near: bool | str = False, text_fields: Sequence[str] | None = None, similarity: float = 0.8, + distance: int = 3, mark: bool = False, ) -> None: + if near not in (False, True, "minhash", "simhash"): + raise ValueError(f"near: True, 'minhash' or 'simhash', not {near!r}") self.key = [key] if isinstance(key, str) else list(key or []) self.fields = list(fields) if fields is not None else None self.near = near @@ -70,7 +76,9 @@ def __init__( self._keys: dict[tuple[str, ...], int] = {} self._hashes: dict[str, int] = {} self.similarity = similarity - self._lsh = MinHashLSH(threshold=similarity, num_perm=_NUM_PERM) if near else None + self.distance = distance + self._lsh = MinHashLSH(threshold=similarity, num_perm=_NUM_PERM) if near in (True, "minhash") else None + self._index = SimHashIndex(distance) if near == "simhash" else None self.seen = 0 self.duplicates: dict[str, int] = {"key": 0, "content": 0, "near": 0} @@ -106,7 +114,7 @@ def check(self, record: Mapping[str, Any]) -> tuple[str, int] | None: digest = content_hash(self._content(record)) if digest in self._hashes: return "content", self._hashes[digest] - signature = None + signature = fingerprint = None if self._lsh is not None: text = self._text(record) if text.strip(): @@ -114,11 +122,20 @@ def check(self, record: Mapping[str, Any]) -> tuple[str, int] | None: matches = self._lsh.query(signature) if matches: return "near", int(matches[0][0]) # type: ignore[call-overload] + elif self._index is not None: + text = self._text(record) + if text.strip(): + fingerprint = simhash(text) + within = self._index.query(fingerprint) + if within: + return "near", int(within[0][0]) # type: ignore[call-overload] if key is not None: self._keys[key] = index self._hashes[digest] = index if signature is not None and self._lsh is not None: self._lsh.insert(index, signature) + if fingerprint is not None and self._index is not None: + self._index.add(index, fingerprint) self.seen += 1 return None diff --git a/src/wintergrab/data/pipeline.py b/src/wintergrab/data/pipeline.py index c72c596..4e3fc5f 100644 --- a/src/wintergrab/data/pipeline.py +++ b/src/wintergrab/data/pipeline.py @@ -1418,7 +1418,8 @@ class Deduplicate(Stage): ``Deduplicate(key="url")`` drops records whose URL was seen; without a key, records with the same content; ``near=True`` also catches lightly edited - copies (``similarity``: the share of shared word 3-grams, default 0.8). + copies (``similarity``: the share of shared word 3-grams, default 0.8), and + ``near="simhash"`` those whose SimHash is within ``distance`` bits (3). """ kind = "dedupe" @@ -1428,23 +1429,29 @@ def __init__( key: str | Sequence[str] | None = None, *, fields: Sequence[str] | None = None, - near: bool = False, + near: bool | str = False, text_fields: Sequence[str] | None = None, similarity: float = 0.8, + distance: int = 3, mark: bool = False, name: str | None = None, ) -> None: super().__init__(name=name) if not 0 < similarity < 1: raise ConfigurationError("similarity must be between 0 and 1", key=self.name) + if near not in (False, True, "minhash", "simhash"): + raise ConfigurationError(f"near: true, 'minhash' or 'simhash', not {near!r}", key=self.name) + if not 0 <= distance < 16: + raise ConfigurationError("distance must be between 0 and 15 bits", key=self.name) self.deduplicator = Deduplicator( - key, fields=fields, near=near, text_fields=text_fields, similarity=similarity, mark=mark + key, fields=fields, near=near, text_fields=text_fields, similarity=similarity, distance=distance, mark=mark ) self._options = { "fields": fields, "near": near, "text_fields": text_fields, "similarity": similarity, + "distance": distance, "mark": mark, } @@ -1470,7 +1477,7 @@ def details(self) -> str: def from_config(cls, options: Any, loader: ConfigLoader) -> Deduplicate: if isinstance(options, (str, list, tuple)): return cls(options) - allowed = {"key", "fields", "near", "text_fields", "similarity", "mark", "name"} + allowed = {"key", "fields", "near", "text_fields", "similarity", "distance", "mark", "name"} return cls(**_options(options, allowed, cls.kind)) def to_config(self) -> dict[str, Any]: diff --git a/src/wintergrab/parser/structured.py b/src/wintergrab/parser/structured.py index 557699b..ddd1341 100644 --- a/src/wintergrab/parser/structured.py +++ b/src/wintergrab/parser/structured.py @@ -13,7 +13,15 @@ from .text import normalize_space, tag_name, text_content -__all__ = ["embedded_json", "find_values", "next_page_url", "structured_data", "table_to_records", "tables"] +__all__ = [ + "canonical_url", + "embedded_json", + "find_values", + "next_page_url", + "structured_data", + "table_to_records", + "tables", +] _FAIL: Any = object() # "could not parse" marker (``None`` is a valid JSON value) _NO_KEY: Any = object() # key of list items in :func:`find_values` @@ -196,6 +204,16 @@ def structured_data(root: etree._Element, base_url: str | None = None) -> dict[s } +def canonical_url(root: etree._Element, base_url: str | None = None) -> str | None: + """The page's canonical URL (````), absolute against ``base_url``; ``None`` without + one.""" + for link in root.iter("link"): + href = link.get("href") + if href and "canonical" in (link.get("rel") or "").lower().split(): + return _absolute(href.strip(), base_url) if href.strip() else None + return None + + def _json_ld(root: etree._Element) -> list[dict[str, Any]]: """Every JSON-LD object on the page as one flat list (``@graph`` and lists expanded).""" out: list[dict[str, Any]] = [] diff --git a/src/wintergrab/spider/engine.py b/src/wintergrab/spider/engine.py index b71338a..6a38777 100644 --- a/src/wintergrab/spider/engine.py +++ b/src/wintergrab/spider/engine.py @@ -15,6 +15,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any +from ..data.similarity import SimHashIndex, simhash from ..errors import ( BrowserNotAvailable, CheckpointError, @@ -33,6 +34,7 @@ from ..fetchers.http import PROXY_FAILURE_STATUSES, AsyncFetcher from ..fetchers.response import Response from ..fetchers.strategy import FetchStrategy +from ..parser.structured import canonical_url from ..proxy import ProxyRotator, proxy_label from ..redact import redact_url from ..request import Request @@ -61,6 +63,7 @@ from .spider import Spider log = logging.getLogger("wintergrab.spider") +_DUPLICATE_TEXT = 50_000 # characters of visible text a near-duplicate fingerprint reads PUSHBACK_STATUSES = frozenset({429, 503}) _HANDLED: Any = object() # a middleware dealt with the request itself (dropped or replaced it) @@ -117,6 +120,14 @@ def __init__( raise ConfigurationError(f"must be 'bfs' or 'dfs', not {spider.crawl_order!r}", key="crawl_order") self._lifo = CRAWL_ORDERS[order] self.scheduler: Scheduler | DiskScheduler | SharedScheduler = Scheduler(dedupe=spider.dedupe, lifo=self._lifo) + if spider.skip_duplicate_pages not in (False, True, "exact", "near"): + raise ConfigurationError( + f"skip_duplicate_pages: True, 'exact' or 'near', not {spider.skip_duplicate_pages!r}", + key="skip_duplicate_pages", + ) + #: Digests of the pages processed (``skip_duplicate_pages``), and their text fingerprints (``"near"``). + self._page_digests: set[bytes] = set() + self._page_index = SimHashIndex(3) if spider.skip_duplicate_pages == "near" else None self._persistent = False # True with the disk frontier, or a shared one self.throttle: AutoThrottle = spider.throttle or AutoThrottle( enabled=spider.autothrottle, @@ -1252,6 +1263,8 @@ async def _process(self, request: Request, slot: Any) -> None: await self._give_up(request, HTTPStatusError(response, detail), quiet=response.status == 404) return self.failures.success(domain) + if (spider.skip_duplicate_pages or spider.canonical_dedupe) and self._duplicate_page(request, response): + return await self._run_callback(request, response) except asyncio.CancelledError: raise @@ -1526,6 +1539,40 @@ async def _errback(self, request: Request, error: BaseException) -> None: # ------------------------------------------------------------------ # # callbacks and outputs # ------------------------------------------------------------------ # + def _duplicate_page(self, request: Request, response: Response) -> bool: + """Whether ``response`` is a page processed already under another URL: one whose canonical URL was seen + (``canonical_dedupe``; a page that names one not seen stands for it, and marks it seen), the same body + (``skip_duplicate_pages``), or nearly the same text (``"near"``).""" + spider = self.spider + if spider.canonical_dedupe and response.is_html: + canonical = canonical_url(response.selector.root, response.url) + if canonical: + if self.url_normalizer is not None: + canonical = self.url_normalizer(canonical) + if canonical not in (response.url, request.url): + fingerprint = Request(canonical).fingerprint() + if fingerprint in self.scheduler.seen: + self.stats.inc("canonical_skipped") + log.debug("skipped %s: its canonical page %s was seen already", request.url, canonical) + return True + self.scheduler.restore_seen([fingerprint]) # this page stands for it + if spider.skip_duplicate_pages and response.body: + digest = hashlib.blake2b(response.body, digest_size=16).digest() + if digest in self._page_digests: + self.stats.inc("duplicate_pages") + log.debug("skipped %s: the same content was processed already", request.url) + return True + self._page_digests.add(digest) + if self._page_index is not None and response.is_html: + text = " ".join(response.get_text().split()) + if text: + same = self._page_index.find_or_add(response.url, simhash(text[:_DUPLICATE_TEXT])) + if same is not None: + self.stats.inc("duplicate_pages") + log.debug("skipped %s: nearly the same content as %s", request.url, same) + return True + return False + async def _run_callback(self, request: Request, response: Response) -> None: callback = request.callback or self.spider.parse if isinstance(callback, str): diff --git a/src/wintergrab/spider/spider.py b/src/wintergrab/spider/spider.py index d882c9b..5a33d3e 100644 --- a/src/wintergrab/spider/spider.py +++ b/src/wintergrab/spider/spider.py @@ -285,6 +285,14 @@ async def parse(self, response): resource_filter: Any = None #: Drop requests for URLs already seen. dedupe: bool = True + #: Skip pages whose content was processed already under another URL: ``True`` (or ``"exact"``) the same + #: bytes, ``"near"`` nearly the same visible text (a SimHash within 3 bits). Counted in + #: ``stats["duplicate_pages"]``; the digests are kept in memory, about 100 bytes a page. + skip_duplicate_pages: bool | str = False + #: A page that names a canonical URL (````) counts as that page: it is skipped when + #: the canonical URL was seen already (``stats["canonical_skipped"]``), else it stands for it and the + #: canonical URL is not fetched. + canonical_dedupe: bool = False #: ``"memory"`` (fastest) or ``"disk"``: an SQLite queue + Bloom filter that keeps #: memory flat for crawls of millions of URLs and survives crashes (needs ``crawl_dir``); #: or a ``postgresql://`` URL: a queue several processes share (see :mod:`wintergrab.spider.shared`). diff --git a/tests/test_data_pipeline.py b/tests/test_data_pipeline.py index 6bcfbf9..f73081b 100644 --- a/tests/test_data_pipeline.py +++ b/tests/test_data_pipeline.py @@ -759,3 +759,22 @@ def test_data_trace_says_where_a_value_came_from(site, tmp_path, capsys) -> None assert capsys.readouterr().out.startswith("no provenance") assert main(["data", "trace", str(out), "--json"]) == 0 assert json.loads(capsys.readouterr().out)["_provenance"]["run"] == "run-1" + + +def test_near_duplicates_by_simhash() -> None: + from wintergrab.data.dedupe import Deduplicator + + words = " ".join(f"w{i}" for i in range(120)) + records = [{"text": words + " one"}, {"text": words + " two"}, {"text": "something else entirely, and short"}] + by_simhash = Deduplicator(near="simhash") + assert [by_simhash.check(r) for r in records] == [None, ("near", 0), None] + strict = Deduplicator(near="simhash", distance=0) + assert [strict.check(r) for r in records] == [None, None, None] # (a word differs: not within 0 bits) + by_minhash = Deduplicator(near=True) + assert [by_minhash.check(r) for r in records] == [None, ("near", 0), None] + with pytest.raises(ValueError, match="near"): + Deduplicator(near="fuzzy") + stage = Deduplicate(near="simhash", distance=2) + assert stage.to_config() == {"dedupe": {"near": "simhash", "distance": 2}} or stage._options["distance"] == 2 + with pytest.raises(ConfigurationError, match="near"): + Deduplicate(near="fuzzy") diff --git a/tests/test_spider.py b/tests/test_spider.py index e620a15..bf852ef 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -53,6 +53,45 @@ def test_crawl_follows_pagination_and_details(site) -> None: assert result.stats["status/200"] == 25 +class Section(Spider): + """Follows the links into the given sections of the site, and yields each page's title.""" + + log_level = None + obey_robots_txt = False + sections: tuple[str, ...] = () + + def parse(self, response): + yield {"url": response.url, "title": response.title} + for link in response.css("a::attr(href)").getall(): + if link.startswith(self.sections): + yield response.follow(link) + + +def test_pages_with_the_same_content_are_processed_once(site) -> None: + same = dict(start_urls=[site.url + "/same/1"], sections=("/same/",)) + plain = Section(**same).run() + assert len(plain.items) == 3 and plain.stats["pages"] == 3 # three URLs, one page: read three times + exact = Section(skip_duplicate_pages=True, **same).run() + assert len(exact.items) == 1 and exact.stats["pages"] == 3 and exact.stats["duplicate_pages"] == 2 + near = dict(start_urls=[site.url + "/near/1"], sections=("/near/",)) + assert len(Section(skip_duplicate_pages="exact", **near).run().items) == 3 # (a number differs: not the same) + nearly = Section(skip_duplicate_pages="near", **near).run() + assert len(nearly.items) == 1 and nearly.stats["duplicate_pages"] == 2 + with pytest.raises(ConfigurationError, match="skip_duplicate_pages"): + Section(skip_duplicate_pages="fuzzy", **same).run() + + +def test_a_page_counts_as_its_canonical_page(site) -> None: + settings = dict(start_urls=[site.url + "/canonical/1"], sections=("/canonical/", "/product/1"), url_normalizer=True) + plain = Section(**settings).run() # /canonical/1, /canonical/1?ref=again and /product/1: the product three times + assert sorted(i["url"] for i in plain.items) == sorted( + [site.url + "/canonical/1", site.url + "/canonical/1?ref=again", site.url + "/product/1"] + ) + canonical = Section(canonical_dedupe=True, **settings).run() + assert [i["url"] for i in canonical.items] == [site.url + "/canonical/1"] # it stands for /product/1 + assert canonical.stats["pages"] == 2 and canonical.stats["canonical_skipped"] == 1 # (/product/1: not fetched) + + def test_callback_styles(site) -> None: class Styles(Spider): log_level = None diff --git a/tests/testsite.py b/tests/testsite.py index 09602e0..4625f1e 100644 --- a/tests/testsite.py +++ b/tests/testsite.py @@ -450,6 +450,24 @@ def _route(self, site: Site) -> None: " = 'leaked'}).catch(e => {document.getElementById('out').textContent = 'refused'});", ) return self.send(200, html) + if path.startswith("/same/"): # the same page under three URLs + links = " ".join(f"{n}" for n in (1, 2, 3)) + return self.send(200, layout("Same", f"

The same page

Nothing here differs.

{links}")) + if path.startswith("/near/"): # nearly the same page under three URLs: one number differs + n = path.rsplit("/", 1)[1] + words = " ".join(f"word{i}" for i in range(200)) + links = " ".join(f"{k}" for k in (1, 2, 3)) + return self.send(200, layout("Near", f"

Almost the same page

{words} Visit {n}.

{links}")) + if path.startswith("/canonical/"): # product N under another URL, naming the product page as canonical + p = product(int(path.rsplit("/", 1)[1].split("?")[0])) + body = ( + f"

{p['name']}

${p['price']:.2f}

" + f"the product again" + ) + html = layout(p["name"], body).replace( + "", f"", 1 + ) + return self.send(200, html) if path.startswith("/item/"): i = path.rsplit("/", 1)[1] delay = float(q("delay", "0")) From 21b47902fe3677b9bd073962a80346d2677534c4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 20:37:08 +0000 Subject: [PATCH 08/30] Live coverage and confidence, and /metrics on the dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §58 asked for a served /metrics and live crawl-coverage and extraction- confidence metrics. spider.metrics() (and a run's metrics.json) now has coverage, the share of the URLs known so far (fetched, queued or in flight) that were fetched, and confidence, the mean _confidence of the last 1,024 records that had one. The dashboard shows both as tiles, and serves /metrics in the Prometheus text format for the running crawls (the last run's when none is running), each metric labelled with its run: to_prometheus(..., labels={"run": ...}). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 6 ++++++ docs/api.md | 1 + docs/dashboard.md | 1 + docs/observability.md | 14 ++++++++++++-- src/wintergrab/dashboard.py | 19 ++++++++++++++++++ src/wintergrab/spider/engine.py | 8 ++++++-- src/wintergrab/spider/metrics.py | 33 +++++++++++++++++++++++++------- tests/test_dashboard.py | 5 +++++ tests/test_observability.py | 11 +++++++++++ 9 files changed, 87 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb139bb..5e7cad2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,12 @@ an error that says so, not a setting silently ignored `Response.blocked_resources` reports what was blocked. - Spider settings can now hold callable values (a URL normalizer, a priority function); only methods are rejected as overrides. +- **Live coverage and confidence.** `spider.metrics()` (and the run's + `metrics.json`, the dashboard's tiles) has `coverage`, the share of the + URLs known so far that were fetched, and `confidence`, the mean + `_confidence` of the last 1,024 records. `wintergrab dashboard` serves + `/metrics`: the running crawls' metrics in the Prometheus text format, + each labelled with its run (`to_prometheus(..., labels=)`). - **The same page under other URLs, once.** `skip_duplicate_pages = True` skips a fetched page whose bytes were processed already, `"near"` one whose visible text nearly was (a SimHash within 3 bits); diff --git a/docs/api.md b/docs/api.md index c446fd5..027bcc7 100644 --- a/docs/api.md +++ b/docs/api.md @@ -1186,6 +1186,7 @@ checks that it is up to date. - **`Dashboard(workspace: str | Path = '.wintergrab', project: Any = None)`** (class). What the dashboard shows: a workspace's runs, and a project's jobs. - `items(self, run: Run, *, offset: int = 0, limit: int = 100) -> dict[str, Any]`: A page of the records ``run`` collected, read from its output file (``/api/runs/RUN/items``): ``{"run", "offset", "limit", "items", "next"}``, ``next`` the next page's offset (``None``: the last page), and a ``note`` when the rest could not be read (a crawl still writing it). - `jobs(self) -> list[dict[str, Any]]`: The project's jobs: what each does, its schedule, when it runs next, and its last run. + - `prometheus(self) -> str`: The running crawls' metrics in the Prometheus text format (``/metrics``), each metric labelled with its ``run``; the last run's when none is running. - `run(self, ref: str) -> Run` - `run_data(self, run: Run) -> dict[str, Any]`: Everything about a run, as JSON values (the run page, and ``/api/runs/RUN``). - `runs(self, limit: int = 200) -> list[Run]` diff --git a/docs/dashboard.md b/docs/dashboard.md index a444127..bab661e 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -73,6 +73,7 @@ The same data is available as JSON, for scripts and other tools: | `/api/runs/RUN` | One run: its record, `state`, `metrics`, a summary of its `events`, and the size of its output | | `/api/runs/RUN/items` | The records the run collected, a page at a time (`?offset=0&limit=100`, 1,000 at most): `{"items": [...], "next": 100}`, `next` being `null` on the last page | | `/api/jobs` | The project's jobs: schedule, next time, last status and run | +| `/metrics` | The running crawls' metrics in the [Prometheus text format](observability.md#prometheus), each metric labelled with its `run`; the last run's when none is running. Point a Prometheus scrape job at it. | A run's records are read from its output file, in any format a crawl writes (JSON Lines, CSV, SQLite, Parquet...), found as it was given or from diff --git a/docs/observability.md b/docs/observability.md index cae375d..6bf8a2b 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -72,6 +72,7 @@ give the spider [`webhooks`](projects.md#webhooks). "pages": 1204, "requests": 1251, "responses": 1240, "items": 980, "bytes": 25_104_331, "errors": 11, "failed": 4, "retries": 47, "blocked": 2, "success_rate": 0.9967, "queued": 311, "in_flight": 16, "active_domains": 3, + "coverage": 0.7861, "confidence": 0.91, "rates": {"pages_per_second": 38.5, "items_per_second": 31.2, "bytes_per_second": 802113.0, ...}, "latency": {"p50": 0.212, "p90": 0.48, "p99": 1.3, "mean": 0.26}, "domains": [ @@ -93,7 +94,13 @@ requests and `target_delay` where healthy responses pull it current settings allow and `current_rate` the measured one. Rates are measured over the last 30 seconds; latency percentiles over the -last 2,048 responses (cache hits excluded). +last 2,048 responses (cache hits excluded). `coverage` is the share of the +URLs the crawl knows of so far (fetched, queued or in flight) that it has +fetched: it falls as a crawl discovers more than it fetches, and reaches 1 +when the queue is empty. `confidence` is the mean extraction confidence +(`_confidence`) of the last 1,024 records that had one, `null` before any: +a drop while the crawl runs says the pages changed, or a section reads +badly. ### Prometheus @@ -105,7 +112,10 @@ text = to_prometheus(spider.metrics(), spider.stats) # serve it on /metrics Counters become `wintergrab_pages_total` and friends, per-domain gauges carry a `domain` label, and stats like `status/404` become -`wintergrab_status_by_label_total{label="404"}`. +`wintergrab_status_by_label_total{label="404"}`. `labels={"run": "run-7"}` +puts a label on every metric. The [dashboard](dashboard.md#json) serves the +running crawls' metrics this way on `/metrics`, each with its `run` label, +for Prometheus to scrape. ## Failure reports diff --git a/src/wintergrab/dashboard.py b/src/wintergrab/dashboard.py index 28a0055..5849fae 100644 --- a/src/wintergrab/dashboard.py +++ b/src/wintergrab/dashboard.py @@ -230,6 +230,18 @@ def summary(self, run: Run) -> dict[str, Any]: data["state"] = self.state(run) return data # type: ignore[no-any-return] + def prometheus(self) -> str: + """The running crawls' metrics in the Prometheus text format (``/metrics``), each metric labelled with + its ``run``; the last run's when none is running.""" + from .spider.metrics import to_prometheus + + runs = self.runs() + live = [run for run in runs if self.state(run) == "running"] + chosen = live or runs[:1] + return "".join( + to_prometheus(run.metrics(), run.stats, labels={"run": run.id}) + "\n" for run in chosen if run.metrics() + ) + def run_data(self, run: Run) -> dict[str, Any]: """Everything about a run, as JSON values (the run page, and ``/api/runs/RUN``).""" data = self.summary(run) @@ -490,8 +502,12 @@ def tile(label: str, value: str, note: str = "") -> str: tiles.append( tile("Data quality", _pct(min(quality)), ", ".join(str(q.get("dataset")) for q in data["quality"])) ) + if m.get("confidence") is not None: + tiles.append(tile("Confidence", _pct(m["confidence"]), "of the last records")) if running: tiles.append(tile("Queued", _num(m.get("queued")), f"{_num(m.get('in_flight'))} in flight")) + if m.get("coverage") is not None: + tiles.append(tile("Coverage", _pct(m["coverage"]), "of the URLs known so far")) return '
' + "".join(tiles) + "
" @@ -702,6 +718,9 @@ def do_GET(self, head: bool = False) -> None: self._json(dashboard.items(dashboard.run(segments[2]), offset=offset, limit=limit), head) elif segments == ["api", "jobs"]: self._json(dashboard.jobs(), head) + elif segments == ["metrics"]: + body = dashboard.prometheus().encode("utf-8") + self._send(HTTPStatus.OK, "text/plain; version=0.0.4; charset=utf-8", body, head) else: self._send(HTTPStatus.NOT_FOUND, "text/plain", b"not found\n", head) except ConfigurationError as exc: # no such run diff --git a/src/wintergrab/spider/engine.py b/src/wintergrab/spider/engine.py index 6a38777..c1778ba 100644 --- a/src/wintergrab/spider/engine.py +++ b/src/wintergrab/spider/engine.py @@ -1870,8 +1870,12 @@ async def _item(self, item: Any) -> Any: processed = await self._run_pipelines(processed) if processed is None: return None - if isinstance(processed, dict) and isinstance(processed.get("_provenance"), dict): - self._stamp(processed["_provenance"]) + if isinstance(processed, dict): + if isinstance(processed.get("_provenance"), dict): + self._stamp(processed["_provenance"]) + confidence = processed.get("_confidence") + if isinstance(confidence, (int, float)) and not isinstance(confidence, bool): + self.metrics.observe_confidence(float(confidence)) if spider.unique_key and self._is_duplicate_item(processed): self.stats.inc("items_duplicate") return None diff --git a/src/wintergrab/spider/metrics.py b/src/wintergrab/spider/metrics.py index 63568c4..2f300ca 100644 --- a/src/wintergrab/spider/metrics.py +++ b/src/wintergrab/spider/metrics.py @@ -87,15 +87,24 @@ def process_usage() -> dict[str, Any]: class CrawlMetrics: """Rolling measurements kept by the engine (cheap to update, computed on demand).""" - def __init__(self, window: float = 30.0, latency_samples: int = 2048) -> None: + def __init__(self, window: float = 30.0, latency_samples: int = 2048, confidence_samples: int = 1024) -> None: self.window = window self.latencies: deque[float] = deque(maxlen=latency_samples) + #: The extraction confidence (``_confidence``) of the most recent records that had one. + self.confidences: deque[float] = deque(maxlen=confidence_samples) self._samples: deque[tuple[float, float, float, float, float]] = deque() # t, pages, items, bytes, requests self._domains: dict[str, deque[tuple[float, int]]] = {} def observe_latency(self, seconds: float) -> None: self.latencies.append(seconds) + def observe_confidence(self, confidence: float) -> None: + self.confidences.append(confidence) + + def confidence(self) -> float | None: + """The mean extraction confidence of the most recent records (``None`` before any record had one).""" + return round(sum(self.confidences) / len(self.confidences), 4) if self.confidences else None + def sample(self, stats: Mapping[str, Any], throttle: AutoThrottle | None, now: float | None = None) -> None: """Record counters for rate computations (the engine calls this about once a second).""" now = time.monotonic() if now is None else now @@ -179,6 +188,7 @@ def snapshot( requests = float(stats.get("requests", 0)) or 0.0 failed = float(stats.get("failed", 0)) pages = float(stats.get("pages", 0)) + known = pages + queued + in_flight # the URLs the crawl knows of so far: fetched, waiting, or in flight return { "elapsed_seconds": round(elapsed, 3), "pages": int(pages), @@ -194,6 +204,8 @@ def snapshot( "success_rate": round(1 - failed / pages, 4) if pages else None, "queued": queued, "in_flight": in_flight, + "coverage": round(pages / known, 4) if known else None, + "confidence": self.confidence(), "active_domains": sum(1 for s in (throttle.slots.values() if throttle else ()) if s.active), "rates": self.rates(), "latency": self.latency(), @@ -214,26 +226,33 @@ def _label(value: Any) -> str: def to_prometheus( - snapshot: Mapping[str, Any], stats: Mapping[str, Any] | None = None, *, prefix: str = "wintergrab_" + snapshot: Mapping[str, Any], + stats: Mapping[str, Any] | None = None, + *, + prefix: str = "wintergrab_", + labels: Mapping[str, Any] | None = None, ) -> str: - """Render a metrics snapshot (and optionally the raw stats counters) in the Prometheus text format.""" + """Render a metrics snapshot (and optionally the raw stats counters) in the Prometheus text format. + ``labels`` go on every metric (``{"run": "run-7"}`` when several crawls' metrics are served together).""" lines: list[str] = [] + shared = dict(labels or {}) - def gauge(name: str, value: Any, labels: Mapping[str, Any] | None = None, kind: str = "gauge") -> None: + def gauge(name: str, value: Any, own: Mapping[str, Any] | None = None, kind: str = "gauge") -> None: if value is None or isinstance(value, bool) or not isinstance(value, (int, float)): return metric = prefix + _METRIC_NAME.sub("_", name) if not any(line.startswith(f"# TYPE {metric} ") for line in lines): lines.append(f"# TYPE {metric} {kind}") label_text = "" - if labels: - label_text = "{" + ",".join(f'{k}="{_label(v)}"' for k, v in labels.items()) + "}" + all_labels = {**shared, **(own or {})} + if all_labels: + label_text = "{" + ",".join(f'{k}="{_label(v)}"' for k, v in all_labels.items()) + "}" lines.append(f"{metric}{label_text} {value}") for key in ("pages", "requests", "responses", "items", "bytes", "browser_pages", "errors", "failed", "retries", "blocked"): # fmt: skip gauge(f"{key}_total", snapshot.get(key), kind="counter") - for key in ("queued", "in_flight", "active_domains", "elapsed_seconds", "success_rate"): + for key in ("queued", "in_flight", "active_domains", "elapsed_seconds", "success_rate", "coverage", "confidence"): gauge(key, snapshot.get(key)) for key, value in (snapshot.get("rates") or {}).items(): gauge(key, value) diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 7284422..d7650ff 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -82,6 +82,9 @@ def crawl() -> None: assert data["metrics"]["pages"] == 5 and data["events"]["counts"]["record_deleted"] == 1 assert data["quality"][0]["dataset"] == "products" and data["quality"][0]["records"] == 3 + status, headers, text = fetch(server, "/metrics") # for Prometheus: the last run's, as none is running + assert status == 200 and headers["Content-Type"].startswith("text/plain") + assert 'wintergrab_pages_total{run="run-2"} 5' in text and 'run="run-1"' not in text assert fetch(server, "/runs/run-9")[0] == 404 and fetch(server, "/nothing")[0] == 404 assert fetch(server, "/", host="evil.example:8710")[0] == 403 # DNS rebinding: not for this host assert fetch(server, "/", method="POST")[0] == 405 # it changes nothing @@ -137,6 +140,8 @@ def parse(self, response: Any) -> Any: assert '
Queued
' in page live = json.loads(fetch(server, "/api/runs/run-1")[2]) assert live["state"] == "running" and 0 < live["metrics"]["pages"] < 12 + assert 0 < live["metrics"]["coverage"] <= 1 and '
Coverage
' in page + assert 'wintergrab_coverage{run="run-1"}' in fetch(server, "/metrics")[2] # (a running crawl's) crawl.join(20) assert json.loads(fetch(server, "/api/runs/run-1")[2])["metrics"]["pages"] == 12 diff --git a/tests/test_observability.py b/tests/test_observability.py index cdda5f4..13757b9 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -146,6 +146,8 @@ def parse(self, response): assert live["pages"] >= 1 and "rates" in live and "latency" in live final = result.metrics assert final["pages"] == 8 and final["items"] == 8 and final["success_rate"] == 1.0 + assert final["coverage"] == 1.0 and final["confidence"] is None # (every URL fetched; no _confidence) + assert 0 < live["coverage"] <= 1.0 assert final["latency"]["p50"] is not None and final["latency"]["p99"] >= final["latency"]["p50"] domain = final["domains"][0] assert domain["domain"] == "127.0.0.1" and domain["mode"] == "normal" and domain["requests"] == 8 @@ -167,6 +169,11 @@ def test_metrics_rates() -> None: for latency in (0.1, 0.2, 0.3, 0.4): metrics.observe_latency(latency) assert metrics.latency()["p50"] == 0.3 and metrics.latency()["mean"] == 0.25 + assert metrics.confidence() is None + for confidence in (0.9, 0.7): + metrics.observe_confidence(confidence) + snapshot = metrics.snapshot({"pages": 20, "requests": 22}, queued=30, in_flight=2) + assert snapshot["coverage"] == round(20 / 52, 4) and snapshot["confidence"] == 0.8 def test_throttle_state_and_modes() -> None: @@ -201,8 +208,12 @@ def parse(self, response): assert 'wintergrab_latency_seconds{quantile="0.50"}' in text assert 'wintergrab_domain_requests{domain="127.0.0.1"} 2' in text assert 'wintergrab_status_by_label_total{label="404"} 1' in text + assert "wintergrab_coverage 1.0" in text and "wintergrab_confidence" not in text # (none: no _confidence) for line in text.splitlines(): assert line.startswith("#") or line.startswith("wintergrab_"), line + labelled = to_prometheus(result.metrics, result.stats, labels={"run": "run-7"}) + assert 'wintergrab_pages_total{run="run-7"} 2' in labelled + assert 'wintergrab_domain_requests{run="run-7",domain="127.0.0.1"} 2' in labelled def test_current_rss_is_plausible() -> None: From 6f4327baf178cb3c1e06d4beece2c0b9d8300ecd Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 09:22:20 +0000 Subject: [PATCH 09/30] More inputs to the throttle: RateLimit headers, the error rate, and pressure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §13 asks AutoThrottle to listen to more than 429s, 503s and Retry-After. - RateLimit headers are the site's own limit and are honoured: the IETF RateLimit field (limit=, remaining=, reset=) and the RateLimit-* fields, or X-RateLimit-* (reset as seconds, a Unix time or an HTTP date; wintergrab.utils.parse_rate_limit). A used-up window pauses the domain until it resets (stats["rate_limited"], a throttle_backoff event), a low remainder spaces requests so that the window lasts. Like Retry-After, they hold with autothrottle off. - A domain whose last responses mostly failed (at least error_rate_backoff, 0.5, of ten or more: timeouts, connection errors) has its concurrency halved, so a site in trouble is not hit in parallel. The domain's state reports error_rate and rate_limit. - max_bytes_per_second, hold_at_memory and hold_at_cpu hold new requests while others are in flight when the download rate over the last three seconds, the process's resident memory or its CPU share is above them (stats["held/bandwidth"], held/memory, held/cpu). A crawl with nothing in flight always starts one request, so a hold never stalls it. metrics.recent() gives the last seconds' download rate and CPU share. Tested with a test-site page that sends each header form, a crawl that waits for a used-up window to reset, and crawls held on memory and on bandwidth that still finish. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 12 +++++++ docs/api.md | 6 ++-- docs/observability.md | 7 ++-- docs/spiders.md | 20 +++++++++++- src/wintergrab/spider/engine.py | 43 ++++++++++++++++++++++-- src/wintergrab/spider/metrics.py | 29 +++++++++++++++-- src/wintergrab/spider/spider.py | 6 ++++ src/wintergrab/spider/throttle.py | 54 +++++++++++++++++++++++++++++-- src/wintergrab/utils.py | 37 +++++++++++++++++++++ tests/test_spider.py | 45 ++++++++++++++++++++++++++ tests/test_throttle_scheduler.py | 46 ++++++++++++++++++++++++++ tests/testsite.py | 8 +++++ 12 files changed, 302 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e7cad2..1759156 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,18 @@ an error that says so, not a setting silently ignored `Response.blocked_resources` reports what was blocked. - Spider settings can now hold callable values (a URL normalizer, a priority function); only methods are rejected as overrides. +- **More inputs to the throttle** (§13). `RateLimit` headers (the IETF + `RateLimit: limit=, remaining=, reset=` and `RateLimit-*` fields, or + `X-RateLimit-*`; `wintergrab.utils.parse_rate_limit`) are honoured: a + used-up window pauses the domain until it resets (`stats["rate_limited"]`, + a `throttle_backoff` event), a low remainder spaces requests so the window + lasts. A domain whose last responses mostly failed (at least + `error_rate_backoff`, 0.5, of ten or more) has its concurrency halved. + `max_bytes_per_second`, `hold_at_memory` and `hold_at_cpu` hold new + requests while others are in flight when the download rate, the resident + memory or the CPU share is above them (`stats["held/..."]`). The domain + state has `error_rate` and `rate_limit`; `metrics.recent()` gives the + last seconds' download rate and CPU share. - **Live coverage and confidence.** `spider.metrics()` (and the run's `metrics.json`, the dashboard's tiles) has `coverage`, the share of the URLs known so far that were fetched, and `confidence`, the mean diff --git a/docs/api.md b/docs/api.md index 027bcc7..1d123bd 100644 --- a/docs/api.md +++ b/docs/api.md @@ -27,13 +27,15 @@ checks that it is up to date. - `post(self, url: str, **kwargs: Any) -> Response` - `put(self, url: str, **kwargs: Any) -> Response` - `request(self, method: str, url: str, *, params: Mapping[str, Any] | None = None, headers: Mapping[str, str] | None = None, cookies: Mapping[str, str] | None = None, data: Any = None, json: Any = None, proxy: str | None = None, timeout: float | None = None, retries: int | None = None, allow_redirects: bool | None = None, request: Request | None = None, **extra: Any) -> Response`: Async version of :meth:`Fetcher.request`. -- **`AutoThrottle(*, enabled: bool = True, base_delay: float = 0.0, max_delay: float = 60.0, max_concurrency: int = 4, target_concurrency: float | None = None, backoff_factor: float = 2.0, min_backoff_delay: float = 1.0, recovery: float = 0.85, increase_every: int = 10, randomize: bool = True)`** (class). Adaptive, per-domain request pacing (AIMD - like TCP congestion control). +- **`AutoThrottle(*, enabled: bool = True, base_delay: float = 0.0, max_delay: float = 60.0, max_concurrency: int = 4, target_concurrency: float | None = None, backoff_factor: float = 2.0, min_backoff_delay: float = 1.0, recovery: float = 0.85, increase_every: int = 10, randomize: bool = True, error_rate_backoff: float = 0.5)`** (class). Adaptive, per-domain request pacing (AIMD - like TCP congestion control). - `can_start(self, slot: DomainSlot, now: float) -> bool` - `describe(self, domain: str) -> str`: One line for reports, e.g. + - `error_rate(slot: DomainSlot) -> float | None`: The share of the last responses (at most 20) that failed; ``None`` before any. - `mode(self, slot: DomainSlot, now: float | None = None) -> str`: ``"paused"`` (honouring Retry-After), ``"backing off"`` (push-back in the last 30 s), ``"recovering"`` (slower than the target after a push-back) or ``"normal"``. - - `on_error(self, domain: str)`: A timeout or connection error: back off gently. + - `on_error(self, domain: str)`: A timeout or connection error: back off gently; errors that keep coming halve concurrency too. - `on_finish(self, slot: DomainSlot)` - `on_pushback(self, domain: str, retry_after: float | None = None)`: The site said "slow down" (429/503/block page). + - `on_rate_limit(self, domain: str, remaining: int | None, reset: float | None) -> bool`: The site's ``RateLimit`` headers: ``remaining`` requests in the window that ends in ``reset`` seconds. - `on_start(self, slot: DomainSlot, now: float)` - `on_success(self, domain: str, latency: float)` - `restore(self, data: dict[str, dict[str, Any]])` diff --git a/docs/observability.md b/docs/observability.md index 6bf8a2b..b171c24 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -78,7 +78,8 @@ give the spider [`webhooks`](projects.md#webhooks). "domains": [ {"domain": "shop.example", "mode": "backing off", "active": 1, "concurrency": 1, "max_concurrency": 4, "delay": 8.0, "target_delay": 0.05, "avg_latency": 0.2, "allowed_rate": 0.125, "current_rate": 0.12, - "paused_for": 0.0, "backoffs": 3, "requests": 402}, + "paused_for": 0.0, "backoffs": 3, "requests": 402, "error_rate": 0.05, + "rate_limit": {"remaining": 40, "reset": 12.0, "pauses": 1}}, ... ], "budget": {"max_requests": {"used": 1251, "limit": 5000, "fraction": 0.25}}, @@ -91,7 +92,9 @@ seconds), `recovering` (still slower than its target after a push-back) or `paused` (honouring a `Retry-After`). `delay` is the current spacing between requests and `target_delay` where healthy responses pull it (`latency / target concurrency`); `allowed_rate` is the request rate the -current settings allow and `current_rate` the measured one. +current settings allow and `current_rate` the measured one. `error_rate` is +the share of the domain's last 20 responses that failed, and `rate_limit` +what its last `RateLimit` headers said, with how often they paused it. Rates are measured over the last 30 seconds; latency percentiles over the last 2,048 responses (cache hits excluded). `coverage` is the share of the diff --git a/docs/spiders.md b/docs/spiders.md index 2b94c4e..64bbc17 100644 --- a/docs/spiders.md +++ b/docs/spiders.md @@ -214,9 +214,26 @@ between requests. AutoThrottle adjusts both from what the site tells it: - **Push-back** (429, 503, a detected block page): concurrency halves and the delay doubles (at least 1 s). A `Retry-After` header pauses the domain for that long. -- **Timeouts and connection errors**: the delay grows by 50%. +- **`RateLimit` headers** (the IETF `RateLimit: limit=, remaining=, reset=` + and `RateLimit-*` fields, or `X-RateLimit-*`) are the site's own limit: a + used-up window pauses the domain until it resets + (`stats["rate_limited"]`, a `throttle_backoff` event), and a low remainder + spaces requests so that the window lasts. Honoured with `autothrottle = + False` too, like `Retry-After`. +- **Timeouts and connection errors**: the delay grows by 50%; when at least + half of a domain's last responses failed (ten or more seen), its + concurrency halves too (`AutoThrottle(error_rate_backoff=0.5)`). - A robots.txt `Crawl-delay` becomes a floor for the delay. +Three more settings hold new requests, for the whole crawl, while others +are in flight (those will free what is short; a crawl with nothing in +flight always starts one): `max_bytes_per_second` when the download rate +over the last three seconds is above it, `hold_at_memory` (bytes) while the +process's resident memory is at least that, `hold_at_cpu` (a share of one +core, e.g. `0.9`) while the process used at least that over the last three +seconds. `stats["held/bandwidth"]`, `held/memory` and `held/cpu` count the +holds. For stopping outright, see [budgets](#budgets). + ```python class Gentle(Spider): concurrency = 32 # overall @@ -557,6 +574,7 @@ wintergrab crawl my_spider.py -o items.jsonl --crawl-dir .crawl/mine -s max_page | `max_pages` / `max_items` / `max_depth` | `None` | Stop after this many pages / items; don't follow deeper than this. With `max_pages`, retries of pages already started still finish. | | `max_requests`, `max_bytes`, `max_runtime`, `max_browser_pages`, `max_errors`, `max_error_rate`, `max_memory`, `max_cpu_seconds`, `max_output_bytes` | `None` | [Budgets](#budgets). | | `budget_soft_limit` / `budget_soft_priority` | `None` / `1` | Past this fraction of a budget, only start requests with at least this priority. | +| `max_bytes_per_second`, `hold_at_memory`, `hold_at_cpu` | `None` | Hold new requests while the download rate, the resident memory or the CPU share is above this ([speed control](#speed-control-autothrottle)). | | `crawl_order` | `"bfs"` | `"bfs"` or `"dfs"` among equal priorities. | | `priority_fn` | `None` | `request -> int` priority for every queued request. | | `run_registry` / `record` | `None` / `False` | Keep a record of each run in a workspace (`True`: `.wintergrab`), and with `record` its pages and items too, to [replay](runs.md) it without the network. `result.run_id` names it. | diff --git a/src/wintergrab/spider/engine.py b/src/wintergrab/spider/engine.py index c1778ba..17102ef 100644 --- a/src/wintergrab/spider/engine.py +++ b/src/wintergrab/spider/engine.py @@ -40,7 +40,15 @@ from ..request import Request from ..runs import RunRecorder, RunRegistry from ..urls import URLNormalizer, URLRules -from ..utils import configure_logging, domain_matches, ensure_scheme, host_of, maybe_await, parse_retry_after +from ..utils import ( + configure_logging, + domain_matches, + ensure_scheme, + host_of, + maybe_await, + parse_rate_limit, + parse_retry_after, +) from ..webhooks import Webhook from .budget import BudgetMonitor, BudgetStatus from .checkpoint import Checkpoint @@ -48,7 +56,7 @@ from .exporters import Exporter, open_exporter, to_dict from .failures import FailureTracker from .frontier import DiskScheduler -from .metrics import CrawlMetrics +from .metrics import CrawlMetrics, current_rss from .middleware import DropItem, IgnoreRequest from .optimizer import CrawlOptimizer from .progress import ProgressDisplay @@ -404,10 +412,33 @@ async def _loop_until_done(self) -> None: timer.cancel() self._wakeup.clear() + def _pressure(self) -> str | None: + """What holds new requests for the moment: ``"bandwidth"``, ``"memory"`` or ``"cpu"`` over the spider's + limit, else ``None``. Only while requests are in flight, which will free what is held.""" + spider = self.spider + if not self._inflight: + return None + if spider.hold_at_memory is not None: + rss = current_rss() + if rss is not None and rss >= spider.hold_at_memory: + return "memory" + if spider.max_bytes_per_second is not None or spider.hold_at_cpu is not None: + recent = self.metrics.recent() + rate, cpu = recent["bytes_per_second"], recent["cpu_fraction"] + if spider.max_bytes_per_second is not None and rate is not None and rate > spider.max_bytes_per_second: + return "bandwidth" + if spider.hold_at_cpu is not None and cpu is not None and cpu >= spider.hold_at_cpu: + return "cpu" + return None + def _dispatch(self) -> float | None: spider = self.spider now = time.monotonic() wait: float | None = None + held = self._pressure() + if held is not None: + self.stats.inc(f"held/{held}") + return 0.25 while len(self._inflight) < spider.concurrency: if self.budget.active and self._counter_budget_hit(): break @@ -1221,6 +1252,14 @@ async def _process(self, request: Request, slot: Any) -> None: if (blocked or response.status in spider.retry_statuses) and response.cache_status is not None: self._uncache(fetcher, request) # never replay a block page or an error from the cache retry_after = parse_retry_after(response.headers.get("retry-after"), cap=600) + _limit, remaining, reset = parse_rate_limit(response.headers, cap=600) + if (remaining is not None or reset is not None) and self.throttle.on_rate_limit(domain, remaining, reset): + self.stats.inc("rate_limited") + log.info("%s: its rate limit is used up (RateLimit headers); waiting %.0fs", domain, reset or 0) + self.events.emit( + "throttle_backoff", domain=domain, delay=round(slot.delay, 3), concurrency=slot.concurrency, + retry_after=reset, + ) # fmt: skip if blocked or response.status in PUSHBACK_STATUSES: self.throttle.on_pushback(domain, retry_after) self.stats.inc("backoffs") diff --git a/src/wintergrab/spider/metrics.py b/src/wintergrab/spider/metrics.py index 2f300ca..4fecdd6 100644 --- a/src/wintergrab/spider/metrics.py +++ b/src/wintergrab/spider/metrics.py @@ -92,7 +92,8 @@ def __init__(self, window: float = 30.0, latency_samples: int = 2048, confidence self.latencies: deque[float] = deque(maxlen=latency_samples) #: The extraction confidence (``_confidence``) of the most recent records that had one. self.confidences: deque[float] = deque(maxlen=confidence_samples) - self._samples: deque[tuple[float, float, float, float, float]] = deque() # t, pages, items, bytes, requests + # t, pages, items, bytes, requests, cpu seconds + self._samples: deque[tuple[float, float, float, float, float, float]] = deque() self._domains: dict[str, deque[tuple[float, int]]] = {} def observe_latency(self, seconds: float) -> None: @@ -105,7 +106,13 @@ def confidence(self) -> float | None: """The mean extraction confidence of the most recent records (``None`` before any record had one).""" return round(sum(self.confidences) / len(self.confidences), 4) if self.confidences else None - def sample(self, stats: Mapping[str, Any], throttle: AutoThrottle | None, now: float | None = None) -> None: + def sample( + self, + stats: Mapping[str, Any], + throttle: AutoThrottle | None, + now: float | None = None, + cpu: float | None = None, + ) -> None: """Record counters for rate computations (the engine calls this about once a second).""" now = time.monotonic() if now is None else now self._samples.append( @@ -115,6 +122,7 @@ def sample(self, stats: Mapping[str, Any], throttle: AutoThrottle | None, now: f float(stats.get("items", 0)), float(stats.get("bytes", 0)), float(stats.get("requests", 0)), + time.process_time() if cpu is None else cpu, ) ) while len(self._samples) > 2 and now - self._samples[0][0] > self.window: @@ -146,6 +154,23 @@ def rates(self) -> dict[str, float]: "requests_per_second": round((last[4] - first[4]) / span, 3), } + def recent(self, seconds: float = 3.0) -> dict[str, float | None]: + """Over the last ``seconds`` of samples: bytes downloaded per second, and the share of one core the + process used (``cpu_fraction``); ``None`` with fewer than two samples in that time.""" + if len(self._samples) < 2: + return {"bytes_per_second": None, "cpu_fraction": None} + last = self._samples[-1] + first = next((s for s in self._samples if last[0] - s[0] <= seconds), None) + if first is None or first is last: + first = self._samples[-2] + span = last[0] - first[0] + if span <= 0: + return {"bytes_per_second": None, "cpu_fraction": None} + return { + "bytes_per_second": round((last[3] - first[3]) / span, 1), + "cpu_fraction": round((last[5] - first[5]) / span, 3), + } + def domain_rate(self, domain: str) -> float: history = self._domains.get(domain) if not history or len(history) < 2: diff --git a/src/wintergrab/spider/spider.py b/src/wintergrab/spider/spider.py index 5a33d3e..d58a96d 100644 --- a/src/wintergrab/spider/spider.py +++ b/src/wintergrab/spider/spider.py @@ -205,6 +205,12 @@ async def parse(self, response): #: start requests with a priority of at least ``budget_soft_priority``. budget_soft_limit: float | None = None budget_soft_priority: int = 1 + #: Pressure that holds new requests while others are in flight (``stats["held/bandwidth"]``, + #: ``held/memory``, ``held/cpu``): bytes downloaded per second over the last three seconds, the process's + #: resident memory (bytes), and the share of one core it used over the last three seconds. + max_bytes_per_second: float | None = None + hold_at_memory: int | None = None + hold_at_cpu: float | None = None # -- queue order -------------------------------------------------------- # #: ``"bfs"`` (breadth-first: oldest first among equal priorities) or ``"dfs"`` (depth-first). diff --git a/src/wintergrab/spider/throttle.py b/src/wintergrab/spider/throttle.py index ec7897b..b2a99f2 100644 --- a/src/wintergrab/spider/throttle.py +++ b/src/wintergrab/spider/throttle.py @@ -4,6 +4,7 @@ import random import time +from collections import deque from dataclasses import dataclass, field from typing import Any @@ -24,6 +25,9 @@ class DomainSlot: latencies: list[float] = field(default_factory=list) requests: int = 0 # requests started (for rate measurements) last_pushback: float = 0.0 # monotonic time of the last push-back + outcomes: deque[bool] = field(default_factory=lambda: deque(maxlen=20)) # the last responses: ok, or failed + rate_limit: tuple[int | None, float | None] | None = None # the site's last RateLimit headers: remaining, reset + rate_limited: int = 0 # pauses because the site's rate limit was used up def ready_at(self) -> float: return max(self.next_start, self.paused_until) @@ -44,6 +48,13 @@ class AutoThrottle: * On push-back (HTTP 429/503, a detected block page, timeouts) the delay is multiplied by ``backoff_factor`` and concurrency is halved. A ``Retry-After`` header pauses the domain for that long. + * ``RateLimit`` headers (and ``X-RateLimit-``) are the site's own limit: a + used-up window pauses the domain until it resets, a low remainder spaces + requests so that the window lasts. They are honoured with ``enabled=False`` + too, like ``Retry-After``. + * Errors that keep coming (timeouts, connection errors: at least + ``error_rate_backoff`` of the last responses, ten or more seen) halve + concurrency too, so a site in trouble is not hit in parallel. Args: enabled: With ``False`` the delay/concurrency stay fixed (``Retry-After`` @@ -59,6 +70,7 @@ class AutoThrottle: recovery: Factor applied to an inflated delay on each success. increase_every: Successes needed before concurrency grows by one. randomize: Jitter each delay by +/-50% so traffic looks less robotic. + error_rate_backoff: The share of failed responses among the last ones from which concurrency halves. """ def __init__( @@ -74,6 +86,7 @@ def __init__( recovery: float = 0.85, increase_every: int = 10, randomize: bool = True, + error_rate_backoff: float = 0.5, ) -> None: self.enabled = enabled self.base_delay = max(0.0, base_delay) @@ -85,6 +98,7 @@ def __init__( self.recovery = min(max(recovery, 0.1), 1.0) self.increase_every = max(1, increase_every) self.randomize = randomize + self.error_rate_backoff = min(max(error_rate_backoff, 0.05), 1.0) self.slots: dict[str, DomainSlot] = {} def slot(self, domain: str) -> DomainSlot: @@ -112,6 +126,7 @@ def on_finish(self, slot: DomainSlot) -> None: def on_success(self, domain: str, latency: float) -> None: slot = self.slot(domain) slot.latencies = [*slot.latencies[-19:], latency] + slot.outcomes.append(True) if not self.enabled: return floor = max(self.base_delay, slot.min_delay) @@ -144,12 +159,41 @@ def on_pushback(self, domain: str, retry_after: float | None = None) -> None: slot.next_start = max(slot.next_start, time.monotonic() + slot.delay) def on_error(self, domain: str) -> None: - """A timeout or connection error: back off gently.""" + """A timeout or connection error: back off gently; errors that keep coming halve concurrency too.""" + slot = self.slot(domain) + slot.outcomes.append(False) if not self.enabled: return - slot = self.slot(domain) slot.successes = 0 slot.delay = min(self.max_delay, max(slot.delay * 1.5, self.min_backoff_delay / 2)) + rate = self.error_rate(slot) + if rate is not None and rate >= self.error_rate_backoff and len(slot.outcomes) >= 10 and slot.concurrency > 1: + slot.concurrency = max(1, slot.concurrency // 2) + slot.outcomes.clear() # (judged again on the next ten) + + def on_rate_limit(self, domain: str, remaining: int | None, reset: float | None) -> bool: + """The site's ``RateLimit`` headers: ``remaining`` requests in the window that ends in ``reset`` seconds. + Used up, the domain pauses until then (``True``); low, requests are spaced so that the window lasts.""" + slot = self.slot(domain) + slot.rate_limit = (remaining, reset) + if remaining is None or not reset: + return False + now = time.monotonic() + if remaining <= 0: + slot.rate_limited += 1 + slot.paused_until = max(slot.paused_until, now + reset) + return True + spacing = reset / remaining + if spacing > slot.delay: + slot.delay = min(self.max_delay, spacing) + slot.next_start = max(slot.next_start, now + slot.delay) + return False + + @staticmethod + def error_rate(slot: DomainSlot) -> float | None: + """The share of the last responses (at most 20) that failed; ``None`` before any.""" + outcomes = slot.outcomes + return round(sum(1 for ok in outcomes if not ok) / len(outcomes), 3) if outcomes else None def set_min_delay(self, domain: str, delay: float) -> None: """Enforce a floor (e.g. a robots.txt ``Crawl-delay``).""" @@ -207,6 +251,12 @@ def state(self, domain: str, now: float | None = None) -> dict[str, Any]: "paused_for": round(max(0.0, slot.paused_until - now), 2), "backoffs": slot.backoffs, "requests": slot.requests, + "error_rate": self.error_rate(slot), + "rate_limit": ( + {"remaining": slot.rate_limit[0], "reset": slot.rate_limit[1], "pauses": slot.rate_limited} + if slot.rate_limit is not None + else None + ), } def describe(self, domain: str) -> str: diff --git a/src/wintergrab/utils.py b/src/wintergrab/utils.py index b840d3b..7e9838a 100644 --- a/src/wintergrab/utils.py +++ b/src/wintergrab/utils.py @@ -130,6 +130,43 @@ def domain_matches(host: str, domains: Iterable[str]) -> bool: return False +def parse_rate_limit(headers: Mapping[str, str], *, cap: float = 3600.0) -> tuple[int | None, int | None, float | None]: + """``(limit, remaining, reset seconds)`` from a site's rate-limit headers, each ``None`` when absent: the IETF + ``RateLimit`` header (``limit=100, remaining=5, reset=30``) and the ``RateLimit-Limit``, ``-Remaining`` and + ``-Reset`` fields, or their ``X-RateLimit-`` forms, whose reset may be seconds, a Unix time or an HTTP date.""" + fields: dict[str, str] = {} + combined: str | None = headers.get("ratelimit") or headers.get("RateLimit") + if combined: + for part in combined.split(","): + key, _, value = part.strip().partition("=") + if value: + fields[key.strip().lower()] = value.strip().strip('"') + for name in ("limit", "remaining", "reset"): + for header in (f"ratelimit-{name}", f"x-ratelimit-{name}"): + given: str | None = headers.get(header) + if given is not None and name not in fields: + fields[name] = given.strip() + + def count(value: str | None) -> int | None: + try: + return max(0, int(float(value))) if value is not None else None + except ValueError: + return None + + reset: float | None = None + raw = fields.get("reset") + if raw: + try: + seconds = float(raw) + except ValueError: + reset = parse_retry_after(raw, cap=cap) # an HTTP date + else: + if seconds > 10**9: # a Unix time, not seconds from now + seconds -= time.time() + reset = max(0.0, min(seconds, cap)) + return count(fields.get("limit")), count(fields.get("remaining")), reset + + def parse_retry_after(value: str | None, *, cap: float = 3600.0) -> float | None: """Seconds to wait from a ``Retry-After`` header (seconds or HTTP date).""" if not value: diff --git a/tests/test_spider.py b/tests/test_spider.py index bf852ef..66010f7 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -4,6 +4,7 @@ import csv import itertools import json +import time from dataclasses import dataclass import pytest @@ -92,6 +93,50 @@ def test_a_page_counts_as_its_canonical_page(site) -> None: assert canonical.stats["pages"] == 2 and canonical.stats["canonical_skipped"] == 1 # (/product/1: not fetched) +def test_rate_limit_headers_pause_the_crawl_of_a_domain(fresh_site) -> None: + class Limited(Spider): + log_level = None + obey_robots_txt = False + concurrency = 1 + events_seen: list[dict] = [] + + def parse(self, response): + yield {"url": response.url} + + urls = [ + fresh_site.url + "/ratelimit?remaining=0&reset=1", + fresh_site.url + "/ratelimit?remaining=5&reset=10&style=x", + ] + spider = Limited(start_urls=urls) + spider.events.subscribe(lambda event: Limited.events_seen.append(event), "throttle_backoff") + started = time.monotonic() + result = spider.run() + assert result.stats["pages"] == 2 and result.stats["rate_limited"] == 1 + assert time.monotonic() - started >= 0.9 # the second page waited for the window to reset + assert Limited.events_seen and Limited.events_seen[0]["retry_after"] == 1.0 + domain = result.metrics["domains"][0] + assert domain["rate_limit"] == {"remaining": 5, "reset": 10.0, "pauses": 1} and domain["error_rate"] == 0.0 + + +def test_pressure_holds_new_requests_while_others_are_in_flight(fresh_site) -> None: + class Held(Spider): + log_level = None + obey_robots_txt = False + concurrency = 4 + + def parse(self, response): + yield {"url": response.url} + + urls = [fresh_site.url + f"/slow?delay=0.2&n={i}" for i in range(6)] + result = Held(start_urls=urls, hold_at_memory=1).run() # (any process is over one byte) + assert result.stats["pages"] == 6 and result.stats["held/memory"] > 0 # held, and still done + started = time.monotonic() + slow = [fresh_site.url + f"/slow?delay=0.4&n={i}" for i in range(10)] + result = Held(start_urls=slow, max_bytes_per_second=1).run() # (a byte a second: over it at once) + assert result.stats["pages"] == 10 and result.stats["held/bandwidth"] > 0 + assert time.monotonic() - started > 1.0 + + def test_callback_styles(site) -> None: class Styles(Spider): log_level = None diff --git a/tests/test_throttle_scheduler.py b/tests/test_throttle_scheduler.py index 5b6bdc7..d1ece39 100644 --- a/tests/test_throttle_scheduler.py +++ b/tests/test_throttle_scheduler.py @@ -61,6 +61,52 @@ def test_crawl_delay_floor_and_snapshot() -> None: assert other.slot("a.test").delay == 2.0 +def test_rate_limit_headers_pause_or_pace_the_domain() -> None: + from wintergrab.utils import parse_rate_limit + + assert parse_rate_limit({"ratelimit": "limit=100, remaining=5, reset=30"}) == (100, 5, 30.0) + assert parse_rate_limit({"ratelimit-remaining": "0", "ratelimit-reset": "12"}) == (None, 0, 12.0) + assert parse_rate_limit({"x-ratelimit-limit": "60", "x-ratelimit-remaining": "7", "x-ratelimit-reset": "9"}) == ( + 60, + 7, + 9.0, + ) + limit, remaining, reset = parse_rate_limit( + {"x-ratelimit-remaining": "3", "x-ratelimit-reset": str(int(time.time()) + 20)} + ) + assert (limit, remaining) == (None, 3) and reset is not None and 18 <= reset <= 20 # (a Unix time) + assert parse_rate_limit({"x-ratelimit-reset": "99999"}, cap=600) == (None, None, 600.0) + assert parse_rate_limit({"content-type": "text/html"}) == (None, None, None) + assert parse_rate_limit({"ratelimit-remaining": "lots"}) == (None, None, None) + + throttle = AutoThrottle(randomize=False) + assert throttle.on_rate_limit("a.test", 0, 5.0) # used up: paused until the window resets + slot = throttle.slot("a.test") + assert not throttle.can_start(slot, time.monotonic()) and throttle.can_start(slot, time.monotonic() + 5.1) + assert throttle.state("a.test")["rate_limit"] == {"remaining": 0, "reset": 5.0, "pauses": 1} + assert not throttle.on_rate_limit("b.test", 4, 8.0) # four left for eight seconds: one every two + assert throttle.slot("b.test").delay == 2.0 + assert not throttle.on_rate_limit("b.test", 40, 8.0) and throttle.slot("b.test").delay == 2.0 # (never faster) + fixed = AutoThrottle(enabled=False) + assert fixed.on_rate_limit("c.test", 0, 3.0) and fixed.slot("c.test").delay == 0.0 # the site's word holds + + +def test_errors_that_keep_coming_halve_concurrency() -> None: + throttle = AutoThrottle(max_concurrency=8, randomize=False) + slot = throttle.slot("a.test") + for _ in range(4): + throttle.on_error("a.test") + assert slot.concurrency == 8 and throttle.state("a.test")["error_rate"] == 1.0 # too few seen to judge + for _ in range(5): + throttle.on_success("a.test", 0.1) + throttle.on_error("a.test") # 5 of 10 failed + assert slot.concurrency == 4 and throttle.error_rate(slot) is None # (judged again on the next ten) + for _ in range(9): + throttle.on_success("a.test", 0.1) + throttle.on_error("a.test") # one of ten: no + assert slot.concurrency == 4 + + def test_scheduler_priority_dedupe_and_domains() -> None: sched = Scheduler() throttle = AutoThrottle(max_concurrency=1, randomize=False) diff --git a/tests/testsite.py b/tests/testsite.py index 4625f1e..ee2a921 100644 --- a/tests/testsite.py +++ b/tests/testsite.py @@ -203,6 +203,14 @@ def _route(self, site: Site) -> None: if path == "/slow": time.sleep(float(q("delay", "0.5"))) return self.send(200, layout("slow", "

finally

")) + if path == "/ratelimit": # a page with the site's rate-limit headers (style: ietf, x or combined) + remaining, reset, style = q("remaining", "5"), q("reset", "10"), q("style", "ietf") + if style == "combined": + headers = {"RateLimit": f"limit=10, remaining={remaining}, reset={reset}"} + else: + prefix = "X-RateLimit-" if style == "x" else "RateLimit-" + headers = {prefix + "Limit": "10", prefix + "Remaining": remaining, prefix + "Reset": reset} + return self.send(200, layout("limited", f"

{remaining} left

"), headers=headers) if path == "/redirect": return self.send(int(q("code", "302")), "", headers={"Location": q("to", "/")}) if path == "/attachment.csv": # a file a browser downloads rather than shows From 75e79bcc5fe840add304470a7af377a1055b935f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 09:27:59 +0000 Subject: [PATCH 10/30] Prepare 0.3.0: the version, the changelog heading, the migration page __version__ is 0.3.0 and the changelog's Unreleased section is 0.3.0, as the release workflow checks against the tag. The migration page now says what behaves differently since 0.2 in this session's work too: a self-healing extractor asks by default (DIR/review.jsonl), --review needs --heal, a crawl's output is closed before its state is saved, an output that fails no longer stops the crawl, structured_data() has an rdfa key, and the additive options of the de-duplicator, to_prometheus and the metrics. Before the tag: the live tests (WINTERGRAB_LIVE=1 pytest -m live) from a network that reaches the practice sites; this session's container could not. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 2 +- docs/api.md | 2 +- docs/migration.md | 16 ++++++++++++++++ src/wintergrab/__init__.py | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1759156..239aebb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 0.3.0 ### Responsible access (breaking) diff --git a/docs/api.md b/docs/api.md index 1d123bd..e6d8bee 100644 --- a/docs/api.md +++ b/docs/api.md @@ -271,7 +271,7 @@ checks that it is up to date. - `search(self, query: str, *, provider: str = 'brave', pages: int = 1, **options: Any) -> SearchAnswer`: A search API's results for ``query`` (:func:`~wintergrab.intel.serp.search`: Brave's, Google's or your SearXNG, with your key from the environment), asked under this WinterGrab's network policy. - `sources(self, page: str | Response) -> DataSources`: Where a page's data is (:func:`~wintergrab.intel.sources.data_sources`): its HTML records, JSON-LD, embedded JSON and, in a browser, the API calls it makes. - **`WintergrabError`** (exception). Base class for every error raised by wintergrab. -- **`__version__`** = `'0.2.0'` +- **`__version__`** = `'0.3.0'` - **`aget(url: str, **kwargs: Any) -> Response`**. Async :func:`get`. - **`apost(url: str, **kwargs: Any) -> Response`**. Async :func:`post`. - **`arender(url: str, **kwargs: Any) -> Response`**. Async :func:`render`. diff --git a/docs/migration.md b/docs/migration.md index 093fbde..c164cb8 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -23,6 +23,16 @@ change, and what wintergrab now writes on its own. The - **A template name is a schema.** `--extract product` and `Extractor("product")` use the product template when no file of that name exists. Before, they failed. +- **A self-healing extractor asks by default.** `--heal DIR` without + `--review` keeps its questions in `DIR/review.jsonl` (before, they were + not asked); `--review FILE` still puts them elsewhere. `--review` without + `--heal` is an error, not an option ignored. +- **A crawl's output is closed before its state is saved**, so what the + output did not take (`export_errors`, `items_not_written`) is in the + counts a paused crawl resumes with, and in `summary.json`. +- **An output that fails no longer stops the crawl.** A write, a flush or a + close that fails is logged and counted, and `wintergrab crawl` and `goal` + exit with status 1 saying how many items were not written. ## Features that are gone @@ -57,6 +67,12 @@ error that says so, not a setting silently ignored: event also gets the new ones (`site_changed`, `record_updated`, `job_finished`...). - `HTTPError` is an alias of `HTTPStatusError`. +- `structured_data()` (and `Response.structured_data()`) has an `rdfa` + key beside `json_ld`, `microdata`, `opengraph`, `twitter` and `meta`. + Code that compares the whole dict, or lists its keys, sees one more. +- `Deduplicator(near=...)` and the `dedupe` stage take `"minhash"` or + `"simhash"` as well as `True`; `to_prometheus()` takes `labels=`; + `CrawlMetrics.sample()` records CPU time too. All additive. ## What wintergrab now writes diff --git a/src/wintergrab/__init__.py b/src/wintergrab/__init__.py index e741aac..6580b17 100644 --- a/src/wintergrab/__init__.py +++ b/src/wintergrab/__init__.py @@ -13,7 +13,7 @@ from __future__ import annotations -__version__ = "0.2.0" +__version__ = "0.3.0" # The parser must be imported before the adaptive package (they reference each other). from .parser import Field, Selector, SelectorList, parse # isort: skip From 64abbb5b8980ba58d1be7cd5c0e02df39aaf6186 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 09:38:57 +0000 Subject: [PATCH 11/30] Measured what the duplicate-page docs claimed The docs said "near" duplicate detection reads a page's text in about a millisecond and that kept digests take about 100 bytes a page; neither was measured. Measured on one core of this 4-vCPU cloud VM: reading and fingerprinting the text takes 1.5 ms for a 2 KB page, 15 ms for 25 KB and 29 ms for 132 KB (the text is cut at 50,000 characters), and 100,000 digests in a set take 9.1 MB, 91 bytes each. The docs, the setting's docstring and the changelog say so. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 4 ++-- docs/spiders.md | 7 +++++-- src/wintergrab/spider/spider.py | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 239aebb..2a48372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,8 +75,8 @@ an error that says so, not a setting silently ignored each labelled with its run (`to_prometheus(..., labels=)`). - **The same page under other URLs, once.** `skip_duplicate_pages = True` skips a fetched page whose bytes were processed already, `"near"` one - whose visible text nearly was (a SimHash within 3 bits); - `stats["duplicate_pages"]`. `canonical_dedupe = True` makes a page that + whose visible text nearly was (a SimHash within 3 bits; 15 ms for a + 25 KB page, measured); `stats["duplicate_pages"]`. `canonical_dedupe = True` makes a page that names a canonical URL count as that page: skipped when the canonical URL was seen, else standing for it, so the canonical URL is not fetched; `stats["canonical_skipped"]`. `wintergrab.parser.structured.canonical_url` diff --git a/docs/spiders.md b/docs/spiders.md index 64bbc17..6783feb 100644 --- a/docs/spiders.md +++ b/docs/spiders.md @@ -531,8 +531,11 @@ fetched, when the URL alone cannot tell: (a SimHash within 3 bits of an earlier page's, as [near-duplicate records](data.md#duplicates) are found). The page is fetched (it counts in `pages`) but no callback runs for it, and - `stats["duplicate_pages"]` counts it. The digests live in memory, about - 100 bytes a page; `"near"` reads each page's text, about a millisecond. + `stats["duplicate_pages"]` counts it. The digests live in memory: 91 + bytes a page (9.1 MB for 100,000, measured). `"near"` reads each page's + text and fingerprints it: 1.5 ms for a 2 KB page, 15 ms for 25 KB and + 29 ms for 132 KB (the text is cut at 50,000 characters), on one core of + a 4-vCPU cloud VM. - `canonical_dedupe = True` makes a page that names a canonical URL (``) count as that page: when the canonical URL was seen already (fetched, or queued), the page is skipped and diff --git a/src/wintergrab/spider/spider.py b/src/wintergrab/spider/spider.py index d58a96d..af30c2b 100644 --- a/src/wintergrab/spider/spider.py +++ b/src/wintergrab/spider/spider.py @@ -293,7 +293,7 @@ async def parse(self, response): dedupe: bool = True #: Skip pages whose content was processed already under another URL: ``True`` (or ``"exact"``) the same #: bytes, ``"near"`` nearly the same visible text (a SimHash within 3 bits). Counted in - #: ``stats["duplicate_pages"]``; the digests are kept in memory, about 100 bytes a page. + #: ``stats["duplicate_pages"]``; the digests are kept in memory, 91 bytes a page (measured). skip_duplicate_pages: bool | str = False #: A page that names a canonical URL (````) counts as that page: it is skipped when #: the canonical URL was seen already (``stats["canonical_skipped"]``), else it stands for it and the From ed2d42d86f7cb15eccb7a346775b098b520e266c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 10:19:50 +0000 Subject: [PATCH 12/30] The type check passes on Windows and macOS too Running mypy on Windows found four errors: the stubs there have no resource.getrusage or RUSAGE_SELF, which peak_rss() and the benchmark's peak-memory helper use behind a try/except ImportError that the type checker cannot see through. Both now return None on win32 before the import, a check mypy narrows on. CI runs mypy with --platform win32 and --platform darwin as well, so a platform-specific stub difference is caught on Linux. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- .github/workflows/ci.yml | 2 ++ CHANGELOG.md | 3 +++ src/wintergrab/bench.py | 2 ++ src/wintergrab/spider/metrics.py | 4 +++- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4954653..d8f2e7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,8 @@ jobs: - run: ruff check . - run: ruff format --check . - run: mypy + - run: mypy --platform win32 # the stubs differ by platform (no resource module on Windows) + - run: mypy --platform darwin audit: name: dependency audit diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a48372..22876c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1178,6 +1178,9 @@ an error that says so, not a setting silently ignored state and summary are saved, so a close that fails is in their counts too (before, a paused crawl resumed without it). +- `mypy` passes on Windows and macOS too: the two uses of the Unix-only + `resource` module are behind a platform check the type checker + understands, and CI checks all three platforms' stubs. - PostgreSQL connections (outputs, and the shared frontier) give up after 10 seconds unless the URL says `connect_timeout` or `PGCONNECT_TIMEOUT` is set. libpq's own default waited more than a minute for a host that had diff --git a/src/wintergrab/bench.py b/src/wintergrab/bench.py index 669cf5b..9a2e576 100644 --- a/src/wintergrab/bench.py +++ b/src/wintergrab/bench.py @@ -182,6 +182,8 @@ def _cpu_seconds() -> float: def _peak_rss_mb() -> float | None: """The process's peak memory, where the platform says (not on Windows).""" + if sys.platform == "win32": # (no resource module; the type checker knows it too) + return None try: import resource except ImportError: diff --git a/src/wintergrab/spider/metrics.py b/src/wintergrab/spider/metrics.py index 4fecdd6..4a6aef2 100644 --- a/src/wintergrab/spider/metrics.py +++ b/src/wintergrab/spider/metrics.py @@ -71,9 +71,11 @@ class Counters(ctypes.Structure): def peak_rss() -> int | None: """Peak resident memory in bytes (``None`` on platforms without ``resource``).""" + if sys.platform == "win32": # pragma: no cover - (no resource module; the type checker knows it too) + return None try: import resource - except ImportError: # pragma: no cover - Windows + except ImportError: # pragma: no cover - a platform without it return None peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss return int(peak if sys.platform == "darwin" else peak * 1024) From 730992f3669a1d8ca5b2827a67a61a7dc89359a8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 12:08:06 +0000 Subject: [PATCH 13/30] A page's menus are not among its data sources data_sources() listed every repeated group detect_records() found as HTML records and ranked the sources by records times fields, so on a page with a sidebar of twelve category links and four product cards the sidebar was the richest source (found when the test site's books section got the real site's category sidebar). detect_records() had scored the sidebar at 0.03 and the cards at 18; the listing ignored the score. RecordGroup.convincing says whether a group's score reaches the one auto_extract() trusts. auto_extract() uses it, and data_sources() lists the convincing groups only. The catalog fixture's table rows, a group scored 0.47, are listed as a table and no longer as HTML records too. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 7 +++++++ docs/power-features.md | 4 +++- docs/sources.md | 4 +++- src/wintergrab/intel/sources.py | 11 +++++++---- src/wintergrab/parser/autoextract.py | 8 +++++++- tests/test_autoextract.py | 1 + tests/test_sources.py | 3 ++- 7 files changed, 30 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22876c3..4b27f73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1178,6 +1178,13 @@ an error that says so, not a setting silently ignored state and summary are saved, so a close that fails is in their counts too (before, a paused crawl resumed without it). +- **A page's menus are not among its data sources.** `data_sources()` + (`wg.sources()`, `wintergrab get --sources`) listed every repeated group + `detect_records()` found as HTML records, and ranked them by records times + fields: a sidebar of twelve category links outranked four product cards as + the richest source. It lists the groups `auto_extract()` would trust now + (`RecordGroup.convincing`); menus, breadcrumbs and pagers score far below + that. - `mypy` passes on Windows and macOS too: the two uses of the Unix-only `resource` module are behind a platform check the type checker understands, and CI checks all three platforms' stubs. diff --git a/docs/power-features.md b/docs/power-features.md index 364196c..0057766 100644 --- a/docs/power-features.md +++ b/docs/power-features.md @@ -72,7 +72,9 @@ page.auto_extract() wintergrab finds the page's main repeating structure (a product grid, search results, table rows) and ignores menus, footers and tag clouds. It then names the fields it finds. `page.detect_records()` returns the candidate groups -with their generated CSS selectors if you want to take it from there. +with their generated CSS selectors if you want to take it from there; a +group's `convincing` says whether `auto_extract` would trust it (a menu or a +breadcrumb trail is a candidate too, scored low). ### Learning by example diff --git a/docs/sources.md b/docs/sources.md index 2286d6c..6c09839 100644 --- a/docs/sources.md +++ b/docs/sources.md @@ -36,7 +36,9 @@ https://shop.example/catalog (The test suite's page, `tests/data/sources/catalog.html`, and its API.) - **HTML**: repeated elements with the same fields (cards, rows), as - [`auto_extract`](power-features.md) finds them. + [`auto_extract`](power-features.md) finds them: the groups it would trust, + so a page's menus, breadcrumbs and pagers, which repeat elements too, are + not listed as records. - **tables**, **JSON-LD**, **microdata**, **RDFa**, **meta**: what `--tables` and `--structured` read, counted (RDFa is listed when the page has some). - **embedded JSON**: the lists of records in each blob `--json-data` reads. diff --git a/src/wintergrab/intel/sources.py b/src/wintergrab/intel/sources.py index 1a66265..82cb8fb 100644 --- a/src/wintergrab/intel/sources.py +++ b/src/wintergrab/intel/sources.py @@ -675,7 +675,8 @@ class DataSources: Attributes: url: The page. - html: Its repeated records in the HTML, the most convincing first. + html: Its repeated records in the HTML (the groups :func:`~wintergrab.auto_extract` would trust: not + its menus, breadcrumbs or pagers), the most convincing first. tables: Its ```` elements: rows and column names. json_ld: JSON-LD objects by ``@type``, and how many. microdata: Microdata items by type, and how many. @@ -869,9 +870,11 @@ def data_sources(response: Any, *, recorded: bool | None = None) -> DataSources: meta = [f"og:{k}" for k in structured.get("opengraph") or {}] meta += [f"twitter:{k}" for k in structured.get("twitter") or {}] meta += [str(k) for k in structured.get("meta") or {}] - html = [] - for group in response.detect_records()[:3]: - html.append(HtmlRecords(group.container_selector, len(group.elements), list(group.fields))) + html = [ # the groups that look like records: a page's menus, breadcrumbs and pagers repeat elements too + HtmlRecords(group.container_selector, len(group.elements), list(group.fields)) + for group in response.detect_records() + if group.convincing + ][:3] tables = [ {"rows": len(table.get("rows") or []), "columns": [str(c) for c in table.get("headers") or []]} for table in response.tables() diff --git a/src/wintergrab/parser/autoextract.py b/src/wintergrab/parser/autoextract.py index 33f2a37..e48d711 100644 --- a/src/wintergrab/parser/autoextract.py +++ b/src/wintergrab/parser/autoextract.py @@ -1008,6 +1008,12 @@ def as_schema(self) -> LearnedSchema: """The group as a reusable :class:`LearnedSchema` (to extract other pages of the site).""" return LearnedSchema(self.container_selector, dict(self.fields)) + @property + def convincing(self) -> bool: + """Whether the group looks like records rather than a menu, a breadcrumb trail, a pager or a footer: its + score reaches the one :func:`auto_extract` trusts.""" + return self.score >= _MIN_AUTO_SCORE + def __repr__(self) -> str: return ( f"RecordGroup({self.container_selector!r}, {len(self.elements)} records, " @@ -1062,7 +1068,7 @@ def auto_extract(root: Any, base_url: str | None = None, *, min_records: int = 3 """ doc, url = _resolve(root) groups = detect_records(doc, min_records=min_records, max_groups=1) - if not groups or groups[0].score < _MIN_AUTO_SCORE or not groups[0].fields: + if not groups or not groups[0].convincing or not groups[0].fields: return [] return groups[0].extract(base_url or url) diff --git a/tests/test_autoextract.py b/tests/test_autoextract.py index 9c09d40..d366015 100644 --- a/tests/test_autoextract.py +++ b/tests/test_autoextract.py @@ -305,6 +305,7 @@ def test_detect_records_prefers_products_over_menus() -> None: assert best.container_selector == "article.product_pod" assert len(page.css(best.container_selector)) == len(best.elements) assert [g.score for g in groups] == sorted((g.score for g in groups), reverse=True) + assert best.convincing and not any(g.convincing for g in groups[1:]) # the menus are candidates, scored low assert {"title", "url", "image", "price", "rating", "availability"} <= set(best.fields) _selectors_are_valid(page, best.container_selector, best.fields) assert "RecordGroup('article.product_pod', 6 records" in repr(best) diff --git a/tests/test_sources.py b/tests/test_sources.py index 279506b..c8af4a1 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -178,7 +178,8 @@ def test_calls_a_page_made() -> None: def test_the_sources_of_a_page_over_http(shop) -> None: sources = data_sources(wg.get(shop + "/catalog")) - assert [(g.count, g.fields) for g in sources.html][:1] == [(6, ["title", "url", "price"])] + # the cards; its table's rows repeat elements too, but are no convincing record group (they are a table) + assert [(g.selector, g.count, g.fields) for g in sources.html] == [("article.card", 6, ["title", "url", "price"])] assert sources.tables == [{"rows": 3, "columns": ["Size", "Chest"]}] assert sources.json_ld == {"Organization": 1, "ItemList": 1} assert sources.meta == ["og:title", "og:type", "title", "description", "language"] From 0b9e6af18cc94b2d225fb09639ee4ad3df074489 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 12:08:23 +0000 Subject: [PATCH 14/30] The survey spreads its sample across URL patterns On books.toscrape.com the live tests' scraper generator gave up: of the 15 pages the survey sampled, none was a book page. Every page there lists fifty categories before its books; the URL classifier saw category pages and book pages alike as categories, and the goal's word "books" matched every URL through the host name, so the sample was the home page and 14 category pages. The survey spider now follows a few links of each URL pattern (url_template) before more of any one: a pattern's links are queued a tier lower every per_pattern links (survey_site(per_pattern=), a fifth of the sample by default, two at least). The start URLs, the start page and the sitemaps' spread, are sampled as given. survey_for looks for the goal's words in the path and query, not the host. The test site's books section has the category sidebar and the category pages the real site has; a new test checks the sample holds book pages and category pages both and that the plan still targets the books. The generator test's counts follow the books sampled instead of assuming five. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 10 ++++++++ docs/api.md | 2 +- docs/goals.md | 6 +++-- src/wintergrab/goals/plan.py | 4 +++- src/wintergrab/intel/survey.py | 33 ++++++++++++++++++++----- tests/test_generate.py | 8 ++++--- tests/test_goals.py | 21 ++++++++++++++++ tests/test_projects.py | 3 ++- tests/test_runs.py | 8 ++++--- tests/testsite.py | 44 +++++++++++++++++++++++++++++++--- 10 files changed, 119 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b27f73..addb022 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1178,6 +1178,16 @@ an error that says so, not a setting silently ignored state and summary are saved, so a close that fails is in their counts too (before, a paused crawl resumed without it). +- **The survey spreads its sample across URL patterns.** On + books.toscrape.com, whose pages list fifty categories before their books, + a 15-page sample was the home page and 14 category pages: the URL + classifier saw every URL there as a category, and the goal's word "books" + matched every URL through the host name. The scraper generator then found + no book page and gave up (found by the live tests). The survey now follows + a few links of each URL pattern before more of any one (`survey_site(..., + per_pattern=)`, a fifth of the sample by default), and the goal's words + are looked for in the path, not the host. The test site's books section + has a category sidebar now, as the real one does. - **A page's menus are not among its data sources.** `data_sources()` (`wg.sources()`, `wintergrab get --sources`) listed every repeated group `detect_records()` found as HTML records, and ranked them by records times diff --git a/docs/api.md b/docs/api.md index e6d8bee..0bcc03e 100644 --- a/docs/api.md +++ b/docs/api.md @@ -906,7 +906,7 @@ checks that it is up to date. - **`pagination_of(url: str, answer: Any = None, *, request: Mapping[str, Any] | None = None, records: int | None = None) -> Pagination | None`**. How the pages of the API ``url`` go, from its query parameters (or ``request``: a GraphQL call's variables, a JSON request body) and its ``answer``, or ``None`` when neither says. - **`read_sitemaps(origin: str, robots_text: str | None = None, *, max_sitemaps: int = 10, max_entries: int = 50000, **fetch_options: Any) -> SitemapRead`**. The pages a site's sitemaps list: those named in robots.txt, or ``/sitemap.xml``, following sitemap indexes, up to ``max_sitemaps`` sitemaps and ``max_entries`` pages. - **`script_endpoints(selector: Any, url: str) -> list[tuple[str | None, str]]`**. The API endpoints a page's inline scripts and ``data-`` attributes name, as ``(method, endpoint)``: calls (``fetch("/api/...")``, axios, jQuery, ``xhr.open``) and API-looking paths (``/api/``, ``/graphql``), query values left out (``https://shop.example/api/products?page=``). -- **`survey_site(url: str, *, pages: int = 30, sitemaps: bool = True, obey_robots: bool = True, browser: bool = False, timeout: float = 20.0, keep_pages: bool = False, prefer: Callable[[str], bool | float] | None = None, extra_urls: Iterable[str] = (), log_level: str | None = 'WARNING', **spider_settings: Any) -> SiteSurvey`**. Read ``url``'s site: robots.txt, sitemaps, and ``pages`` pages, into a :class:`SiteSurvey`. +- **`survey_site(url: str, *, pages: int = 30, sitemaps: bool = True, obey_robots: bool = True, browser: bool = False, timeout: float = 20.0, keep_pages: bool = False, prefer: Callable[[str], bool | float] | None = None, extra_urls: Iterable[str] = (), per_pattern: int | None = None, log_level: str | None = 'WARNING', **spider_settings: Any) -> SiteSurvey`**. Read ``url``'s site: robots.txt, sitemaps, and ``pages`` pages, into a :class:`SiteSurvey`. ## `wintergrab.intel.serp`: Search results diff --git a/docs/goals.md b/docs/goals.md index 9be1b31..32b743a 100644 --- a/docs/goals.md +++ b/docs/goals.md @@ -95,8 +95,10 @@ steam wand", "remote"). `plan_goal(goal)` surveys each site: robots.txt, the sitemaps, and 30 pages. Those are the start page, pages spread across the sitemaps, and the pages their links lead to. Pages in the part of the site the goal names come -first, then pages that look like the goal's records, then their listings. It -learns from the sample: +first, then pages that look like the goal's records, then their listings; +and a few pages of each URL pattern come before more of any one pattern, so +a site that lists fifty categories before its products still has product +pages in the sample. It learns from the sample: - which pages hold the records: classified as the kind's pages (product, article, job...), or, for pages that list nothing, giving a record with its diff --git a/src/wintergrab/goals/plan.py b/src/wintergrab/goals/plan.py index 185426f..562a44d 100644 --- a/src/wintergrab/goals/plan.py +++ b/src/wintergrab/goals/plan.py @@ -465,7 +465,9 @@ def prefer(url: str) -> int: score = 2 if section and (path == section or path.startswith(section + "/")) else 0 seen_as = classify_url(url).type score += 2 if seen_as in kind.page_types else 1 if seen_as in kind.listing_types else 0 - return score + (1 if any(w in url.lower() for w in words) else 0) + where = urlsplit(url) + text = f"{where.path}?{where.query}".lower() # (not the host: books.example names every page "books") + return score + (1 if any(w in text for w in words) else 0) return survey_site( site, diff --git a/src/wintergrab/intel/survey.py b/src/wintergrab/intel/survey.py index b6c20a6..699fdbb 100644 --- a/src/wintergrab/intel/survey.py +++ b/src/wintergrab/intel/survey.py @@ -10,12 +10,15 @@ ``wintergrab inspect`` prints a survey's profile; the goal planner (:mod:`wintergrab.goals`) starts from one. The sample is the start page, pages spread across the sitemaps (the ones ``prefer`` likes first) and the pages they -link to, fetched politely: robots.txt is obeyed unless told otherwise. +link to, a few of each URL pattern before more of any one pattern, so that a +site's record pages are sampled however many category or tag links come first +on its pages. Fetched politely: robots.txt is obeyed unless told otherwise. """ from __future__ import annotations import logging +from collections import Counter from collections.abc import Callable, Iterable, Mapping from dataclasses import dataclass, field from typing import Any @@ -26,6 +29,7 @@ from ..request import Request from ..sitemaps import SitemapEntry, parse_sitemap, robots_sitemaps from ..spider import Spider +from ..urls import url_template from ..utils import ensure_scheme, host_of from .profile import SiteProfile, SiteProfiler @@ -140,22 +144,34 @@ class _SurveySpider(Spider): url_rules = True # skip media, archives and crawler traps #: ``prefer(url) -> score``: links worth more are followed first. prefer: Any = None + #: Links of one URL pattern (:func:`~wintergrab.url_template`) followed before the other patterns get their + #: turn: the sample is spread across the patterns a site has, so its record pages are among the pages sampled + #: however many category or tag links come first on a page. ``0``: page order alone. + per_pattern: int = 3 def __init__(self, **settings: Any) -> None: super().__init__(**settings) self.kept: list[Response] = [] + self._patterns: Counter[str] = Counter() # URLs queued so far, by pattern + self._queued: set[str] = set() - def _request(self, url: str) -> Request: + def _request(self, url: str, *, spread: bool = True) -> Request: request = Request(url, dont_filter=False) if self.capture_api: request.options["capture"] = True - if self.prefer is not None: - request.priority = int(10 * float(self.prefer(url))) + priority = int(10 * float(self.prefer(url))) if self.prefer is not None else 0 + if self.per_pattern and url not in self._queued: + self._queued.add(url) + pattern = url_template(url) + if spread: # the n-th link of a pattern waits for the first of every other: a tier lower each time + priority -= 100 * (self._patterns[pattern] // self.per_pattern) + self._patterns[pattern] += 1 + request.priority = priority return request def start_requests(self) -> Any: - for url in self.start_urls: - yield self._request(str(url)) + for url in self.start_urls: # (the start page and the sitemaps' spread: sampled as given) + yield self._request(str(url), spread=False) def parse(self, response: Response) -> Any: if self.keep_pages and response.is_html: @@ -188,6 +204,7 @@ def survey_site( keep_pages: bool = False, prefer: Callable[[str], bool | float] | None = None, extra_urls: Iterable[str] = (), + per_pattern: int | None = None, log_level: str | None = "WARNING", **spider_settings: Any, ) -> SiteSurvey: @@ -202,6 +219,9 @@ def survey_site( prefer: How much a page is worth sampling (``prefer(url) -> score``, ``True``/``False`` too): the sitemaps' pages are sampled best first, and links are followed best first. extra_urls: Pages to visit besides the start page and the sitemap sample. + per_pattern: Links of one URL pattern followed before the other patterns get their turn (by default a + fifth of ``pages``, two at least), so the sample is spread across the site's patterns; ``0``: page + order alone. """ from ..fetchers import Fetcher @@ -249,6 +269,7 @@ def survey_site( capture_api=browser, keep_pages=keep_pages, prefer=prefer, + per_pattern=max(2, pages // 5) if per_pattern is None else per_pattern, timeout=timeout, output=None, keep_items=False, diff --git a/tests/test_generate.py b/tests/test_generate.py index 5110e78..489a175 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -88,9 +88,11 @@ def test_a_scraper_generated_tested_and_accepted(site, tmp_path) -> None: sites=[site.url + "/books/"], sample=15, on_stage=seen.append) # fmt: skip assert result.accepted, result.describe() assert [s.name for s in seen] == ["plan", "generate", "lint", "test", "sample", "validate", "benchmark"] - assert result.stage("test").summary.startswith("5/5 pages give the expected values") + trained = min(5, result.stage("plan").details["record_pages"]) # generated from the sampled books, 5 at most + assert trained >= 3 # (the sample is spread over the site's URL patterns: category pages are in it too) + assert result.stage("test").summary.startswith(f"{trained}/{trained} pages give the expected values") sample = result.stage("sample").details - assert sample["unseen"] == 7 and sample["record_pages"] == 12 # the 12 books, 5 of them the samples + assert sample["record_pages"] == 12 and sample["unseen"] == 12 - trained # all 12 books, less the samples validate = result.stage("validate") assert validate.details["agreed"] == validate.details["compared"] > 0 benchmark = result.stage("benchmark").details @@ -110,7 +112,7 @@ def test_a_scraper_generated_tested_and_accepted(site, tmp_path) -> None: assert FixtureSuite(directory / "fixtures").run().ok # again in the same directory: its own files replaced, the tests too again = generate_scraper("books with title and price", directory, sites=[site.url + "/books/"], sample=15) - assert again.accepted and len(FixtureSuite(directory / "fixtures").fixtures()) == 5 + assert again.accepted and len(FixtureSuite(directory / "fixtures").fixtures()) == trained def test_a_model_while_generating_and_none_after(site, tmp_path) -> None: diff --git a/tests/test_goals.py b/tests/test_goals.py index 97fd359..5a6c8d3 100644 --- a/tests/test_goals.py +++ b/tests/test_goals.py @@ -220,6 +220,27 @@ def test_plans_run_and_replay(site, tmp_path) -> None: assert limited.run(log_level=None).counts["records"] == 2 +def test_the_survey_samples_every_url_pattern(site) -> None: + """The books index links its twelve categories before its books, as books.toscrape.com lists fifty. A sample + is not spent on the first pattern seen: a few pages of each pattern come before more of any, so the record + pages are among them, and the plan knows their pattern (found by the live tests: the scraper generator saw 15 + category pages and no book).""" + from collections import Counter + + from wintergrab import url_template + from wintergrab.goals.plan import survey_for + + goal = read("books with title and price", sites=[site.url + "/books/"]) + survey = survey_for(goal, site.url + "/books/", sample=8, log_level=None) + patterns = Counter(url_template(p.url, include_host=False) for p in survey.pages) + assert len(survey.pages) == 8 + assert patterns["/books/catalogue/{slug}/index.html"] >= 2 # the books + assert patterns["/books/catalogue/category/books/{slug}/index.html"] >= 2 # and the categories + plan = plan_goal(goal, sample=8) + assert plan.sites[0].target == ["/books/catalogue/*/index.html"] + assert not any("looked like a product page" in w for w in plan.sites[0].warnings) + + def test_the_whole_loop_in_one_run(site, tmp_path, capsys) -> None: """provenance=True, heal=DIR: records say where each value came from, the plan's schema is read by a self-healing extractor kept in DIR (a fixture per site, questions in DIR/review.jsonl), and a redesign is diff --git a/tests/test_projects.py b/tests/test_projects.py index bf927c6..127c5fa 100644 --- a/tests/test_projects.py +++ b/tests/test_projects.py @@ -12,6 +12,7 @@ import pytest from test_history import ShopSpider, product_page, shop # noqa: F401 (the fixture) +from testsite import BOOK_SECTION_PAGES from wintergrab.errors import ConfigurationError from wintergrab.project import Project, Scheduler from wintergrab.schedules import Cron, Interval, Once, parse_schedule @@ -224,7 +225,7 @@ def test_a_project(site, receiver, tmp_path, capsys, monkeypatch) -> None: assert main(["run", "--project", str(path)]) == 0 err = capsys.readouterr().err assert "books: finished in" in err and "rated: finished in" in err - assert len((tmp_path / "data" / "books.jsonl").read_text().splitlines()) == 16 + assert len((tmp_path / "data" / "books.jsonl").read_text().splitlines()) == BOOK_SECTION_PAGES assert len((tmp_path / "data" / "rated.jsonl").read_text().splitlines()) == 4 from wintergrab.runs import RunRegistry diff --git a/tests/test_runs.py b/tests/test_runs.py index 6a086db..820eb0a 100644 --- a/tests/test_runs.py +++ b/tests/test_runs.py @@ -7,6 +7,7 @@ import pytest +from testsite import BOOK_SECTION_PAGES from wintergrab import Spider from wintergrab.errors import ConfigurationError from wintergrab.fetchers.cache import HTTPCache @@ -49,11 +50,12 @@ def test_a_recorded_crawl_replays_without_the_network(fresh_site, tmp_path) -> N str(tmp_path / "books.jsonl"), ) assert run.recipe == {"spider": "test_runs:Books"} and run.settings["start_urls"] == [fresh_site.url + "/books/"] - assert len(list(run.items())) == 12 and run.stats["pages"] == 16 + assert len(list(run.items())) == 12 and run.stats["pages"] == BOOK_SECTION_PAGES kinds = [e["event"] for e in run.events()] - assert kinds[0] == "crawl_started" and kinds[-1] == "crawl_finished" and kinds.count("response") == 16 + assert kinds[0] == "crawl_started" and kinds[-1] == "crawl_finished" + assert kinds.count("response") == BOOK_SECTION_PAGES assert all("latency" in e for e in run.events("response")) # the timings - assert len(HTTPCache(run.archive, mode="offline")) == 17 # every page, and robots.txt + assert len(HTTPCache(run.archive, mode="offline")) == BOOK_SECTION_PAGES + 1 # every page, and robots.txt before = sum(fresh_site.site.hits.values()) again = replay("run-1", Books, registry=workspace) diff --git a/tests/testsite.py b/tests/testsite.py index ee2a921..ff0f4e1 100644 --- a/tests/testsite.py +++ b/tests/testsite.py @@ -502,8 +502,47 @@ def _book(i: int) -> dict[str, Any]: } +BOOK_CATEGORIES = ("Travel", "Mystery", "Historical Fiction", "Sequential Art", "Classics", "Philosophy", "Romance", + "Womens Fiction", "Fiction", "Childrens", "Religion", "Nonfiction") # fmt: skip +# Every page under /books/ a crawl that follows each link reaches: the listing and its index.html alias (the +# breadcrumbs name it), the other listing pages, the books, and the categories, the root "Books" one included. +BOOK_SECTION_PAGES = 2 + (BOOK_PAGES - 1) + BOOK_PAGES * BOOKS_PER_PAGE + len(BOOK_CATEGORIES) + 1 + + +def _book_pods(numbers: Any, prefix: str) -> str: + return "".join( + f"
    • " + for i, b in ((i, _book(i)) for i in numbers) + ) + + +def _book_categories(prefix: str) -> str: + """The sidebar of categories that comes before the books on every page, as on books.toscrape.com.""" + links = "".join( + f"
  • {name}
    • " + for n, name in enumerate(BOOK_CATEGORIES, 2) + ) + return f"
    " + + def _books_page(handler: Handler, path: str) -> None: """A miniature of books.toscrape.com (same markup, fewer books).""" + if path.startswith("/books/catalogue/category/books_1/"): # "Books", the root category: the first page of all + body = ( + _book_categories("../../") + + f"

    Books

      {_book_pods(range(1, BOOKS_PER_PAGE + 1), '../../')}
    " + ) + return handler.send(200, layout("Books | Books to Scrape", body)) + if path.startswith("/books/catalogue/category/books/"): # a category: two of the books, and the sidebar + n = int(path.rsplit("_", 1)[1].split("/")[0]) + name = BOOK_CATEGORIES[(n - 2) % len(BOOK_CATEGORIES)] + books = [(n - 2) % (BOOK_PAGES * BOOKS_PER_PAGE) + 1, (n - 1) % (BOOK_PAGES * BOOKS_PER_PAGE) + 1] + body = _book_categories("../../../") + f"

    {name}

      {_book_pods(books, '../../../')}
    " + return handler.send(200, layout(f"{name} | Books to Scrape", body)) if path in ("/books/", "/books/index.html") or path.startswith("/books/catalogue/page-"): n = 1 if not path.startswith("/books/catalogue/page-") else int(path.rsplit("-", 1)[1].split(".")[0]) prefix = "catalogue/" if not path.startswith("/books/catalogue/") else "" @@ -516,9 +555,8 @@ def _books_page(handler: Handler, path: str) -> None: for i, b in ((i, _book(i)) for i in range((n - 1) * BOOKS_PER_PAGE + 1, n * BOOKS_PER_PAGE + 1)) ) nxt = f"" if n < BOOK_PAGES else "" - return handler.send( - 200, layout("All products | Books to Scrape", f"
      {pods}
      {nxt}
    ") - ) + body = _book_categories(prefix) + f"
      {pods}
      {nxt}
    " + return handler.send(200, layout("All products | Books to Scrape", body)) if path.startswith("/books/catalogue/book-"): i = int(path.split("book-")[1].split("/")[0]) b = _book(i) From 53cec344c8d71ee7ad4199ebaf8b3b6e62911100 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 12:08:23 +0000 Subject: [PATCH 15/30] The PyPI browser live test accepts a bot check that stays pypi.org puts a JavaScript check in front of its pages for a browser at times. The test asserted the browser ends up with the whole page, which asks for a way past a check that does not clear; the responsible-access rules say a bot check is the site's answer and is never worked around. The test now skips when bot_check(page) still holds after the wait, and checks the page only when the check cleared by itself or was not there. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- tests/test_live.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_live.py b/tests/test_live.py index 2dfc990..fcc79db 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -128,9 +128,11 @@ def test_pypi_feed_and_cache(tmp_path) -> None: @pytest.mark.browser def test_browser_gets_the_whole_pypi_page() -> None: - # pypi.org may put a JavaScript bot check in front of the page. Either way - # the browser must end up with the complete real page (about 1.2 MB). + # pypi.org may put a JavaScript bot check in front of the page. A check that clears by itself is waited + # for; one that does not is the site's answer, and the page counts as blocked (nothing is done to get past + # it). Otherwise the browser must end up with the complete real page (about 1.2 MB). page = wg.render("https://pypi.org/project/lxml/") - assert not looks_blocked(page) + if bot_check(page): + pytest.skip("pypi.org's bot check did not clear for the browser (detected correctly, not worked around)") assert page.css("h1.project-header__name::text").get("").strip().startswith("lxml") assert len(page.body) > 100_000 From 5f45b43ada2cefacd03080830d51fe0e6d07747f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 12:22:16 +0000 Subject: [PATCH 16/30] The inspect docs say how the sample is drawn The CLI and intelligence pages described inspect's sample as the start page and pages spread across the sitemaps; the pages their links lead to, a few of each URL pattern before more of any one, are part of it too. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- docs/cli.md | 4 +++- docs/intelligence.md | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 222b7c9..1f49c97 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -249,7 +249,9 @@ wintergrab inspect URL [--pages N] [--browser] [--no-robots] [--no-sitemaps] [-o ``` Reads the site's robots.txt and sitemaps, visits `--pages` pages (30): the -start page and a sample spread across the sitemaps, obeying robots.txt. It +start page, a sample spread across the sitemaps, and the pages their links +lead to, a few of each URL pattern before more of any one, obeying +robots.txt. It prints the site's technologies, languages, page types, templates, structured data, links, API endpoints, sitemaps and crawlability, then its sections as a tree (`--depth` levels, 2), its navigation, feeds, paginated listings, diff --git a/docs/intelligence.md b/docs/intelligence.md index 4c71dd8..0a1c9a3 100644 --- a/docs/intelligence.md +++ b/docs/intelligence.md @@ -198,8 +198,9 @@ crawlability: robots.txt allows crawling, crawl-delay 1; 8 page(s) with a canoni ``` (The profile of eleven generated pages of a WordPress shop, with a -robots.txt and sitemaps. On a live site, `inspect` visits the start page and -a sample spread across the sitemaps, so that more templates show up.) +robots.txt and sitemaps. On a live site, `inspect` visits the start page, a +sample spread across the sitemaps, and the pages their links lead to, a few +of each URL pattern before more of any one, so that more templates show up.) | Part | From | |---|---| From 808d8d6bff244ab0ff6f72da597f0ed8c735eec6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 17:18:39 +0000 Subject: [PATCH 17/30] The cards of a page's other records are not its own values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A book page on books.toscrape.com lists six other books ("Products you recently viewed") with their prices and ratings, as most shops' product pages do ("customers also bought"). Running the whole loop there, the owner found the extractor taking those as candidates for the page's own price and rating: confidence fell to 0.35, a rating could be dropped for low confidence, and six of fourteen records raised questions for the reviewer. The page classifier read such a page as a category ("many prices"), so the planner counted one product page in fifteen sampled, and the generated scraper learned a related book's availability and a one-book rating class (p.Three) as the page's. other_records() (parser.autoextract) names the cards of a page's lists of repeated records: the convincing groups detect_records() finds whose records link somewhere, none holding the page's

    . PageContext keeps them, and everything that reads a page leaves them out: the DOM heuristics (_marked, the image finder), the text patterns (the text outside the cards), the model's main text (Selector.markdown(skip=), find_main_content() never picks a card), and the selector learner (_locate demotes matches inside them; learn_schema and the generator's proposals use it). In the visible text an amount of zero ("Tax £0.00") is no price: the real book pages' information table had dragged the page's own price to 0.69, under the confidence a generated scraper learns from. The classifier looks at the price in the title's block: one price in the smallest element around the page's single

    that holds a price says the page is about one priced thing, and "many prices" and "repeated cards with prices" do not count against it. The test site's book pages carry the real ones' cards, information table, "Add to basket" button and article wrapper; the tests that read them expect the selectors that name the book's own block. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 21 ++++++++++ docs/api.md | 2 +- docs/extraction.md | 7 +++- docs/generate.md | 4 +- docs/intelligence.md | 2 +- src/wintergrab/extraction/generate.py | 12 ++++-- src/wintergrab/extraction/page.py | 19 +++++++-- src/wintergrab/extraction/strategies.py | 46 +++++++++++++++++--- src/wintergrab/intel/classify.py | 30 +++++++++++-- src/wintergrab/parser/autoextract.py | 41 ++++++++++++++++-- src/wintergrab/parser/selector.py | 7 ++-- src/wintergrab/parser/text.py | 56 ++++++++++++++++++------- tests/test_builder.py | 7 +++- tests/test_examples.py | 2 +- tests/test_extraction.py | 25 +++++++++++ tests/test_generate.py | 11 +++-- tests/test_intel.py | 26 ++++++++++++ tests/test_text.py | 15 +++++++ tests/testsite.py | 13 +++++- 19 files changed, 297 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index addb022..7a1bc1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1188,6 +1188,27 @@ an error that says so, not a setting silently ignored per_pattern=)`, a fifth of the sample by default), and the goal's words are looked for in the path, not the host. The test site's books section has a category sidebar now, as the real one does. +- **The cards of a page's other records are not its own values.** A book + page on books.toscrape.com lists six other books ("Products you recently + viewed") with their prices and ratings, and so do most shops ("customers + also bought"). The extractor took those as candidates for the page's own + price and rating: confidence fell to 0.35, a rating could be dropped, and + six of fourteen records raised questions for the reviewer (found by the + owner's sandbox run). The cards of a page's lists of repeated records (with + links, not holding the page's `

    `) are left out now, for every field + and whatever the list is called, by the DOM heuristics and the text + patterns alike; a model is not shown them (`PageContext.other_records`, + `Selector.markdown(skip=)`), the selector learner ranks matches inside them + last (a generated scraper read a related book's availability as the + page's), and a card is never taken for the page's main content. In the + visible text an amount of zero ("Tax £0.00") is no price: the real book + pages' information table had dragged the page's own price to 0.69, under + the confidence a generated scraper learns from, so no price selector was + learned. The page classifier reads such a page as a product too: one + price in the title's block says what the page is about, however many prices + the cards below it show (before, "many prices" made it a category, and the + planner then saw one product page in fifteen sampled). The test site's book + pages carry the cards and the "Add to basket" button the real ones have. - **A page's menus are not among its data sources.** `data_sources()` (`wg.sources()`, `wintergrab get --sources`) listed every repeated group `detect_records()` found as HTML records, and ranked them by records times diff --git a/docs/api.md b/docs/api.md index 0bcc03e..b238535 100644 --- a/docs/api.md +++ b/docs/api.md @@ -205,7 +205,7 @@ checks that it is up to date. - `getall(self) -> list[str]` - `learn(self, examples: Mapping[str, str] | list[Mapping[str, str]]) -> Any`: Learn an extraction schema from example values ("scraping by example"). - `links(self, css: str | None = None, *, allow: str | Iterable[str] | None = None, deny: str | Iterable[str] | None = None, domains: str | Iterable[str] | None = None, same_domain: bool = False, unique: bool = True) -> list[str]`: Absolute http(s) URLs of the links on the page (fragments removed). - - `markdown(self, *, main_content: bool = False) -> str`: Convert the element to Markdown (links and images made absolute). + - `markdown(self, *, main_content: bool = False, skip: Collection[Any] = ()) -> str`: Convert the element to Markdown (links and images made absolute). - `next_page(self) -> str | None`: URL of the "next page" link (rel=next, "Next", arrows, numbered pagination...), if any. - `re(self, pattern: str | Pattern[str], flags: int = 0) -> list[str]`: Apply a regex to the text and return every match. - `re_first(self, pattern: str | Pattern[str], default: str | None = None, flags: int = 0) -> str | None` diff --git a/docs/extraction.md b/docs/extraction.md index dbb794e..b02d749 100644 --- a/docs/extraction.md +++ b/docs/extraction.md @@ -109,7 +109,12 @@ names a structured-data path explicitly. The heuristics avoid the usual traps: a struck-through price is the old price (a field named `list_price`, `old_price` or `was_price` takes it, `price` does not), and prices inside related-product blocks, carts, headers and -footers are ignored. +footers are ignored. The cards of a page's lists of other records ("Products +you recently viewed", "customers also bought": a list of repeated cards with +links that does not hold the page's `

    `) are left out for every field, +whatever the list is called: their prices, ratings and pictures are not the +page's own, and a model is not shown them either. In the visible text, an +amount of zero ("Tax £0.00", free shipping) is no price. ## Confidence diff --git a/docs/generate.md b/docs/generate.md index 44d6d27..8b56580 100644 --- a/docs/generate.md +++ b/docs/generate.md @@ -76,8 +76,8 @@ The directory holds: - the tag; - its place under an ancestor with a class; - its place on the page. - - An element in the page's content is preferred to one in a menu or a - breadcrumb. + - An element in the page's content is preferred to one in a menu, a + breadcrumb or a related product's card. - Each selector is tried on every sample page, through the same code the extractor reads selectors with. It is kept only when it reads the value found on **all** of them. diff --git a/docs/intelligence.md b/docs/intelligence.md index 0a1c9a3..1e67497 100644 --- a/docs/intelligence.md +++ b/docs/intelligence.md @@ -36,7 +36,7 @@ It adds up evidence. Each rule that applies gives its type a weight: | a 404 status, "Page not found" in the title | | 8, 5 | | a password field in a small form | | 6 | | the URL | `/p/...`, `/category/`, `/blog/...`, `/careers/`, `?q=`, the site root | 2-4 | -| layout | an add-to-cart button, many repeated cards with prices, pagination, a long `
    `, a byline and a date, code blocks | 1-3 | +| layout | an add-to-cart button, one price in the title's block (a product, whatever the related-product cards below it list), many repeated cards with prices, pagination, a long `
    `, a byline and a date, code blocks | 1-3 | | wording | "Results for", "Sign in", "Apply now", "Get tickets" | 1.5-3 | The weights are evidence strengths chosen by hand, not trained diff --git a/src/wintergrab/extraction/generate.py b/src/wintergrab/extraction/generate.py index 961430b..d5e1779 100644 --- a/src/wintergrab/extraction/generate.py +++ b/src/wintergrab/extraction/generate.py @@ -359,28 +359,34 @@ def _proposals( doc = root.getroottree().getroot() holders: list[tuple[etree._Element, str | None]] = [] raw = fv.raw if isinstance(fv.raw, str) else None if fv.raw is None else str(fv.raw) + others = ctx.other_records # the cards of related products: another record's values, not this page's if raw and len(raw) <= 300: # where the page shows the value as it was found - holders += [(m.element, m.attr) for m in sorted(_locate(doc, raw, ctx.url), key=lambda m: m.rank)[:4]] + located = sorted(_locate(doc, raw, ctx.url, others), key=lambda m: m.rank) + holders += [(m.element, m.attr) for m in located[:4]] value = _plain(fv.value) if not holders or f.type in _WRITTEN_VARIOUSLY: # where the page shows it written another way holders += [(el, None) for el in _holding(doc, value, lambda text: _read(schema, f.name, text))[:3]] if field_kind(f) == "rating": # "star-rating Three": the rating is in the class for el in doc.iter(etree.Element): word = _class_rating(el.get("class") or "") - if word and _same(_read(schema, f.name, word), fv.value): + if word and _same(_read(schema, f.name, word), fv.value) and not _in(el, others): holders.append((el, "class")) break out: dict[str, tuple[int, int, int]] = {} for element, attr in holders: if tag_name(element) in ("html", "head", "body", "title"): continue # the title is the meta tags' to read - aside = int(_context_penalty(element) < 1) # in a menu, a breadcrumb, a footer... + aside = int(_context_penalty(element) < 1 or _in(element, others)) # a menu, a footer, another record suffix = f"::attr({attr})" if attr else "" for query, rank in _selectors_for(element, doc): out.setdefault(query + suffix, (aside, rank, len(out))) return list(out.items()) +def _in(element: etree._Element, nodes: set[Any]) -> bool: + return bool(nodes) and (element in nodes or any(a in nodes for a in element.iterancestors())) + + def _selectors_for(element: etree._Element, doc: etree._Element) -> list[tuple[str, int]]: """Selectors whose first match on the page is ``element``, with how stable each is, the most descriptive first.""" diff --git a/src/wintergrab/extraction/page.py b/src/wintergrab/extraction/page.py index 5c073e1..f74c5d4 100644 --- a/src/wintergrab/extraction/page.py +++ b/src/wintergrab/extraction/page.py @@ -159,13 +159,26 @@ def lines(self) -> list[str]: """Visible text, one line per block element (for ``Label: value`` patterns).""" return [line.strip() for line in self.root.get_text().splitlines() if line.strip()] + @cached_property + def other_records(self) -> set[Any]: + """The cards of the page's lists of other records ("Products you recently viewed", "customers also + bought": see :func:`~wintergrab.parser.autoextract.other_records`), lxml elements. Their values are not + the page's own: the strategies, the model's text and the selector learner leave them out. Empty for a + record of a listing (its element holds nothing else).""" + if self.scope is not None or self.selector.root is None: + return set() + from ..parser.autoextract import other_records + + return other_records(self.selector.root) + @cached_property def main_text(self) -> str: - """The main content as Markdown, trimmed for an extraction model.""" + """The main content as Markdown, trimmed for an extraction model (the other records' cards left out).""" root = self.root - text = root.markdown(main_content=self.scope is None) + skip = self.other_records + text = root.markdown(main_content=self.scope is None, skip=skip) if len(text) < 200 and self.scope is None: # the "main" element was a small part of the page - text = root.markdown() + text = root.markdown(skip=skip) return text[:_MAX_MODEL_TEXT] def __repr__(self) -> str: diff --git a/src/wintergrab/extraction/strategies.py b/src/wintergrab/extraction/strategies.py index f7aae2e..2e4060f 100644 --- a/src/wintergrab/extraction/strategies.py +++ b/src/wintergrab/extraction/strategies.py @@ -516,9 +516,20 @@ def _markers(page: PageContext) -> list[tuple[Selector, str]]: def _marked(page: PageContext, kind: str) -> list[Selector]: - """Elements whose class, id, itemprop, data-testid or rel mentions one of ``kind``'s words, in page order.""" + """Elements whose class, id, itemprop, data-testid or rel mentions one of ``kind``'s words, in page order; + on a whole page, those outside its lists of other records (related products, "recently viewed").""" words = _DOM_WORDS[kind] - return [el for el, marker in _markers(page) if any(word in marker for word in words)] + found = [el for el, marker in _markers(page) if any(word in marker for word in words)] + if found and page.scope is None: + cards = _cards(page) + if cards: + found = [el for el in found if not _inside(el.root, cards)] + return found + + +def _inside(node: Any, nodes: set[Any]) -> bool: + """Whether ``node`` is one of ``nodes`` or inside one.""" + return node is not None and (node in nodes or any(a in nodes for a in node.iterancestors())) def _asides(page: PageContext) -> set[Any]: @@ -537,10 +548,29 @@ def _asides(page: PageContext) -> set[Any]: for header in root.iter("header") if not any(tag_name(a) in ("main", "article") for a in header.iterancestors()) ) + found.update(_cards(page)) cache[key] = found return cache[key] # type: ignore[no-any-return] +def _cards(page: PageContext) -> set[Any]: + """The cards of the page's lists of other records ("Products you recently viewed", "customers also bought": + :attr:`PageContext.other_records`): about other records than the page's own.""" + return page.other_records + + +def _text_outside_cards(page: PageContext) -> str: + """The page's visible text without its lists of other records (a record's element has none).""" + if page.scope is not None: + return page.text + cache = page.__dict__ + if "_text_outside_cards" not in cache: + cards = _cards(page) + root = page.root.root + cache["_text_outside_cards"] = page.text if not cards or root is None else text_content(root, skip_nodes=cards) + return cache["_text_outside_cards"] # type: ignore[no-any-return] + + _CART_BUTTON = re.compile( r"add to (?:cart|bag|basket)|buy now|in den warenkorb|ajouter au panier|añadir al carrito", re.I ) @@ -675,8 +705,11 @@ def _image(self, page: PageContext, f: SchemaField) -> list[tuple[Any, str]]: "main img, article img", "img", ) + cards = _cards(page) if page.scope is None else set() # (related products' pictures are not this page's) for query in queries if page.scope is None else ("img",): for match in page.root.css(query): + if cards and _inside(match.root, cards): + continue value = _element_value(match, "image") if value and not value.startswith("data:"): out.append((value, f"dom:{query.split(',')[0]}")) @@ -861,6 +894,9 @@ def _class_rating(classes: str) -> str | None: ) +_ZERO_AMOUNT = re.compile(r"^\D*0+(?:[.,]0+)?\D*$") + + class Patterns(Strategy): """Regular expressions over the visible text, for values with a recognisable shape. @@ -874,13 +910,13 @@ class Patterns(Strategy): def candidates(self, page: PageContext, f: SchemaField, schema: Schema) -> list[Candidate]: kind = field_kind(f) - text = page.text + text = _text_outside_cards(page) # (a related product's price is not this page's) if not text: return [] found: list[str] = [] label = kind - if kind == "price": - found = [m.group(0).strip() for m in _MONEY_TEXT.finditer(text)] + if kind == "price": # (an amount of zero is a tax or a shipping line, not the record's price) + found = [m.group(0).strip() for m in _MONEY_TEXT.finditer(text) if not _ZERO_AMOUNT.match(m.group(0))] elif kind == "email": found = _EMAIL_TEXT.findall(text) if "@" in text else [] elif kind == "phone": diff --git a/src/wintergrab/intel/classify.py b/src/wintergrab/intel/classify.py index 4d1cb44..d759074 100644 --- a/src/wintergrab/intel/classify.py +++ b/src/wintergrab/intel/classify.py @@ -31,7 +31,7 @@ from urllib.parse import parse_qs, urlsplit from ..extraction.page import STRUCTURED_KINDS, PageContext, schema_types -from ..parser.text import tag_name +from ..parser.text import tag_name, text_content __all__ = ["PAGE_TYPES", "PageClassifier", "PageFeatures", "PageType", "classify_page", "classify_url"] @@ -224,6 +224,22 @@ def text(self) -> str: def prices(self) -> int: return len(set(_MONEY.findall(self.text))) + @cached_property + def title_prices(self) -> int: + """Distinct prices in the title's block: the smallest element around the page's one ``

    `` that holds a + price. One: the page is about one priced thing, whatever the related items or "recently viewed" cards + below it list. A listing's title stands above its cards, so its block holds all their prices.""" + headings = self.page.root.css("h1") + if len(headings) != 1 or headings[0].root is None: + return 0 + for ancestor in headings[0].root.iterancestors(): + found = set(_MONEY.findall(text_content(ancestor))) + if found: + return len(found) + if tag_name(ancestor) == "body": + break + return 0 + @cached_property def password_inputs(self) -> int: return len(self.page.root.css("input[type=password]")) @@ -343,9 +359,15 @@ def schema_rule(f: PageFeatures) -> list[tuple[str, float, str]]: # layout and wording add("product", "add-to-cart button", lambda f: 3.0 if f.cart_button else None) - add("product", "one price near the title", lambda f: 1.5 if 1 <= f.prices <= 3 and len(f.h1) == 1 else None) - add("category", "many prices", lambda f: 3.0 if f.prices >= 5 and f.repeated >= 5 else None) - add("category", "repeated cards with prices", lambda f: 2.0 if f.repeated >= 8 and f.prices >= 3 else None) + # One price in the title's block: a page about one priced thing; the prices of the related products + # or "recently viewed" cards below it are not the page's own. + add("product", "one price near the title", lambda f: 2.5 if f.title_prices == 1 else None) + add("category", "many prices", lambda f: 3.0 if f.prices >= 5 and f.repeated >= 5 and f.title_prices != 1 else None) + add( + "category", + "repeated cards with prices", + lambda f: 2.0 if f.repeated >= 8 and f.prices >= 3 and f.title_prices != 1 else None, + ) add("listing", "repeated cards", lambda f: 3.0 if f.repeated >= 8 and f.prices < 3 else None) add("listing", "pagination", lambda f: 1.0 if f.pagination and f.repeated >= 5 else None) add( diff --git a/src/wintergrab/parser/autoextract.py b/src/wintergrab/parser/autoextract.py index e48d711..7d93a52 100644 --- a/src/wintergrab/parser/autoextract.py +++ b/src/wintergrab/parser/autoextract.py @@ -31,7 +31,15 @@ from .selector import Selector, parse_document from .text import SKIP_TAGS, normalize_space, own_text, tag_name, text_content -__all__ = ["LearnedSchema", "RecordGroup", "auto_extract", "detect_records", "is_stable_class", "learn_schema"] +__all__ = [ + "LearnedSchema", + "RecordGroup", + "auto_extract", + "detect_records", + "is_stable_class", + "learn_schema", + "other_records", +] # Minimum score for auto_extract() to trust the best group (tuned on the test fixtures). _MIN_AUTO_SCORE = 2.0 @@ -1056,6 +1064,25 @@ def detect_records(root: Any, *, min_records: int = 3, max_groups: int = 5) -> l return results +def other_records(root: Any) -> set[etree._Element]: + """The cards of a page's lists of repeated records ("Products you recently viewed", "customers also bought", + "more from this seller"...): the convincing groups :func:`detect_records` finds whose records link somewhere + and none of which holds the page's ``

    ``. On a page about one thing they are about other things: their + prices, ratings and pictures are not the page's own. A list holding the ``

    `` is the page's own (a listing + read as a whole). + """ + doc, _ = _resolve(root) + own: set[etree._Element] = set() + for h1 in doc.iter("h1"): + own.add(h1) + own.update(h1.iterancestors()) + found: set[etree._Element] = set() + for group in detect_records(doc, min_records=2): + if group.convincing and "url" in group.fields and not any(el in own for el in group.elements): + found.update(group.elements) + return found + + def auto_extract(root: Any, base_url: str | None = None, *, min_records: int = 3) -> list[dict[str, Any]]: """Records of the page's main repeating list, as dicts - no selectors needed. @@ -1152,8 +1179,11 @@ def _deepest(matches: list[_Match]) -> list[_Match]: return [m for m in matches if m.element not in covered] -def _locate(root: etree._Element, value: str, base: str | None) -> list[_Match]: - """Every element whose text or attribute holds ``value``, ranked (exact text < attribute < contains).""" +def _locate( + root: etree._Element, value: str, base: str | None, demote: Collection[etree._Element] = () +) -> list[_Match]: + """Every element whose text or attribute holds ``value``, ranked (exact text < attribute < contains). + Matches inside ``demote`` (the cards of the page's other records) rank after every other.""" needle = normalize_space(value) if not needle: return [] @@ -1203,6 +1233,8 @@ def _locate(root: etree._Element, value: str, base: str | None) -> list[_Match]: for m in found: if _context_penalty(m.element) < 1: # breadcrumbs, menus, footers m.rank += 0.4 + if demote and (m.element in demote or any(a in demote for a in m.element.iterancestors())): + m.rank += 5 # another record's value: after even a partial match outside the cards return found @@ -1372,11 +1404,12 @@ def learn_schema(root: Any, examples: Any, *, base_url: str | None = None) -> Le doc, url = _resolve(root) base = _document_base(doc, base_url or url) records = _normalize_examples(examples) + cards = other_records(doc) if len(records) == 1 else set() # one record: the page's, not a related item's located: list[dict[str, _Match]] = [] for record in records: pools: dict[str, list[_Match]] = {} for name, value in record.items(): - found = _locate(doc, value, base) + found = _locate(doc, value, base, cards) if not found: raise ValueError( f"Example value for field {name!r} not found on the page: {value!r}. It must equal (or be " diff --git a/src/wintergrab/parser/selector.py b/src/wintergrab/parser/selector.py index cbdfd9b..6bd7878 100644 --- a/src/wintergrab/parser/selector.py +++ b/src/wintergrab/parser/selector.py @@ -5,7 +5,7 @@ import logging import re import threading -from collections.abc import Callable, Iterable, Iterator, Mapping +from collections.abc import Callable, Collection, Iterable, Iterator, Mapping from re import Pattern from typing import Any, SupportsIndex, Union, overload from urllib.parse import urldefrag, urljoin, urlsplit @@ -247,12 +247,13 @@ def get_text(self) -> str: return self._value or "" return _text.to_text(self._root) - def markdown(self, *, main_content: bool = False) -> str: + def markdown(self, *, main_content: bool = False, skip: Collection[Any] = ()) -> str: """Convert the element to Markdown (links and images made absolute). Args: main_content: Only convert the page's main content (``
    ``, ``
    ``...) instead of the whole element. + skip: Elements (lxml) whose subtrees are left out. """ if self._root is None: return self._value or "" @@ -260,7 +261,7 @@ def markdown(self, *, main_content: bool = False) -> str: if self._doc.type == "html" and _text.tag_name(el) == "html": body = el.find("body") el = body if body is not None else el - return _text.to_markdown(el, self._doc.base_url(self._top())) + return _text.to_markdown(el, self._doc.base_url(self._top()), skip=skip) # ------------------------------------------------------------------ # # regular expressions diff --git a/src/wintergrab/parser/text.py b/src/wintergrab/parser/text.py index d412b2b..f21f8c0 100644 --- a/src/wintergrab/parser/text.py +++ b/src/wintergrab/parser/text.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +from collections.abc import Container from urllib.parse import urljoin from lxml import etree @@ -78,8 +79,14 @@ def tag_name(el: etree._Element) -> str: return tag.lower() -def iter_text(el: etree._Element, skip: frozenset[str] = SKIP_TAGS, block_separator: str = "") -> list[str]: - """All text fragments inside ``el`` in document order, skipping invisible elements. +def iter_text( + el: etree._Element, + skip: frozenset[str] = SKIP_TAGS, + block_separator: str = "", + skip_nodes: Container[etree._Element] = (), +) -> list[str]: + """All text fragments inside ``el`` in document order, skipping invisible elements and the subtrees of + ``skip_nodes``. ``block_separator`` is inserted around block-level elements (and ``
    ``, table cells) so that text from different blocks does not run together. @@ -94,7 +101,7 @@ def iter_text(el: etree._Element, skip: frozenset[str] = SKIP_TAGS, block_separa if not isinstance(node.tag, str): continue name = tag_name(node) - if node is not el and name in skip: + if node is not el and (name in skip or node in skip_nodes): continue is_block = block_separator and (name in BLOCK_TAGS or name in ("br", "td", "th")) if is_block: @@ -109,9 +116,10 @@ def iter_text(el: etree._Element, skip: frozenset[str] = SKIP_TAGS, block_separa return out -def text_content(el: etree._Element) -> str: - """Visible text of ``el`` with whitespace normalized (blocks separated by spaces).""" - return normalize_space("".join(iter_text(el, block_separator=" "))) +def text_content(el: etree._Element, *, skip_nodes: Container[etree._Element] = ()) -> str: + """Visible text of ``el`` with whitespace normalized (blocks separated by spaces), the subtrees of + ``skip_nodes`` left out.""" + return normalize_space("".join(iter_text(el, block_separator=" ", skip_nodes=skip_nodes))) def own_text(el: etree._Element) -> str: @@ -122,14 +130,33 @@ def own_text(el: etree._Element) -> str: def find_main_content(root: etree._Element) -> etree._Element: - """Best guess at the element holding a page's main content.""" - candidates = root.xpath("//main | //*[@role='main'] | //article") + """Best guess at the element holding a page's main content: the longest ``
    ``, ``role="main"`` or + ``
    ``, one of a row of repeated cards (``
  • ``, the related products) + aside; else the body.""" + candidates = [e for e in root.xpath("//main | //*[@role='main'] | //article") if not _repeated(e)] if candidates: return max(candidates, key=lambda e: len(text_content(e))) body = root.find(".//body") if tag_name(root) != "body" else root return body if body is not None else root +def _repeated(el: etree._Element) -> bool: + """Whether ``el`` is one of two or more siblings with the same tag and class (a card in a list), looking + through a wrapper that holds nothing else (``
  • ``).""" + node = el + for _ in range(2): + parent = node.getparent() + if parent is None: + return False + key = (node.tag, node.get("class")) + if sum(1 for s in parent if isinstance(s.tag, str) and (s.tag, s.get("class")) == key) >= 2: + return True + if sum(1 for s in parent if isinstance(s.tag, str)) != 1: + return False + node = parent + return False + + # --------------------------------------------------------------------------- # # Block-aware plain text # --------------------------------------------------------------------------- # @@ -181,8 +208,9 @@ def to_text(el: etree._Element) -> str: class _MarkdownRenderer: - def __init__(self, base_url: str | None) -> None: + def __init__(self, base_url: str | None, skip: Container[etree._Element] = ()) -> None: self.base_url = base_url + self.skip = skip # subtrees left out self.code_blocks: list[str] = [] # -- helpers ---------------------------------------------------------- # @@ -210,7 +238,7 @@ def protect(self, code: str) -> str: # -- dispatch --------------------------------------------------------- # def node(self, el: etree._Element) -> str: name = tag_name(el) - if not name or name in SKIP_TAGS: + if not name or name in SKIP_TAGS or el in self.skip: return "" if name in ("h1", "h2", "h3", "h4", "h5", "h6"): text = normalize_space(self.inline(el)) @@ -279,7 +307,7 @@ def list(self, el: etree._Element, ordered: bool) -> str: lines: list[str] = [] number = int(el.get("start", "1")) if (el.get("start") or "").isdigit() else 1 for li in el: - if tag_name(li) != "li": + if tag_name(li) != "li" or li in self.skip: continue marker = f"{number}. " if ordered else "- " number += 1 @@ -323,9 +351,9 @@ def _clean(text: str) -> str: return "\n".join(out).strip("\n") -def to_markdown(el: etree._Element, base_url: str | None = None) -> str: - """Convert an element (usually ```` or ``
    ``) to Markdown.""" +def to_markdown(el: etree._Element, base_url: str | None = None, *, skip: Container[etree._Element] = ()) -> str: + """Convert an element (usually ```` or ``
    ``) to Markdown, the subtrees in ``skip`` left out.""" try: - return _MarkdownRenderer(base_url).render(el) + return _MarkdownRenderer(base_url, skip).render(el) except RecursionError: # absurdly deep documents return to_text(el) diff --git a/tests/test_builder.py b/tests/test_builder.py index 4e07df3..96aca15 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -77,9 +77,14 @@ def test_what_it_proposes(site, tmp_path) -> None: assert rating["type"] == "rating" and rating["candidates"][0]["selector"].endswith("::attr(class)") specs = detail.table(_number(detail, lambda el: el.tag == "td")) # a record's properties, by their labels assert specs["kind"] == "properties" - assert [(f["name"], f["selector"]) for f in specs["fields"]] == [ + assert [(f["name"], f["selector"]) for f in specs["fields"]] == [ # every row, as the real site's table has ("upc", '//tr[th[normalize-space()="UPC"]]/td'), ("product_type", '//tr[th[normalize-space()="Product Type"]]/td'), + ("price_excl_tax", '//tr[th[normalize-space()="Price (excl. tax)"]]/td'), + ("price_incl_tax", '//tr[th[normalize-space()="Price (incl. tax)"]]/td'), + ("tax", '//tr[th[normalize-space()="Tax"]]/td'), + ("availability", '//tr[th[normalize-space()="Availability"]]/td'), + ("number_of_reviews", '//tr[th[normalize-space()="Number of reviews"]]/td'), ] with pytest.raises(ConfigurationError, match="not in a table"): detail.table(_number(detail, lambda el: el.tag == "h1")) diff --git a/tests/test_examples.py b/tests/test_examples.py index 9af0686..0057546 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -158,7 +158,7 @@ def test_13_record_and_replay(fresh_site, tmp_path) -> None: def test_14_generate_scraper(site, tmp_path, capsys) -> None: result = load("14_generate_scraper").main(site.url + "/books/", directory=str(tmp_path / "books")) - assert result.accepted and result.generated.fields["price"].selector == "p.price_color" + assert result.accepted and result.generated.fields["price"].selector == "div.product_main > p.price_color" printed = capsys.readouterr().out assert "accepted:" in printed and "record(s); the first:" in printed diff --git a/tests/test_extraction.py b/tests/test_extraction.py index 590b714..33f02d1 100644 --- a/tests/test_extraction.py +++ b/tests/test_extraction.py @@ -399,6 +399,31 @@ def test_page_context_accepts_pages_and_rejects_others() -> None: # --------------------------------------------------------------------------- # # command line and crawls # --------------------------------------------------------------------------- # +def test_other_records_cards_are_not_the_pages_own_values(site) -> None: + """A book page lists six other books with their prices and ratings ("Products you recently viewed", as on + books.toscrape.com). The book's own price and rating are read as surely as on a page without them, and the + other books' values are no alternatives (on the real site they were: prices at 0.35 confidence, a rating + dropped, six questions for the reviewer in fourteen records).""" + import wintergrab + + page = wintergrab.get(site.url + "/books/catalogue/book-3/index.html") + extractor = Extractor("product", provenance=True) + record = extractor.extract(page).to_dict() + assert (record["name"], record["price"], record["rating"]) == ("Book number 3", 14.5, 4) + fields = record["_provenance"]["fields"] + assert fields["price"]["confidence"] >= 0.85 and "alternatives" not in fields["price"] + assert fields["rating"]["confidence"] >= 0.7 and "alternatives" not in fields["rating"] + candidates = extractor.candidates(page) + assert {c.raw for c in candidates["price"]} == {"£14.50"} # the cards' prices are not even candidates + assert {c.raw for c in candidates["rating"]} == {"four"} + ctx = PageContext(page) # a model is shown the page without them, and the product's own details + assert len(ctx.other_records) == 6 + assert "UPC" in ctx.main_text and "Book number 3" in ctx.main_text and "Book number 4" not in ctx.main_text + # a listing page read record by record still reads each card + listing = wintergrab.get(site.url + "/books/") + assert len(Extractor("product").extract_all(listing)) == 4 + + def test_cli_get_extract(site, tmp_path, capsys) -> None: schema = tmp_path / "product.json" schema.write_text( diff --git a/tests/test_generate.py b/tests/test_generate.py index 489a175..4d8f8c3 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -24,14 +24,15 @@ def test_selectors_learned_from_sample_pages(site) -> None: generated = generate_schema(_books(site, 1, 2, 3, 5), "product") fields = generated.fields assert fields["name"].selector == "h1" and fields["name"].found_by == "meta" and fields["name"].reproduced == 4 - assert fields["price"].selector == "p.price_color" - assert fields["availability"].selector == "p.availability" # not "p.instock": the class naming the field - assert fields["rating"].selector == "p.star-rating::attr(class)" # "star-rating Three" reads as 3 + assert fields["price"].selector == "div.product_main > p.price_color" # the book's, not a related book's + assert fields["availability"].selector == "div.product_main > p.availability" # not "p.instock": the class + assert fields["rating"].selector == "div.product_main > p.star-rating::attr(class)" # naming the field; the assert fields["category"].selector == "ul.breadcrumb > li:nth-of-type(3) > a" assert fields["url"].status == "skipped" and fields["url"].note == "the page's own address" assert fields["currency"].status == "skipped" and fields["currency"].note == "read from price" assert fields["brand"].status == "not found" - assert generated.schema["price"].selectors == ["p.price_color"] and not generated.base["price"].selectors + assert generated.schema["price"].selectors == ["div.product_main > p.price_color"] # class "Three" reads as 3 + assert not generated.base["price"].selectors assert generated.values[0]["category"] == "Poetry" and generated.methods[0]["price"] == "dom" assert "selectors learned for 5 of 14 fields, from 4 pages" in generated.describe() # a page it was not generated from, read without anything else: the selectors agree with the page's @@ -106,6 +107,8 @@ def test_a_scraper_generated_tested_and_accepted(site, tmp_path) -> None: # the plan names its schema, beside it; its records are read with it plan = GoalPlan.load(directory / "plan.json") assert plan.schema == "schema.json" and plan.extraction_schema()["name"].selectors == ["h1"] + rating = result.generated.fields["rating"].selector # learned on books rated differently: the class they + assert rating.endswith("p.star-rating::attr(class)"), rating # share, not one book's "Three" records = plan.run(max_pages=4, log_level=None).records assert records and all(r["_confidence"] > 0.9 for r in records) # the samples are extraction tests of the scraper diff --git a/tests/test_intel.py b/tests/test_intel.py index 0d904ef..977241a 100644 --- a/tests/test_intel.py +++ b/tests/test_intel.py @@ -110,6 +110,32 @@ def test_a_short_listing_with_a_pager() -> None: assert classify_page(one).type != "category" # one price and a pager: no list +def test_a_product_page_that_shows_other_products_is_a_product() -> None: + """A product page lists other products too ("Products you recently viewed" on books.toscrape.com, "customers + also bought"): their prices and cards are not the page's, whose title's block holds one price. Found on the + real site, where the planner then saw one product page in fifteen sampled.""" + cards = "".join( + f'

  • Book {i}

    ' + f'

    £{10 + i}.50

    • ' + for i in range(6) + ) + product = page( + '

    Sharp Objects

    £47.82

    ' + f'

    Products you recently viewed

      {cards}
    ', + url="https://shop.example/catalogue/sharp-objects_997/index.html", + ) + result = classify_page(product) + assert result.type == "product" and "one price near the title" in result.evidence, result.scores + assert result.scores.get("category", 0) < result.scores["product"] # "many prices" did not count + # the same cards under a listing's title: its block holds all their prices + listing = page( + f'

    Travel

      {cards}
    ', + url="https://shop.example/catalogue/category/books/travel_2/index.html", + ) + result = classify_page(listing) + assert result.type == "category" and "many prices" in result.evidence, result.scores + + def test_related_types_do_not_compete() -> None: # A news page is also an article: "article" scoring high must not make "news" unsure. result = classify_page(news_page()) diff --git a/tests/test_text.py b/tests/test_text.py index c6de017..fe1e4bd 100644 --- a/tests/test_text.py +++ b/tests/test_text.py @@ -42,6 +42,21 @@ def test_markdown_main_content_only() -> None: assert "Home" not in md +def test_main_content_is_not_one_of_the_cards() -> None: + """A product page's "recently viewed" cards are
    s too: the main content is the page's, not the + longest card (as on books.toscrape.com, whose product page is one
    holding the cards).""" + cards = "".join( + f'
    • ' + for i in range(4) + ) + html = f'

    Own title

    Own text.

      {cards}
    ' + md = wg.parse(html).markdown(main_content=True) + assert md.startswith("# Own title") and "Other 1" in md # the body: no card is the main content + parsed = wg.parse(html) + skipped = parsed.markdown(skip=[el.root for el in parsed.css("article")]) + assert "Own text." in skipped and "Other" not in skipped and "£" not in skipped + + def test_get_text_is_block_aware() -> None: text = wg.parse(DOC).css("body").first.get_text() lines = text.splitlines() diff --git a/tests/testsite.py b/tests/testsite.py index ff0f4e1..a8d47c1 100644 --- a/tests/testsite.py +++ b/tests/testsite.py @@ -563,11 +563,20 @@ def _books_page(handler: Handler, path: str) -> None: body = ( f"" + f"
    " # as on the real site: the page's content, the related books included f"

    {b['title']}

    {b['price']}

    " f"

    In stock ({b['stock']} available)

    " - f"

    " + f"

    " f"

    • " - f"
    UPC{b['upc']}
    Product TypeBooks
    " + f"Product TypeBooksPrice (excl. tax){b['price']}" + f"Price (incl. tax){b['price']}Tax£0.00" + f"AvailabilityIn stock ({b['stock']} available)" + f"Number of reviews0" + # the real site's "Products you recently viewed": six other books' cards, prices and ratings included + f"

    Products you recently viewed

    " + f"
      {_book_pods([(i + k) % (BOOK_PAGES * BOOKS_PER_PAGE) + 1 for k in range(6)], '../')}
    " + f"
    " ) return handler.send(200, layout(f"{b['title']} | Books to Scrape", body)) return handler.send(404, layout("Not found", "

    nope

    ")) From 14bee5cb67f1a6ac4dcc9173a440c4755b80c71d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 17:20:31 +0000 Subject: [PATCH 18/30] The survey samples more of the pages the goal wants On books.toscrape.com a goal's 15-page survey held one book page: the pattern spreading gives each URL pattern a few pages, and with fifty categories, a root category, pagination and the site's index alias all taking their turn, the books' second turn never came. The plan's estimates rested on that one page ("records: about 0"). Two rules now let the record pages through. The links of a page's record cards (the convincing repeated groups detect_records() finds, with links: what a listing lists) are followed in preference order alone, before the other patterns' next turn, until a page of their pattern turns out not to be wanted; and once a sampled page is one of the goal's record pages (survey_site(wanted=), which survey_for answers with the page classifier), the links of its pattern are followed freely too. The first rule is structural, so it does not depend on which page returns first. On a real-scale copy of the site a 15-page sample holds ten book pages instead of one; on the test site's books section, nine. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 6 ++++++ docs/api.md | 2 +- docs/goals.md | 4 +++- src/wintergrab/goals/plan.py | 9 ++++++++ src/wintergrab/intel/survey.py | 39 ++++++++++++++++++++++++++++++---- tests/test_goals.py | 19 +++++++++++++++++ 6 files changed, 73 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a1bc1a..b7baebe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1209,6 +1209,12 @@ an error that says so, not a setting silently ignored the cards below it show (before, "many prices" made it a category, and the planner then saw one product page in fifteen sampled). The test site's book pages carry the cards and the "Add to basket" button the real ones have. +- **The survey samples more of the pages the goal wants.** The links of a + page's record cards (what a listing lists) and, once a sampled page is one + of the goal's record pages, the links of its URL pattern are followed in + preference order alone, before the other patterns' next turn + (`survey_site(wanted=)`): a 15-page sample of books.toscrape.com holds + ten book pages instead of one, and the plan's estimates rest on them. - **A page's menus are not among its data sources.** `data_sources()` (`wg.sources()`, `wintergrab get --sources`) listed every repeated group `detect_records()` found as HTML records, and ranked them by records times diff --git a/docs/api.md b/docs/api.md index b238535..da08368 100644 --- a/docs/api.md +++ b/docs/api.md @@ -906,7 +906,7 @@ checks that it is up to date. - **`pagination_of(url: str, answer: Any = None, *, request: Mapping[str, Any] | None = None, records: int | None = None) -> Pagination | None`**. How the pages of the API ``url`` go, from its query parameters (or ``request``: a GraphQL call's variables, a JSON request body) and its ``answer``, or ``None`` when neither says. - **`read_sitemaps(origin: str, robots_text: str | None = None, *, max_sitemaps: int = 10, max_entries: int = 50000, **fetch_options: Any) -> SitemapRead`**. The pages a site's sitemaps list: those named in robots.txt, or ``/sitemap.xml``, following sitemap indexes, up to ``max_sitemaps`` sitemaps and ``max_entries`` pages. - **`script_endpoints(selector: Any, url: str) -> list[tuple[str | None, str]]`**. The API endpoints a page's inline scripts and ``data-`` attributes name, as ``(method, endpoint)``: calls (``fetch("/api/...")``, axios, jQuery, ``xhr.open``) and API-looking paths (``/api/``, ``/graphql``), query values left out (``https://shop.example/api/products?page=``). -- **`survey_site(url: str, *, pages: int = 30, sitemaps: bool = True, obey_robots: bool = True, browser: bool = False, timeout: float = 20.0, keep_pages: bool = False, prefer: Callable[[str], bool | float] | None = None, extra_urls: Iterable[str] = (), per_pattern: int | None = None, log_level: str | None = 'WARNING', **spider_settings: Any) -> SiteSurvey`**. Read ``url``'s site: robots.txt, sitemaps, and ``pages`` pages, into a :class:`SiteSurvey`. +- **`survey_site(url: str, *, pages: int = 30, sitemaps: bool = True, obey_robots: bool = True, browser: bool = False, timeout: float = 20.0, keep_pages: bool = False, prefer: Callable[[str], bool | float] | None = None, extra_urls: Iterable[str] = (), per_pattern: int | None = None, wanted: Callable[[Response], bool] | None = None, log_level: str | None = 'WARNING', **spider_settings: Any) -> SiteSurvey`**. Read ``url``'s site: robots.txt, sitemaps, and ``pages`` pages, into a :class:`SiteSurvey`. ## `wintergrab.intel.serp`: Search results diff --git a/docs/goals.md b/docs/goals.md index 32b743a..dff54f3 100644 --- a/docs/goals.md +++ b/docs/goals.md @@ -98,7 +98,9 @@ pages their links lead to. Pages in the part of the site the goal names come first, then pages that look like the goal's records, then their listings; and a few pages of each URL pattern come before more of any one pattern, so a site that lists fifty categories before its products still has product -pages in the sample. It learns from the sample: +pages in the sample; the records a listing lists, and more pages of a +pattern once a sampled page of it holds one of the goal's records, come +before the other patterns' next turn. It learns from the sample: - which pages hold the records: classified as the kind's pages (product, article, job...), or, for pages that list nothing, giving a record with its diff --git a/src/wintergrab/goals/plan.py b/src/wintergrab/goals/plan.py index 562a44d..7ad98d3 100644 --- a/src/wintergrab/goals/plan.py +++ b/src/wintergrab/goals/plan.py @@ -56,6 +56,7 @@ from .goal import Goal if TYPE_CHECKING: + from ..fetchers.response import Response from ..intel.survey import SiteSurvey from .run import GoalResult @@ -469,6 +470,13 @@ def prefer(url: str) -> int: text = f"{where.path}?{where.query}".lower() # (not the host: books.example names every page "books") return score + (1 if any(w in text for w in words) else 0) + def wanted(page: Response) -> bool: + """A sampled page that is one of the goal's record pages (as :func:`plan_goal` will see it): the URLs + of its pattern can look like anything (``/catalogue/{slug}_{id}/index.html`` reads as a category), so + more of them are sampled before the other patterns get their next turn.""" + classified = classify_page(PageContext(page)) + return classified.confidence >= _SURE and classified.type in kind.page_types + return survey_site( site, pages=sample, @@ -477,6 +485,7 @@ def prefer(url: str) -> int: timeout=timeout, keep_pages=True, prefer=prefer, + wanted=wanted, log_level=log_level, **dict(settings or {}), ) diff --git a/src/wintergrab/intel/survey.py b/src/wintergrab/intel/survey.py index 699fdbb..68bc1d3 100644 --- a/src/wintergrab/intel/survey.py +++ b/src/wintergrab/intel/survey.py @@ -12,7 +12,8 @@ spread across the sitemaps (the ones ``prefer`` likes first) and the pages they link to, a few of each URL pattern before more of any one pattern, so that a site's record pages are sampled however many category or tag links come first -on its pages. Fetched politely: robots.txt is obeyed unless told otherwise. +on its pages; and once a sampled page is what the survey looks for (``wanted``), +more of its pattern. Fetched politely: robots.txt is obeyed unless told otherwise. """ from __future__ import annotations @@ -148,14 +149,20 @@ class _SurveySpider(Spider): #: turn: the sample is spread across the patterns a site has, so its record pages are among the pages sampled #: however many category or tag links come first on a page. ``0``: page order alone. per_pattern: int = 3 + #: ``wanted(response) -> bool``: a sampled page that is what the survey looks for. Links of its URL pattern + #: are followed in ``prefer`` order alone from then on, before the other patterns' next turn; so are the + #: links of a page's record cards (what a listing lists), until a page of their pattern is not wanted. + wanted: Any = None def __init__(self, **settings: Any) -> None: super().__init__(**settings) self.kept: list[Response] = [] self._patterns: Counter[str] = Counter() # URLs queued so far, by pattern self._queued: set[str] = set() + self._wanted_patterns: set[str] = set() + self._unwanted_patterns: set[str] = set() - def _request(self, url: str, *, spread: bool = True) -> Request: + def _request(self, url: str, *, spread: bool = True, listed: bool = False) -> Request: request = Request(url, dont_filter=False) if self.capture_api: request.options["capture"] = True @@ -163,7 +170,9 @@ def _request(self, url: str, *, spread: bool = True) -> Request: if self.per_pattern and url not in self._queued: self._queued.add(url) pattern = url_template(url) - if spread: # the n-th link of a pattern waits for the first of every other: a tier lower each time + free = pattern in self._wanted_patterns or (listed and pattern not in self._unwanted_patterns) + if spread and not free: + # the n-th link of a pattern waits for the first of every other: a tier lower each time priority -= 100 * (self._patterns[pattern] // self.per_pattern) self._patterns[pattern] += 1 request.priority = priority @@ -178,13 +187,28 @@ def parse(self, response: Response) -> Any: self.kept.append(response) if not response.is_html: return + listed: set[str] = set() + if self.wanted is not None: + pattern = url_template(response.url) + (self._wanted_patterns if self.wanted(response) else self._unwanted_patterns).add(pattern) + listed = _listed(response) for link in response.links(same_domain=True): - yield self._request(link) + yield self._request(link, listed=link in listed) next_url = response.next_page() if next_url: yield self._request(next_url) +def _listed(response: Response) -> set[str]: + """The links of the page's lists of records (its cards' own links): what a listing lists, before it is + known what kind of page they lead to.""" + found: set[str] = set() + for group in response.detect_records(): + if group.convincing and "url" in group.fields: + found.update(url for record in group.extract(response.url) if isinstance(url := record.get("url"), str)) + return found + + def _spread(urls: list[str], count: int) -> list[str]: """``count`` URLs spread evenly across ``urls``.""" if count <= 0 or not urls: @@ -205,6 +229,7 @@ def survey_site( prefer: Callable[[str], bool | float] | None = None, extra_urls: Iterable[str] = (), per_pattern: int | None = None, + wanted: Callable[[Response], bool] | None = None, log_level: str | None = "WARNING", **spider_settings: Any, ) -> SiteSurvey: @@ -222,6 +247,11 @@ def survey_site( per_pattern: Links of one URL pattern followed before the other patterns get their turn (by default a fifth of ``pages``, two at least), so the sample is spread across the site's patterns; ``0``: page order alone. + wanted: ``wanted(page) -> bool``: a sampled page that is what the survey looks for (a goal's record + page). Links of its URL pattern are then followed in ``prefer`` order alone, before the other + patterns' next turn, and so are the links of a page's record cards (what a listing lists) until a + page of their pattern is not wanted: a site's fifty categories no longer crowd its record pages out + of the sample. """ from ..fetchers import Fetcher @@ -270,6 +300,7 @@ def survey_site( keep_pages=keep_pages, prefer=prefer, per_pattern=max(2, pages // 5) if per_pattern is None else per_pattern, + wanted=wanted, timeout=timeout, output=None, keep_items=False, diff --git a/tests/test_goals.py b/tests/test_goals.py index 5a6c8d3..00c17ff 100644 --- a/tests/test_goals.py +++ b/tests/test_goals.py @@ -241,6 +241,25 @@ def test_the_survey_samples_every_url_pattern(site) -> None: assert not any("looked like a product page" in w for w in plan.sites[0].warnings) +def test_the_survey_samples_more_of_the_pages_the_goal_wants(site) -> None: + """Once a sampled page is one of the goal's record pages, more links of its URL pattern are followed before + the other patterns' next turn, so a site's many categories do not crowd its record pages out of the sample + (on books.toscrape.com a 15-page sample held one book page; the plan's estimates rested on it).""" + from collections import Counter + + from wintergrab import url_template + from wintergrab.goals.plan import survey_for + + goal = read("Find all books rated 4 stars or more with title, price and rating", sites=[site.url + "/books/"]) + survey = survey_for(goal, site.url + "/books/", sample=15, log_level=None) + patterns = Counter(url_template(p.url, include_host=False) for p in survey.pages) + assert len(survey.pages) == 15 + assert patterns["/books/catalogue/{slug}/index.html"] >= 6 # the books: more than a pattern's share + assert patterns["/books/catalogue/category/books/{slug}/index.html"] == 3 # the categories: their share + plan = plan_goal(goal, surveys={site.url + "/books/": survey}) + assert plan.sites[0].sample["record_pages"] >= 6 + + def test_the_whole_loop_in_one_run(site, tmp_path, capsys) -> None: """provenance=True, heal=DIR: records say where each value came from, the plan's schema is read by a self-healing extractor kept in DIR (a fixture per site, questions in DIR/review.jsonl), and a redesign is From 73817ddf11780b28a9bb84851d3f35d066918c81 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 17:20:31 +0000 Subject: [PATCH 19/30] A plan starts from a page that exists The part of a site a goal is about may have no page of its own: on books.toscrape.com the categories live under /catalogue/category/books/, and /catalogue/category/books answers 403. _section_url() made that URL up from the section's path when no sampled page or link had it, and the crawl started with an error and a wasted request (found by the owner's sandbox run). It now takes a page whose path is the section's, its index.html included; else the first sampled page or link under the section, from which the crawl follows the section's own links; else the survey's start page. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 4 ++++ src/wintergrab/goals/plan.py | 16 ++++++++++++---- tests/test_goals.py | 17 +++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7baebe..6d59f84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1215,6 +1215,10 @@ an error that says so, not a setting silently ignored preference order alone, before the other patterns' next turn (`survey_site(wanted=)`): a 15-page sample of books.toscrape.com holds ten book pages instead of one, and the plan's estimates rest on them. +- **A plan starts from a page that exists.** The section a goal is about may + have no page of its own (`/catalogue/category/books` on books.toscrape.com + answers 403, its categories live below it): the crawl starts from a + sampled page under the section now, never from a URL made up from the path. - **A page's menus are not among its data sources.** `data_sources()` (`wg.sources()`, `wintergrab get --sources`) listed every repeated group `detect_records()` found as HTML records, and ranked them by records times diff --git a/src/wintergrab/goals/plan.py b/src/wintergrab/goals/plan.py index 7ad98d3..7bb938d 100644 --- a/src/wintergrab/goals/plan.py +++ b/src/wintergrab/goals/plan.py @@ -725,16 +725,24 @@ def _api_estimate(goal: Goal, plan: SitePlan, source: ApiSource, survey: SiteSur def _section_url(survey: SiteSurvey, path: str) -> str: - """A real URL for a section path (``/books`` may only answer as ``/books/``): the start URL, or one - of the sampled pages or their links, when their path is the section's.""" + """A URL that exists for a section path: the start URL, a sampled page or one of their links whose path is + the section's (``/books`` may only answer as ``/books/`` or ``/books/index.html``); else the first sampled + page under the section, or one of the links, from which the crawl follows the section's own links; else + the start URL. Never a URL made up from the path: a site whose products live under ``/catalogue/category/ + books/`` has no page at ``/catalogue/category/books`` (a 403 the crawl would give up on).""" + section = path.rstrip("/") or "/" candidates = [survey.url, *(page.url for page in survey.pages)] for page in survey.pages: if page.is_html: candidates.extend(page.links(same_domain=True)) for url in candidates: - if (urlsplit(url).path.rstrip("/") or "/") == path.rstrip("/"): + page_path = urlsplit(url).path.rstrip("/") or "/" + if page_path == section or page_path in (f"{section}/index.html", f"{section}/index.htm"): return url.split("#")[0] - return survey.origin + path + for url in candidates: + if _under(urlsplit(url).path, [section]): + return url.split("#")[0] + return survey.url def _holds(condition: Expression, record: dict[str, Any]) -> bool: diff --git a/tests/test_goals.py b/tests/test_goals.py index 00c17ff..570b947 100644 --- a/tests/test_goals.py +++ b/tests/test_goals.py @@ -260,6 +260,23 @@ def test_the_survey_samples_more_of_the_pages_the_goal_wants(site) -> None: assert plan.sites[0].sample["record_pages"] >= 6 +def test_a_plan_starts_from_a_page_that_exists(site) -> None: + """The part of a site a goal is about may have no page of its own: on books.toscrape.com the categories + live under /catalogue/category/books/, and /catalogue/category/books answers 403. The crawl starts from a + page under the section then, never from a URL made up from the path.""" + from wintergrab.goals.plan import _section_url + from wintergrab.intel.survey import survey_site + + survey = survey_site(site.url + "/books/", pages=6, keep_pages=True, log_level=None) + start = _section_url(survey, "/books/catalogue/category/books") + assert start.startswith(site.url + "/books/catalogue/category/books/") and start.endswith("/index.html") + assert _section_url(survey, "/books/catalogue/category/books_1") == ( + site.url + "/books/catalogue/category/books_1/index.html" + ) + assert _section_url(survey, "/books") == site.url + "/books/" # the start page itself + assert _section_url(survey, "/nowhere") == site.url + "/books/" # not origin + path + + def test_the_whole_loop_in_one_run(site, tmp_path, capsys) -> None: """provenance=True, heal=DIR: records say where each value came from, the plan's schema is read by a self-healing extractor kept in DIR (a fixture per site, questions in DIR/review.jsonl), and a redesign is From fc0792e9c939861a39b21415768a6c4ae97b7e7a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 19:19:27 +0000 Subject: [PATCH 20/30] The DuckDB output waits for a held file on Windows too A DuckDB file another program has open is waited for a few seconds before a crawl gives up on it. The retry recognised DuckDB's own messages ("Could not set lock on file", "File is already open in") but not the system's on Windows, "The process cannot access the file because it is being used by another process", so a crawl starting while a reader had the file open failed at once (the Windows CI job, on a test that holds the file for half a second). Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 4 ++++ src/wintergrab/storage/duckdb.py | 6 +++--- tests/test_storage.py | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d59f84..3959784 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1219,6 +1219,10 @@ an error that says so, not a setting silently ignored have no page of its own (`/catalogue/category/books` on books.toscrape.com answers 403, its categories live below it): the crawl starts from a sampled page under the section now, never from a URL made up from the path. +- **The DuckDB output waits for a held file on Windows too.** A file another + program has open is waited for a few seconds; on Windows the system's + message for it ("being used by another process") was not read as a hold, + so a crawl starting while a reader had the file open failed at once. - **A page's menus are not among its data sources.** `data_sources()` (`wg.sources()`, `wintergrab get --sources`) listed every repeated group `detect_records()` found as HTML records, and ranked them by records times diff --git a/src/wintergrab/storage/duckdb.py b/src/wintergrab/storage/duckdb.py index 6143742..5463409 100644 --- a/src/wintergrab/storage/duckdb.py +++ b/src/wintergrab/storage/duckdb.py @@ -70,10 +70,10 @@ def _connect(path: Path, *, read_only: bool) -> Any: def _held(exc: BaseException) -> bool: - """Whether DuckDB could not open a file because another program has it ("Could not set lock on file"; - on Windows, "File is already open in").""" + """Whether DuckDB could not open a file because another program has it ("Could not set lock on file"; on + Windows, "File is already open in", or the system's "being used by another process").""" message = str(exc).lower() - return "could not set lock" in message or "already open" in message + return "could not set lock" in message or "already open" in message or "used by another process" in message def _objects(connection: Any) -> dict[str, str]: diff --git a/tests/test_storage.py b/tests/test_storage.py index aa77363..85bff87 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -235,7 +235,7 @@ def hold(seconds: float) -> subprocess.Popen[str]: try: with pytest.raises( ConfigurationError, - match=r"(?s)cannot write items to .*: IO Error: .*(Could not set lock|File is already open)", + match=r"(?s)cannot write items to .*: IO Error: .*(Could not set lock|File is already open|being used by another process)", ): open_exporter(path, append=True) # (a crawl starting now is told at once) with pytest.raises(ExportError, match=r"could not write .*items are kept in .*\.items\.duckdb\.spool\.jsonl"): From 40f78ea69e354f4a76c11cdf09cd393ff161b923 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 19:19:27 +0000 Subject: [PATCH 21/30] The frontier's volume test checks scaling, not the runner's speed The test pushed and popped 200,000 requests through the disk frontier and asserted the whole took under 120 seconds. On this session's machine it takes 23 s; the Windows CI runner took 173 s one day after passing the day before, and the job failed on nothing in the code. The bound stood for "throughput stays flat as the queue grows", so the test now asserts that: the last quarter of the pushes, and of the pops with their acks, costs no more than twice the first (here 3.8 s against 4.1 s, and 1.9 s against 1.8 s). The rates are still printed. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 4 ++++ tests/test_frontier.py | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3959784..70a14c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1223,6 +1223,10 @@ an error that says so, not a setting silently ignored program has open is waited for a few seconds; on Windows the system's message for it ("being used by another process") was not read as a hold, so a crawl starting while a reader had the file open failed at once. +- The disk frontier's volume test checks that the cost per request stays + flat over 200,000 requests, in place of a wall-clock bound a slow shared + CI runner cannot meet (the Windows job took 173 s for what takes 23 s on a + developer machine). - **A page's menus are not among its data sources.** `data_sources()` (`wg.sources()`, `wintergrab get --sources`) listed every repeated group `detect_records()` found as HTML records, and ranked them by records times diff --git a/tests/test_frontier.py b/tests/test_frontier.py index a38bc97..649f19e 100644 --- a/tests/test_frontier.py +++ b/tests/test_frontier.py @@ -465,13 +465,45 @@ def _traced_peak(run: Callable[[], object]) -> int: tracemalloc.stop() +def _quarters(n: int, step: Callable[[int], None]) -> list[float]: + """How long each quarter of ``n`` calls of ``step`` took.""" + out: list[float] = [] + start = time.perf_counter() + for i in range(n): + step(i) + if (i + 1) % (n // 4) == 0: + out.append(time.perf_counter() - start) + start = time.perf_counter() + return out + + def test_volume_throughput_and_flat_memory(db: Path) -> None: - n = 200_000 # 20 domains + """200,000 requests over 20 domains: the cost of a push, and of a pop with its ack, stays flat as the queue + grows (the last quarter of the run costs no more than twice the first), and the memory stays small. No + wall-clock bound: a shared CI runner can be three times slower one day than the next (the Windows job took + 173 s for what takes 23 s here, and failed a 120 s bound).""" + n = 200_000 disk = DiskScheduler(db, FakeSpider()) - push_s, pop_s = _volume_run(disk, n) + push_quarters = _quarters( + n, lambda i: disk.push(Request(f"https://site{i % 20}.test/item/{i}?page={i // 20}", meta={"depth": 1})) + ) + assert len(disk) == n + throttle = AutoThrottle(randomize=False) + now = time.monotonic() + + def pop(_: int) -> None: + request, _wait = disk.pop_ready(throttle, now) + assert request is not None + disk.ack(request) + + pop_quarters = _quarters(n, pop) + assert disk.pop_ready(throttle, now)[0] is None and len(disk) == 0 disk.close() + push_s, pop_s = sum(push_quarters), sum(pop_quarters) print(f"\nDiskScheduler, {n:,} requests: push {n / push_s:,.0f}/s, pop+ack {n / pop_s:,.0f}/s") - assert push_s + pop_s < 120 + print(f"quarters: push {[round(q, 2) for q in push_quarters]} s, pop+ack {[round(q, 2) for q in pop_quarters]} s") + assert push_quarters[-1] < 2 * push_quarters[0], push_quarters + assert pop_quarters[-1] < 2 * pop_quarters[0], pop_quarters # tracemalloc makes these runs ~4x slower, so compare memory on a smaller load. The in-memory Scheduler peaks # once everything is pushed; the DiskScheduler run includes the pops (refills) and its 1M-key Bloom filter. From b6f988b53cf6d1e8fa50514298068984f613af32 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 20:12:30 +0000 Subject: [PATCH 22/30] A page the classifier is unsure about says nothing to the survey The owner's second sandbox run still sampled three book pages in fifteen on books.toscrape.com. A diagnostic on the real pages showed why: the site's product pages have no "Add to basket" button of their own, and the first one sampled had no related cards either, so the classifier was unsure (2.5 for the price near the title against 2.0 for the URL's "catalogue", confidence 0.09). The survey took "unsure" for "not wanted", and stopped following the book links freely after that one page. The other book pages only looked sure because the cart buttons of their related cards counted as evidence, which is the wrong reason. The survey's wanted callback now answers True, False or None, and None teaches it nothing about the pattern; what a listing lists stays followed freely until a page of the pattern is surely something else. The classifier reads a product page from its own evidence: one price in the title's block (its other records' cards aside, so a category listing two books is no product) and the page's own availability line next to it count, a related card's cart button does not, and a section name in the URL ("catalogue", "shop", "collections") is weaker evidence of a listing than "category". The planner's grid of record cards must link somewhere (a book's information table repeats rows but is no listing), and a listing of two is a listing. The test site's book pages have no cart button of their own now, as the real ones. On a real-scale copy of the site a 15-page sample holds ten book pages. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 14 +++++--- docs/api.md | 2 +- docs/intelligence.md | 4 +-- src/wintergrab/goals/plan.py | 23 ++++++++----- src/wintergrab/intel/classify.py | 58 +++++++++++++++++++++++++------- src/wintergrab/intel/survey.py | 26 +++++++------- tests/test_goals.py | 31 +++++++++++++++-- tests/test_intel.py | 26 ++++++++++++++ tests/testsite.py | 3 +- 9 files changed, 142 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70a14c8..a04b612 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1205,15 +1205,19 @@ an error that says so, not a setting silently ignored pages' information table had dragged the page's own price to 0.69, under the confidence a generated scraper learns from, so no price selector was learned. The page classifier reads such a page as a product too: one - price in the title's block says what the page is about, however many prices - the cards below it show (before, "many prices" made it a category, and the - planner then saw one product page in fifteen sampled). The test site's book - pages carry the cards and the "Add to basket" button the real ones have. + price in the title's block says what the page is about, and so does the + page's own availability line next to it, however many prices the cards + below show (before, "many prices" made it a category, and the planner then + saw one product page in fifteen sampled); a related card's cart button is + not the page's, and a section name in the URL (`/catalogue/`, `/shop/`) is + weaker evidence of a listing than `/category/`. The test site's book pages + carry the cards and the information table the real ones have. - **The survey samples more of the pages the goal wants.** The links of a page's record cards (what a listing lists) and, once a sampled page is one of the goal's record pages, the links of its URL pattern are followed in preference order alone, before the other patterns' next turn - (`survey_site(wanted=)`): a 15-page sample of books.toscrape.com holds + (`survey_site(wanted=)`); a page the classifier is unsure about says + nothing about its pattern. A 15-page sample of books.toscrape.com holds ten book pages instead of one, and the plan's estimates rest on them. - **A plan starts from a page that exists.** The section a goal is about may have no page of its own (`/catalogue/category/books` on books.toscrape.com diff --git a/docs/api.md b/docs/api.md index da08368..ddad903 100644 --- a/docs/api.md +++ b/docs/api.md @@ -906,7 +906,7 @@ checks that it is up to date. - **`pagination_of(url: str, answer: Any = None, *, request: Mapping[str, Any] | None = None, records: int | None = None) -> Pagination | None`**. How the pages of the API ``url`` go, from its query parameters (or ``request``: a GraphQL call's variables, a JSON request body) and its ``answer``, or ``None`` when neither says. - **`read_sitemaps(origin: str, robots_text: str | None = None, *, max_sitemaps: int = 10, max_entries: int = 50000, **fetch_options: Any) -> SitemapRead`**. The pages a site's sitemaps list: those named in robots.txt, or ``/sitemap.xml``, following sitemap indexes, up to ``max_sitemaps`` sitemaps and ``max_entries`` pages. - **`script_endpoints(selector: Any, url: str) -> list[tuple[str | None, str]]`**. The API endpoints a page's inline scripts and ``data-`` attributes name, as ``(method, endpoint)``: calls (``fetch("/api/...")``, axios, jQuery, ``xhr.open``) and API-looking paths (``/api/``, ``/graphql``), query values left out (``https://shop.example/api/products?page=``). -- **`survey_site(url: str, *, pages: int = 30, sitemaps: bool = True, obey_robots: bool = True, browser: bool = False, timeout: float = 20.0, keep_pages: bool = False, prefer: Callable[[str], bool | float] | None = None, extra_urls: Iterable[str] = (), per_pattern: int | None = None, wanted: Callable[[Response], bool] | None = None, log_level: str | None = 'WARNING', **spider_settings: Any) -> SiteSurvey`**. Read ``url``'s site: robots.txt, sitemaps, and ``pages`` pages, into a :class:`SiteSurvey`. +- **`survey_site(url: str, *, pages: int = 30, sitemaps: bool = True, obey_robots: bool = True, browser: bool = False, timeout: float = 20.0, keep_pages: bool = False, prefer: Callable[[str], bool | float] | None = None, extra_urls: Iterable[str] = (), per_pattern: int | None = None, wanted: Callable[[Response], bool | None] | None = None, log_level: str | None = 'WARNING', **spider_settings: Any) -> SiteSurvey`**. Read ``url``'s site: robots.txt, sitemaps, and ``pages`` pages, into a :class:`SiteSurvey`. ## `wintergrab.intel.serp`: Search results diff --git a/docs/intelligence.md b/docs/intelligence.md index 1e67497..8afaaf2 100644 --- a/docs/intelligence.md +++ b/docs/intelligence.md @@ -35,8 +35,8 @@ It adds up evidence. Each rule that applies gives its type a weight: | `og:type` | `product`, `article`, `profile` | 3 | | a 404 status, "Page not found" in the title | | 8, 5 | | a password field in a small form | | 6 | -| the URL | `/p/...`, `/category/`, `/blog/...`, `/careers/`, `?q=`, the site root | 2-4 | -| layout | an add-to-cart button, one price in the title's block (a product, whatever the related-product cards below it list), many repeated cards with prices, pagination, a long `
    `, a byline and a date, code blocks | 1-3 | +| the URL | `/p/...`, `/category/` (`/shop/`, `/catalogue/`: a section that holds the products too, weaker), `/blog/...`, `/careers/`, `?q=`, the site root | 1-4 | +| layout | an add-to-cart button of the page's own, one price in the title's block and an availability line next to it (a product, whatever the related-product cards below it list), many repeated cards with prices, pagination, a long `
    `, a byline and a date, code blocks | 1-3 | | wording | "Results for", "Sign in", "Apply now", "Get tickets" | 1.5-3 | The weights are evidence strengths chosen by hand, not trained diff --git a/src/wintergrab/goals/plan.py b/src/wintergrab/goals/plan.py index 7bb938d..3f3f9fa 100644 --- a/src/wintergrab/goals/plan.py +++ b/src/wintergrab/goals/plan.py @@ -470,12 +470,15 @@ def prefer(url: str) -> int: text = f"{where.path}?{where.query}".lower() # (not the host: books.example names every page "books") return score + (1 if any(w in text for w in words) else 0) - def wanted(page: Response) -> bool: - """A sampled page that is one of the goal's record pages (as :func:`plan_goal` will see it): the URLs - of its pattern can look like anything (``/catalogue/{slug}_{id}/index.html`` reads as a category), so - more of them are sampled before the other patterns get their next turn.""" + def wanted(page: Response) -> bool | None: + """Whether a sampled page is one of the goal's record pages (as :func:`plan_goal` will see it), or + ``None`` when the classifier is unsure: the URLs of its pattern can look like anything + (``/catalogue/{slug}_{id}/index.html`` reads as a category), so more of them are sampled before the + other patterns get their next turn.""" classified = classify_page(PageContext(page)) - return classified.confidence >= _SURE and classified.type in kind.page_types + if classified.confidence < _SURE: + return None + return classified.type in kind.page_types return survey_site( site, @@ -533,11 +536,15 @@ def _plan_site(goal: Goal, survey: SiteSurvey, rendered: list[Any] | None = None ctx = PageContext(page) classified = classify_page(ctx) page_type = classified.type if classified.confidence >= _SURE else "unknown" # a weak verdict decides nothing - # A list of records: classified as one, or a grid of cards (a strong repeating group; a - # breadcrumb or a small table repeats too, weakly). The classifier's record verdict wins. + # A list of records: classified as one, or a grid of record cards (a repeating group whose records link + # somewhere; a breadcrumb or a menu repeats too, weakly, and a table of properties links nowhere). The + # classifier's record verdict wins. lists = page_type in kind.listing_types or ( page_type not in kind.page_types - and any(len(g.elements) >= 3 and g.score >= _GRID_SCORE for g in ctx.selector.detect_records()) + and any( + len(g.elements) >= 3 and g.score >= _GRID_SCORE and "url" in g.fields + for g in ctx.selector.detect_records() + ) ) parse_seconds.append(time.perf_counter() - started) if page.source != "browser" and needs_javascript(page): diff --git a/src/wintergrab/intel/classify.py b/src/wintergrab/intel/classify.py index d759074..abd9c7b 100644 --- a/src/wintergrab/intel/classify.py +++ b/src/wintergrab/intel/classify.py @@ -30,6 +30,7 @@ from typing import Any from urllib.parse import parse_qs, urlsplit +from ..data.normalize import normalize_availability from ..extraction.page import STRUCTURED_KINDS, PageContext, schema_types from ..parser.text import tag_name, text_content @@ -64,7 +65,9 @@ _URL_RULES: list[tuple[str, float, re.Pattern[str]]] = [ ("product", 2.0, re.compile(r"/(?:products?|p|dp|item|items|sku|pd|gp/product)/(?!page(?:/|$))[^/]+|[-_/]p[-_]?\d{3,}(?:\.html?)?$|/\d{5,}\.html?$")), - ("category", 2.0, re.compile(r"/(?:category|categories|c|collections?|shop|department|departments|catalog|catalogue|browse)(?:/|$)")), + ("category", 2.0, re.compile(r"/(?:category|categories|c)(?:/|$)")), + # a section that holds the products too (shop.example/catalogue/blue-widget_123): weaker evidence + ("category", 1.0, re.compile(r"/(?:collections?|shop|departments?|catalog|catalogue|browse)(?:/|$)")), ("news", 2.0, re.compile(r"/(?:news|press|press-releases?|newsroom)(?:/|$)")), ("article", 2.0, re.compile(r"/(?:blog|article|articles|post|posts|stories|story|insights|magazine)/[^/]+|/\d{4}/\d{2}/(?:\d{2}/)?[^/]+")), ("job", 3.0, re.compile(r"/(?:jobs?|careers?|vacanc(?:y|ies)|positions?|openings?)(?:/|$)")), @@ -225,20 +228,43 @@ def prices(self) -> int: return len(set(_MONEY.findall(self.text))) @cached_property - def title_prices(self) -> int: - """Distinct prices in the title's block: the smallest element around the page's one ``

    `` that holds a - price. One: the page is about one priced thing, whatever the related items or "recently viewed" cards - below it list. A listing's title stands above its cards, so its block holds all their prices.""" + def title_block(self) -> Any: + """The title's block: the smallest element (lxml) around the page's one ``

    `` that holds a price, or + ``None``. A page about one priced thing has its price there; a listing's title stands above its cards, + so its block holds all their prices.""" headings = self.page.root.css("h1") if len(headings) != 1 or headings[0].root is None: - return 0 + return None for ancestor in headings[0].root.iterancestors(): - found = set(_MONEY.findall(text_content(ancestor))) - if found: - return len(found) + if _MONEY.search(text_content(ancestor)): + return ancestor if tag_name(ancestor) == "body": break - return 0 + return None + + @cached_property + def title_text(self) -> str: + """The title's block's text, the cards of the page's other records left out (a listing's two cards, a + product's "recently viewed" strip: their prices and stock lines are not the page's own).""" + block = self.title_block + return text_content(block, skip_nodes=self.page.other_records) if block is not None else "" + + @cached_property + def title_prices(self) -> int: + """Prices written in the title's block, its other records' cards aside (mentions, not distinct values: a + listing of same-priced items writes the price once per card).""" + return len(_MONEY.findall(self.title_text)) + + @cached_property + def own_price(self) -> bool: + """Whether the title's block holds one price of the page's own (or two: an old price beside the new one): + the page is about one priced thing, whatever the cards below it list.""" + return 1 <= self.title_prices <= 2 + + @cached_property + def title_availability(self) -> bool: + """Whether the title's block says if the thing is in stock: a product page's own line, not a card's.""" + return normalize_availability(self.title_text) is not None @cached_property def password_inputs(self) -> int: @@ -250,7 +276,12 @@ def form_inputs(self) -> int: @cached_property def cart_button(self) -> bool: + """An add-to-cart button of the page's own (a related product's card has one too, and says nothing).""" + others = self.page.other_records for el in self.page.root.css("button, input[type=submit], a[class*=cart], a[class*=button], a[class*=btn]"): + node = el.root + if others and node is not None and (node in others or any(a in others for a in node.iterancestors())): + continue label = el.text or el.attr("value") or el.attr("aria-label") or "" if label and _CART.search(label): return True @@ -361,12 +392,13 @@ def schema_rule(f: PageFeatures) -> list[tuple[str, float, str]]: add("product", "add-to-cart button", lambda f: 3.0 if f.cart_button else None) # One price in the title's block: a page about one priced thing; the prices of the related products # or "recently viewed" cards below it are not the page's own. - add("product", "one price near the title", lambda f: 2.5 if f.title_prices == 1 else None) - add("category", "many prices", lambda f: 3.0 if f.prices >= 5 and f.repeated >= 5 and f.title_prices != 1 else None) + add("product", "one price near the title", lambda f: 2.5 if f.own_price else None) + add("product", "availability near the title", lambda f: 1.5 if f.own_price and f.title_availability else None) + add("category", "many prices", lambda f: 3.0 if f.prices >= 5 and f.repeated >= 5 and not f.own_price else None) add( "category", "repeated cards with prices", - lambda f: 2.0 if f.repeated >= 8 and f.prices >= 3 and f.title_prices != 1 else None, + lambda f: 2.0 if f.repeated >= 8 and f.prices >= 3 and not f.own_price else None, ) add("listing", "repeated cards", lambda f: 3.0 if f.repeated >= 8 and f.prices < 3 else None) add("listing", "pagination", lambda f: 1.0 if f.pagination and f.repeated >= 5 else None) diff --git a/src/wintergrab/intel/survey.py b/src/wintergrab/intel/survey.py index 68bc1d3..e8eec68 100644 --- a/src/wintergrab/intel/survey.py +++ b/src/wintergrab/intel/survey.py @@ -149,9 +149,10 @@ class _SurveySpider(Spider): #: turn: the sample is spread across the patterns a site has, so its record pages are among the pages sampled #: however many category or tag links come first on a page. ``0``: page order alone. per_pattern: int = 3 - #: ``wanted(response) -> bool``: a sampled page that is what the survey looks for. Links of its URL pattern - #: are followed in ``prefer`` order alone from then on, before the other patterns' next turn; so are the - #: links of a page's record cards (what a listing lists), until a page of their pattern is not wanted. + #: ``wanted(response) -> bool | None``: ``True``, a sampled page that is what the survey looks for; ``False``, + #: one that is surely not; ``None``, unsure (nothing learned). Links of a wanted page's URL pattern are + #: followed in ``prefer`` order alone from then on, before the other patterns' next turn; so are the links of + #: a page's record cards (what a listing lists), until a page of their pattern is surely not wanted. wanted: Any = None def __init__(self, **settings: Any) -> None: @@ -189,8 +190,9 @@ def parse(self, response: Response) -> Any: return listed: set[str] = set() if self.wanted is not None: - pattern = url_template(response.url) - (self._wanted_patterns if self.wanted(response) else self._unwanted_patterns).add(pattern) + verdict = self.wanted(response) + if verdict is not None: # (unsure: nothing learned about the pattern) + (self._wanted_patterns if verdict else self._unwanted_patterns).add(url_template(response.url)) listed = _listed(response) for link in response.links(same_domain=True): yield self._request(link, listed=link in listed) @@ -203,7 +205,7 @@ def _listed(response: Response) -> set[str]: """The links of the page's lists of records (its cards' own links): what a listing lists, before it is known what kind of page they lead to.""" found: set[str] = set() - for group in response.detect_records(): + for group in response.detect_records(min_records=2): # (a small category lists two) if group.convincing and "url" in group.fields: found.update(url for record in group.extract(response.url) if isinstance(url := record.get("url"), str)) return found @@ -229,7 +231,7 @@ def survey_site( prefer: Callable[[str], bool | float] | None = None, extra_urls: Iterable[str] = (), per_pattern: int | None = None, - wanted: Callable[[Response], bool] | None = None, + wanted: Callable[[Response], bool | None] | None = None, log_level: str | None = "WARNING", **spider_settings: Any, ) -> SiteSurvey: @@ -247,11 +249,11 @@ def survey_site( per_pattern: Links of one URL pattern followed before the other patterns get their turn (by default a fifth of ``pages``, two at least), so the sample is spread across the site's patterns; ``0``: page order alone. - wanted: ``wanted(page) -> bool``: a sampled page that is what the survey looks for (a goal's record - page). Links of its URL pattern are then followed in ``prefer`` order alone, before the other - patterns' next turn, and so are the links of a page's record cards (what a listing lists) until a - page of their pattern is not wanted: a site's fifty categories no longer crowd its record pages out - of the sample. + wanted: ``wanted(page) -> bool | None``: whether a sampled page is what the survey looks for (a goal's + record page), ``None`` when unsure. Links of a wanted page's URL pattern are then followed in + ``prefer`` order alone, before the other patterns' next turn, and so are the links of a page's + record cards (what a listing lists) until a page of their pattern is surely not wanted: a site's + fifty categories no longer crowd its record pages out of the sample. """ from ..fetchers import Fetcher diff --git a/tests/test_goals.py b/tests/test_goals.py index 570b947..4b04a6a 100644 --- a/tests/test_goals.py +++ b/tests/test_goals.py @@ -251,15 +251,42 @@ def test_the_survey_samples_more_of_the_pages_the_goal_wants(site) -> None: from wintergrab.goals.plan import survey_for goal = read("Find all books rated 4 stars or more with title, price and rating", sites=[site.url + "/books/"]) - survey = survey_for(goal, site.url + "/books/", sample=15, log_level=None) + # (without the site's sitemap: a third of the sample would come from it, other sections included) + survey = survey_for(goal, site.url + "/books/", sample=15, log_level=None, settings={"sitemaps": False}) patterns = Counter(url_template(p.url, include_host=False) for p in survey.pages) - assert len(survey.pages) == 15 + assert len(survey.pages) == 15, patterns assert patterns["/books/catalogue/{slug}/index.html"] >= 6 # the books: more than a pattern's share assert patterns["/books/catalogue/category/books/{slug}/index.html"] == 3 # the categories: their share plan = plan_goal(goal, surveys={site.url + "/books/": survey}) assert plan.sites[0].sample["record_pages"] >= 6 +def test_the_survey_learns_nothing_from_an_unsure_page(site) -> None: + """A page the classifier is unsure about says nothing about its URL pattern: what the listings list stays + followed freely. Only a page surely not wanted puts its pattern back into the queue's turns (on + books.toscrape.com the first book page sampled was unsure, and a 15-page sample fell back to three books).""" + from collections import Counter + + from wintergrab import url_template + from wintergrab.intel.survey import _SurveySpider, survey_site + + def categories_first(url: str) -> int: # the goal's order: the listings rank, the site's other pages last + return 2 if "/category/" in url else 1 if "/books/" in url else 0 + + survey = survey_site(site.url + "/books/", pages=15, sitemaps=False, keep_pages=True, prefer=categories_first, + wanted=lambda page: None, log_level=None) # fmt: skip + patterns = Counter(url_template(p.url, include_host=False) for p in survey.pages) + assert patterns["/books/catalogue/{slug}/index.html"] >= 6, patterns + spider = _SurveySpider(start_urls=["https://s.example/"], per_pattern=2, wanted=lambda page: None) + assert [spider._request(f"https://s.example/p/{i}", listed=True).priority for i in range(3)] == [0, 0, 0] + spider._unwanted_patterns.add(url_template("https://s.example/p/9")) # surely not wanted: turns again + assert [spider._request(f"https://s.example/p/{i}", listed=True).priority for i in range(3, 6)] == [ + -100, + -200, + -200, + ] + + def test_a_plan_starts_from_a_page_that_exists(site) -> None: """The part of a site a goal is about may have no page of its own: on books.toscrape.com the categories live under /catalogue/category/books/, and /catalogue/category/books answers 403. The crawl starts from a diff --git a/tests/test_intel.py b/tests/test_intel.py index 977241a..78ca682 100644 --- a/tests/test_intel.py +++ b/tests/test_intel.py @@ -136,6 +136,32 @@ def test_a_product_page_that_shows_other_products_is_a_product() -> None: assert result.type == "category" and "many prices" in result.evidence, result.scores +def test_a_product_page_without_a_cart_button_of_its_own() -> None: + """books.toscrape.com's product pages have no "Add to basket" of their own, and the first one sampled had no + related cards either: the classifier was unsure (2.5 for the price near the title against 2.0 for the URL's + "catalogue"), so the survey learned nothing from it. A product page's own availability line next to its one + price is product evidence; a section name in the URL ("catalogue", "shop") is weaker evidence of a listing + than "category"; and a related card's cart button is not the page's.""" + own = ( + '

    A Light in the Attic

    £51.77

    ' + '

    In stock (22 available)

    ' + "
    UPCa897fe39b1053632
    Tax£0.00
    " + ) + url = "https://shop.example/catalogue/a-light-in-the-attic_1000/index.html" + result = classify_page(page(own, url=url)) + assert result.type == "product" and result.confidence >= 0.25, result.scores + assert "availability near the title" in result.evidence and "add-to-cart button" not in result.evidence + cards = "".join( + f'

  • B {i}

    ' + f'

    £{i}.50

    • ' + for i in range(3) + ) + result = classify_page(page(own + f"

    Recently viewed

      {cards}
    ", url=url)) + assert result.type == "product" and "add-to-cart button" not in result.evidence, result.scores + assert classify_url("https://shop.example/category/phones").type == "category" + assert classify_url("https://shop.example/catalogue/blue-widget_123/index.html").type == "unknown" # a section + + def test_related_types_do_not_compete() -> None: # A news page is also an article: "article" scoring high must not make "news" unsure. result = classify_page(news_page()) diff --git a/tests/testsite.py b/tests/testsite.py index a8d47c1..a0010f3 100644 --- a/tests/testsite.py +++ b/tests/testsite.py @@ -566,8 +566,7 @@ def _books_page(handler: Handler, path: str) -> None: f"
    " # as on the real site: the page's content, the related books included f"

    {b['title']}

    {b['price']}

    " f"

    In stock ({b['stock']} available)

    " - f"

    " + f"

    " # (no cart button of its own, as on the real site) f"" f"" f"" From f0fad032e4af3220c25405ab8742c8b75be1bf01 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 20:47:26 +0000 Subject: [PATCH 23/30] What a listing lists comes before a listing found later The third sandbox run on books.toscrape.com still sampled three book pages in fifteen, and the diagnostic showed the order: after the home page, the root category and three categories came their own pagination pages (/catalogue/category/books_1/page-2.html, .../mystery_3/page-2.html: six of them, two URL patterns found on the category pages, three turns each), and only then the books. The book links were free of the turns, but the goal's preference ranks a category-looking URL above a book URL, and a free link was still ranked by its own preference alone. A free link is now worth the best preference of the links that take turns, so it comes before any pattern's next turn and before the first turn of a listing found later that prefer ranks higher. On a real-scale copy of the site with the categories' pagination, a 15-page sample holds ten book pages (four before), a 30-page one twenty, with the listings and their pagination still represented. The test site's categories have two pages each and the root category its own pages, as the real ones, which made the survey test fail before the change. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- CHANGELOG.md | 11 ++++++----- src/wintergrab/intel/survey.py | 22 +++++++++++++++------- tests/testsite.py | 23 ++++++++++++++++------- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a04b612..f36965a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1214,11 +1214,12 @@ an error that says so, not a setting silently ignored carry the cards and the information table the real ones have. - **The survey samples more of the pages the goal wants.** The links of a page's record cards (what a listing lists) and, once a sampled page is one - of the goal's record pages, the links of its URL pattern are followed in - preference order alone, before the other patterns' next turn - (`survey_site(wanted=)`); a page the classifier is unsure about says - nothing about its pattern. A 15-page sample of books.toscrape.com holds - ten book pages instead of one, and the plan's estimates rest on them. + of the goal's record pages, the links of its URL pattern are followed + before the other patterns' next turn, ranked with the best of the listings + (`survey_site(wanted=)`), so a category's own pagination found later does + not take their place; a page the classifier is unsure about says nothing + about its pattern. A 15-page sample of books.toscrape.com holds ten book + pages instead of one, and the plan's estimates rest on them. - **A plan starts from a page that exists.** The section a goal is about may have no page of its own (`/catalogue/category/books` on books.toscrape.com answers 403, its categories live below it): the crawl starts from a diff --git a/src/wintergrab/intel/survey.py b/src/wintergrab/intel/survey.py index e8eec68..78a79e6 100644 --- a/src/wintergrab/intel/survey.py +++ b/src/wintergrab/intel/survey.py @@ -150,9 +150,10 @@ class _SurveySpider(Spider): #: however many category or tag links come first on a page. ``0``: page order alone. per_pattern: int = 3 #: ``wanted(response) -> bool | None``: ``True``, a sampled page that is what the survey looks for; ``False``, - #: one that is surely not; ``None``, unsure (nothing learned). Links of a wanted page's URL pattern are - #: followed in ``prefer`` order alone from then on, before the other patterns' next turn; so are the links of - #: a page's record cards (what a listing lists), until a page of their pattern is surely not wanted. + #: one that is surely not; ``None``, unsure (nothing learned). Links of a wanted page's URL pattern are free + #: from then on: followed before the other patterns' next turn, ranked with the best of the listings; so are + #: the links of a page's record cards (what a listing lists), until a page of their pattern is surely not + #: wanted. wanted: Any = None def __init__(self, **settings: Any) -> None: @@ -162,6 +163,7 @@ def __init__(self, **settings: Any) -> None: self._queued: set[str] = set() self._wanted_patterns: set[str] = set() self._unwanted_patterns: set[str] = set() + self._top = 0 # the best ``prefer`` priority of the links that take turns: what a free link is worth def _request(self, url: str, *, spread: bool = True, listed: bool = False) -> Request: request = Request(url, dont_filter=False) @@ -174,7 +176,12 @@ def _request(self, url: str, *, spread: bool = True, listed: bool = False) -> Re free = pattern in self._wanted_patterns or (listed and pattern not in self._unwanted_patterns) if spread and not free: # the n-th link of a pattern waits for the first of every other: a tier lower each time + self._top = max(self._top, priority) priority -= 100 * (self._patterns[pattern] // self.per_pattern) + elif spread: + # a free link is worth the best listing queued so far: before any pattern's next turn, and before + # the first turn of a listing found later (a category's own pagination) that prefer ranks higher + priority = max(priority, self._top) self._patterns[pattern] += 1 request.priority = priority return request @@ -250,10 +257,11 @@ def survey_site( fifth of ``pages``, two at least), so the sample is spread across the site's patterns; ``0``: page order alone. wanted: ``wanted(page) -> bool | None``: whether a sampled page is what the survey looks for (a goal's - record page), ``None`` when unsure. Links of a wanted page's URL pattern are then followed in - ``prefer`` order alone, before the other patterns' next turn, and so are the links of a page's - record cards (what a listing lists) until a page of their pattern is surely not wanted: a site's - fifty categories no longer crowd its record pages out of the sample. + record page), ``None`` when unsure. Links of a wanted page's URL pattern are then free: followed + before the other patterns' next turn, ranked with the best of the listings; and so are the links of + a page's record cards (what a listing lists) until a page of their pattern is surely not wanted. A + site's fifty categories, and their own pagination, no longer crowd its record pages out of the + sample. """ from ..fetchers import Fetcher diff --git a/tests/testsite.py b/tests/testsite.py index a0010f3..2b6d370 100644 --- a/tests/testsite.py +++ b/tests/testsite.py @@ -505,8 +505,9 @@ def _book(i: int) -> dict[str, Any]: BOOK_CATEGORIES = ("Travel", "Mystery", "Historical Fiction", "Sequential Art", "Classics", "Philosophy", "Romance", "Womens Fiction", "Fiction", "Childrens", "Religion", "Nonfiction") # fmt: skip # Every page under /books/ a crawl that follows each link reaches: the listing and its index.html alias (the -# breadcrumbs name it), the other listing pages, the books, and the categories, the root "Books" one included. -BOOK_SECTION_PAGES = 2 + (BOOK_PAGES - 1) + BOOK_PAGES * BOOKS_PER_PAGE + len(BOOK_CATEGORIES) + 1 +# breadcrumbs name it), the other listing pages, the books, the categories (two pages each), and the root "Books" +# category with its own other pages. +BOOK_SECTION_PAGES = 2 + (BOOK_PAGES - 1) + BOOK_PAGES * BOOKS_PER_PAGE + 2 * len(BOOK_CATEGORIES) + BOOK_PAGES def _book_pods(numbers: Any, prefix: str) -> str: @@ -531,17 +532,25 @@ def _book_categories(prefix: str) -> str: def _books_page(handler: Handler, path: str) -> None: """A miniature of books.toscrape.com (same markup, fewer books).""" - if path.startswith("/books/catalogue/category/books_1/"): # "Books", the root category: the first page of all + if path.startswith("/books/catalogue/category/books_1/"): # "Books", the root category: every book, paginated + n = int(path.rsplit("-", 1)[1].split(".")[0]) if "/page-" in path else 1 # (its own page-N.html, as the + nxt = f"" # real one) body = ( _book_categories("../../") - + f"

    Books

      {_book_pods(range(1, BOOKS_PER_PAGE + 1), '../../')}
    " + + f"

    Books

      {_book_pods(range((n - 1) * BOOKS_PER_PAGE + 1, n * BOOKS_PER_PAGE + 1), '../../')}
    " + + (nxt if n < BOOK_PAGES else "") ) return handler.send(200, layout("Books | Books to Scrape", body)) - if path.startswith("/books/catalogue/category/books/"): # a category: two of the books, and the sidebar + if path.startswith("/books/catalogue/category/books/"): # a category: two books a page, two pages, the sidebar n = int(path.rsplit("_", 1)[1].split("/")[0]) + page = int(path.rsplit("-", 1)[1].split(".")[0]) if "/page-" in path else 1 # (its own page-2.html) name = BOOK_CATEGORIES[(n - 2) % len(BOOK_CATEGORIES)] - books = [(n - 2) % (BOOK_PAGES * BOOKS_PER_PAGE) + 1, (n - 1) % (BOOK_PAGES * BOOKS_PER_PAGE) + 1] - body = _book_categories("../../../") + f"

    {name}

      {_book_pods(books, '../../../')}
    " + first = n - 2 + 2 * (page - 1) + books = [first % (BOOK_PAGES * BOOKS_PER_PAGE) + 1, (first + 1) % (BOOK_PAGES * BOOKS_PER_PAGE) + 1] + nxt = "" if page == 1 else "" + body = ( + _book_categories("../../../") + f"

    {name}

      {_book_pods(books, '../../../')}
    {nxt}" + ) return handler.send(200, layout(f"{name} | Books to Scrape", body)) if path in ("/books/", "/books/index.html") or path.startswith("/books/catalogue/page-"): n = 1 if not path.startswith("/books/catalogue/page-") else int(path.rsplit("-", 1)[1].split(".")[0]) From 590f389f1b020c2855756f9cdb9b9fcc5d8f0172 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 20:59:19 +0000 Subject: [PATCH 24/30] The generator test counts the second run's own training pages The test generated a scraper twice in one directory and expected the second run's fixtures to number the first run's training pages. Each run surveys the site anew, and the survey fetches concurrently: with the sitemap's share, the home page and the listings taking ten of fifteen pages, the books that fill the rest come in FIFO order among equals, and a run can hold one book more or less than the other (four against five on the Python 3.10 CI job). The fixtures are the second run's training pages: the test counts those. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HWaLhkonw8qvXJX6vSWNfY --- tests/test_generate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_generate.py b/tests/test_generate.py index 4d8f8c3..c38c19f 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -115,7 +115,9 @@ def test_a_scraper_generated_tested_and_accepted(site, tmp_path) -> None: assert FixtureSuite(directory / "fixtures").run().ok # again in the same directory: its own files replaced, the tests too again = generate_scraper("books with title and price", directory, sites=[site.url + "/books/"], sample=15) - assert again.accepted and len(FixtureSuite(directory / "fixtures").fixtures()) == trained + trained_again = min(5, again.stage("plan").details["record_pages"]) # (its own survey: fetched concurrently, + assert again.accepted and len(FixtureSuite(directory / "fixtures").fixtures()) == trained_again # its sample + assert trained_again >= 3 # can hold a book more or less than the first one's) def test_a_model_while_generating_and_none_after(site, tmp_path) -> None: From 99469f6056412b7b0a7b5ce9288c3e8577bcf39b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 08:50:07 +0000 Subject: [PATCH 25/30] A product's long description is no article The owner's fourth sandbox run on books.toscrape.com sampled ten book pages in fifteen, and the plan still counted three: "records: about 0" rated 4 or more, where the crawl then found 21 in 54. Their diagnostic showed eight of the ten books at confidence 0.16, product 4.0 against article 3.0: "long article text". The site keeps a book's description, its information table and its "recently viewed" cards in one
    , and past 1,500 characters that read as an article's text. The planner leaves a verdict under 0.25 out, and seven of the eight unsure books, with three or more cards, were read as listings of their cards; the survey learned nothing from them. A page that offers one thing describes it at length. When the title's block holds the page's own price and a stock line of its own (a short element: "In stock (22 available)", not a sentence that mentions "available"), in a block shorter than an article, "long article text" and "many paragraphs" do not count against it. An article that names a price and "available" in its body is no offer: the smallest block around its title holding the price is the article itself. And the cards of the page's other records are no part of its article's text. On a copy of the real book page (Oscar's markup, the warning box, the information table), a description of 1,200 characters or more gave 0.16 with or without cards, and gives 0.47 now; a news or review page reads as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qv1sKo4WDr8zQ8wFuW24k8 --- CHANGELOG.md | 11 +++++++++ docs/intelligence.md | 2 +- src/wintergrab/intel/classify.py | 38 ++++++++++++++++++++++++++++---- tests/test_intel.py | 38 ++++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f36965a..19f70fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1224,6 +1224,17 @@ an error that says so, not a setting silently ignored have no page of its own (`/catalogue/category/books` on books.toscrape.com answers 403, its categories live below it): the crawl starts from a sampled page under the section now, never from a URL made up from the path. +- **A product's long description is no article.** books.toscrape.com keeps + a book's description, its information table and its "recently viewed" + cards in one `
    `. Past 1,500 characters the + classifier counted that as an article's text (3.0 against the book's 4.0, + confidence 0.16): in the owner's fourth sandbox run eight of the ten books + sampled were unsure, seven of them read as listings of their cards, and the + plan counted three books and expected "about 0" rated 4 or more (the crawl + found 21 in 54). A page that offers one thing (its own price and a stock + line in the title's block, a block shorter than an article) describes it at + length: "long article text" and "many paragraphs" do not count against it, + and the cards of the page's other records are no part of its article. - **The DuckDB output waits for a held file on Windows too.** A file another program has open is waited for a few seconds; on Windows the system's message for it ("being used by another process") was not read as a hold, diff --git a/docs/intelligence.md b/docs/intelligence.md index 8afaaf2..9c36f48 100644 --- a/docs/intelligence.md +++ b/docs/intelligence.md @@ -36,7 +36,7 @@ It adds up evidence. Each rule that applies gives its type a weight: | a 404 status, "Page not found" in the title | | 8, 5 | | a password field in a small form | | 6 | | the URL | `/p/...`, `/category/` (`/shop/`, `/catalogue/`: a section that holds the products too, weaker), `/blog/...`, `/careers/`, `?q=`, the site root | 1-4 | -| layout | an add-to-cart button of the page's own, one price in the title's block and an availability line next to it (a product, whatever the related-product cards below it list), many repeated cards with prices, pagination, a long `
    `, a byline and a date, code blocks | 1-3 | +| layout | an add-to-cart button of the page's own, one price in the title's block and an availability line next to it (a product, whatever the related-product cards below it list), many repeated cards with prices, pagination, a long `
    ` (not on a page offering one thing by its title, price and stock line: its long text is that thing's description), a byline and a date, code blocks | 1-3 | | wording | "Results for", "Sign in", "Apply now", "Get tickets" | 1.5-3 | The weights are evidence strengths chosen by hand, not trained diff --git a/src/wintergrab/intel/classify.py b/src/wintergrab/intel/classify.py index abd9c7b..5315f1d 100644 --- a/src/wintergrab/intel/classify.py +++ b/src/wintergrab/intel/classify.py @@ -118,6 +118,8 @@ ) ) _MAX_TEXT = 100_000 # wording rules read this much of the page's text +_ARTICLE_TEXT = 1500 # characters of an
    that make it an article's text +_STOCK_LINE = 40 # characters of a stock line ("In stock (22 available)"), not a sentence _NOT_FOUND = re.compile(r"\b(?:404|page not found|not found|nicht gefunden|introuvable|no encontrada)\b", re.I) _LOGIN_WORDS = re.compile(r"\b(?:log ?in|sign ?in|anmelden|connexion|iniciar sesi[oó]n)\b", re.I) _SEARCH_WORDS = re.compile(r"\b(?:search results|results for|no results|resultados|suchergebnisse|résultats)\b", re.I) @@ -266,6 +268,28 @@ def title_availability(self) -> bool: """Whether the title's block says if the thing is in stock: a product page's own line, not a card's.""" return normalize_availability(self.title_text) is not None + @cached_property + def stock_line(self) -> bool: + """Whether the title's block has a line of its own saying whether the thing is in stock ("In stock (22 + available)", "Sold out"): a short element, not a sentence that mentions "available".""" + block, others = self.title_block, self.page.other_records + if block is None: + return False + for el in block.iter(): + if not isinstance(el.tag, str) or el in others or any(a in others for a in el.iterancestors()): + continue + text = text_content(el) + if 0 < len(text) <= _STOCK_LINE and normalize_availability(text) is not None: + return True + return False + + @cached_property + def offer(self) -> bool: + """Whether the title's block offers the page's one thing: its own price and stock line, in a block shorter + than an article's text, so that the page's long text lies outside it (the thing's description). A block + as long as an article is the article itself, whose words may name a price and "available".""" + return self.own_price and len(self.title_text) < _ARTICLE_TEXT and self.stock_line + @cached_property def password_inputs(self) -> int: return len(self.page.root.css("input[type=password]")) @@ -289,9 +313,13 @@ def cart_button(self) -> bool: @cached_property def article_text(self) -> int: - """Characters of text in the page's longest ``
    `` (or ``
    ``) element.""" + """Characters of text in the page's longest ``
    `` (or ``
    ``) element, the cards of the page's + other records left out (books.toscrape.com's "recently viewed" strip is inside the book's ``
    ``).""" + others = self.page.other_records lengths = [ - len(el.text) for el in self.page.root.css("article, [itemprop=articleBody], .article-body, .post-content") + len(text_content(el.root, skip_nodes=others)) + for el in self.page.root.css("article, [itemprop=articleBody], .article-body, .post-content") + if el.root is not None ] return max(lengths, default=0) @@ -407,8 +435,10 @@ def schema_rule(f: PageFeatures) -> list[tuple[str, float, str]]: "priced cards and a pager", lambda f: 2.5 if f.pagination and f.repeated >= 3 and f.prices >= 3 else None, ) - add("article", "long article text", lambda f: 3.0 if f.article_text >= 1500 else None) - add("article", "many paragraphs", lambda f: 1.5 if f.paragraphs >= 6 else None) + # A page that offers one thing (its price and stock line by its title) describes it at length: its long text is + # the thing's description, not an article's. + add("article", "long article text", lambda f: 3.0 if f.article_text >= _ARTICLE_TEXT and not f.offer else None) + add("article", "many paragraphs", lambda f: 1.5 if f.paragraphs >= 6 and not f.offer else None) add("article", "byline and date", lambda f: 2.0 if f.byline and f.time_elements else None) add("documentation", "code blocks", lambda f: 3.0 if f.code_blocks >= 3 else None) add("login", "password field", lambda f: 6.0 if f.password_inputs and f.form_inputs <= 6 else None) diff --git a/tests/test_intel.py b/tests/test_intel.py index 78ca682..4ee470f 100644 --- a/tests/test_intel.py +++ b/tests/test_intel.py @@ -162,6 +162,44 @@ def test_a_product_page_without_a_cart_button_of_its_own() -> None: assert classify_url("https://shop.example/catalogue/blue-widget_123/index.html").type == "unknown" # a section +def test_a_product_page_with_a_long_description_is_a_product() -> None: + """A book page on books.toscrape.com keeps its description, its information table and its "recently viewed" + cards in one
    . Past 1,500 characters that read as an article's text: 3.0 + against the book's 4.0, confidence 0.16: in the owner's fourth sandbox run eight of the ten books sampled + were unsure, seven of them read as listings of their cards, so the plan counted three books and expected + "about 0" rated 4 or more. A page that offers one thing (its price and stock line by its title) describes it at length, and + the cards of the page's other records are no part of its article.""" + cards = "".join( + f'

  • Book {i}

    ' + f'

    £{20 + i}.99

    In stock

    ' + f"
    • " + for i in range(6) + ) + viewed = f'

    Products you recently viewed

      {cards}
    ' + table = "
    UPC{b['upc']}
    Product TypeBooks
    Price (excl. tax){b['price']}
    Price (incl. tax){b['price']}
    Tax£0.00
    UPC6957f44c3847a760
    Tax£0.00
    " + title = '

    Sharp Objects

    £47.82

    ' + url = "https://shop.example/catalogue/sharp-objects_997/index.html" + + stock = '

    In stock (20 available)

    ' + book = page( + f'
    {title}{stock}

    {LOREM * 10}

    {table}{viewed}
    ', url=url + ) + result = classify_page(book) + assert result.type == "product" and result.confidence >= 0.25, result.scores + assert "article" not in result.scores + # without a stock line, the cards alone had taken a shorter description past an article's length + book = page(f'
    {title}

    {LOREM * 7}

    {table}{viewed}
    ', url=url) + assert "article" not in classify_page(book).scores + # an article that names a price and says "available" in its text is no offer: its title's block is the article + news = page( + f"

    Budget approved

    {''.join(f'

    {LOREM}

    ' for _ in range(8))}" + "

    Each household gets £120, available from May.

    ", + url="https://news.example/budget-approved", + ) + result = classify_page(news) + assert result.type == "article" and "long article text" in result.evidence, result.scores + + def test_related_types_do_not_compete() -> None: # A news page is also an article: "article" scoring high must not make "news" unsure. result = classify_page(news_page()) From 20a966326642033a5b12581d9ac3ac7dfc815cfd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 08:50:15 +0000 Subject: [PATCH 26/30] A description is not its section's title On books.toscrape.com a book's description follows its section's title,

    Product Description

    ...

    . The DOM heuristics took the element marked "description" whole and read "Product Description" on every book page, and the scraper generator learned #product_description from it (4/4 pages). An element marked as the description that holds nothing but a heading titles the text after it now, unless that is another section's title. The selector learner does not look for texts longer than 300 characters (_locate and _holding stop there). It said "the value is not in the pages' text or attributes" for such a description; it says it is a text longer than 300 characters, read from each page's own evidence, and a generated scraper reads it so. The test site's book pages have the real ones' description block, with a description as long as many of theirs: with the classifier before the previous commit every book page was unsure there, read as a listing of its six other books, and the plan for "books rated 4 stars or more" counted no book and expected "about 0" (ten goal and generator tests failed). The survey test now checks that estimate: 4 of the site's 12. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qv1sKo4WDr8zQ8wFuW24k8 --- CHANGELOG.md | 10 +++++++++ docs/extraction.md | 5 ++++- src/wintergrab/extraction/generate.py | 11 ++++++++-- src/wintergrab/extraction/strategies.py | 29 ++++++++++++++++++++++++- tests/test_extraction.py | 23 ++++++++++++++++++++ tests/test_generate.py | 3 +++ tests/test_goals.py | 3 +++ tests/testsite.py | 11 ++++++++++ 8 files changed, 91 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19f70fc..a63275d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1235,6 +1235,16 @@ an error that says so, not a setting silently ignored line in the title's block, a block shorter than an article) describes it at length: "long article text" and "many paragraphs" do not count against it, and the cards of the page's other records are no part of its article. +- **A description is not its section's title.** On books.toscrape.com + `

    Product Description

    ` comes + before the `

    ` that holds the description, and the DOM heuristics read + "Product Description" on every book page; the scraper generator learned + `#product_description` from it. An element marked as a description that + holds nothing but a heading titles the text after it now. The selector + learner does not look for texts longer than 300 characters: it says so, + and a generated scraper reads them from each page's own evidence. The test + site's book pages have the real ones' description, as long as many of + theirs. - **The DuckDB output waits for a held file on Windows too.** A file another program has open is waited for a few seconds; on Windows the system's message for it ("being used by another process") was not read as a hold, diff --git a/docs/extraction.md b/docs/extraction.md index b02d749..160d173 100644 --- a/docs/extraction.md +++ b/docs/extraction.md @@ -114,7 +114,10 @@ you recently viewed", "customers also bought": a list of repeated cards with links that does not hold the page's `

    `) are left out for every field, whatever the list is called: their prices, ratings and pictures are not the page's own, and a model is not shown them either. In the visible text, an -amount of zero ("Tax £0.00", free shipping) is no price. +amount of zero ("Tax £0.00", free shipping) is no price. An element marked +as the description that holds nothing but a heading (`

    Product Description

    `) is the title +of the text after it, and that text is the description. ## Confidence diff --git a/src/wintergrab/extraction/generate.py b/src/wintergrab/extraction/generate.py index d5e1779..db03fcf 100644 --- a/src/wintergrab/extraction/generate.py +++ b/src/wintergrab/extraction/generate.py @@ -57,6 +57,8 @@ _MODEL_CONFIDENCE = 0.5 #: Pages whose elements are searched for a field's value (the others only check the selectors). _SEARCHED_PAGES = 4 +#: The longest text searched for on a page: a description longer than this is read from the page's evidence. +_SEARCHED_TEXT = 300 #: How stable each kind of selector is (lower is better): an attribute meant for machines, a class #: or an id, a label beside the element, its tag, its place under a classed ancestor, its place on the page. _ATTRIBUTE, _CLASS, _LABEL, _TAG, _ANCHORED, _POSITION = range(6) @@ -272,7 +274,12 @@ def _choose( proposed[query] = min(preference, proposed.get(query, preference)) if not proposed: out.status = "not learned" - out.note = "the value is not in the pages' text or attributes" + texts = [found[i][f.name].raw for i in pages[:_SEARCHED_PAGES]] + out.note = ( + f"a text longer than {_SEARCHED_TEXT} characters: read from each page's own evidence" + if all(isinstance(t, str) and len(t) > _SEARCHED_TEXT for t in texts) + else "the value is not in the pages' text or attributes" + ) return best: tuple[tuple[int, ...], str, _Check] | None = None closest = 0 @@ -360,7 +367,7 @@ def _proposals( holders: list[tuple[etree._Element, str | None]] = [] raw = fv.raw if isinstance(fv.raw, str) else None if fv.raw is None else str(fv.raw) others = ctx.other_records # the cards of related products: another record's values, not this page's - if raw and len(raw) <= 300: # where the page shows the value as it was found + if raw and len(raw) <= _SEARCHED_TEXT: # where the page shows the value as it was found located = sorted(_locate(doc, raw, ctx.url, others), key=lambda m: m.rank) holders += [(m.element, m.attr) for m in located[:4]] value = _plain(fv.value) diff --git a/src/wintergrab/extraction/strategies.py b/src/wintergrab/extraction/strategies.py index 2e4060f..20aaf8b 100644 --- a/src/wintergrab/extraction/strategies.py +++ b/src/wintergrab/extraction/strategies.py @@ -527,6 +527,26 @@ def _marked(page: PageContext, kind: str) -> list[Selector]: return found +_HEADINGS = ("h1", "h2", "h3", "h4", "h5", "h6") + + +def _section_title(node: Any) -> bool: + """Whether an element holds nothing but a heading: the title of the section that follows it + (``

    Product Description

    `` on books.toscrape.com).""" + if tag_name(node) in _HEADINGS: + return True + headings = list(node.iter(*_HEADINGS)) + return len(headings) == 1 and text_content(node) == text_content(headings[0]) + + +def _next_text(node: Any) -> str: + """The text of the element after ``node``, unless that is another section's title.""" + following = next((el for el in node.itersiblings() if isinstance(el.tag, str)), None) + if following is None or _section_title(following): + return "" + return text_content(following) + + def _inside(node: Any, nodes: set[Any]) -> bool: """Whether ``node`` is one of ``nodes`` or inside one.""" return node is not None and (node in nodes or any(a in nodes for a in node.iterancestors())) @@ -770,7 +790,14 @@ def _coordinates(self, page: PageContext, f: SchemaField) -> list[tuple[Any, str return [(f"{lat}, {lon}", source) for (lat, lon), source in _map_points(page)] def _description(self, page: PageContext, f: SchemaField) -> list[tuple[Any, str]]: - found = self._texts(page, "description", "[class*=description]", max_len=20_000) + found = [] + for match in _marked(page, "description"): + text = match.text + if text and match.root is not None and _section_title(match.root): + #

    Product Description

    ...

    : the text follows + text = _next_text(match.root) + if text and len(text) <= 20_000: + found.append((text, "dom:[class*=description]")) return sorted(found, key=lambda pair: -len(pair[0]))[:1] def _sku(self, page: PageContext, f: SchemaField) -> list[tuple[Any, str]]: diff --git a/tests/test_extraction.py b/tests/test_extraction.py index 33f02d1..1e931ea 100644 --- a/tests/test_extraction.py +++ b/tests/test_extraction.py @@ -424,6 +424,29 @@ def test_other_records_cards_are_not_the_pages_own_values(site) -> None: assert len(Extractor("product").extract_all(listing)) == 4 +def test_a_description_under_its_section_title(site) -> None: + """books.toscrape.com titles a book's description in an element of its own (
    +

    Product Description

    ) and writes it in the

    after that: the description is the text, not + its title (read from the real markup, the generator learned #product_description, "Product Description" on + every page).""" + import wintergrab + + page = wintergrab.get(site.url + "/books/catalogue/book-3/index.html") + description = Extractor("product").extract(page).data["description"] + assert description.startswith("Book number 3 is the story of a town") and len(description) > 300 + + def read(body: str) -> object: + doc = wintergrab.parse(f"

    Lamp

    {body}") + return Extractor("product").extract(doc).data["description"] + + assert read('

    Description

    A brass lamp with a linen shade.
    ') == ( + "A brass lamp with a linen shade." + ) + assert read('

    About

    A brass lamp.

    ') == "About A brass lamp." + # a section's title followed by another section's: no text of its own + assert read('

    Product Description

    Details

    ') is None + + def test_cli_get_extract(site, tmp_path, capsys) -> None: schema = tmp_path / "product.json" schema.write_text( diff --git a/tests/test_generate.py b/tests/test_generate.py index c38c19f..acf4056 100644 --- a/tests/test_generate.py +++ b/tests/test_generate.py @@ -31,6 +31,8 @@ def test_selectors_learned_from_sample_pages(site) -> None: assert fields["url"].status == "skipped" and fields["url"].note == "the page's own address" assert fields["currency"].status == "skipped" and fields["currency"].note == "read from price" assert fields["brand"].status == "not found" + # not "#product_description", the title of the description's section: the text, too long to look for + assert fields["description"].status == "not learned" and "longer than 300 characters" in fields["description"].note assert generated.schema["price"].selectors == ["div.product_main > p.price_color"] # class "Three" reads as 3 assert not generated.base["price"].selectors assert generated.values[0]["category"] == "Poetry" and generated.methods[0]["price"] == "dom" @@ -40,6 +42,7 @@ def test_selectors_learned_from_sample_pages(site) -> None: [other] = _books(site, 11) record = Extractor(generated.schema).extract(other) assert record.data["name"] == "Book number 11" and record.data["price"] == 26.5 and record.data["rating"] == 2 + assert record.data["description"].startswith("Book number 11 is the story") # read from the page's evidence assert record.fields["price"].method == "selector" and "dom" in record.fields["price"].agreed assert record.confidence > Extractor("product").extract(other).confidence diff --git a/tests/test_goals.py b/tests/test_goals.py index 4b04a6a..d1f2e69 100644 --- a/tests/test_goals.py +++ b/tests/test_goals.py @@ -259,6 +259,9 @@ def test_the_survey_samples_more_of_the_pages_the_goal_wants(site) -> None: assert patterns["/books/catalogue/category/books/{slug}/index.html"] == 3 # the categories: their share plan = plan_goal(goal, surveys={site.url + "/books/": survey}) assert plan.sites[0].sample["record_pages"] >= 6 + # the books rated 4 or more among them: 4 of the site's 12 (a book's long description had made every book + # page an unsure one, read as a listing of its six other books: "records: about 0") + assert plan.sites[0].sample["passing"] >= 1 and plan.sites[0].estimate.records >= 2 def test_the_survey_learns_nothing_from_an_unsure_page(site) -> None: diff --git a/tests/testsite.py b/tests/testsite.py index 2b6d370..7cf58c4 100644 --- a/tests/testsite.py +++ b/tests/testsite.py @@ -491,6 +491,14 @@ def _route(self, site: Site) -> None: return self.send(404, layout("Not found", "

    nope

    ")) +# A book's description, as long as many on books.toscrape.com: with the page's information table it takes the +# book's
    past an article's length of text. +_BLURB = ( + "It is told over one summer by the people who lived through it, each chapter taking one of them from the " + "harbour to the hills and back. " +) * 9 + + def _book(i: int) -> dict[str, Any]: return { "title": f"Book number {i}", @@ -499,6 +507,7 @@ def _book(i: int) -> dict[str, Any]: "stock": 3 + i, "upc": f"upc{i:04d}", "category": "Poetry" if i % 2 else "Travel", + "description": f"Book number {i} is the story of a town and the family that kept its secrets. " + _BLURB, } @@ -576,6 +585,8 @@ def _books_page(handler: Handler, path: str) -> None: f"

    {b['title']}

    {b['price']}

    " f"

    In stock ({b['stock']} available)

    " f"

    " # (no cart button of its own, as on the real site) + f"

    Product Description

    " + f"

    {b['description']}

    Product Information

    " f"" f"" f"" From 5aae989924997834809c74d18f22c5dc2aa9170b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 12:17:15 +0000 Subject: [PATCH 27/30] Estimates that say what the sample cannot The owner's run on books.toscrape.com planned on three sampled books, none rated 4 or more, and said "records: about 0"; the crawl found 21. A share the sample never saw is not zero: under 3/n of n records, 95 times in 100 (the rule of three). When no sampled record meets the conditions, the estimate says "few if any" now, with the most the sample allows (Estimate.records_at_most: three sampled books leave room for every record page, ten for three in ten). The estimates also ignored --max-pages. "requests: 73" printed over a run capped at 60, and the confirmation before a big crawl counted the plan's requests, not the capped crawl's: a 10,000-page plan asked to confirm 10,000 requests for a 60-page run. The plan now says where the cap stops the crawl and about how many records that leaves, and the confirmation counts the requests the crawl will make. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qv1sKo4WDr8zQ8wFuW24k8 --- CHANGELOG.md | 7 +++++++ docs/api.md | 2 +- docs/goals.md | 7 ++++++- src/wintergrab/cli.py | 25 ++++++++++++++++++++++--- src/wintergrab/goals/plan.py | 30 ++++++++++++++++++++++++++++-- tests/test_goals.py | 21 +++++++++++++++++++++ 6 files changed, 85 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a63275d..8d929ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1245,6 +1245,13 @@ an error that says so, not a setting silently ignored and a generated scraper reads them from each page's own evidence. The test site's book pages have the real ones' description, as long as many of theirs. +- **Estimates that say what the sample cannot.** A plan whose sampled + records all failed its conditions said "records: about 0" (three books on + books.toscrape.com, none rated 4 or more: the crawl found 21). It says + "few if any" now, with the most the sample allows by the rule of three + (`Estimate.records_at_most`). `wintergrab goal --max-pages` under the pages + the plan needs is shown with the records it leaves, and the confirmation + asked before a big crawl counts the requests the crawl will make. - **The DuckDB output waits for a held file on Windows too.** A file another program has open is waited for a few seconds; on Windows the system's message for it ("being used by another process") was not read as a hold, diff --git a/docs/api.md b/docs/api.md index ddad903..4cad3a6 100644 --- a/docs/api.md +++ b/docs/api.md @@ -770,7 +770,7 @@ checks that it is up to date. - **`ENTITIES`**: a dict - **`EntityKind(name: str, words: tuple[str, ...], page_types: tuple[str, ...], listing_types: tuple[str, ...], fields: Mapping[str, Any], default_fields: tuple[str, ...], date_field: str | None = None)`** (class). A kind of record a goal can ask for. - `schema(self, fields: list[str]) -> Schema`: An extraction schema for these fields (types from :attr:`fields`, strings otherwise). -- **`Estimate(pages: int = 0, exact: bool = False, listing_pages: int = 0, requests: int = 0, browser_pages: int = 0, bytes: int = 0, seconds: float = 0.0, records: int = 0, cpu_seconds: float = 0.0, storage_bytes: int = 0, basis: list[str] = ...)`** (class). What a plan will cost, and what the numbers rest on (``basis``). +- **`Estimate(pages: int = 0, exact: bool = False, listing_pages: int = 0, requests: int = 0, browser_pages: int = 0, bytes: int = 0, seconds: float = 0.0, records: int = 0, cpu_seconds: float = 0.0, storage_bytes: int = 0, basis: list[str] = ..., records_at_most: int | None = None)`** (class). What a plan will cost, and what the numbers rest on (``basis``). - `describe(self) -> str` - **`GenerationResult(directory: Path, goal: Goal, stages: list[Stage] = ..., accepted: bool = False, reasons: list[str] = ..., plan: GoalPlan | None = None, generated: GeneratedSchema | None = None)`** (class). What :func:`generate_scraper` made, each step, and the verdict. - `describe(self) -> str`: Each step in a line (its problems and warnings under it), then the verdict. diff --git a/docs/goals.md b/docs/goals.md index dff54f3..a295cc3 100644 --- a/docs/goals.md +++ b/docs/goals.md @@ -127,7 +127,12 @@ CPU, storage) rest on the sample: its latency, page sizes, extraction time and the share of records meeting the conditions, robots.txt's crawl delay, and the spider's concurrency. `--explain` (or `plan.explain()`) prints what each rests on. Pages that need a browser are assumed to take 2 s each, -eight at a time: that one is not measured. +eight at a time: that one is not measured. When none of the sampled records +meets the conditions, the plan does not promise none: it says "few if any" +and the most the sample allows (a share never seen in n records is under +3/n, the rule of three; `Estimate.records_at_most`). A `--max-pages` under +the pages the plan needs is shown under the estimates, with the records it +leaves, and the confirmation below counts the requests the crawl will make. A plan is JSON: save it, read it, edit it (the patterns, the start URLs, the conditions), and run it later: diff --git a/src/wintergrab/cli.py b/src/wintergrab/cli.py index 43ffdc1..efceacf 100644 --- a/src/wintergrab/cli.py +++ b/src/wintergrab/cli.py @@ -720,6 +720,21 @@ def cmd_inspect(args: argparse.Namespace) -> int: return 0 +def _max_pages_note(estimate: Any, max_pages: int | None) -> str: + """What ``--max-pages`` leaves of a plan that needs more pages than that ("" when it needs fewer).""" + planned = estimate.pages + estimate.listing_pages + if not max_pages or max_pages >= planned: + return "" + share = max_pages / planned + more = "" if estimate.exact else " or more" + records = ( + "" + if estimate.records_at_most is not None + else f": about {round(estimate.records * share):,} of the {estimate.records:,} records" + ) + return f" --max-pages {max_pages:,} stops the crawl before the {planned:,}{more} pages the plan needs{records}" + + def cmd_goal(args: argparse.Namespace) -> int: from .goals import GoalPlan, parse_goal, plan_goal @@ -776,6 +791,9 @@ def cmd_goal(args: argparse.Namespace) -> int: if args.plan_only or args.verbose >= 0: whole = bool(args.plan) or args.plan_only # else "Understood:" said what the goal is print(plan.describe() if whole else "\n" + plan.describe(goal=False), file=shown) + cut = _max_pages_note(plan.estimate, args.max_pages) + if cut: + print(cut, file=shown) if args.explain: print("\nWhat the estimates rest on:\n" + plan.explain(), file=shown) if args.plan_only: @@ -784,14 +802,15 @@ def cmd_goal(args: argparse.Namespace) -> int: if not any(site.allowed for site in plan.sites): print("robots.txt keeps crawlers out: nothing to collect", file=sys.stderr) return 1 - big = estimate.requests > args.confirm_over or estimate.browser_pages > 50 + requests = min(estimate.requests, args.max_pages) if args.max_pages else estimate.requests + big = requests > args.confirm_over or min(estimate.browser_pages, requests) > 50 if big and not args.yes: if sys.stdin.isatty(): - answer = input(f"Run it? About {estimate.requests:,} requests. [y/N] ") + answer = input(f"Run it? About {requests:,} requests. [y/N] ") if answer.strip().lower() not in ("y", "yes"): return 0 else: - print(f"the plan makes about {estimate.requests:,} requests: add --yes to run it", file=sys.stderr) + print(f"the plan makes about {requests:,} requests: add --yes to run it", file=sys.stderr) return 0 output = args.output or "-" if args.review and not args.heal: diff --git a/src/wintergrab/goals/plan.py b/src/wintergrab/goals/plan.py index 3f3f9fa..37e4432 100644 --- a/src/wintergrab/goals/plan.py +++ b/src/wintergrab/goals/plan.py @@ -144,6 +144,9 @@ class Estimate: records: Records expected after the conditions. cpu_seconds: Processor time for parsing and extraction. storage_bytes: Size of the records as JSON Lines. + records_at_most: When none of the sampled records met the conditions, the most records the sample + allows (a share never seen in n records is under 3/n, 95 times in 100: the rule of three); + else ``None``. ``records`` is 0 then, and says less than this. """ pages: int = 0 @@ -157,6 +160,7 @@ class Estimate: cpu_seconds: float = 0.0 storage_bytes: int = 0 basis: list[str] = field(default_factory=list) + records_at_most: int | None = None def __add__(self, other: Estimate) -> Estimate: return Estimate( @@ -171,6 +175,11 @@ def __add__(self, other: Estimate) -> Estimate: cpu_seconds=self.cpu_seconds + other.cpu_seconds, storage_bytes=self.storage_bytes + other.storage_bytes, basis=self.basis + other.basis, + records_at_most=( + None + if self.records_at_most is None and other.records_at_most is None + else _at_most(self) + _at_most(other) + ), ) def describe(self) -> str: @@ -181,11 +190,18 @@ def describe(self) -> str: f"requests: {self.requests:,}" + (f" ({self.browser_pages:,} in a browser)" if self.browser_pages else " (none in a browser)"), f"download: {_size(self.bytes)}; time: {_duration(self.seconds)}; CPU: {_duration(self.cpu_seconds)}", - f"records: about {self.records:,} ({_size(self.storage_bytes)} as JSON Lines)", + f"records: few if any (none of the sampled records meet the conditions): at most about " + f"{self.records_at_most:,}" + if self.records_at_most is not None + else f"records: about {self.records:,} ({_size(self.storage_bytes)} as JSON Lines)", ] return "\n".join(lines) +def _at_most(estimate: Estimate) -> int: + return estimate.records if estimate.records_at_most is None else estimate.records_at_most + + def _size(n: float) -> str: for unit in ("bytes", "KB", "MB", "GB"): if n < 1000 or unit == "GB": @@ -845,7 +861,16 @@ def _estimate( expected = round(pages * min(1.0, record_rate) * pass_rate) if goal.limit: expected = min(expected, goal.limit) - if records: + at_most = None + if records and not pass_rate: # a share the sample never saw can still be up to 3 in its size + at_most = round(pages * min(1.0, record_rate) * min(1.0, 3 / len(records))) + if goal.limit: + at_most = min(at_most, goal.limit) + basis.append( + f"records: none of the {len(records)} sampled records meet the goal's conditions, so at most " + f"{min(1.0, 3 / len(records)):.0%} of the record pages should (the rule of three)" + ) + elif records: basis.append(f"records: {pass_rate:.0%} of the sampled records meet the goal's conditions") record_size = statistics.fmean(len(json.dumps(r, default=str)) for r in records) if records else 300.0 return Estimate( @@ -860,6 +885,7 @@ def _estimate( cpu_seconds=round((pages + listing) * per_page, 1), storage_bytes=int(expected * (record_size + 1)), basis=basis, + records_at_most=at_most, ) diff --git a/tests/test_goals.py b/tests/test_goals.py index d1f2e69..f51600b 100644 --- a/tests/test_goals.py +++ b/tests/test_goals.py @@ -398,3 +398,24 @@ def test_goal_command(site, tmp_path, capsys) -> None: assert len(capsys.readouterr().out.splitlines()) == 4 assert main(["goal", "products with name and price"]) == 2 # no site assert "which site?" in capsys.readouterr().err + + +def test_estimates_say_what_the_sample_cannot(site, tmp_path, capsys) -> None: + """A condition none of the sampled records met is no proof that none will: on books.toscrape.com a + sample of three books, none rated 4 or more, made the plan expect "about 0" records, and the crawl found + 21. The estimate says "few if any" then, with the most the sample allows (the rule of three). And a + --max-pages under the pages the plan needs says where the crawl will stop, and the confirmation counts + the requests it will make.""" + from wintergrab.cli import main + + goal = read("books costing more than £500 with title and price", sites=[site.url + "/books/"]) + estimate = plan_goal(goal, sample=15).sites[0].estimate + assert estimate.records == 0 and estimate.records_at_most is not None and estimate.records_at_most >= 1 + assert "few if any" in estimate.describe() and "about 0" not in estimate.describe() + + request = ["goal", "books rated 4 stars or more with title and price", "--site", site.url + "/books/"] + assert main([*request, "--sample", "15", "--plan-only", "--max-pages", "5"]) == 0 + assert "--max-pages 5 stops the crawl before the 15 or more pages the plan needs" in capsys.readouterr().out + out = tmp_path / "books.jsonl" + assert main([*request, "--sample", "15", "--max-pages", "5", "--confirm-over", "8", "-o", str(out)]) == 0 + assert "add --yes to run it" not in capsys.readouterr().err and out.exists() # 5 requests: under the bar From a82504f8213d5e01684adc67137d66a8106f3c53 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 12:18:22 +0000 Subject: [PATCH 28/30] README: why wintergrab, first The README listed twenty features in one run, with its strongest evidence eighth: the reproducible benchmark (about 1,000 pages/s on one core, 1.8x Crawlee and 3.8x Scrapy at the same concurrency, under half their memory). A "Why wintergrab" section now comes first, each point one the repository shows: the goal command on books.toscrape.com, the benchmark, adaptive selectors and healing extractors, the disk frontier and replay, polite crawling, and the test matrix CI runs. The full list follows under "Everything it does". Badges for the PyPI version, the Python versions, CI and the license; the install section says the project is in beta and where the changes between versions are listed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qv1sKo4WDr8zQ8wFuW24k8 --- README.md | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 90b4592..a96d9e1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ # wintergrab +[![PyPI](https://img.shields.io/pypi/v/wintergrab)](https://pypi.org/project/wintergrab/) +[![Python](https://img.shields.io/pypi/pyversions/wintergrab)](https://pypi.org/project/wintergrab/) +[![CI](https://github.com/opensourcewinter/wintergrab/actions/workflows/ci.yml/badge.svg)](https://github.com/opensourcewinter/wintergrab/actions/workflows/ci.yml) +[![License: MIT](https://img.shields.io/badge/license-MIT-blue)](https://github.com/opensourcewinter/wintergrab/blob/main/LICENSE) + **Friendly web scraping that scales from one page to big crawls.** ```python @@ -14,6 +19,36 @@ wintergrab is a Python toolkit for grabbing data from websites. Simple things are one line. When a site is harder (JavaScript, rate limits, thousands of pages), the same API scales up. +## Why wintergrab + +- **Say what you want, get clean records.** No selectors to write: + + ```bash + wintergrab goal "Find all books rated 4 stars or more with title, price and rating" \ + --site https://books.toscrape.com/ -o books.jsonl + ``` + + It surveys the site, shows you a plan with what it will cost, then + collects typed, validated, de-duplicated records. +- **Fast and light.** About 1,000 pages/s on one core in a + [reproducible benchmark](https://github.com/opensourcewinter/wintergrab/blob/main/benchmarks/README.md): + 1.8× Crawlee and 3.8× Scrapy at the same concurrency, with under half + their memory. +- **Keeps working when sites change.** Selectors that find their elements + again after a redesign, and extractors that repair themselves, with + versions you can roll back and a review queue when they are not sure. +- **One API from one page to millions.** `wg.get()` for a page, a `Spider` + for a site, a disk-backed queue that survives `kill -9` for millions of + URLs, and a cache that replays whole crawls offline. +- **Polite by default.** robots.txt, per-site throttling that backs off + when asked, and an honest browser: a refusal is reported, never worked + around. +- **Tested everywhere it runs.** Over 1,200 tests on Linux, macOS and + Windows with Python 3.10 to 3.14, against a local test site, so they need + no internet. + +## Everything it does + - **Fetch like a real browser.** HTTP requests carry Chrome/Firefox/Safari TLS and HTTP/2 fingerprints (via [curl_cffi](https://github.com/lexiforest/curl_cffi)). A headless Chromium (via [Playwright](https://playwright.dev/python/)) is @@ -106,7 +141,9 @@ playwright install chromium # one-time browser download (browser extr wintergrab doctor # check what is installed ``` -Python 3.10+ on Linux, macOS and Windows. On a fresh Linux machine, use +wintergrab is in beta: until 1.0 an API may still change between minor +versions, and [Upgrading from 0.2](https://github.com/opensourcewinter/wintergrab/blob/main/docs/migration.md) +lists each change. Python 3.10+ on Linux, macOS and Windows. On a fresh Linux machine, use `playwright install --with-deps chromium` to get the browser's system libraries too. The development version installs straight from GitHub: `pip install "wintergrab @ git+https://github.com/opensourcewinter/wintergrab"`. From 62f66f89d5fb150698f7347efe0c7c30a57f2694 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 12:40:54 +0000 Subject: [PATCH 29/30] The frontier's volume test measures the processor time it spends The release candidate's Windows job failed the test on a commit whose Windows job had passed minutes before: the last quarter of 200,000 pushes took 16.7 s against 4.2 s for the first (4.2, 5.0, 7.0, 16.7), while the pops, which read and delete, stayed flat (4.4, 4.2, 4.2, 4.0). Nothing in a push grows with the queue (a Bloom filter sized for a million keys, bisect into a bounded per-domain buffer, inserts batched 500 to a commit), and on Linux both measures are flat (3.3, 3.4, 3.2, 3.0 s of processor time). A shared runner's disk slowing down while the test writes 150 MB is wall time the process spends waiting. The test is about the frontier's cost, so it now compares the processor time each quarter takes (time.process_time). A cost that grows with the queue, a scan per push, shows there on every system; a slow disk does not. The wall-clock quarters are still printed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qv1sKo4WDr8zQ8wFuW24k8 --- tests/test_frontier.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/tests/test_frontier.py b/tests/test_frontier.py index 649f19e..4ad04b1 100644 --- a/tests/test_frontier.py +++ b/tests/test_frontier.py @@ -465,26 +465,31 @@ def _traced_peak(run: Callable[[], object]) -> int: tracemalloc.stop() -def _quarters(n: int, step: Callable[[int], None]) -> list[float]: - """How long each quarter of ``n`` calls of ``step`` took.""" - out: list[float] = [] - start = time.perf_counter() +def _quarters(n: int, step: Callable[[int], None]) -> tuple[list[float], list[float]]: + """How long each quarter of ``n`` calls of ``step`` took: the wall-clock seconds, and the processor seconds + this process spent (what grows when a step's own cost does; not the time spent waiting for a slow disk).""" + wall: list[float] = [] + cpu: list[float] = [] + start, start_cpu = time.perf_counter(), time.process_time() for i in range(n): step(i) if (i + 1) % (n // 4) == 0: - out.append(time.perf_counter() - start) - start = time.perf_counter() - return out + wall.append(time.perf_counter() - start) + cpu.append(time.process_time() - start_cpu) + start, start_cpu = time.perf_counter(), time.process_time() + return wall, cpu def test_volume_throughput_and_flat_memory(db: Path) -> None: """200,000 requests over 20 domains: the cost of a push, and of a pop with its ack, stays flat as the queue - grows (the last quarter of the run costs no more than twice the first), and the memory stays small. No - wall-clock bound: a shared CI runner can be three times slower one day than the next (the Windows job took - 173 s for what takes 23 s here, and failed a 120 s bound).""" + grows (the last quarter of the run costs no more processor time than twice the first), and the memory stays + small. No wall-clock bound: a shared CI runner can be three times slower one day than the next (the Windows + job took 173 s for what takes 23 s here, and failed a 120 s bound), and its disk can slow down while the test + writes (a Windows job's last quarter of pushes took 16.7 s against 4.2 s for the first, on a commit whose + Windows job had passed minutes before). A cost that grows with the queue shows in the processor time.""" n = 200_000 disk = DiskScheduler(db, FakeSpider()) - push_quarters = _quarters( + push_wall, push_quarters = _quarters( n, lambda i: disk.push(Request(f"https://site{i % 20}.test/item/{i}?page={i // 20}", meta={"depth": 1})) ) assert len(disk) == n @@ -496,12 +501,15 @@ def pop(_: int) -> None: assert request is not None disk.ack(request) - pop_quarters = _quarters(n, pop) + pop_wall, pop_quarters = _quarters(n, pop) assert disk.pop_ready(throttle, now)[0] is None and len(disk) == 0 disk.close() - push_s, pop_s = sum(push_quarters), sum(pop_quarters) + push_s, pop_s = sum(push_wall), sum(pop_wall) print(f"\nDiskScheduler, {n:,} requests: push {n / push_s:,.0f}/s, pop+ack {n / pop_s:,.0f}/s") - print(f"quarters: push {[round(q, 2) for q in push_quarters]} s, pop+ack {[round(q, 2) for q in pop_quarters]} s") + print(f"quarters (wall): push {[round(q, 2) for q in push_wall]} s, pop+ack {[round(q, 2) for q in pop_wall]} s") + print( + f"quarters (CPU): push {[round(q, 2) for q in push_quarters]} s, pop+ack {[round(q, 2) for q in pop_quarters]} s" + ) assert push_quarters[-1] < 2 * push_quarters[0], push_quarters assert pop_quarters[-1] < 2 * pop_quarters[0], pop_quarters From 1e6732bf4dc21657bf15702c38f54c1b54a9c19e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 12:53:43 +0000 Subject: [PATCH 30/30] The estimates test does not count the pages a sample finds test_estimates_say_what_the_sample_cannot expected "--max-pages 5 stops the crawl before the 15 or more pages the plan needs", the number this machine's survey gave. The survey fetches concurrently, so the pages it samples, and the record pages they link to, differ between runs: the Python 3.10 CI job's plan needed 18 or more. The test reads the note's shape now, and checks that the same plan without the cap asks to be confirmed, so the cap is what lets it run. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qv1sKo4WDr8zQ8wFuW24k8 --- tests/test_goals.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_goals.py b/tests/test_goals.py index f51600b..545da9d 100644 --- a/tests/test_goals.py +++ b/tests/test_goals.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re from datetime import datetime, timezone import pytest @@ -415,7 +416,11 @@ def test_estimates_say_what_the_sample_cannot(site, tmp_path, capsys) -> None: request = ["goal", "books rated 4 stars or more with title and price", "--site", site.url + "/books/"] assert main([*request, "--sample", "15", "--plan-only", "--max-pages", "5"]) == 0 - assert "--max-pages 5 stops the crawl before the 15 or more pages the plan needs" in capsys.readouterr().out + # (the pages the plan needs depend on the sample, which the survey fetches concurrently) + note = r"--max-pages 5 stops the crawl before the \d+ or more pages the plan needs: about \d+ of the \d+ records" + assert re.search(note, capsys.readouterr().out) out = tmp_path / "books.jsonl" + assert main([*request, "--sample", "15", "--confirm-over", "8", "-o", str(out)]) == 0 + assert "add --yes to run it" in capsys.readouterr().err and not out.exists() # the whole plan: over the bar assert main([*request, "--sample", "15", "--max-pages", "5", "--confirm-over", "8", "-o", str(out)]) == 0 - assert "add --yes to run it" not in capsys.readouterr().err and out.exists() # 5 requests: under the bar + assert "add --yes to run it" not in capsys.readouterr().err and out.exists() # 5 requests: under it
    UPC{b['upc']}
    Product TypeBooks
    Price (excl. tax){b['price']}
    Price (incl. tax){b['price']}
    Tax£0.00