From da3b9a3e12c4bc549c7a3a50cd9ea7345446aae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abraham=20P=C3=A9rez?= Date: Fri, 11 Sep 2026 08:04:45 -0600 Subject: [PATCH 1/6] =?UTF-8?q?release:=201.3.0=20=E2=80=94=20split=20CI?= =?UTF-8?q?=20checks=20and=20automate=20PyPI=20publishing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split the single CI "test" job into separate top-level checks — lint (ruff check), format (ruff format --check), type check (mypy) and security (bandit) — alongside the unit-test matrix, so a red check points straight at the cause instead of hiding inside one job. Releases now publish to PyPI automatically via GitHub Actions Trusted Publishing (OIDC): a push to main with a new __version__ runs the tests, tags the release, and uploads to PyPI — nothing publishes without a version bump. Also apply ruff format across the codebase, single-source the version from __version__ (guard tests pin the CHANGELOG top and both READMEs), mypy fixes across scraping/crawl, and one audited # nosec on the MCP HTTP bind. No behaviour change. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 50 ++++++++++- .github/workflows/release.yml | 26 ++++++ CHANGELOG.md | 20 +++++ README.es.md | 4 +- README.md | 4 +- pyproject.toml | 2 +- src/bytecrawl/__init__.py | 17 +++- src/bytecrawl/crawl/__init__.py | 13 ++- src/bytecrawl/crawl/base.py | 33 ++++--- src/bytecrawl/crawl/compare.py | 31 ++++--- src/bytecrawl/crawl/graph.py | 5 +- src/bytecrawl/crawl/result.py | 2 +- src/bytecrawl/crawl/strategies.py | 6 +- src/bytecrawl/crawl/text.py | 4 +- src/bytecrawl/mcp_http.py | 142 +++++++++++++++++------------- src/bytecrawl/mcp_server.py | 40 ++++----- src/bytecrawl/scraping/page.py | 54 ++++++++---- src/bytecrawl/scraping/scraper.py | 29 ++++-- src/bytecrawl/scraping/session.py | 9 +- src/bytecrawl/security.py | 4 +- tests/conftest.py | 30 +++++-- tests/test_browser_live.py | 1 + tests/test_core.py | 36 ++++---- tests/test_crawler_strategies.py | 25 ++---- tests/test_graph_utils.py | 20 +++-- tests/test_markdown_clean.py | 20 +++-- tests/test_mcp_http.py | 75 ++++++++++------ tests/test_mcp_server.py | 91 +++++++++++-------- tests/test_packaging.py | 92 ++++++++++++++----- tests/test_pep8_vercel.py | 8 +- tests/test_selectors.py | 44 +++++---- tests/test_session.py | 23 +++-- tests/test_smoke_production.py | 109 +++++++++++++++-------- 33 files changed, 705 insertions(+), 364 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53468ab..b46e53f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,55 @@ on: pull_request: jobs: + # Quality gates run once on the current Python — they don't vary by runtime. + # Each is its own job so it shows as a separate check on the PR. + lint: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install ruff + - run: ruff check src/bytecrawl tests + + format: + name: format + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install ruff + - run: ruff format --check src/bytecrawl tests + + typecheck: + name: type check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install (package + llm/mcp deps so mypy resolves imports) + run: pip install -e ".[llm,dev,mcp]" + - run: mypy src/bytecrawl + + security: + name: security + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install bandit + - run: bandit -r src/bytecrawl -ll + test: + name: unit test ${{ matrix.python-version }} runs-on: ubuntu-latest strategy: matrix: @@ -23,7 +71,5 @@ jobs: - name: Install MCP extra (needs Python 3.10+) if: matrix.python-version != '3.9' run: pip install -e ".[mcp]" - - name: Lint - run: ruff check src/bytecrawl tests - name: Tests run: pytest --cov=bytecrawl --cov-report=term-missing diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3bb1148..bee6e9d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,10 @@ jobs: runs-on: ubuntu-latest permissions: contents: write + outputs: + # true only when this push introduced a new version (no tag yet) — + # the publish job below keys off it so PyPI never gets a duplicate. + publish: ${{ steps.tag.outputs.exists == 'false' }} steps: - uses: actions/checkout@v4 with: @@ -66,3 +70,25 @@ jobs: gh release create "v${{ steps.v.outputs.version }}" \ --title "v${{ steps.v.outputs.version }}" \ --notes-file release-notes.md + + # Runs in the SAME workflow run right after `release`, only for a new version. + # (A separate workflow keyed on the tag/release would NOT fire: events made + # with GITHUB_TOKEN don't trigger further workflow runs.) + publish: + needs: release + if: needs.release.outputs.publish == 'true' + runs-on: ubuntu-latest + permissions: + # OIDC token for PyPI Trusted Publishing — no API token/secret needed. + id-token: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Build sdist + wheel + run: | + pip install build + python -m build + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index cede886..0be6d0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,24 @@ # Changelog +## 1.3.0 — 2026-09-11 + +### Changed +- **CI is split into separate checks instead of one "test" job.** Lint + (`ruff check`), format (`ruff format --check`), type check (`mypy`) and + security (`bandit`) each run as their own top-level check alongside the + unit-test matrix (`unit test 3.9` / `unit test 3.12`). A red check now points + straight at the cause — a formatting slip no longer hides behind a failing + type check inside a single collapsed job. +- **Releases publish to PyPI automatically** through GitHub Actions Trusted + Publishing (OIDC) — no API tokens or stored secrets. A push to `main` with a + new `__version__` runs the tests, tags the release, and uploads to PyPI; + nothing publishes without a version bump. + +### Internal +- Version is single-sourced from `__version__`: a test now pins the top + CHANGELOG entry and both READMEs to it, so they can't drift. +- Type-checking fixes across the scraping and crawl modules and one audited + `# nosec` on the MCP HTTP bind. No behaviour change. + ## 1.2.2 — 2026-09-10 ### Fixed diff --git a/README.es.md b/README.es.md index c10d350..ec8097d 100644 --- a/README.es.md +++ b/README.es.md @@ -10,8 +10,8 @@ Shark-Search y OPIC en Python puro. - **Web**: https://bytecrawl.vercel.app/ - **Endpoint MCP hosteado**: https://bytecrawl.vercel.app/mcp - **Skill para agentes**: https://bytecrawl.vercel.app/agent-onboarding/SKILL.md -- **Última versión**: **1.2.2** — seis herramientas MCP, comparación de - estrategias, `Page.links()` absolutos ([changelog](https://github.com/abrahamperz/ByteCrawl/blob/main/CHANGELOG.md)) +- **Última versión**: **1.3.0** — publicación automática a PyPI y checks de CI + separados (lint, formato, tipos, seguridad) ([changelog](https://github.com/abrahamperz/ByteCrawl/blob/main/CHANGELOG.md)) - **PyPI**: https://pypi.org/project/bytecrawl/ - **GitHub**: https://github.com/abrahamperz/ByteCrawl diff --git a/README.md b/README.md index 1db6f2d..702fd92 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ and OPIC in pure Python. - **Webpage**: https://bytecrawl.vercel.app/ - **Hosted MCP endpoint**: https://bytecrawl.vercel.app/mcp - **Agent skill**: https://bytecrawl.vercel.app/agent-onboarding/SKILL.md -- **Latest release**: **1.2.2** — six MCP tools, strategy comparison, - absolute `Page.links()` ([changelog](https://github.com/abrahamperz/ByteCrawl/blob/main/CHANGELOG.md)) +- **Latest release**: **1.3.0** — automated PyPI publishing and separated CI + gates (lint, format, types, security) ([changelog](https://github.com/abrahamperz/ByteCrawl/blob/main/CHANGELOG.md)) ## Why focused crawling? diff --git a/pyproject.toml b/pyproject.toml index 397df76..9981017 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ browser = ["playwright>=1.40"] llm = ["markdownify>=0.11", "trafilatura>=1.6"] mcp = ["mcp>=2; python_version >= '3.10'", "markdownify>=0.11", "trafilatura>=1.6"] all = ["playwright>=1.40", "markdownify>=0.11", "trafilatura>=1.6", "mcp>=2; python_version >= '3.10'"] -dev = ["pytest>=7", "pytest-cov", "ruff"] +dev = ["pytest>=7", "pytest-cov", "ruff", "mypy", "bandit"] [project.scripts] bytecrawl-mcp = "bytecrawl.mcp_server:main" diff --git a/src/bytecrawl/__init__.py b/src/bytecrawl/__init__.py index 8a15b45..beca973 100644 --- a/src/bytecrawl/__init__.py +++ b/src/bytecrawl/__init__.py @@ -36,8 +36,17 @@ ) __all__ = [ - "Scraper", "Page", "Session", - "Crawler", "BFS", "SharkSearch", "OPIC", "CrawlResult", "STRATEGIES", - "pagerank", "cosine", "compare", + "Scraper", + "Page", + "Session", + "Crawler", + "BFS", + "SharkSearch", + "OPIC", + "CrawlResult", + "STRATEGIES", + "pagerank", + "cosine", + "compare", ] -__version__ = "1.2.2" +__version__ = "1.3.0" diff --git a/src/bytecrawl/crawl/__init__.py b/src/bytecrawl/crawl/__init__.py index 45bef4e..30bab17 100644 --- a/src/bytecrawl/crawl/__init__.py +++ b/src/bytecrawl/crawl/__init__.py @@ -27,6 +27,15 @@ from .text import cosine, relevance __all__ = [ - "Crawler", "BFS", "SharkSearch", "OPIC", "CrawlResult", "Frontier", - "STRATEGIES", "cosine", "relevance", "pagerank", "compare", + "Crawler", + "BFS", + "SharkSearch", + "OPIC", + "CrawlResult", + "Frontier", + "STRATEGIES", + "cosine", + "relevance", + "pagerank", + "compare", ] diff --git a/src/bytecrawl/crawl/base.py b/src/bytecrawl/crawl/base.py index 47f471b..090399d 100644 --- a/src/bytecrawl/crawl/base.py +++ b/src/bytecrawl/crawl/base.py @@ -31,15 +31,17 @@ class Crawler: name = "bfs" - def __init__(self, query: str = "", delay: float = 0.2, timeout: int = 10, - same_domain: bool = True): + def __init__( + self, query: str = "", delay: float = 0.2, timeout: int = 10, same_domain: bool = True + ): self.query = query self.same_domain = same_domain self.scraper = Scraper(delay=delay, timeout=timeout) # --- extension point ------------------------------------------------------- - def score_links(self, url: str, links: list[dict], page_relevance: float, - depth: int) -> list[tuple[str, float]]: + def score_links( + self, url: str, links: list[dict], page_relevance: float, depth: int + ) -> list[tuple[str, float]]: """BFS: the score only encodes depth (shallower = visited sooner). links: [{url, anchor}]. Returns [(url, score)] for the frontier. @@ -89,7 +91,7 @@ def crawl(self, start: str, max_pages: int = 50, max_depth: int = 10) -> CrawlRe links = [] for a in page.soup.select("a[href]"): - child = normalize(a["href"], url) + child = normalize(str(a["href"]), url) if not child or child == url: continue if self.same_domain and _root_domain(urlparse(child).netloc) != domain: @@ -97,11 +99,16 @@ def crawl(self, start: str, max_pages: int = 50, max_depth: int = 10) -> CrawlRe links.append({"url": child, "anchor": a.get_text(" ", strip=True)}) result.graph[url] = [link["url"] for link in links] - result.pages.append({ - "url": url, "title": title[:120], "score": round(score, 4), - "relevance": round(page_relevance, 4), "depth": depth, - "order": len(result.pages) + 1, - }) + result.pages.append( + { + "url": url, + "title": title[:120], + "score": round(score, 4), + "relevance": round(page_relevance, 4), + "depth": depth, + "order": len(result.pages) + 1, + } + ) self.on_visit(url, links) @@ -117,8 +124,8 @@ def crawl(self, start: str, max_pages: int = 50, max_depth: int = 10) -> CrawlRe "elapsed": round(time.perf_counter() - t0, 2), "frontier_left": len(frontier), "relevant_found": len(result.relevant()) if self.query else None, - "avg_relevance": round( - sum(p["relevance"] for p in result.pages) / len(result.pages), 4 - ) if result.pages else 0.0, + "avg_relevance": round(sum(p["relevance"] for p in result.pages) / len(result.pages), 4) + if result.pages + else 0.0, } return result diff --git a/src/bytecrawl/crawl/compare.py b/src/bytecrawl/crawl/compare.py index 432e650..7aca8d9 100644 --- a/src/bytecrawl/crawl/compare.py +++ b/src/bytecrawl/crawl/compare.py @@ -7,13 +7,15 @@ from .strategies import STRATEGIES -def _leg(name: str, start: str, query: str, max_pages: int, max_depth: int, - delay: float, timeout: int) -> dict: +def _leg( + name: str, start: str, query: str, max_pages: int, max_depth: int, delay: float, timeout: int +) -> dict: """One strategy's leg of a comparison. Never raises: a strategy that fails reports inside its own entry so the other two still come back.""" try: result = STRATEGIES[name](query=query, delay=delay, timeout=timeout).crawl( - start, max_pages=max_pages, max_depth=max_depth) + start, max_pages=max_pages, max_depth=max_depth + ) except Exception as e: return {"error": f"{type(e).__name__}: {e}"} return { @@ -23,8 +25,14 @@ def _leg(name: str, start: str, query: str, max_pages: int, max_depth: int, } -def compare(start: str, query: str, max_pages: int = 20, max_depth: int = 4, - delay: float = 0.2, timeout: int = 10) -> dict: +def compare( + start: str, + query: str, + max_pages: int = 20, + max_depth: int = 4, + delay: float = 0.2, + timeout: int = 10, +) -> dict: """Run every strategy over one site on the same budget, side by side. A single ordering looks like any other crawler's. The case for having @@ -49,13 +57,16 @@ def compare(start: str, query: str, max_pages: int = 20, max_depth: int = 4, if not query: raise ValueError( "compare needs a query: with nothing to be relevant to, the " - "strategies aren't comparable (use a single crawler instead)") + "strategies aren't comparable (use a single crawler instead)" + ) names = list(STRATEGIES) with ThreadPoolExecutor(max_workers=len(names)) as pool: - legs = list(pool.map( - lambda n: _leg(n, start, query, max_pages, max_depth, - delay * len(names), timeout), - names)) + legs = list( + pool.map( + lambda n: _leg(n, start, query, max_pages, max_depth, delay * len(names), timeout), + names, + ) + ) results = dict(zip(names, legs)) ok = {n: r for n, r in results.items() if "error" not in r} diff --git a/src/bytecrawl/crawl/graph.py b/src/bytecrawl/crawl/graph.py index 780068d..f10d2da 100644 --- a/src/bytecrawl/crawl/graph.py +++ b/src/bytecrawl/crawl/graph.py @@ -8,8 +8,9 @@ from __future__ import annotations -def pagerank(graph: dict[str, list[str]], damping: float = 0.85, - iterations: int = 30) -> dict[str, float]: +def pagerank( + graph: dict[str, list[str]], damping: float = 0.85, iterations: int = 30 +) -> dict[str, float]: """PageRank via power iteration over the crawled graph. Offline version of the same concept OPIC approximates online: comparing diff --git a/src/bytecrawl/crawl/result.py b/src/bytecrawl/crawl/result.py index 5fe6f75..9b400e8 100644 --- a/src/bytecrawl/crawl/result.py +++ b/src/bytecrawl/crawl/result.py @@ -10,7 +10,7 @@ class CrawlResult: """What a crawl returns: pages, graph and numbers for comparison.""" strategy: str - pages: list[dict] = field(default_factory=list) # url, title, score, relevance, depth, order + pages: list[dict] = field(default_factory=list) # url, title, score, relevance, depth, order graph: dict[str, list[str]] = field(default_factory=dict) stats: dict = field(default_factory=dict) diff --git a/src/bytecrawl/crawl/strategies.py b/src/bytecrawl/crawl/strategies.py index 02e1fdb..86a998b 100644 --- a/src/bytecrawl/crawl/strategies.py +++ b/src/bytecrawl/crawl/strategies.py @@ -51,7 +51,7 @@ def score_links(self, url, links, page_relevance, depth): scored = [] for link in links: url_words = " ".join(_tokens(urlparse(link["url"]).path)) - local = relevance(f'{link["anchor"]} {url_words}', self.query) + local = relevance(f"{link['anchor']} {url_words}", self.query) score = self.gamma * inherited + (1 - self.gamma) * local self._inherited[link["url"]] = inherited scored.append((link["url"], score)) @@ -104,5 +104,7 @@ def score_links(self, url, links, page_relevance, depth): # strategy, so a crawl requested over HTTP, over MCP or in Python means the # same thing everywhere. STRATEGIES: dict[str, type[Crawler]] = { - "bfs": BFS, "shark": SharkSearch, "opic": OPIC, + "bfs": BFS, + "shark": SharkSearch, + "opic": OPIC, } diff --git a/src/bytecrawl/crawl/text.py b/src/bytecrawl/crawl/text.py index 1783768..7a7473b 100644 --- a/src/bytecrawl/crawl/text.py +++ b/src/bytecrawl/crawl/text.py @@ -38,7 +38,7 @@ def cosine(text_a: str, text_b: str) -> float: @lru_cache(maxsize=4096) def _trigrams(word: str) -> frozenset: padded = f"~{word}~" - return frozenset(padded[i:i + 3] for i in range(len(padded) - 2)) + return frozenset(padded[i : i + 3] for i in range(len(padded) - 2)) def _dice(a: frozenset, b: frozenset) -> float: @@ -81,7 +81,7 @@ def _resolve_terms(query: str, vocabulary: set) -> str: score = _dice(grams, _trigrams(word)) if score > best_score: best, best_score = word, score - resolved.append(best if best_score >= _FUZZY_MIN else term) + resolved.append(best if best is not None and best_score >= _FUZZY_MIN else term) return " ".join(resolved) diff --git a/src/bytecrawl/mcp_http.py b/src/bytecrawl/mcp_http.py index 110c8f6..b51bcdb 100644 --- a/src/bytecrawl/mcp_http.py +++ b/src/bytecrawl/mcp_http.py @@ -38,13 +38,13 @@ from . import mcp_server as local from .security import assert_public_url -MAX_PAGES = 10 # hard cap per focused_crawl call (local allows 50) +MAX_PAGES = 10 # hard cap per focused_crawl call (local allows 50) # compare_strategies is three crawls that the rate limiter charges as one # request, so it gets a smaller budget: 3 x 6 = 18 fetches, about two # focused_crawls' worth. Same bounded-cost rule, applied to a heavier call. MAX_COMPARE_PAGES = 6 -RATE_LIMIT = 20 # requests per window per client IP -RATE_WINDOW = 60.0 # seconds +RATE_LIMIT = 20 # requests per window per client IP +RATE_WINDOW = 60.0 # seconds def _guarded(url: str) -> None: @@ -93,16 +93,19 @@ def _client_ip(scope) -> str: async def __call__(self, scope, receive, send): if scope["type"] == "http" and not self.limiter.allow(self._client_ip(scope)): - await send({ - "type": "http.response.start", - "status": 429, - "headers": [(b"content-type", b"application/json"), - (b"retry-after", b"60")], - }) - await send({ - "type": "http.response.body", - "body": b'{"error": "rate limit exceeded, try again in a minute"}', - }) + await send( + { + "type": "http.response.start", + "status": 429, + "headers": [(b"content-type", b"application/json"), (b"retry-after", b"60")], + } + ) + await send( + { + "type": "http.response.body", + "body": b'{"error": "rate limit exceeded, try again in a minute"}', + } + ) return await self.app(scope, receive, send) @@ -119,64 +122,75 @@ async def __call__(self, scope, receive, send): ) -@server.tool(description=( - "Fetch a public web page and return it as clean Markdown for LLM " - "consumption, with tokens_estimate against tokens_html so you can " - "state the saving. Static HTML only on the hosted server." -)) +@server.tool( + description=( + "Fetch a public web page and return it as clean Markdown for LLM " + "consumption, with tokens_estimate against tokens_html so you can " + "state the saving. Static HTML only on the hosted server." + ) +) def fetch_markdown(url: str) -> dict: _guarded(url) return local.fetch_markdown(url) -@server.tool(description=( - "Extract data from a public page with CSS selectors. Call it with ONLY the " - "url to discover what is extractable — it returns the page's repeated " - "blocks with ready-to-use item + fields and a sample record each, which is " - "what you need when you have not seen the markup. Then 'item'+'fields' for " - "structured records (::text / ::attr(name); a field name ending in [] " - "collects a list), or 'select' on its own for a flat list of one " - "selector's values." -)) -def extract(url: str, item: str = "", fields: dict[str, str] | None = None, - select: str = "") -> dict: +@server.tool( + description=( + "Extract data from a public page with CSS selectors. Call it with ONLY the " + "url to discover what is extractable — it returns the page's repeated " + "blocks with ready-to-use item + fields and a sample record each, which is " + "what you need when you have not seen the markup. Then 'item'+'fields' for " + "structured records (::text / ::attr(name); a field name ending in [] " + "collects a list), or 'select' on its own for a flat list of one " + "selector's values." + ) +) +def extract( + url: str, item: str = "", fields: dict[str, str] | None = None, select: str = "" +) -> dict: _guarded(url) return local.extract(url, item=item, fields=fields, select=select) -@server.tool(description=( - "Every outbound link on a public page as an absolute, deduplicated URL " - "(#fragments, mailto: and asset files dropped). raw=true returns the " - "untouched href values." -)) +@server.tool( + description=( + "Every outbound link on a public page as an absolute, deduplicated URL " + "(#fragments, mailto: and asset files dropped). raw=true returns the " + "untouched href values." + ) +) def list_links(url: str, raw: bool = False) -> dict: _guarded(url) return local.list_links(url, raw=raw) -@server.tool(description=( - "Focused crawl of a public site: visit the pages most relevant to 'query' " - "first. strategy: shark (default) | opic | bfs. Capped at " - f"{MAX_PAGES} pages per call on the hosted server." -)) -def focused_crawl(url: str, query: str = "", strategy: str = "shark", - max_pages: int = MAX_PAGES) -> dict: +@server.tool( + description=( + "Focused crawl of a public site: visit the pages most relevant to 'query' " + "first. strategy: shark (default) | opic | bfs. Capped at " + f"{MAX_PAGES} pages per call on the hosted server." + ) +) +def focused_crawl( + url: str, query: str = "", strategy: str = "shark", max_pages: int = MAX_PAGES +) -> dict: _guarded(url) - return local.focused_crawl(url, query=query, strategy=strategy, - max_pages=min(max_pages, MAX_PAGES)) - - -@server.tool(description=( - "Run Shark-Search, OPIC and BFS over the same public site on the same " - "budget and return them side by side with a winner. Three crawls in one " - f"call, so it is capped at {MAX_COMPARE_PAGES} pages per strategy on the " - "hosted server." -)) -def compare_strategies(url: str, query: str, - max_pages: int = MAX_COMPARE_PAGES) -> dict: + return local.focused_crawl( + url, query=query, strategy=strategy, max_pages=min(max_pages, MAX_PAGES) + ) + + +@server.tool( + description=( + "Run Shark-Search, OPIC and BFS over the same public site on the same " + "budget and return them side by side with a winner. Three crawls in one " + f"call, so it is capped at {MAX_COMPARE_PAGES} pages per strategy on the " + "hosted server." + ) +) +def compare_strategies(url: str, query: str, max_pages: int = MAX_COMPARE_PAGES) -> dict: _guarded(url) - return local.compare_strategies(url, query=query, - max_pages=min(max_pages, MAX_COMPARE_PAGES)) + return local.compare_strategies(url, query=query, max_pages=min(max_pages, MAX_COMPARE_PAGES)) @server.tool(description="Fetch a public JSON API endpoint.") @@ -204,12 +218,14 @@ def _transport_security() -> TransportSecuritySettings: and BYTECRAWL_ALLOWED_ORIGINS alongside it) to turn it back on — worth doing if you self-host on a domain that also serves authenticated apps. """ - hosts = [h.strip() for h in - os.environ.get("BYTECRAWL_ALLOWED_HOSTS", "").split(",") if h.strip()] + hosts = [ + h.strip() for h in os.environ.get("BYTECRAWL_ALLOWED_HOSTS", "").split(",") if h.strip() + ] if not hosts: return TransportSecuritySettings(enable_dns_rebinding_protection=False) - origins = [o.strip() for o in - os.environ.get("BYTECRAWL_ALLOWED_ORIGINS", "").split(",") if o.strip()] + origins = [ + o.strip() for o in os.environ.get("BYTECRAWL_ALLOWED_ORIGINS", "").split(",") if o.strip() + ] return TransportSecuritySettings( enable_dns_rebinding_protection=True, allowed_hosts=hosts, @@ -221,7 +237,9 @@ def create_app(limiter: RateLimiter | None = None): """ASGI app: streamable-HTTP MCP at /mcp, rate-limited. Stateless for serverless (json_response avoids long-lived SSE streams on Vercel).""" app = server.streamable_http_app( - streamable_http_path="/mcp", stateless_http=True, json_response=True, + streamable_http_path="/mcp", + stateless_http=True, + json_response=True, transport_security=_transport_security(), ) return RateLimitMiddleware(app, limiter) @@ -231,7 +249,9 @@ def main() -> None: """Entry point for bytecrawl-mcp-http (self-hosted).""" import uvicorn - uvicorn.run(create_app(), host="0.0.0.0", port=8000) + # nosec B104 — an MCP HTTP server must bind all interfaces to be reachable + # inside a container or serverless host. + uvicorn.run(create_app(), host="0.0.0.0", port=8000) # nosec B104 if __name__ == "__main__": diff --git a/src/bytecrawl/mcp_server.py b/src/bytecrawl/mcp_server.py index 57381b9..12931a2 100644 --- a/src/bytecrawl/mcp_server.py +++ b/src/bytecrawl/mcp_server.py @@ -16,9 +16,7 @@ try: from mcp.server.mcpserver import MCPServer except ImportError as e: # pragma: no cover - exercised only without the extra - raise ImportError( - "The MCP server needs the 'mcp' extra: pip install bytecrawl[mcp]" - ) from e + raise ImportError("The MCP server needs the 'mcp' extra: pip install bytecrawl[mcp]") from e from .core import Scraper from .crawler import STRATEGIES as _STRATEGIES @@ -79,8 +77,9 @@ def fetch_markdown(url: str) -> dict: "selector's values across the page, e.g. 'h3 a::attr(title)'." ) ) -def extract(url: str, item: str = "", fields: dict[str, str] | None = None, - select: str = "") -> dict: +def extract( + url: str, item: str = "", fields: dict[str, str] | None = None, select: str = "" +) -> dict: # Three shapes, one tool. Agents reach for a single selector far more # often than for a record schema, and an agent that has never seen the # page can produce neither: fetch_markdown strips exactly the classes a @@ -91,26 +90,28 @@ def extract(url: str, item: str = "", fields: dict[str, str] | None = None, if select and (item or fields): raise ValueError("pass either 'select' or 'item'+'fields', not both") if bool(item) != bool(fields): - raise ValueError("'item' and 'fields' go together — or pass neither " - "to see what this page offers") + raise ValueError( + "'item' and 'fields' go together — or pass neither to see what this page offers" + ) # Checked before the fetch: a malformed call shouldn't cost the site a # request. page = _scraper.fetch(url) if select: values = page.css_all(select) - return {"url": url, "select": select, "count": len(values), - "values": values} + return {"url": url, "select": select, "count": len(values), "values": values} if item: - records = page.extract(item, fields) + records = page.extract(item, fields or {}) return {"url": url, "count": len(records), "records": records} candidates = page.selectors() return { "url": url, "candidates": candidates, - "hint": ("call extract again with the item and fields of whichever " - "candidate holds what you want" if candidates else - "no repeated blocks here — this page is probably not a " - "listing; pass 'select' with a specific selector instead"), + "hint": ( + "call extract again with the item and fields of whichever candidate holds what you want" + if candidates + else "no repeated blocks here — this page is probably not a " + "listing; pass 'select' with a specific selector instead" + ), } @@ -136,8 +137,7 @@ def list_links(url: str, raw: bool = False) -> dict: "(level by level). Returns pages ranked by relevance plus crawl stats." ) ) -def focused_crawl(url: str, query: str = "", strategy: str = "shark", - max_pages: int = 20) -> dict: +def focused_crawl(url: str, query: str = "", strategy: str = "shark", max_pages: int = 20) -> dict: if strategy not in _STRATEGIES: raise ValueError(f"strategy must be one of {sorted(_STRATEGIES)}") max_pages = min(max_pages, 50) # keep agent calls bounded and polite @@ -148,9 +148,7 @@ def focused_crawl(url: str, query: str = "", strategy: str = "shark", "strategy": strategy, "stats": result.stats, "pages": result.top(max_pages) if query else result.pages, - "pagerank_top": [ - {"url": u, "rank": round(r, 4)} for u, r in list(ranks.items())[:10] - ], + "pagerank_top": [{"url": u, "rank": round(r, 4)} for u, r in list(ranks.items())[:10]], } @@ -170,9 +168,7 @@ def compare_strategies(url: str, query: str, max_pages: int = 20) -> dict: return {"url": url, "query": query, "max_pages": max_pages, **out} -@server.tool( - description="Fetch a JSON API endpoint (the 'hidden API' scraping technique)." -) +@server.tool(description="Fetch a JSON API endpoint (the 'hidden API' scraping technique).") def fetch_json_api(url: str, params: dict | None = None) -> dict: page = _scraper.api(url, params=params) return {"url": url, "status": page.status, "data": page.json()} diff --git a/src/bytecrawl/scraping/page.py b/src/bytecrawl/scraping/page.py index 92a4aa9..aef61bc 100644 --- a/src/bytecrawl/scraping/page.py +++ b/src/bytecrawl/scraping/page.py @@ -63,7 +63,7 @@ def extract(self, item: str, fields: dict[str, str]) -> list[dict]: """ records = [] for node in self.soup.select(item): - row = {} + row: dict[str, Any] = {} for name, spec in fields.items(): sel, op, arg = _split_selector(spec) if name.endswith("[]"): @@ -138,8 +138,11 @@ def sig(node) -> str | None: items = {s for s, _, _, _ in found} out = [] for s, count, fields, node in found: - if any(count == c2 and set(fields) <= set(f2) and node in n2.parents - for s2, c2, f2, n2 in found if s2 != s): + if any( + count == c2 and set(fields) <= set(f2) and node in n2.parents + for s2, c2, f2, n2 in found + if s2 != s + ): continue # The other tell of a wrapper: one of its columns is an entire # other record. `li.col-xs-6` offers `article.product_pod::text` @@ -147,8 +150,9 @@ def sig(node) -> str | None: if any(spec.split("::")[0] in items for spec in fields.values()): continue sample = self.extract(s, fields) - out.append({"item": s, "count": count, "fields": fields, - "sample": sample[0] if sample else {}}) + out.append( + {"item": s, "count": count, "fields": fields, "sample": sample[0] if sample else {}} + ) if len(out) == limit: break return out @@ -180,8 +184,10 @@ def _fields_of(nodes: list, coverage: float = 0.6) -> dict[str, str]: need = max(1, int(len(nodes) * coverage)) # Named classes before bare tags, then shortest: `p.price_color::text` # is a column, `p::text` is whichever paragraph happened to be first. - keep = sorted((s for s, n in hits.items() if n >= need), - key=lambda s: ("." not in s.split("::")[0], len(s))) + keep = sorted( + (s for s, n in hits.items() if n >= need), + key=lambda s: ("." not in s.split("::")[0], len(s)), + ) fields, used = {}, set() for spec in keep: @@ -195,8 +201,9 @@ def _fields_of(nodes: list, coverage: float = 0.6) -> dict[str, str]: # A selector that resolves everywhere but is blank everywhere is # not a column. Check against the instances, not the markup. op, arg = _split_selector(spec)[1:] - if not any(_value_from(n.select_one(sel), op, arg) - for n in nodes[:5] if n.select_one(sel)): + if not any( + _value_from(n.select_one(sel), op, arg) for n in nodes[:5] if n.select_one(sel) + ): continue used.add(sel) fields[name] = spec @@ -208,13 +215,19 @@ def _fields_of(nodes: list, coverage: float = 0.6) -> dict[str, str]: # "£51.77In stockAdd to basket" — three fields glued into one, and # never what the caller wanted from any of them. probe = nodes[0] - inner = {sel for sel in used - if (n := probe.select_one(sel)) is not None - and any(n is not m and m in n.descendants - for other in used - if (m := probe.select_one(other)) is not None)} - return {k: v for k, v in fields.items() - if _split_selector(v)[0] not in inner or "::attr" in v} + inner = { + sel + for sel in used + if (n := probe.select_one(sel)) is not None + and any( + n is not m and m in n.descendants + for other in used + if (m := probe.select_one(other)) is not None + ) + } + return { + k: v for k, v in fields.items() if _split_selector(v)[0] not in inner or "::attr" in v + } def links(self, raw: bool = False) -> list[str]: """Every link on the page as an absolute URL, deduplicated in order. @@ -225,7 +238,7 @@ def links(self, raw: bool = False) -> list[str]: use, so what you get back is the set of pages you could visit next. Pass raw=True for the untouched attribute values. """ - hrefs = [a["href"] for a in self.soup.select("a[href]")] + hrefs = [str(a["href"]) for a in self.soup.select("a[href]")] if raw: return hrefs out: list[str] = [] @@ -261,8 +274,11 @@ def _extraction_gutted_it(self, md: str) -> bool: visible = len(self.soup.get_text(" ", strip=True)) if visible and len(md) / visible < self._MAIN_CONTENT_FLOOR: return True - headings = [h.get_text(strip=True) - for h in self.soup.select("h1,h2,h3,h4") if h.get_text(strip=True)] + headings = [ + h.get_text(strip=True) + for h in self.soup.select("h1,h2,h3,h4") + if h.get_text(strip=True) + ] if len(headings) < self._MANY_HEADINGS: return False kept = sum(1 for h in headings if h[:30] in md) diff --git a/src/bytecrawl/scraping/scraper.py b/src/bytecrawl/scraping/scraper.py index 2631707..fd7129b 100644 --- a/src/bytecrawl/scraping/scraper.py +++ b/src/bytecrawl/scraping/scraper.py @@ -34,8 +34,13 @@ def static(self, url: str) -> Page: r = self._session.get(url, timeout=self.timeout) r.raise_for_status() self._wait() - return Page(url=url, html=_decoded_html(r), method="static", - elapsed=round(time.perf_counter() - t0, 3), status=r.status_code) + return Page( + url=url, + html=_decoded_html(r), + method="static", + elapsed=round(time.perf_counter() - t0, 3), + status=r.status_code, + ) def api(self, url: str, params: dict | None = None) -> Page: """Technique 3: requests an API and stores the JSON.""" @@ -48,8 +53,13 @@ def api(self, url: str, params: dict | None = None) -> Page: except ValueError as e: ctype = r.headers.get("content-type", "unknown") raise ValueError(f"{url} did not return JSON (content-type: {ctype})") from e - return Page(url=url, data=data, method="api", - elapsed=round(time.perf_counter() - t0, 3), status=r.status_code) + return Page( + url=url, + data=data, + method="api", + elapsed=round(time.perf_counter() - t0, 3), + status=r.status_code, + ) def browser(self, url: str, wait: str | None = None, scroll: bool = False) -> Page: """Technique 2: real browser (Playwright) for JS-rendered sites.""" @@ -75,8 +85,13 @@ def browser(self, url: str, wait: str | None = None, scroll: bool = False) -> Pa status = resp.status if resp else 0 nav.close() self._wait() - return Page(url=url, html=html, method="browser", - elapsed=round(time.perf_counter() - t0, 3), status=status) + return Page( + url=url, + html=html, + method="browser", + elapsed=round(time.perf_counter() - t0, 3), + status=status, + ) def fetch(self, url: str, strategy: str = "auto") -> Page: """Fetches the page. strategy: auto | static | browser. @@ -116,7 +131,7 @@ def crawl( """ from urllib.parse import urljoin - url = start + url: str | None = start results: list[dict] = [] n = 0 while url: diff --git a/src/bytecrawl/scraping/session.py b/src/bytecrawl/scraping/session.py index 61753ef..981a15f 100644 --- a/src/bytecrawl/scraping/session.py +++ b/src/bytecrawl/scraping/session.py @@ -49,5 +49,10 @@ def fetch(self, url: str) -> Page: t0 = time.perf_counter() r = self._s.get(url, timeout=self._scraper.timeout) r.raise_for_status() - return Page(url=url, html=_decoded_html(r), method="static", - elapsed=round(time.perf_counter() - t0, 3), status=r.status_code) + return Page( + url=url, + html=_decoded_html(r), + method="static", + elapsed=round(time.perf_counter() - t0, 3), + status=r.status_code, + ) diff --git a/src/bytecrawl/security.py b/src/bytecrawl/security.py index d0af9a2..437bd3d 100644 --- a/src/bytecrawl/security.py +++ b/src/bytecrawl/security.py @@ -32,6 +32,4 @@ def assert_public_url(url: str) -> None: for info in infos: ip = ipaddress.ip_address(info[4][0]) if not ip.is_global or ip.is_multicast: - raise ValueError( - f"URL resolves to a non-public address ({ip}) — refusing to fetch it" - ) + raise ValueError(f"URL resolves to a non-public address ({ip}) — refusing to fetch it") diff --git a/tests/conftest.py b/tests/conftest.py index 7620afe..d6ca5d4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,8 +16,15 @@ class FakeResponse: """Minimal stand-in for requests.Response.""" - def __init__(self, text="", status=200, json_data=None, headers=None, - encoding="utf-8", apparent_encoding="utf-8"): + def __init__( + self, + text="", + status=200, + json_data=None, + headers=None, + encoding="utf-8", + apparent_encoding="utf-8", + ): self.status_code = status self.content = text.encode(apparent_encoding) self.encoding = encoding @@ -70,16 +77,22 @@ def _post(self, url, **kw): # /cats/* is not. One external link tests the same_domain filter. SITE = { "https://s.test/": ( - "Home", "welcome to the machine learning intro site", - [("/ml", "machine learning"), ("/cats", "cat pictures"), - ("https://other.test/x", "external")], + "Home", + "welcome to the machine learning intro site", + [ + ("/ml", "machine learning"), + ("/cats", "cat pictures"), + ("https://other.test/x", "external"), + ], ), "https://s.test/ml": ( - "ML", "machine learning and deep learning models machine learning", + "ML", + "machine learning and deep learning models machine learning", [("/ml/deep", "deep learning"), ("/", "home")], ), "https://s.test/cats": ( - "Cats", "cats cats cats pictures of cats", + "Cats", + "cats cats cats pictures of cats", [("/cats/more", "more cats")], ), "https://s.test/ml/deep": ("Deep", "deep neural networks training", []), @@ -90,8 +103,7 @@ def _post(self, url, **kw): def render(url: str) -> str: title, text, links = SITE[url] anchors = "".join(f'{a}' for h, a in links) - return (f"{title}" - f"

{text}

{anchors}") + return f"{title}

{text}

{anchors}" @pytest.fixture diff --git a/tests/test_browser_live.py b/tests/test_browser_live.py index f6e8992..5962aab 100644 --- a/tests/test_browser_live.py +++ b/tests/test_browser_live.py @@ -21,6 +21,7 @@ @pytest.fixture(scope="module") def chromium_available(): from playwright.sync_api import sync_playwright + try: with sync_playwright() as p: p.chromium.launch(headless=True).close() diff --git a/tests/test_core.py b/tests/test_core.py index 4ba5878..eaba67c 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -36,8 +36,7 @@ class TestEncoding: def test_iso_8859_1_fallback_uses_apparent_encoding(self): # requests defaults to ISO-8859-1 when the header has no charset # (RFC 2616); for a UTF-8 body that turns £ into £. - r = FakeResponse(text="price £10", encoding="iso-8859-1", - apparent_encoding="utf-8") + r = FakeResponse(text="price £10", encoding="iso-8859-1", apparent_encoding="utf-8") assert _decoded_html(r) == "price £10" def test_declared_encoding_is_respected(self): @@ -48,8 +47,8 @@ def test_declared_encoding_is_respected(self): class TestApi: def test_json_page(self, http): http.routes["https://api.test/items"] = FakeResponse( - json_data={"items": [1, 2, 3]}, - headers={"content-type": "application/json"}) + json_data={"items": [1, 2, 3]}, headers={"content-type": "application/json"} + ) page = Scraper().api("https://api.test/items") assert page.method == "api" assert page.json() == {"items": [1, 2, 3]} @@ -64,23 +63,24 @@ class TestFetchAuto: def test_content_rich_page_stays_static(self, http, monkeypatch): http.routes["https://x.test/"] = FakeResponse(text=LONG_HTML) called = [] - monkeypatch.setattr(Scraper, "browser", - lambda self, url, **kw: called.append(url)) + monkeypatch.setattr(Scraper, "browser", lambda self, url, **kw: called.append(url)) page = Scraper().fetch("https://x.test/") assert page.method == "static" assert called == [] def test_sparse_page_falls_back_to_browser(self, http, monkeypatch): http.routes["https://spa.test/"] = FakeResponse( - text="
") + text="
" + ) sentinel = object() monkeypatch.setattr(Scraper, "browser", lambda self, url, **kw: sentinel) assert Scraper().fetch("https://spa.test/") is sentinel def test_explicit_static_never_falls_back(self, http, monkeypatch): http.routes["https://spa.test/"] = FakeResponse(text="") - monkeypatch.setattr(Scraper, "browser", - lambda self, url, **kw: pytest.fail("browser called")) + monkeypatch.setattr( + Scraper, "browser", lambda self, url, **kw: pytest.fail("browser called") + ) page = Scraper().fetch("https://spa.test/", strategy="static") assert page.method == "static" @@ -101,21 +101,27 @@ def test_follows_next_page_link(self, http): http.routes["https://x.test/page1"] = FakeResponse(text=ITEMS_P1) http.routes["https://x.test/page2"] = FakeResponse(text=ITEMS_P2) rows = Scraper().crawl( - "https://x.test/page1", item="article.item", + "https://x.test/page1", + item="article.item", fields={"name": "h3::text"}, - next_page="li.next a::attr(href)") + next_page="li.next a::attr(href)", + ) assert [r["name"] for r in rows] == ["A", "B", "C"] def test_pages_limit_stops_early(self, http): http.routes["https://x.test/page1"] = FakeResponse(text=ITEMS_P1) rows = Scraper().crawl( - "https://x.test/page1", item="article.item", + "https://x.test/page1", + item="article.item", fields={"name": "h3::text"}, - next_page="li.next a::attr(href)", pages=1) + next_page="li.next a::attr(href)", + pages=1, + ) assert [r["name"] for r in rows] == ["A", "B"] def test_no_next_page_selector_single_page(self, http): http.routes["https://x.test/page1"] = FakeResponse(text=ITEMS_P1) - rows = Scraper().crawl("https://x.test/page1", item="article.item", - fields={"name": "h3::text"}) + rows = Scraper().crawl( + "https://x.test/page1", item="article.item", fields={"name": "h3::text"} + ) assert len(rows) == 2 diff --git a/tests/test_crawler_strategies.py b/tests/test_crawler_strategies.py index f432135..05a42b8 100644 --- a/tests/test_crawler_strategies.py +++ b/tests/test_crawler_strategies.py @@ -22,8 +22,7 @@ def test_visits_level_by_level(self, fake_site): # Seed first, then both depth-1 pages before any depth-2 page. assert fake_site[0] == SEED assert set(fake_site[1:3]) == {"https://s.test/ml", "https://s.test/cats"} - assert set(fake_site[3:5]) == {"https://s.test/ml/deep", - "https://s.test/cats/more"} + assert set(fake_site[3:5]) == {"https://s.test/ml/deep", "https://s.test/cats/more"} def test_max_depth_excludes_deeper_pages(self, fake_site): result = BFS(**crawler_kwargs()).crawl(SEED, max_pages=10, max_depth=1) @@ -53,29 +52,23 @@ def test_invalid_url_raises(self): class TestSharkSearch: def test_relevant_branch_visited_before_irrelevant(self, fake_site): - SharkSearch(query="machine learning", **crawler_kwargs()).crawl( - SEED, max_pages=4) - assert fake_site.index("https://s.test/ml") < \ - fake_site.index("https://s.test/cats") + SharkSearch(query="machine learning", **crawler_kwargs()).crawl(SEED, max_pages=4) + assert fake_site.index("https://s.test/ml") < fake_site.index("https://s.test/cats") def test_irrelevant_branch_decays_geometrically(self, fake_site): - shark = SharkSearch(query="machine learning", delta=0.5, - **crawler_kwargs()) + shark = SharkSearch(query="machine learning", delta=0.5, **crawler_kwargs()) shark.crawl(SEED, max_pages=5) # /cats has no query overlap, so /cats/more inherits delta * inherited(/cats). cats = shark._inherited["https://s.test/cats"] - assert shark._inherited["https://s.test/cats/more"] == \ - pytest.approx(shark.delta * cats) + assert shark._inherited["https://s.test/cats/more"] == pytest.approx(shark.delta * cats) def test_relevance_scores_favor_ml_pages(self, fake_site): - result = SharkSearch(query="machine learning", - **crawler_kwargs()).crawl(SEED, max_pages=5) + result = SharkSearch(query="machine learning", **crawler_kwargs()).crawl(SEED, max_pages=5) by_url = {p["url"]: p["relevance"] for p in result.pages} assert by_url["https://s.test/ml"] > by_url["https://s.test/cats"] def test_top_orders_by_relevance(self, fake_site): - result = SharkSearch(query="machine learning", - **crawler_kwargs()).crawl(SEED, max_pages=5) + result = SharkSearch(query="machine learning", **crawler_kwargs()).crawl(SEED, max_pages=5) top = result.top(3) assert top[0]["relevance"] >= top[1]["relevance"] >= top[2]["relevance"] @@ -101,13 +94,13 @@ def test_history_accumulates_importance(self, fake_site): def test_visits_all_reachable_pages(self, fake_site): result = OPIC(**crawler_kwargs()).crawl(SEED, max_pages=10) - assert {p["url"] for p in result.pages} == set( - u for u in fake_site) + assert {p["url"] for p in result.pages} == set(u for u in fake_site) class TestErrorHandling: def test_fetch_errors_counted_not_fatal(self, fake_site, monkeypatch): from bytecrawl.crawler import Crawler + original = Crawler.score_links def with_broken_link(self, url, links, relevance, depth): diff --git a/tests/test_graph_utils.py b/tests/test_graph_utils.py index 84a06f2..ea28998 100644 --- a/tests/test_graph_utils.py +++ b/tests/test_graph_utils.py @@ -1,6 +1,5 @@ """Pure graph/text utilities: cosine, normalize, _root_domain, pagerank.""" - import pytest from bytecrawl.crawler import ( @@ -43,7 +42,8 @@ def test_one_letter_typos_still_match(self, typo): # A slipped keystroke used to score 0.0, which reads as "broken" # rather than "no match". assert relevance(self.TEXT, typo, self.URL) == pytest.approx( - relevance(self.TEXT, "mystery", self.URL)) + relevance(self.TEXT, "mystery", self.URL) + ) @pytest.mark.parametrize("miss", ["travel", "poetry", "zzzzzz"]) def test_absent_topics_stay_zero(self, miss): @@ -67,8 +67,7 @@ def test_resolves_relative(self): assert normalize("/docs", "https://x.test/a/b") == "https://x.test/docs" def test_strips_fragment(self): - assert normalize("https://x.test/p#section", "https://x.test") == \ - "https://x.test/p" + assert normalize("https://x.test/p#section", "https://x.test") == "https://x.test/p" def test_bare_host_and_root_are_the_same_page(self): # Otherwise the crawler burns two budget slots on one page. @@ -124,11 +123,14 @@ def test_symmetric_graph_equal_ranks(self): class TestCrawlResult: def _result(self): - return CrawlResult(strategy="test", pages=[ - {"url": "a", "relevance": 0.9}, - {"url": "b", "relevance": 0.05}, - {"url": "c", "relevance": 0.5}, - ]) + return CrawlResult( + strategy="test", + pages=[ + {"url": "a", "relevance": 0.9}, + {"url": "b", "relevance": 0.05}, + {"url": "c", "relevance": 0.5}, + ], + ) def test_relevant_filters_by_threshold(self): assert [p["url"] for p in self._result().relevant(0.1)] == ["a", "c"] diff --git a/tests/test_markdown_clean.py b/tests/test_markdown_clean.py index 6269122..a6a4c0b 100644 --- a/tests/test_markdown_clean.py +++ b/tests/test_markdown_clean.py @@ -9,8 +9,9 @@ class TestCarouselDedup: def test_repeated_long_block_kept_once(self): - md = "\n\n".join([LONG_BLOCK, "Something else entirely different here", - LONG_BLOCK, LONG_BLOCK]) + md = "\n\n".join( + [LONG_BLOCK, "Something else entirely different here", LONG_BLOCK, LONG_BLOCK] + ) out = _clean_markdown(md) assert out.count(LONG_BLOCK) == 1 @@ -55,12 +56,19 @@ class TestMainContentFallback: that as a 144x saving. """ - LISTING = ("" + "".join( - f"" for i in range(20)) + "") + LISTING = ( + "" + + "".join( + f"" + for i in range(20) + ) + + "" + ) def test_falls_back_when_extraction_guts_the_page(self, monkeypatch): from bytecrawl.core import Page + page = Page(url="https://x.test/", html=self.LISTING) trafilatura = pytest.importorskip("trafilatura") # stand in for what it really does here: return a sliver of the page @@ -70,6 +78,7 @@ def test_falls_back_when_extraction_guts_the_page(self, monkeypatch): def test_keeps_extraction_when_it_looks_sane(self, monkeypatch): from bytecrawl.core import Page + page = Page(url="https://x.test/", html=self.LISTING) trafilatura = pytest.importorskip("trafilatura") full = page.soup.get_text(" ", strip=True) @@ -82,4 +91,5 @@ def test_the_floor_sits_well_below_real_pages(self): would fire on an article buried in navigation and hand back the navigation that was correctly removed.""" from bytecrawl.core import Page + assert Page._MAIN_CONTENT_FLOOR < 0.5 diff --git a/tests/test_mcp_http.py b/tests/test_mcp_http.py index ff064e1..087019a 100644 --- a/tests/test_mcp_http.py +++ b/tests/test_mcp_http.py @@ -14,22 +14,26 @@ class TestSSRFGuard: """IP literals only — no DNS needed except localhost (hosts file).""" - @pytest.mark.parametrize("url", [ - "http://127.0.0.1/admin", # loopback - "http://localhost:8080/", # loopback via hosts file - "http://10.0.0.5/internal", # RFC1918 - "http://172.16.3.1/", # RFC1918 - "http://192.168.1.1/router", # RFC1918 - "http://169.254.169.254/latest/meta-data/", # cloud metadata - "http://[::1]/", # IPv6 loopback - "http://0.0.0.0/", # unspecified - ]) + @pytest.mark.parametrize( + "url", + [ + "http://127.0.0.1/admin", # loopback + "http://localhost:8080/", # loopback via hosts file + "http://10.0.0.5/internal", # RFC1918 + "http://172.16.3.1/", # RFC1918 + "http://192.168.1.1/router", # RFC1918 + "http://169.254.169.254/latest/meta-data/", # cloud metadata + "http://[::1]/", # IPv6 loopback + "http://0.0.0.0/", # unspecified + ], + ) def test_rejects_non_public_addresses(self, url): with pytest.raises(ValueError, match="non-public|resolve"): assert_public_url(url) - @pytest.mark.parametrize("url", ["ftp://example.com/x", "file:///etc/passwd", - "javascript:alert(1)"]) + @pytest.mark.parametrize( + "url", ["ftp://example.com/x", "file:///etc/passwd", "javascript:alert(1)"] + ) def test_rejects_non_http_schemes(self, url): with pytest.raises(ValueError, match="http"): assert_public_url(url) @@ -43,28 +47,31 @@ def test_allows_global_address(self): def test_tools_guard_before_fetching(self, monkeypatch): from mcp.server.mcpserver.exceptions import ToolError + called = [] - monkeypatch.setattr(mcp_http.local, "fetch_markdown", - lambda url: called.append(url)) + monkeypatch.setattr(mcp_http.local, "fetch_markdown", lambda url: called.append(url)) # tools re-raise as ToolError so the client sees the reason with pytest.raises(ToolError, match="non-public"): mcp_http.fetch_markdown("http://169.254.169.254/") assert called == [] # the underlying fetch never ran - @pytest.mark.parametrize("tool,kwargs", [ - ("list_links", {}), - ("compare_strategies", {"query": "anything"}), - ("extract", {"select": "h3"}), - ("fetch_json_api", {}), - ("focused_crawl", {}), - ]) + @pytest.mark.parametrize( + "tool,kwargs", + [ + ("list_links", {}), + ("compare_strategies", {"query": "anything"}), + ("extract", {"select": "h3"}), + ("fetch_json_api", {}), + ("focused_crawl", {}), + ], + ) def test_every_tool_guards_before_fetching(self, monkeypatch, tool, kwargs): """A new tool that forgets _guarded() would be an open proxy into the host's network, so every one of them is checked, not just the first.""" from mcp.server.mcpserver.exceptions import ToolError + called = [] - monkeypatch.setattr(mcp_http.local, tool, - lambda *a, **kw: called.append(a)) + monkeypatch.setattr(mcp_http.local, tool, lambda *a, **kw: called.append(a)) with pytest.raises(ToolError, match="non-public"): getattr(mcp_http, tool)("http://10.0.0.5/internal", **kwargs) assert called == [] @@ -99,8 +106,11 @@ async def inner(scope, receive, send): await send({"type": "http.response.body", "body": b"ok"}) mw = RateLimitMiddleware(inner, RateLimiter(limit=1, window=60)) - scope = {"type": "http", "headers": [(b"x-forwarded-for", b"1.2.3.4")], - "client": ("9.9.9.9", 1234)} + scope = { + "type": "http", + "headers": [(b"x-forwarded-for", b"1.2.3.4")], + "client": ("9.9.9.9", 1234), + } async def receive(): # pragma: no cover - never called return {"type": "http.request"} @@ -110,6 +120,7 @@ async def run_once(): async def send(ev): events.append(ev) + await mw(scope, receive, send) return events[0]["status"] @@ -124,6 +135,7 @@ def test_focused_crawl_clamped(self, monkeypatch): def spy(url, query="", strategy="shark", max_pages=0): seen["max_pages"] = max_pages return {} + monkeypatch.setattr(mcp_http.local, "focused_crawl", spy) mcp_http.focused_crawl("http://8.8.8.8/", max_pages=9999) assert seen["max_pages"] == mcp_http.MAX_PAGES @@ -135,6 +147,7 @@ def test_compare_strategies_clamped_lower(self, monkeypatch): def spy(url, query="", max_pages=0): seen["max_pages"] = max_pages return {} + monkeypatch.setattr(mcp_http.local, "compare_strategies", spy) mcp_http.compare_strategies("http://8.8.8.8/", query="x", max_pages=9999) assert seen["max_pages"] == mcp_http.MAX_COMPARE_PAGES @@ -146,9 +159,16 @@ async def test_hosted_server_registers_same_tools(): """Same six as the local server: an agent that learned the hosted surface can point at a local install and every call still resolves.""" from bytecrawl import mcp_server as local_server + hosted = {t.name for t in await mcp_http.server.list_tools()} - assert hosted == {"fetch_markdown", "extract", "list_links", - "focused_crawl", "compare_strategies", "fetch_json_api"} + assert hosted == { + "fetch_markdown", + "extract", + "list_links", + "focused_crawl", + "compare_strategies", + "fetch_json_api", + } assert hosted == {t.name for t in await local_server.server.list_tools()} @@ -184,6 +204,7 @@ def test_app_passes_the_settings_through(self, monkeypatch): def spy(**kw): seen.update(kw) return orig(**kw) + monkeypatch.setattr(mcp_http.server, "streamable_http_app", spy) mcp_http.create_app() assert "transport_security" in seen, "app built without Host settings" diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 669fff9..5e64374 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -24,16 +24,26 @@ async def test_all_tools_registered(): names = {t.name for t in tools} # One per playground mode, with graph crawl split into the cheap # single-strategy call and the three-way comparison. - assert names == {"fetch_markdown", "extract", "list_links", "focused_crawl", - "compare_strategies", "fetch_json_api"} + assert names == { + "fetch_markdown", + "extract", + "list_links", + "focused_crawl", + "compare_strategies", + "fetch_json_api", + } def test_extract_tool(http): http.routes["https://x.test/"] = FakeResponse( - text="

" + "enough visible text to stay static " * 10 + "

" - + '

Widget

9
' * 20) - out = mcp_server.extract("https://x.test/", item="article.p", - fields={"name": "h3::text", "cost": "span.c::text"}) + text="

" + + "enough visible text to stay static " * 10 + + "

" + + '

Widget

9
' * 20 + ) + out = mcp_server.extract( + "https://x.test/", item="article.p", fields={"name": "h3::text", "cost": "span.c::text"} + ) assert out["count"] == 20 assert out["records"][0] == {"name": "Widget", "cost": "9"} @@ -41,8 +51,11 @@ def test_extract_tool(http): def test_extract_by_single_selector(http): """The playground's Extract chip: one selector, a flat list of values.""" http.routes["https://x.test/"] = FakeResponse( - text="

" + "enough visible text to stay static " * 10 + "

" - + "".join(f'

W{i}

' for i in range(5))) + text="

" + + "enough visible text to stay static " * 10 + + "

" + + "".join(f'

W{i}

' for i in range(5)) + ) out = mcp_server.extract("https://x.test/", select="article.p h3::text") assert out["values"] == ["W0", "W1", "W2", "W3", "W4"] assert out["count"] == 5 @@ -51,15 +64,15 @@ def test_extract_by_single_selector(http): def test_extract_rejects_both_shapes(http): with pytest.raises(ValueError, match="not both"): - mcp_server.extract("https://x.test/", item="a", fields={"x": "y"}, - select="h3") + mcp_server.extract("https://x.test/", item="a", fields={"x": "y"}, select="h3") def test_extract_with_nothing_is_discovery_not_an_error(http): """This used to raise. Refusing a bare call was refusing the only question an agent can ask about a page it has not seen.""" http.routes["https://x.test/"] = FakeResponse( - text="

" + "enough visible text to stay static " * 10 + "

") + text="

" + "enough visible text to stay static " * 10 + "

" + ) out = mcp_server.extract("https://x.test/") assert "candidates" in out @@ -68,7 +81,8 @@ def test_list_links_tool(http): http.routes["https://x.test/"] = FakeResponse( text="

" + "enough visible text to stay static " * 10 + "

" 'adupefrag' - 'mailasset') + 'mailasset' + ) out = mcp_server.list_links("https://x.test/") # Absolute and deduplicated; mailto: and assets gone. A bare "#cite" # collapses onto the page itself rather than becoming its own entry — @@ -80,19 +94,19 @@ def test_list_links_tool(http): def test_list_links_raw_passes_hrefs_through(http): http.routes["https://x.test/"] = FakeResponse( text="

" + "enough visible text to stay static " * 10 + "

" - 'afrag') + 'afrag' + ) out = mcp_server.list_links("https://x.test/", raw=True) assert out["links"] == ["/a", "#cite"] and out["raw"] is True def test_compare_strategies_tool(fake_site): - out = mcp_server.compare_strategies("https://s.test/", - query="machine learning", max_pages=3) + out = mcp_server.compare_strategies("https://s.test/", query="machine learning", max_pages=3) assert set(out["strategies"]) == {"bfs", "shark", "opic"} assert out["query"] == "machine learning" for leg in out["strategies"].values(): - assert leg["relevant"] == 2 # "/" and "/ml" on this site - assert leg["pages"][0]["url"] == "https://s.test/ml" # ranked, not visit order + assert leg["relevant"] == 2 # "/" and "/ml" on this site + assert leg["pages"][0]["url"] == "https://s.test/ml" # ranked, not visit order # Five pages is too small a site to separate them: all three find the same # two relevant pages, so this is a real tie and the tool has to say so # rather than crown whichever came first. @@ -103,10 +117,8 @@ def test_compare_strategies_tool(fake_site): def test_compare_strategies_spends_its_budget_differently(fake_site): """The tie is in the score, not the behaviour: with 3 pages Shark-Search stays in the relevant branch (/ml/deep) where BFS takes the level (/cats).""" - out = mcp_server.compare_strategies("https://s.test/", - query="machine learning", max_pages=3) - visited = {n: {p["url"] for p in leg["pages"]} - for n, leg in out["strategies"].items()} + out = mcp_server.compare_strategies("https://s.test/", query="machine learning", max_pages=3) + visited = {n: {p["url"] for p in leg["pages"]} for n, leg in out["strategies"].items()} assert "https://s.test/ml/deep" in visited["shark"] assert "https://s.test/cats" in visited["bfs"] @@ -125,35 +137,36 @@ def boom(*a, **kw): raise RuntimeError("frontier exploded") monkeypatch.setitem(crawler.STRATEGIES, "opic", boom) - out = mcp_server.compare_strategies("https://s.test/", - query="machine learning", max_pages=3) + out = mcp_server.compare_strategies("https://s.test/", query="machine learning", max_pages=3) assert "frontier exploded" in out["strategies"]["opic"]["error"] assert "stats" in out["strategies"]["shark"] - assert out["winner"] in ("bfs", "shark") # the failed leg can't win + assert out["winner"] in ("bfs", "shark") # the failed leg can't win def test_focused_crawl_tool(fake_site): - out = mcp_server.focused_crawl("https://s.test/", query="machine learning", - strategy="shark", max_pages=5) + out = mcp_server.focused_crawl( + "https://s.test/", query="machine learning", strategy="shark", max_pages=5 + ) assert out["strategy"] == "shark" assert out["stats"]["requests"] == len(SITE) assert out["pages"][0]["relevance"] >= out["pages"][-1]["relevance"] assert out["pagerank_top"] # pagerank computed over the crawled graph + def test_focused_crawl_rejects_unknown_strategy(): with pytest.raises(ValueError, match="strategy"): mcp_server.focused_crawl("https://s.test/", strategy="dfs") def test_focused_crawl_caps_max_pages(fake_site): - out = mcp_server.focused_crawl("https://s.test/", strategy="bfs", - max_pages=9999) + out = mcp_server.focused_crawl("https://s.test/", strategy="bfs", max_pages=9999) assert out["stats"]["requests"] <= 50 def test_fetch_json_api_tool(http): http.routes["https://api.test/x"] = FakeResponse( - json_data={"ok": True}, headers={"content-type": "application/json"}) + json_data={"ok": True}, headers={"content-type": "application/json"} + ) out = mcp_server.fetch_json_api("https://api.test/x") assert out["data"] == {"ok": True} @@ -163,9 +176,14 @@ def test_extract_with_only_a_url_discovers(http): an agent that has not seen the markup cannot invent a selector, and no tool here returns HTML for it to read.""" http.routes["https://x.test/"] = FakeResponse( - text="

" + "enough visible text to stay static " * 10 + "

" - + "".join(f"

W{i}

" - f"

£{i}

" for i in range(5))) + text="

" + + "enough visible text to stay static " * 10 + + "

" + + "".join( + f"

W{i}

£{i}

" + for i in range(5) + ) + ) out = mcp_server.extract("https://x.test/") assert "candidates" in out and out["candidates"] top = out["candidates"][0] @@ -181,10 +199,13 @@ def test_extract_rejects_item_without_fields(http): def test_fetch_markdown_tool(http): """The one tool with no offline test at all. The smoke suite only ever called it to check the SSRF guard refused it, never that it worked.""" - body = "".join(f"

Book {i}

" - f"

£{i}.00

" for i in range(20)) + body = "".join( + f"

Book {i}

£{i}.00

" + for i in range(20) + ) http.routes["https://x.test/"] = FakeResponse( - text=f"{body}") + text=f"{body}" + ) out = mcp_server.fetch_markdown("https://x.test/") assert out["method"] == "static" and out["status"] == 200 # a listing is all repetition, which is what an article extractor discards; diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 676ae14..70c16d7 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,14 +1,38 @@ """Package surface: version single-sourcing and top-level exports.""" import importlib.metadata +import re +from pathlib import Path import bytecrawl +_ROOT = Path(__file__).resolve().parent.parent + def test_version_matches_installed_metadata(): assert bytecrawl.__version__ == importlib.metadata.version("bytecrawl") +def test_changelog_top_entry_matches_version(): + """The newest CHANGELOG heading is the version being shipped — + __init__.py is the single source, everything else tracks it.""" + text = (_ROOT / "CHANGELOG.md").read_text(encoding="utf-8") + top = re.search(r"^## (\d+\.\d+\.\d+)", text, re.M) + assert top, "no `## X.Y.Z` heading found in CHANGELOG.md" + assert top.group(1) == bytecrawl.__version__, ( + f"CHANGELOG top is {top.group(1)} but __version__ is {bytecrawl.__version__}" + ) + + +def test_readmes_advertise_current_version(): + """Both READMEs' 'latest release' line must name the shipped version.""" + for name in ("README.md", "README.es.md"): + text = (_ROOT / name).read_text(encoding="utf-8") + assert bytecrawl.__version__ in text, ( + f"{name} does not mention the current version {bytecrawl.__version__}" + ) + + def test_top_level_exports(): from bytecrawl import ( # noqa: F401 BFS, @@ -41,11 +65,24 @@ def test_serverless_entrypoints_do_not_shadow_dependencies(): from pathlib import Path root = Path(__file__).resolve().parent.parent - third_party = {"mcp", "flask", "requests", "bs4", "lxml", "uvicorn", - "starlette", "anyio", "pydantic", "posthog", "dotenv", - "markdownify", "trafilatura", "playwright", "bytecrawl"} - clashes = [p.name for p in (root / "web/mcp").glob("*.py") - if p.stem in third_party] + third_party = { + "mcp", + "flask", + "requests", + "bs4", + "lxml", + "uvicorn", + "starlette", + "anyio", + "pydantic", + "posthog", + "dotenv", + "markdownify", + "trafilatura", + "playwright", + "bytecrawl", + } + clashes = [p.name for p in (root / "web/mcp").glob("*.py") if p.stem in third_party] assert not clashes, f"web/mcp/ entrypoints shadow installed packages: {clashes}" # and every path vercel.json points at has to exist @@ -71,14 +108,18 @@ def test_serverless_requirements_stay_in_sync(): root = Path(__file__).resolve().parent.parent def pkgs(path): - return [ln.strip() for ln in (root / path).read_text().splitlines() - if ln.strip() and not ln.startswith("#")] + return [ + ln.strip() + for ln in (root / path).read_text().splitlines() + if ln.strip() and not ln.startswith("#") + ] demo, api = pkgs("web/landing/requirements.txt"), pkgs("web/mcp/requirements.txt") assert demo == api, ( "web/landing/requirements.txt and web/mcp/requirements.txt must match — they " f"install into the same venv.\n demo only: {set(demo) - set(api)}\n" - f" api only: {set(api) - set(demo)}") + f" api only: {set(api) - set(demo)}" + ) # the hosted MCP server cannot start without these assert any(p.startswith("mcp") for p in api) assert "trafilatura" in api and "flask" in api @@ -96,8 +137,9 @@ def test_agent_skill_points_at_the_real_repo(): from pathlib import Path root = Path(__file__).resolve().parent.parent - canonical = re.search(r'Homepage = "(https://github\.com/[^"]+)"', - (root / "pyproject.toml").read_text()).group(1) + canonical = re.search( + r'Homepage = "(https://github\.com/[^"]+)"', (root / "pyproject.toml").read_text() + ).group(1) skill = (root / "web/landing/static/SKILL.md").read_text() found = set(re.findall(r"https://github\.com/[A-Za-z0-9._/-]+", skill)) wrong = {u for u in found if not u.startswith(canonical)} @@ -125,12 +167,13 @@ def test_the_skill_does_not_order_the_agent_around(): # nothing about writing to the machine before the reader knows what this is for probe in ("~/.claude/skills", "mcpServers", "mcp add"): - assert probe not in head + opening, \ + assert probe not in head + opening, ( f"{probe!r} appears before the skill has said what it does" + ) # the option still has to exist, and be framed as the person's choice assert "~/.claude/skills/bytecrawl" in skill, "no way to install it at all" - flat = " ".join(skill.split()) # the prose is hard-wrapped + flat = " ".join(skill.split()) # the prose is hard-wrapped assert "do not change their configuration on your own" in flat assert "ask first" in flat @@ -143,12 +186,14 @@ def test_the_skill_does_not_order_the_agent_around(): landing = (root / "web/landing/templates/landing.html").read_text() setup = re.search(r"const SETUP_CMD = (.+?);\n", landing, re.S).group(1) assert "agent-onboarding/SKILL.md" in setup, "the button lost the URL" - assert "MCP server" in setup and "skill" in setup, \ + assert "MCP server" in setup and "skill" in setup, ( "the button must ask for both — the server alone leaves /bytecrawl missing" + ) def _repo_version(): import bytecrawl + return bytecrawl.__version__ @@ -172,8 +217,10 @@ def test_every_version_in_the_repo_agrees(): wrong["CHANGELOG.md"] = top.group(1) # both READMEs advertise it to anyone who never opens the changelog - for name, label in (("README.md", r"\*\*Latest release\*\*: \*\*([\d.]+)\*\*"), - ("README.es.md", r"\*\*Última versión\*\*: \*\*([\d.]+)\*\*")): + for name, label in ( + ("README.md", r"\*\*Latest release\*\*: \*\*([\d.]+)\*\*"), + ("README.es.md", r"\*\*Última versión\*\*: \*\*([\d.]+)\*\*"), + ): m = re.search(label, (root / name).read_text()) assert m, f"{name} no longer states a version" if m.group(1) != version: @@ -208,9 +255,10 @@ def test_the_skill_can_tell_it_is_stale(): flat = " ".join(skill.split()) assert "skill.json" in skill, "the skill never mentions how to check" assert "shasum -a 256" in skill - assert "do not fetch it yourself" in flat, \ - "an agent overwriting a file in someone's home because a hash differed " \ + assert "do not fetch it yourself" in flat, ( + "an agent overwriting a file in someone's home because a hash differed " "is what the rest of this document tells it to refuse" + ) # and it has to look on its own, not wait for a symptom: the failure that # prompted this was a link that 404s, which errors nothing assert "Check once, the first time you use this in a session" in flat @@ -219,12 +267,14 @@ def test_the_skill_can_tell_it_is_stale(): # route rather than running it: importing the landing app drags in Flask, # which the package does not depend on and CI does not install — the landing # site is a separate application that happens to live in this repository. - route = app_py[app_py.index('"/agent-onboarding/skill.json"'):] - route = route[:route.index("@pages_bp.route", 1)] - assert 'read_bytes()' in route and "hashlib.sha256" in route, \ + route = app_py[app_py.index('"/agent-onboarding/skill.json"') :] + route = route[: route.index("@pages_bp.route", 1)] + assert "read_bytes()" in route and "hashlib.sha256" in route, ( "the endpoint must hash the file it serves, not a stored value" + ) assert "SKILL.md" in route # a stored hash would be the drift this is meant to prevent - assert hashlib.sha256(body).hexdigest()[:8] not in app_py, \ + assert hashlib.sha256(body).hexdigest()[:8] not in app_py, ( "the current hash is hardcoded somewhere — it has to be computed" + ) diff --git a/tests/test_pep8_vercel.py b/tests/test_pep8_vercel.py index 4f30737..d15ae1b 100644 --- a/tests/test_pep8_vercel.py +++ b/tests/test_pep8_vercel.py @@ -11,6 +11,7 @@ length (pyproject.toml, 100). Reading the file list from vercel.json means a new entrypoint is covered the moment it's deployed, with no edit here. """ + from __future__ import annotations import json @@ -34,7 +35,8 @@ def test_vercel_entrypoints_follow_pep8(): assert sources, "vercel.json declares no Python builds — nothing to check" proc = subprocess.run( - ["ruff", "check", "--select", "E,W", *sources], - cwd=ROOT, capture_output=True, text=True) + ["ruff", "check", "--select", "E,W", *sources], cwd=ROOT, capture_output=True, text=True + ) assert proc.returncode == 0, ( - "Vercel-deployed code violates PEP 8:\n" + proc.stdout + proc.stderr) + "Vercel-deployed code violates PEP 8:\n" + proc.stdout + proc.stderr + ) diff --git a/tests/test_selectors.py b/tests/test_selectors.py index c2ed2a0..13f806c 100644 --- a/tests/test_selectors.py +++ b/tests/test_selectors.py @@ -54,16 +54,17 @@ def test_css_all_no_match_is_empty(self): assert page().css_all("h1.missing") == [] def test_extract_records_with_list_field(self): - rows = page().extract("div.quote", { - "quote": "span.text::text", - "author": "small.author::text", - "tags[]": "a.tag::text", - }) + rows = page().extract( + "div.quote", + { + "quote": "span.text::text", + "author": "small.author::text", + "tags[]": "a.tag::text", + }, + ) assert rows == [ - {"quote": "To be or not to be", "author": "Shakespeare", - "tags": ["life", "doubt"]}, - {"quote": "I think therefore I am", "author": "Descartes", - "tags": ["mind"]}, + {"quote": "To be or not to be", "author": "Shakespeare", "tags": ["life", "doubt"]}, + {"quote": "I think therefore I am", "author": "Descartes", "tags": ["mind"]}, ] def test_extract_missing_field_is_none(self): @@ -81,12 +82,15 @@ def test_links_raw_keeps_the_attribute_values(self): assert page().links(raw=True) == ["/tag/life", "/tag/doubt", "/tag/mind"] def test_links_drops_fragments_mailto_and_duplicates(self): - p = Page(url="https://x.test/a/", html=""" + p = Page( + url="https://x.test/a/", + html=""" 1top maillogo bb again c - """) + """, + ) assert p.links() == ["https://x.test/a/", "https://x.test/b", "https://x.test/c"] def test_tokens_estimate(self): @@ -97,14 +101,21 @@ def test_tokens_estimate(self): class TestSelectorDiscovery: """page.selectors(): the answer to "I have never seen this markup".""" - LISTING = ("
" + "".join( - f"
  • " - f"

    Book {i}

    " - f"

    £{i}.00

    In stock

    " - f"
    • " for i in range(6)) + "
    ") + LISTING = ( + "
    " + + "".join( + f"
  • " + f"

    Book {i}

    " + f"

    £{i}.00

    In stock

    " + f"
    • " + for i in range(6) + ) + + "
    " + ) def _page(self): from bytecrawl.core import Page + return Page(url="https://x.test/", html=self.LISTING) def test_finds_the_repeating_record(self): @@ -136,5 +147,6 @@ def test_no_field_is_always_empty(self): def test_a_page_with_no_listing_returns_nothing(self): from bytecrawl.core import Page + page = Page(url="https://x.test/", html="

    Just prose.

    ") assert page.selectors() == [] diff --git a/tests/test_session.py b/tests/test_session.py index 8117a7a..8d44070 100644 --- a/tests/test_session.py +++ b/tests/test_session.py @@ -17,23 +17,20 @@ def test_csrf_token_read_and_posted(self, http): http.routes["https://x.test/login"] = FakeResponse(text=LOGIN_FORM) http.routes["POST https://x.test/login"] = FakeResponse(status=200) - Scraper().session().login("https://x.test/login", - {"user": "ana", "pass": "s3cret"}, - csrf_field="csrf_token") + Scraper().session().login( + "https://x.test/login", {"user": "ana", "pass": "s3cret"}, csrf_field="csrf_token" + ) method, url, kw = http.calls[-1] assert (method, url) == ("POST", "https://x.test/login") - assert kw["data"] == {"user": "ana", "pass": "s3cret", - "csrf_token": "tok123"} + assert kw["data"] == {"user": "ana", "pass": "s3cret", "csrf_token": "tok123"} def test_csrf_input_without_value_posts_empty(self, http): # Regression: token["value"] used to KeyError on with no value. - http.routes["https://x.test/login"] = FakeResponse( - text='') + http.routes["https://x.test/login"] = FakeResponse(text='') http.routes["POST https://x.test/login"] = FakeResponse(status=200) - Scraper().session().login("https://x.test/login", {"user": "ana"}, - csrf_field="csrf_token") + Scraper().session().login("https://x.test/login", {"user": "ana"}, csrf_field="csrf_token") assert http.calls[-1][2]["data"]["csrf_token"] == "" def test_failed_post_raises(self, http): @@ -42,8 +39,9 @@ def test_failed_post_raises(self, http): http.routes["POST https://x.test/login"] = FakeResponse(status=403) with pytest.raises(requests.HTTPError): - Scraper().session().login("https://x.test/login", {"user": "ana"}, - csrf_field="csrf_token") + Scraper().session().login( + "https://x.test/login", {"user": "ana"}, csrf_field="csrf_token" + ) def test_login_without_csrf_skips_get(self, http): http.routes["POST https://x.test/login"] = FakeResponse(status=200) @@ -58,7 +56,8 @@ def test_sets_authorization_header(self): def test_fetch_returns_page(self, http): http.routes["https://x.test/private"] = FakeResponse( - text="secret") + text="secret" + ) page = Scraper().session().fetch("https://x.test/private") assert "secret" in page.html assert page.status == 200 diff --git a/tests/test_smoke_production.py b/tests/test_smoke_production.py index e7f308c..f5af23f 100644 --- a/tests/test_smoke_production.py +++ b/tests/test_smoke_production.py @@ -39,12 +39,17 @@ def _get(path: str, **params) -> tuple[dict, str]: def _rpc(method: str, params: dict | None = None) -> dict: - body = json.dumps({"jsonrpc": "2.0", "id": 1, "method": method, - "params": params or {}}).encode() + body = json.dumps( + {"jsonrpc": "2.0", "id": 1, "method": method, "params": params or {}} + ).encode() req = urllib.request.Request( - f"{BASE}/mcp", data=body, - headers={"Content-Type": "application/json", - "Accept": "application/json, text/event-stream"}) + f"{BASE}/mcp", + data=body, + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }, + ) with urllib.request.urlopen(req, timeout=TIMEOUT) as r: # noqa: S310 return json.loads(r.read().decode()) @@ -62,13 +67,15 @@ def test_served_as_markdown(self): def test_every_link_it_gives_out_resolves(self): """It shipped pointing at a repo that 404s, and nothing caught it.""" import re + req = urllib.request.Request(f"{BASE}/agent-onboarding/SKILL.md") with urllib.request.urlopen(req, timeout=TIMEOUT) as r: # noqa: S310 body = r.read().decode() for url in sorted(set(re.findall(r"https://github\.com/[A-Za-z0-9._/-]+", body))): try: urllib.request.urlopen( # noqa: S310 - urllib.request.Request(url, method="HEAD"), timeout=TIMEOUT) + urllib.request.Request(url, method="HEAD"), timeout=TIMEOUT + ) except urllib.error.HTTPError as e: pytest.fail(f"SKILL.md points at {url} — HTTP {e.code}") @@ -80,12 +87,15 @@ class TestHttpApi: `elapsed`, which no method has ever returned. """ - @pytest.mark.parametrize("method,extra,expected", [ - ("markdown", {}, {"markdown", "tokens"}), - ("text", {}, {"text"}), - ("links", {}, {"links"}), - ("extract", {"select": "h3 a::attr(title)"}, {"values", "select"}), - ]) + @pytest.mark.parametrize( + "method,extra,expected", + [ + ("markdown", {}, {"markdown", "tokens"}), + ("text", {}, {"text"}), + ("links", {}, {"links"}), + ("extract", {"select": "h3 a::attr(title)"}, {"values", "select"}), + ], + ) def test_documented_fields(self, method, extra, expected): out, _ = _get("/api", url="books.toscrape.com", method=method, **extra) assert {"url", "method"} <= set(out), "url and method are on every response" @@ -93,8 +103,7 @@ def test_documented_fields(self, method, extra, expected): assert "status" not in out and "elapsed" not in out def test_compare_returns_a_verdict(self): - out, _ = _get("/api", url="books.toscrape.com", method="compare", - query="fiction", pages=3) + out, _ = _get("/api", url="books.toscrape.com", method="compare", query="fiction", pages=3) assert set(out["strategies"]) == {"bfs", "shark", "opic"} assert out["winner"] in out["tied"] @@ -108,20 +117,35 @@ class TestHostedMcp: def test_six_tools(self): """Deployed once answering 421 to every request, and once not at all.""" names = {t["name"] for t in _rpc("tools/list")["result"]["tools"]} - assert names == {"fetch_markdown", "extract", "list_links", - "focused_crawl", "compare_strategies", "fetch_json_api"} - - @pytest.mark.parametrize("tool,args,expect", [ - ("fetch_markdown", {"url": BOOKS}, {"markdown", "tokens_html"}), - ("extract", {"url": BOOKS}, {"candidates"}), - ("extract", {"url": BOOKS, "select": "h3 a::attr(title)"}, {"values"}), - ("list_links", {"url": BOOKS}, {"links"}), - ("fetch_json_api", {"url": QUOTES_API}, {"data"}), - ("focused_crawl", {"url": BOOKS, "query": "fiction", "max_pages": 3}, - {"pages", "stats"}), - ("compare_strategies", {"url": BOOKS, "query": "fiction", "max_pages": 3}, - {"strategies", "winner"}), - ]) + assert names == { + "fetch_markdown", + "extract", + "list_links", + "focused_crawl", + "compare_strategies", + "fetch_json_api", + } + + @pytest.mark.parametrize( + "tool,args,expect", + [ + ("fetch_markdown", {"url": BOOKS}, {"markdown", "tokens_html"}), + ("extract", {"url": BOOKS}, {"candidates"}), + ("extract", {"url": BOOKS, "select": "h3 a::attr(title)"}, {"values"}), + ("list_links", {"url": BOOKS}, {"links"}), + ("fetch_json_api", {"url": QUOTES_API}, {"data"}), + ( + "focused_crawl", + {"url": BOOKS, "query": "fiction", "max_pages": 3}, + {"pages", "stats"}, + ), + ( + "compare_strategies", + {"url": BOOKS, "query": "fiction", "max_pages": 3}, + {"strategies", "winner"}, + ), + ], + ) def test_every_tool_actually_runs(self, tool, args, expect): """Listing six tools is not the same as six tools working. @@ -139,8 +163,10 @@ def test_every_tool_actually_runs(self, tool, args, expect): def test_ssrf_guard_is_live(self): """The one property that must never regress: a hosted scraper that forwards to link-local addresses is an open door to cloud metadata.""" - out = _rpc("tools/call", {"name": "fetch_markdown", - "arguments": {"url": "http://169.254.169.254/"}}) + out = _rpc( + "tools/call", + {"name": "fetch_markdown", "arguments": {"url": "http://169.254.169.254/"}}, + ) text = json.dumps(out) assert "non-public" in text, text[:200] @@ -167,7 +193,8 @@ def _mm(v: str) -> tuple[int, int]: def _pypi(self) -> str: with urllib.request.urlopen( # noqa: S310 - "https://pypi.org/pypi/bytecrawl/json", timeout=TIMEOUT) as r: + "https://pypi.org/pypi/bytecrawl/json", timeout=TIMEOUT + ) as r: return json.loads(r.read())["info"]["version"] def test_the_readme_version_is_installable(self): @@ -180,23 +207,27 @@ def test_the_readme_version_is_installable(self): import bytecrawl root = Path(__file__).resolve().parent.parent - advertised = re.search(r"\*\*Latest release\*\*: \*\*([\d.]+)\*\*", - (root / "README.md").read_text()).group(1) - assert advertised == bytecrawl.__version__ # offline suite covers this + advertised = re.search( + r"\*\*Latest release\*\*: \*\*([\d.]+)\*\*", (root / "README.md").read_text() + ).group(1) + assert advertised == bytecrawl.__version__ # offline suite covers this published = self._pypi() assert self._mm(advertised) == self._mm(published), ( f"README and the package say {advertised}, PyPI serves {published} — " - "publish it, or the docs describe a release nobody can install") + "publish it, or the docs describe a release nobody can install" + ) assert self._mm(published) <= self._mm(advertised), ( - f"PyPI is ahead of the repo: {published} vs {advertised}") + f"PyPI is ahead of the repo: {published} vs {advertised}" + ) def test_the_site_shows_the_published_version(self): """/docs renders it from the package, so a deploy behind PyPI shows a version people cannot get.""" import re + req = urllib.request.Request(f"{BASE}/docs") with urllib.request.urlopen(req, timeout=TIMEOUT) as r: # noqa: S310 - shown = re.search(r'v([\d.]+)', - r.read().decode()).group(1) - assert self._mm(shown) == self._mm(self._pypi()), \ + shown = re.search(r'v([\d.]+)', r.read().decode()).group(1) + assert self._mm(shown) == self._mm(self._pypi()), ( f"/docs shows v{shown}, PyPI serves {self._pypi()}" + ) From f813f82d99eea454a576074fd138b79e8b0ff63a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abraham=20P=C3=A9rez?= Date: Fri, 11 Sep 2026 08:14:34 -0600 Subject: [PATCH 2/6] fix(ci): ignore playwright missing stubs in mypy type check The type-check job installs .[llm,dev,mcp] but not the optional [browser] extra, so mypy could not resolve playwright.sync_api (imported lazily in Scraper.browser() behind a try/except) and failed with import-not-found. Add a [tool.mypy] override so playwright.* is exempt from missing-import errors, keeping the heavy browser stack out of CI. Co-Authored-By: Claude Opus 4.8 --- pyproject.toml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9981017..79f2862 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,5 +60,14 @@ target-version = "py39" [tool.ruff.lint] select = ["E", "F", "W", "I", "UP", "B"] +[tool.mypy] + +[[tool.mypy.overrides]] +# playwright is the optional [browser] extra, not installed for the type-check +# job — Scraper.browser() imports it lazily behind a try/except. Ignore its +# missing stubs instead of pulling the heavy browser stack into CI. +module = ["playwright.*"] +ignore_missing_imports = true + [tool.vercel] entrypoint = "landing.app:app" From 8d152c7896c2bc1b02163e54fad5638b43b145ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abraham=20P=C3=A9rez?= Date: Fri, 11 Sep 2026 13:46:35 -0600 Subject: [PATCH 3/6] feat: name rate limits and unreachable seeds, surface them on every surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A genuine HTTP 429 now raises bytecrawl.RateLimitError (a requests.HTTPError subclass) instead of folding into "unreachable" — carrying Retry-After in both RFC 7231 forms (delta-seconds and HTTP-date). Unreachable seeds raise a single UnreachableError. A 429 with a bot-wall signature still reports as BlockedError. Every surface reports these the same way: the hosted API answers 429 with a Retry-After header and retry_after field (403 for blocked); the playground renders one calm card per failure across all modes; and the MCP tools (local and hosted) re-raise as ToolError so the agent sees the real message instead of a masked "Error executing tool", the MCP counterpart of the API's typed body. All rolled into the 1.4.0 changelog entry. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 37 ++ README.es.md | 4 +- README.md | 4 +- src/bytecrawl/__init__.py | 20 +- src/bytecrawl/core.py | 19 +- src/bytecrawl/crawl/base.py | 17 +- src/bytecrawl/crawl/compare.py | 11 +- src/bytecrawl/crawl/result.py | 46 +++ src/bytecrawl/mcp_server.py | 61 +++- src/bytecrawl/scraping/__init__.py | 4 +- src/bytecrawl/scraping/scraper.py | 6 +- src/bytecrawl/scraping/session.py | 8 +- src/bytecrawl/scraping/url.py | 221 ++++++++++++ src/bytecrawl/security.py | 13 +- tests/conftest.py | 5 +- tests/test_blocked.py | 141 ++++++++ tests/test_seed_failures.py | 524 +++++++++++++++++++++++++++++ web/landing/routes.py | 26 +- web/landing/services.py | 60 +++- web/landing/static/i18n.js | 15 + web/landing/templates/landing.html | 73 +++- web/landing/templates/try.html | 228 ++++++++++--- 22 files changed, 1457 insertions(+), 86 deletions(-) create mode 100644 tests/test_blocked.py create mode 100644 tests/test_seed_failures.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0be6d0c..67270b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,41 @@ # Changelog +## 1.4.0 — 2026-09-11 + +### Added +- **A bot wall now says so.** When a site's protection refuses the fetch, the + error names the wall instead of a bare `403 Client Error: Forbidden` — e.g. + *"Blocked by Cloudflare — tapology.com uses bot protection and refused this + request (HTTP 403). Automated access isn't allowed."* Cloudflare, DataDome, + PerimeterX/HUMAN, Akamai and Imperva + Incapsula are recognised by their response headers and challenge-page markers. + A new `bytecrawl.BlockedError` (a subclass of `requests.HTTPError`) carries it, + so `except requests.HTTPError` still catches it and you can catch the block + specifically. Every surface benefits at once — the library, the crawlers, the + hosted API and the playground — since the detection lives at the fetch layer. + Detection is gated on a block-looking status, so a normal page served from + behind Cloudflare is never mislabelled. +- **A rate limit now says so, too.** A genuine `HTTP 429 Too Many Requests` + (no bot-wall signature) no longer reads as "couldn't reach the site" — it gets + its own calm message: *"The site is rate-limiting us… it's up and reachable, + just asking for fewer requests. Wait a moment and try again."* A new + `bytecrawl.RateLimitError` (also a `requests.HTTPError` subclass) carries the + server's `Retry-After` when present — both forms, a delta in seconds and an + HTTP date. The hosted API answers `429` with a `Retry-After` header and a + `retry_after` field, and the playground shows a wait hint. A 429 that *does* + carry a bot-wall signature still reports as blocked — the more specific answer + wins. +- **One error card everywhere.** Unreachable seeds (bad DNS, refused + connection, timeout) now raise a single `bytecrawl.UnreachableError` across the + library, crawlers, hosted API and playground, so every playground mode — + Markdown, links, JSON API, extract, auto and compare — renders the same card + for the same failure instead of three different-looking errors. +- **MCP tools now tell the agent what went wrong.** A blocked, rate-limited or + unreachable seed used to reach an MCP client as a masked *"Error executing + tool …"* — the SDK treats a stray `requests` exception as a crash and withholds + its text. Every tool (local and hosted) now re-raises these as a `ToolError`, + so the agent receives the real message — the wall's name, or the wait hint for + a 429 — the MCP counterpart of the API's typed 403/429 body. + ## 1.3.0 — 2026-09-11 ### Changed diff --git a/README.es.md b/README.es.md index ec8097d..78d336a 100644 --- a/README.es.md +++ b/README.es.md @@ -10,8 +10,8 @@ Shark-Search y OPIC en Python puro. - **Web**: https://bytecrawl.vercel.app/ - **Endpoint MCP hosteado**: https://bytecrawl.vercel.app/mcp - **Skill para agentes**: https://bytecrawl.vercel.app/agent-onboarding/SKILL.md -- **Última versión**: **1.3.0** — publicación automática a PyPI y checks de CI - separados (lint, formato, tipos, seguridad) ([changelog](https://github.com/abrahamperz/ByteCrawl/blob/main/CHANGELOG.md)) +- **Última versión**: **1.4.0** — errores más claros cuando la protección + anti-bot de un sitio (Cloudflare, DataDome, …) bloquea el fetch ([changelog](https://github.com/abrahamperz/ByteCrawl/blob/main/CHANGELOG.md)) - **PyPI**: https://pypi.org/project/bytecrawl/ - **GitHub**: https://github.com/abrahamperz/ByteCrawl diff --git a/README.md b/README.md index 702fd92..b82fe6e 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ and OPIC in pure Python. - **Webpage**: https://bytecrawl.vercel.app/ - **Hosted MCP endpoint**: https://bytecrawl.vercel.app/mcp - **Agent skill**: https://bytecrawl.vercel.app/agent-onboarding/SKILL.md -- **Latest release**: **1.3.0** — automated PyPI publishing and separated CI - gates (lint, format, types, security) ([changelog](https://github.com/abrahamperz/ByteCrawl/blob/main/CHANGELOG.md)) +- **Latest release**: **1.4.0** — clearer errors when a site's bot protection + (Cloudflare, DataDome, …) blocks a fetch ([changelog](https://github.com/abrahamperz/ByteCrawl/blob/main/CHANGELOG.md)) ## Why focused crawling? diff --git a/src/bytecrawl/__init__.py b/src/bytecrawl/__init__.py index beca973..2ca8588 100644 --- a/src/bytecrawl/__init__.py +++ b/src/bytecrawl/__init__.py @@ -22,7 +22,17 @@ LLM: page.markdown() · page.tokens() """ -from .core import Page, Scraper, Session +from .core import ( + RATE_LIMIT_MESSAGE, + UNREACHABLE_MESSAGE, + BlockedError, + Page, + RateLimitError, + Scraper, + Session, + UnreachableError, + open_seed, +) from .crawler import ( BFS, OPIC, @@ -39,6 +49,12 @@ "Scraper", "Page", "Session", + "BlockedError", + "RateLimitError", + "UnreachableError", + "RATE_LIMIT_MESSAGE", + "UNREACHABLE_MESSAGE", + "open_seed", "Crawler", "BFS", "SharkSearch", @@ -49,4 +65,4 @@ "cosine", "compare", ] -__version__ = "1.3.0" +__version__ = "1.4.0" diff --git a/src/bytecrawl/core.py b/src/bytecrawl/core.py index 3b06f1d..9d911cc 100644 --- a/src/bytecrawl/core.py +++ b/src/bytecrawl/core.py @@ -14,12 +14,29 @@ from .scraping.scraper import Scraper from .scraping.selectors import _split_selector, _value_from from .scraping.session import Session -from .scraping.url import _SKIP_EXT, DEFAULT_UA, _decoded_html, normalize +from .scraping.url import ( + _SKIP_EXT, + DEFAULT_UA, + RATE_LIMIT_MESSAGE, + UNREACHABLE_MESSAGE, + BlockedError, + RateLimitError, + UnreachableError, + _decoded_html, + normalize, + open_seed, +) __all__ = [ "Page", "Session", "Scraper", + "BlockedError", + "RateLimitError", + "UnreachableError", + "RATE_LIMIT_MESSAGE", + "UNREACHABLE_MESSAGE", + "open_seed", "normalize", "DEFAULT_UA", "_SKIP_EXT", diff --git a/src/bytecrawl/crawl/base.py b/src/bytecrawl/crawl/base.py index 090399d..a87c7ed 100644 --- a/src/bytecrawl/crawl/base.py +++ b/src/bytecrawl/crawl/base.py @@ -15,7 +15,7 @@ # crawlers agree on what counts as a link; the crawler module re-exports it # because that is where the tests and docs import it from. from ..scraping.scraper import Scraper -from ..scraping.url import normalize +from ..scraping.url import BlockedError, RateLimitError, normalize from .frontier import Frontier, _root_domain from .result import CrawlResult from .text import relevance @@ -81,6 +81,21 @@ def crawl(self, start: str, max_pages: int = 50, max_depth: int = 10) -> CrawlRe try: page = self.scraper.static(url) + except BlockedError as e: + # A single walled page shouldn't abort the crawl, but remember + # the first wall: if the crawl comes back empty, the caller can + # say "blocked" instead of the misleading "nothing relevant". + errors += 1 + if result.blocked is None: + result.blocked = e + continue + except RateLimitError as e: + # Likewise for a 429: keep crawling other pages, but remember the + # first throttle so an empty crawl reports "slow down and retry". + errors += 1 + if result.rate_limited is None: + result.rate_limited = e + continue except Exception: errors += 1 continue diff --git a/src/bytecrawl/crawl/compare.py b/src/bytecrawl/crawl/compare.py index 7aca8d9..31bc716 100644 --- a/src/bytecrawl/crawl/compare.py +++ b/src/bytecrawl/crawl/compare.py @@ -4,18 +4,25 @@ from concurrent.futures import ThreadPoolExecutor +from ..scraping.url import BlockedError, RateLimitError, UnreachableError from .strategies import STRATEGIES def _leg( name: str, start: str, query: str, max_pages: int, max_depth: int, delay: float, timeout: int ) -> dict: - """One strategy's leg of a comparison. Never raises: a strategy that fails - reports inside its own entry so the other two still come back.""" + """One strategy's leg of a comparison. A strategy that fails on its own + reports inside its own entry so the other two still come back — but a + seed-level failure (a walled or unreachable start URL) is not that: it + fails every leg identically, so re-raise it and let :func:`compare` surface + it once instead of burying three copies in three error strings.""" try: result = STRATEGIES[name](query=query, delay=delay, timeout=timeout).crawl( start, max_pages=max_pages, max_depth=max_depth ) + result.raise_for_seed() + except (BlockedError, RateLimitError, UnreachableError): + raise except Exception as e: return {"error": f"{type(e).__name__}: {e}"} return { diff --git a/src/bytecrawl/crawl/result.py b/src/bytecrawl/crawl/result.py index 9b400e8..33edcce 100644 --- a/src/bytecrawl/crawl/result.py +++ b/src/bytecrawl/crawl/result.py @@ -4,6 +4,8 @@ from dataclasses import dataclass, field +from ..scraping.url import BlockedError, RateLimitError, UnreachableError + @dataclass class CrawlResult: @@ -13,9 +15,53 @@ class CrawlResult: pages: list[dict] = field(default_factory=list) # url, title, score, relevance, depth, order graph: dict[str, list[str]] = field(default_factory=dict) stats: dict = field(default_factory=dict) + # The bot wall that turned the crawl away, if any. A crawl swallows a + # per-page block and keeps going, but when the seed is walled it comes + # back empty — callers read this to tell "walled" apart from "nothing here". + blocked: BlockedError | None = None + # Same idea for a throttled seed: the crawl swallows a per-page 429 and + # keeps going, but a rate-limited *seed* comes back empty — remembered here + # so the caller can say "slow down and retry" instead of "nothing here". + rate_limited: RateLimitError | None = None def relevant(self, threshold: float = 0.1) -> list[dict]: return [p for p in self.pages if p["relevance"] >= threshold] def top(self, n: int = 10) -> list[dict]: return sorted(self.pages, key=lambda p: p["relevance"], reverse=True)[:n] + + @property + def unreachable(self) -> bool: + """Every fetch failed: the crawl never read a single page. + + A crawl swallows per-page errors and keeps going, so an empty result + is ambiguous — the seed was unreachable (a domain that doesn't + resolve, a refused connection, a 404, or two URLs pasted into one), or + it read fine and simply matched nothing. When every request failed + (``requests - errors <= 0``) it is the former. A walled seed also + lands here, but is reported through :attr:`blocked` first. + """ + reqs = self.stats.get("requests", 0) + return reqs >= 1 and reqs - self.stats.get("errors", 0) <= 0 + + def raise_for_seed(self) -> None: + """Raise if the crawl never got off the ground; return quietly otherwise. + + Turns a dead-on-arrival crawl into the same exception a single fetch + would have raised, so every caller (the HTTP API, the MCP tools) can + report a walled or unreachable seed uniformly instead of dressing an + empty crawl up as "found nothing relevant". A crawl that read at least + one page returns quietly — even one that matched nothing, which is a + real answer, not a failure. + + Precedence, most specific first: a bot wall (blocked), then a rate limit + (throttled but reachable), then a dead seed (unreachable). A walled seed + can set both blocked and unreachable; "a bot wall refused you" is the + more actionable answer, and a 429 sits between the two. + """ + if self.blocked is not None and not self.pages: + raise self.blocked + if self.rate_limited is not None and not self.pages: + raise self.rate_limited + if self.unreachable: + raise UnreachableError() # default message: the one shared wording diff --git a/src/bytecrawl/mcp_server.py b/src/bytecrawl/mcp_server.py index 12931a2..409e9c1 100644 --- a/src/bytecrawl/mcp_server.py +++ b/src/bytecrawl/mcp_server.py @@ -15,18 +15,60 @@ try: from mcp.server.mcpserver import MCPServer + from mcp.server.mcpserver.exceptions import ToolError except ImportError as e: # pragma: no cover - exercised only without the extra raise ImportError("The MCP server needs the 'mcp' extra: pip install bytecrawl[mcp]") from e -from .core import Scraper +from contextlib import contextmanager + +from .core import ( + BlockedError, + RateLimitError, + Scraper, + UnreachableError, + open_seed, +) from .crawler import STRATEGIES as _STRATEGIES from .crawler import compare, pagerank DELAY = 0.2 # seconds between requests, per crawler + +@contextmanager +def _seed_errors_as_tool_errors(): + """Report a seed failure as an anticipated :class:`ToolError`. + + Our seed errors are ``requests`` exceptions. To the MCP SDK an escaping + ``requests`` exception is a *crash*: it withholds the message and hands the + agent a bare ``Error executing tool ``. Re-raised as ``ToolError`` they + come back as an ``is_error`` result carrying our text — the MCP counterpart + of the HTTP API's typed body. The specifics fold into the message (the wall's + name, the wait hint) because an MCP client reads the sentence, not a JSON + field, so this is the same information the API returns as ``retry_after``. + """ + try: + yield + except RateLimitError as e: + wait = f" (retry after ~{e.retry_after}s)" if e.retry_after is not None else "" + raise ToolError(f"{e}{wait}") from e + except (BlockedError, UnreachableError) as e: + raise ToolError(str(e)) from e + # One polite scraper shared by every tool call. _scraper = Scraper(delay=DELAY) + +def _open(url: str, *, api: bool = False, **kw): + """Fetch the seed through the shared :func:`bytecrawl.open_seed` guard. + + Keeps the technique choice here (``api`` -> ``Scraper.api``, otherwise the + auto ``Scraper.fetch``) while the block/unreachable translation lives in the + core, so a walled or unopenable URL reports the same way here, in the HTTP + API, and in the crawl tools' ``CrawlResult.raise_for_seed()``. + """ + with _seed_errors_as_tool_errors(): + return open_seed(lambda: _scraper.api(url, **kw) if api else _scraper.fetch(url)) + server = MCPServer( "bytecrawl", instructions=( @@ -47,7 +89,7 @@ ) ) def fetch_markdown(url: str) -> dict: - page = _scraper.fetch(url) + page = _open(url) md = page.markdown() # tokens_html alongside it, the same pair the playground shows. On its own # tokens_estimate is a number with nothing to compare against; next to the @@ -95,7 +137,7 @@ def extract( ) # Checked before the fetch: a malformed call shouldn't cost the site a # request. - page = _scraper.fetch(url) + page = _open(url) if select: values = page.css_all(select) return {"url": url, "select": select, "count": len(values), "values": values} @@ -124,7 +166,7 @@ def extract( ) ) def list_links(url: str, raw: bool = False) -> dict: - page = _scraper.fetch(url) + page = _open(url) links = page.links(raw=raw) return {"url": url, "count": len(links), "raw": raw, "links": links} @@ -143,6 +185,12 @@ def focused_crawl(url: str, query: str = "", strategy: str = "shark", max_pages: max_pages = min(max_pages, 50) # keep agent calls bounded and polite crawler = _STRATEGIES[strategy](query=query, delay=DELAY) result = crawler.crawl(url, max_pages=max_pages) + # A walled, throttled or unreachable seed comes back as an empty crawl; raise + # it as the same blocked/rate-limited/unreachable error a single fetch would + # (as a ToolError, so the agent sees it), rather than returning an empty page + # list dressed up as "nothing relevant". + with _seed_errors_as_tool_errors(): + result.raise_for_seed() ranks = pagerank(result.graph) return { "strategy": strategy, @@ -164,13 +212,14 @@ def focused_crawl(url: str, query: str = "", strategy: str = "shark", max_pages: ) def compare_strategies(url: str, query: str, max_pages: int = 20) -> dict: max_pages = min(max_pages, 50) # same bound as focused_crawl, per strategy - out = compare(url, query, max_pages=max_pages, delay=DELAY) + with _seed_errors_as_tool_errors(): + out = compare(url, query, max_pages=max_pages, delay=DELAY) return {"url": url, "query": query, "max_pages": max_pages, **out} @server.tool(description="Fetch a JSON API endpoint (the 'hidden API' scraping technique).") def fetch_json_api(url: str, params: dict | None = None) -> dict: - page = _scraper.api(url, params=params) + page = _open(url, api=True, params=params) return {"url": url, "status": page.status, "data": page.json()} diff --git a/src/bytecrawl/scraping/__init__.py b/src/bytecrawl/scraping/__init__.py index 65f3ec0..bf12b90 100644 --- a/src/bytecrawl/scraping/__init__.py +++ b/src/bytecrawl/scraping/__init__.py @@ -10,6 +10,6 @@ from .page import Page from .scraper import Scraper from .session import Session -from .url import normalize +from .url import BlockedError, RateLimitError, normalize -__all__ = ["Page", "Session", "Scraper", "normalize"] +__all__ = ["Page", "Session", "Scraper", "normalize", "BlockedError", "RateLimitError"] diff --git a/src/bytecrawl/scraping/scraper.py b/src/bytecrawl/scraping/scraper.py index fd7129b..f5e7d89 100644 --- a/src/bytecrawl/scraping/scraper.py +++ b/src/bytecrawl/scraping/scraper.py @@ -14,7 +14,7 @@ from .page import Page from .selectors import _split_selector, _value_from from .session import Session -from .url import DEFAULT_UA, _decoded_html +from .url import DEFAULT_UA, _decoded_html, _raise_for_status class Scraper: @@ -32,7 +32,7 @@ def static(self, url: str) -> Page: """Technique 1: static HTML with requests.""" t0 = time.perf_counter() r = self._session.get(url, timeout=self.timeout) - r.raise_for_status() + _raise_for_status(r) self._wait() return Page( url=url, @@ -46,7 +46,7 @@ def api(self, url: str, params: dict | None = None) -> Page: """Technique 3: requests an API and stores the JSON.""" t0 = time.perf_counter() r = self._session.get(url, params=params, timeout=self.timeout) - r.raise_for_status() + _raise_for_status(r) self._wait() try: data = r.json() diff --git a/src/bytecrawl/scraping/session.py b/src/bytecrawl/scraping/session.py index 981a15f..4b9138f 100644 --- a/src/bytecrawl/scraping/session.py +++ b/src/bytecrawl/scraping/session.py @@ -14,7 +14,7 @@ from bs4 import BeautifulSoup from .page import Page -from .url import _decoded_html +from .url import _decoded_html, _raise_for_status if TYPE_CHECKING: from .scraper import Scraper @@ -32,13 +32,13 @@ def login(self, url: str, data: dict, csrf_field: str | None = None) -> Session: """Logs in. If csrf_field is given, reads it from the form first.""" if csrf_field: r = self._s.get(url, timeout=self._scraper.timeout) - r.raise_for_status() + _raise_for_status(r) form = BeautifulSoup(_decoded_html(r), "lxml") token = form.select_one(f'input[name="{csrf_field}"]') if token: data = {**data, csrf_field: token.get("value", "")} r = self._s.post(url, data=data, timeout=self._scraper.timeout) - r.raise_for_status() + _raise_for_status(r) return self def bearer(self, token: str) -> Session: @@ -48,7 +48,7 @@ def bearer(self, token: str) -> Session: def fetch(self, url: str) -> Page: t0 = time.perf_counter() r = self._s.get(url, timeout=self._scraper.timeout) - r.raise_for_status() + _raise_for_status(r) return Page( url=url, html=_decoded_html(r), diff --git a/src/bytecrawl/scraping/url.py b/src/bytecrawl/scraping/url.py index 4123886..f6c83b3 100644 --- a/src/bytecrawl/scraping/url.py +++ b/src/bytecrawl/scraping/url.py @@ -7,6 +7,8 @@ from __future__ import annotations import re +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime from urllib.parse import urldefrag, urljoin, urlparse import requests @@ -51,3 +53,222 @@ def _decoded_html(r: requests.Response) -> str: if r.encoding is None or r.encoding.lower() == "iso-8859-1": r.encoding = r.apparent_encoding return r.text + + +class BlockedError(requests.HTTPError): + """A site's bot-protection / WAF refused the request. + + Subclasses :class:`requests.HTTPError`, so code that already does + ``except requests.HTTPError`` keeps catching it. Catch ``BlockedError`` + specifically to tell a bot wall (Cloudflare, DataDome, …) apart from an + ordinary 404/500. + + Carries the ``wall`` name and ``status`` so a caller can render a + structured message without re-parsing the string. + """ + + def __init__(self, *args: object, wall: str | None = None, status: int | None = None) -> None: + super().__init__(*args) + self.wall = wall + self.status = status + + +# The one wording for "the seed never loaded", so the API, the MCP tools and +# the crawl path (via CrawlResult.raise_for_seed) all say the same thing and a +# reword lands everywhere at once. UnreachableError() with no message uses it. +UNREACHABLE_MESSAGE = ( + "Couldn't open that page — the site didn't respond. Check the URL is real " + "and complete." +) + + +# The one wording for "the site is throttling us", shared like UNREACHABLE_MESSAGE +# so a 429 reads the same on every surface. RateLimitError() with no message uses +# it; the actual seconds-to-wait ride alongside on `.retry_after`. +RATE_LIMIT_MESSAGE = ( + "The site is rate-limiting us (HTTP 429 Too Many Requests). It's up and " + "reachable — just asking for fewer requests. Wait a moment and try again." +) + + +def _retry_after_seconds(value: str | None) -> int | None: + """Seconds to wait from a ``Retry-After`` header, or ``None``. + + RFC 7231 allows either a delta in seconds ("120") or an HTTP date. We read + both: the numeric form directly, and the date form as the delta from now + (clamped at zero, so a past date reads as "retry now" rather than negative). + """ + if not value: + return None + value = value.strip() + if value.isdigit(): + return int(value) + try: + when = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + if when is None: + return None + if when.tzinfo is None: + when = when.replace(tzinfo=timezone.utc) + delta = (when - datetime.now(timezone.utc)).total_seconds() + return max(0, int(delta)) + + +class RateLimitError(requests.HTTPError): + """The seed answered 429 Too Many Requests — reachable, just throttled. + + Distinct from :class:`BlockedError` (a bot wall refusing automated access + outright) and :class:`UnreachableError` (the page never loaded at all): a + rate limit is temporary and often carries a ``Retry-After`` saying how long + to wait. Subclasses :class:`requests.HTTPError`, so ``except + requests.HTTPError`` keeps catching it; catch ``RateLimitError`` specifically + to tell "slow down and retry" apart from a hard block or a dead seed. + + Raised with no message it carries :data:`RATE_LIMIT_MESSAGE`, the single + wording every surface shares. ``retry_after`` is the seconds to wait when + the site said so (else ``None``); ``status`` is 429. + """ + + def __init__( + self, + *args: object, + retry_after: int | None = None, + status: int = 429, + url: str | None = None, + ) -> None: + if not args: + args = (RATE_LIMIT_MESSAGE,) + super().__init__(*args) + self.retry_after = retry_after + self.status = status + self.url = url + + +class UnreachableError(requests.RequestException): + """The seed URL couldn't be opened at all — the page never loaded. + + A domain that doesn't resolve, a refused connection, a 404, or two URLs + pasted into one. Distinct from :class:`BlockedError` (a bot wall said no) + and from a crawl that read the page fine but matched nothing (a real + answer, ``pages`` non-empty). Subclasses :class:`requests.RequestException` + so ``except requests.RequestException`` keeps catching it; catch + ``UnreachableError`` specifically to tell "couldn't open it" apart from an + ordinary request failure. + + Raised with no message it carries :data:`UNREACHABLE_MESSAGE`, the single + wording every surface shares. Carries the offending ``url`` when a caller + has it, for a structured message without re-parsing the string. + """ + + def __init__(self, *args: object, url: str | None = None) -> None: + if not args: + args = (UNREACHABLE_MESSAGE,) + super().__init__(*args) + self.url = url + + +def open_seed(fetch): + """Run one seed fetch, turning a non-block failure into UnreachableError. + + A bot wall stays a :class:`BlockedError` (the caller still gets the named + wall); anything else that stopped the page loading — a domain that doesn't + resolve, a refused connection, a 404, two URLs pasted into one — becomes a + single "couldn't open that page" answer instead of a raw traceback. The one + place the single-fetch surfaces (the HTTP API, the MCP tools) share, so the + fix lands in all of them at once; the crawl surfaces reach the same + :class:`UnreachableError` through :meth:`CrawlResult.raise_for_seed`. + + ``fetch`` is a zero-arg callable returning a Page — e.g. + ``lambda: scraper.static(url)`` or ``lambda: scraper.api(url)`` — so the + caller keeps choosing the technique. + """ + try: + return fetch() + except (BlockedError, RateLimitError): + # A named wall and a 429 are both reachable-but-refused answers, more + # specific than "couldn't open it" — let them through unchanged. + raise + except requests.RequestException as e: + raise UnreachableError() from e + + +# WAF / bot-protection fingerprints: (label, response-header keys, body markers). +# Only consulted on a block-looking status, so serving a normal page from behind +# Cloudflare never trips it — just a challenge/deny does. +_BOT_WALLS = ( + ( + "Cloudflare", + ("cf-mitigated", "cf-ray"), + ("just a moment", "attention required", "cf-browser-verification", "cf_chl", "_cf_chl"), + ), + ("DataDome", ("x-datadome", "x-dd-b"), ("datadome",)), + ("PerimeterX / HUMAN", ("x-px",), ("px-captcha", "perimeterx")), + # "access denied" / "reference #" alone are too generic (a plain auth + # failure says them too); "akamaighost" is Akamai's own edge signature. + ("Akamai", (), ("akamaighost",)), + ("Imperva Incapsula", ("x-iinfo",), ("incapsula", "_incap_", "incap_ses")), +) + +# Statuses a bot wall actually uses to turn a request away. +_BLOCK_STATUS = frozenset({401, 403, 406, 429, 503}) + + +def _detect_bot_wall(r: requests.Response) -> str | None: + """Return the name of the bot wall that blocked ``r``, or ``None``. + + Gated on a block-looking status first, then sniffs response headers, the + ``Server`` header and a slice of the body for known WAF signatures. + """ + if r.status_code not in _BLOCK_STATUS: + return None + headers = {k.lower(): str(v).lower() for k, v in r.headers.items()} + server = headers.get("server", "") + try: + body = r.text[:4096].lower() + except Exception: # a challenge page may not decode cleanly + body = "" + for label, header_keys, markers in _BOT_WALLS: + vendor = label.split()[0].lower() # "cloudflare", "datadome", "imperva", … + if any(k in headers for k in header_keys): + return label + if vendor in server: + return label + if any(m in body for m in markers): + return label + return None + + +def _raise_for_status(r: requests.Response) -> None: + """Like ``Response.raise_for_status()`` but clearer about bot walls. + + On an error status, if the response carries a bot-protection signature, + raise :class:`BlockedError` with a human message naming the wall instead + of a bare ``403 Client Error: Forbidden``. Everything else raises the + ordinary :class:`requests.HTTPError`, unchanged. + """ + try: + r.raise_for_status() + except requests.HTTPError as e: + wall = _detect_bot_wall(r) + if wall: + host = urlparse(getattr(r, "url", "") or "").hostname or "the site" + host = host.removeprefix("www.") + raise BlockedError( + f"Blocked by {wall} — {host} uses bot protection and refused " + f"this request (HTTP {r.status_code}). Automated access isn't " + f"allowed.", + wall=wall, + status=r.status_code, + ) from e + if r.status_code == 429: + # A bare 429 (no WAF signature) is an honest "too many requests": + # the site is up and answered, it just wants fewer of them. That is + # not "couldn't open the page" — surface it as its own retryable + # state, carrying the Retry-After wait when the site gave one. + raise RateLimitError( + retry_after=_retry_after_seconds(r.headers.get("Retry-After")), + status=r.status_code, + url=getattr(r, "url", None), + ) from e + raise diff --git a/src/bytecrawl/security.py b/src/bytecrawl/security.py index 437bd3d..fd1130a 100644 --- a/src/bytecrawl/security.py +++ b/src/bytecrawl/security.py @@ -11,6 +11,8 @@ import socket from urllib.parse import urlparse +from .scraping.url import UnreachableError + def assert_public_url(url: str) -> None: """Reject URLs that could reach private infrastructure. @@ -18,6 +20,15 @@ def assert_public_url(url: str) -> None: Resolves the hostname and requires every returned address to be globally routable. Blocks localhost, RFC1918 ranges, link-local (cloud metadata endpoints like 169.254.169.254) and other reserved space. + + A host that doesn't resolve isn't a security refusal — it just can't be + reached — so it raises :class:`UnreachableError` (the same exception a + failed fetch would), not ``ValueError``. That way an unresolvable or + mistyped domain gets the one shared "couldn't open that page" answer + across every surface, instead of a raw "Cannot resolve host" on the one + path that happens to pre-check. The genuine refusals — a non-http(s) + scheme, a missing hostname, or an address that resolves into private + space — stay ``ValueError``. """ parsed = urlparse(url) if parsed.scheme not in ("http", "https"): @@ -28,7 +39,7 @@ def assert_public_url(url: str) -> None: try: infos = socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP) except socket.gaierror as e: - raise ValueError(f"Cannot resolve host: {host}") from e + raise UnreachableError(url=url) from e for info in infos: ip = ipaddress.ip_address(info[4][0]) if not ip.is_global or ip.is_multicast: diff --git a/tests/conftest.py b/tests/conftest.py index d6ca5d4..76610f1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,7 +30,10 @@ def __init__( self.encoding = encoding self.apparent_encoding = apparent_encoding self._json = json_data - self.headers = headers or {"content-type": "text/html"} + # A real Response.headers is case-insensitive; mirror that so code that + # reads e.g. "Retry-After" finds a header sent as "retry-after". + self.headers = requests.structures.CaseInsensitiveDict( + headers or {"content-type": "text/html"}) @property def text(self): diff --git a/tests/test_blocked.py b/tests/test_blocked.py new file mode 100644 index 0000000..2f394b7 --- /dev/null +++ b/tests/test_blocked.py @@ -0,0 +1,141 @@ +"""Bot-wall detection: a WAF block becomes a clear BlockedError, not a bare 403. + +Covers the two things that matter: known walls (Cloudflare, DataDome, …) are +named in a human message, and ordinary errors (a plain 404, an unfingerprinted +403, a normal page served from behind Cloudflare) are left exactly as they were. +""" + +from __future__ import annotations + +import pytest +import requests + +import bytecrawl +from bytecrawl import BlockedError, Scraper +from bytecrawl.scraping.url import _detect_bot_wall, _raise_for_status + +from .conftest import FakeResponse + + +def _resp(status, headers=None, text="", url="https://blocked.test/page"): + r = FakeResponse(status=status, headers=headers or {"content-type": "text/html"}, text=text) + r.url = url + return r + + +# --- detection: each vendor, by header and by body --------------------------- +@pytest.mark.parametrize( + "headers, body, expected", + [ + ({"server": "cloudflare", "cf-ray": "8a2b"}, "", "Cloudflare"), + ({"cf-mitigated": "challenge"}, "", "Cloudflare"), + ({}, "Just a moment...", "Cloudflare"), + ({}, "Attention Required! | Cloudflare", "Cloudflare"), + ({"x-datadome": "protected"}, "", "DataDome"), + ({}, "please enable JS — datadome", "DataDome"), + ({"x-px": "1"}, "", "PerimeterX / HUMAN"), + ({}, "px-captcha challenge", "PerimeterX / HUMAN"), + ({}, "Access Denied — AkamaiGHost", "Akamai"), + ({"x-iinfo": "9-1"}, "", "Imperva Incapsula"), + ], +) +def test_detects_named_walls(headers, body, expected): + assert _detect_bot_wall(_resp(403, headers, body)) == expected + + +def test_ignores_walls_on_a_success_status(): + # A normal page served from behind Cloudflare must not look like a block. + ok = _resp(200, {"server": "cloudflare", "cf-ray": "8a2b"}, "real content") + assert _detect_bot_wall(ok) is None + + +def test_unfingerprinted_403_is_not_a_wall(): + assert _detect_bot_wall(_resp(403, {"server": "nginx"}, "Forbidden")) is None + + +def test_plain_access_denied_is_not_a_wall(): + # A credential/permission failure says "Access Denied" too — must NOT be + # mislabelled as a bot wall without a real vendor signature. + r = _resp(403, {"server": "Apache"}, "Access Denied. You lack permission.") + assert _detect_bot_wall(r) is None + + +# --- _raise_for_status: what actually gets raised ---------------------------- +def test_blocked_status_raises_blockederror_with_a_clear_message(): + r = _resp(403, {"server": "cloudflare", "cf-ray": "8a2b"}, "Just a moment...") + with pytest.raises(BlockedError) as exc: + _raise_for_status(r) + msg = str(exc.value) + assert "Cloudflare" in msg + assert "HTTP 403" in msg + assert "blocked.test" in msg # host pulled from the response URL + + +def test_blockederror_carries_wall_and_status(): + # The frontend renders a structured notice from these, not a re-parsed string. + r = _resp(429, {"x-datadome": "protected"}, "") + err = _capture(r) + assert err.wall == "DataDome" + assert err.status == 429 + + +def test_message_strips_www_prefix(): + r = _resp(403, {"cf-mitigated": "block"}, "", "https://www.example.com/x") + with pytest.raises(BlockedError, match=r"example\.com"): + _raise_for_status(r) + assert "www." not in str(_capture(r)) + + +def _capture(r): + try: + _raise_for_status(r) + except BlockedError as e: + return e + return None + + +def test_blockederror_is_an_httperror(): + # Existing `except requests.HTTPError` handlers keep catching it. + assert issubclass(BlockedError, requests.HTTPError) + assert bytecrawl.BlockedError is BlockedError + + +def test_plain_404_stays_a_plain_httperror(): + r = _resp(404, {"server": "nginx"}, "Not Found") + with pytest.raises(requests.HTTPError) as exc: + _raise_for_status(r) + assert not isinstance(exc.value, BlockedError) + + +def test_success_passes_through(): + _raise_for_status(_resp(200)) # does not raise + + +# --- through a crawl --------------------------------------------------------- +def test_crawl_records_the_wall_and_comes_back_empty(http): + # A walled seed yields no pages, but the crawl remembers the wall so the + # caller can say "blocked" instead of the misleading "nothing relevant". + from bytecrawl.crawl import BFS + + url = "https://walled.test/" + http.routes[url] = _resp(403, {"server": "cloudflare", "cf-ray": "1"}, "Just a moment...", url) + result = BFS(query="anything").crawl(url, max_pages=5) + assert result.pages == [] + assert isinstance(result.blocked, BlockedError) + assert result.blocked.wall == "Cloudflare" + + +# --- through the public Scraper surface -------------------------------------- +def test_scraper_static_surfaces_the_block(http): + url = "https://walled.test/" + http.routes[url] = _resp(403, {"server": "cloudflare", "cf-ray": "1"}, "Just a moment...", url) + with pytest.raises(BlockedError, match="Cloudflare"): + Scraper().static(url) + + +def test_scraper_static_still_raises_plain_404(http): + url = "https://gone.test/" + http.routes[url] = _resp(404, {"server": "nginx"}, "nope", url) + with pytest.raises(requests.HTTPError) as exc: + Scraper().static(url) + assert not isinstance(exc.value, BlockedError) diff --git a/tests/test_seed_failures.py b/tests/test_seed_failures.py new file mode 100644 index 0000000..15ba593 --- /dev/null +++ b/tests/test_seed_failures.py @@ -0,0 +1,524 @@ +"""The two seed-failure states, reported identically on every surface. + +A crawl swallows per-page failures and keeps going, so an empty result is +ambiguous: the seed was *walled* (a bot wall said no), *unreachable* (the page +never loaded — bad domain, refused connection, 404, two URLs pasted into one), +or read fine and simply matched nothing. The first two are failures the caller +should hear about plainly; the third is a real answer. + +The point of this file is that the *same* two failures come back the *same* +way whether you go through the core primitives, the MCP tools, or the HTTP +API — so the tests are written as parallel triples: one core assertion, the MCP +tool, the /api method, against the same fake walled and dead seeds. If a +surface ever drifts (an empty crawl dressed up as "nothing relevant" again, a +raw traceback for a dead domain), exactly one of these fails. + +The single seam is ``requests.Session.get`` (the ``http`` fixture): every +surface — ``Scraper.static``/``.api``/``.fetch``, and the crawlers on top of +them — funnels through it, so one routing table drives all three. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest +import requests + +import bytecrawl +from bytecrawl import BlockedError, RateLimitError, UnreachableError +from bytecrawl.crawl import BFS +from bytecrawl.crawl.result import CrawlResult +from bytecrawl.crawler import compare + +from .conftest import FakeResponse + +# The Flask landing app lives under web/, outside the importable package tree. +# create_app() puts web/ and src/ on sys.path at import, but importing the +# module to *reach* create_app needs web/ on the path first. +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "web")) + + +SEED = "https://seed.test/" + + +def _walled(url=SEED): + """A Cloudflare wall on `url` — a real vendor signature, not a bare 403.""" + r = FakeResponse( + status=403, + headers={"server": "cloudflare", "cf-ray": "1a2b"}, + text="Just a moment...", + ) + r.url = url + return r + + +def _dead(url=SEED): + """A seed that 404s: the page never loaded. Distinct from a wall.""" + r = FakeResponse(status=404, headers={"server": "nginx"}, text="Not Found") + r.url = url + return r + + +def _throttled(url=SEED, retry_after="30"): + """A bare 429 with a Retry-After: reachable, just asking us to slow down. + + No WAF signature — a plain "Too Many Requests" from a live site, which must + read as a rate limit, not a dead seed and not a bot wall. + """ + r = FakeResponse( + status=429, + headers={"server": "nginx", "retry-after": retry_after}, + text="Too Many Requests", + ) + r.url = url + return r + + +def _walled_429(url=SEED): + """A 429 that also carries a Cloudflare signature — a wall, not a rate limit. + + Locks the precedence: a bot wall answer is more specific than "slow down". + """ + r = FakeResponse( + status=429, + headers={"server": "cloudflare", "cf-ray": "9z9z"}, + text="Just a moment...", + ) + r.url = url + return r + + +# --------------------------------------------------------------------------- # +# Core: the shared primitives every surface is built on. +# --------------------------------------------------------------------------- # +class TestCorePrimitives: + def test_unreachable_is_true_when_every_fetch_failed(self): + r = CrawlResult(strategy="bfs", stats={"requests": 3, "errors": 3}) + assert r.unreachable is True + + def test_unreachable_is_false_when_a_page_was_read(self): + # One good read among failures is a real (if thin) answer, not a + # dead seed. + r = CrawlResult(strategy="bfs", stats={"requests": 3, "errors": 2}) + assert r.unreachable is False + + def test_unreachable_is_false_before_any_request(self): + # An empty stats dict (nothing ran) must not read as "unreachable". + assert CrawlResult(strategy="bfs", stats={}).unreachable is False + + def test_raise_for_seed_raises_unreachable_on_a_dead_seed(self): + r = CrawlResult(strategy="bfs", stats={"requests": 1, "errors": 1}) + with pytest.raises(UnreachableError): + r.raise_for_seed() + + def test_raise_for_seed_is_quiet_when_a_page_was_read(self): + r = CrawlResult( + strategy="bfs", + pages=[{"url": SEED, "relevance": 0.0}], + stats={"requests": 1, "errors": 0}, + ) + r.raise_for_seed() # does not raise + + def test_block_wins_over_unreachable(self): + # A walled seed sets both flags; "a bot wall refused you" is the more + # specific, more actionable answer, so it takes precedence. + wall = BlockedError("Cloudflare wall", wall="Cloudflare", status=403) + r = CrawlResult(strategy="bfs", stats={"requests": 1, "errors": 1}, blocked=wall) + with pytest.raises(BlockedError): + r.raise_for_seed() + + def test_unreachable_error_is_a_requestexception(self): + # `except requests.RequestException` handlers keep catching it. + assert issubclass(UnreachableError, requests.RequestException) + assert bytecrawl.UnreachableError is UnreachableError + + def test_rate_limit_error_is_an_httperror(self): + # `except requests.HTTPError` keeps catching a 429, and it is exported + # at the top level like the other two seed errors. + assert issubclass(RateLimitError, requests.HTTPError) + assert bytecrawl.RateLimitError is RateLimitError + + def test_raise_for_seed_raises_rate_limit_on_a_throttled_seed(self): + throttle = RateLimitError(retry_after=30) + r = CrawlResult(strategy="bfs", stats={"requests": 1, "errors": 1}, + rate_limited=throttle) + with pytest.raises(RateLimitError) as exc: + r.raise_for_seed() + assert exc.value.retry_after == 30 + + def test_block_wins_over_rate_limit(self): + # A seed can look both walled and throttled; the bot-wall answer is the + # more specific, more actionable one, so it takes precedence. + wall = BlockedError("Cloudflare wall", wall="Cloudflare", status=429) + r = CrawlResult(strategy="bfs", stats={"requests": 1, "errors": 1}, + blocked=wall, rate_limited=RateLimitError()) + with pytest.raises(BlockedError): + r.raise_for_seed() + + def test_rate_limit_wins_over_unreachable(self): + # A throttled seed also has every fetch failing (so `unreachable` is + # true), but "slow down and retry" is the more specific answer than + # "couldn't open it". + r = CrawlResult(strategy="bfs", stats={"requests": 1, "errors": 1}, + rate_limited=RateLimitError()) + assert r.unreachable is True # every fetch failed… + with pytest.raises(RateLimitError): # …but the rate limit is reported + r.raise_for_seed() + + def test_crawl_over_a_throttled_seed_reports_rate_limited(self, http): + http.routes[SEED] = _throttled() + result = BFS(query="anything").crawl(SEED, max_pages=5) + assert result.pages == [] + assert result.rate_limited is not None + assert result.rate_limited.retry_after == 30 + with pytest.raises(RateLimitError): + result.raise_for_seed() + + def test_compare_surfaces_a_throttled_seed_once(self, http): + http.routes[SEED] = _throttled() + with pytest.raises(RateLimitError): + compare(SEED, query="machine learning", max_pages=3, delay=0.0) + + def test_retry_after_parses_numeric_and_http_date(self): + # Both RFC 7231 forms: a delta in seconds, and an absolute HTTP date + # read as the delta from now (clamped at zero for a past date). + from datetime import datetime, timedelta, timezone + from email.utils import format_datetime + + from bytecrawl.scraping.url import _retry_after_seconds + + assert _retry_after_seconds("30") == 30 + assert _retry_after_seconds(None) is None + assert _retry_after_seconds("soon") is None + + future = datetime.now(timezone.utc) + timedelta(seconds=120) + secs = _retry_after_seconds(format_datetime(future)) + assert 110 <= secs <= 120 # ~120s out, allowing for clock drift + + past = datetime.now(timezone.utc) - timedelta(hours=1) + assert _retry_after_seconds(format_datetime(past)) == 0 + + def test_throttle_with_http_date_retry_after_is_reported(self, http): + # A 429 whose Retry-After is an HTTP date still yields a positive wait. + from datetime import datetime, timedelta, timezone + from email.utils import format_datetime + + when = format_datetime(datetime.now(timezone.utc) + timedelta(seconds=60)) + http.routes[SEED] = _throttled(retry_after=when) + result = BFS(query="anything").crawl(SEED, max_pages=5) + assert result.rate_limited is not None + assert 50 <= result.rate_limited.retry_after <= 60 + + def test_crawl_over_a_dead_seed_reports_unreachable(self, http): + http.routes[SEED] = _dead() + result = BFS(query="anything").crawl(SEED, max_pages=5) + assert result.pages == [] + assert result.unreachable is True + with pytest.raises(UnreachableError): + result.raise_for_seed() + + def test_compare_surfaces_a_dead_seed_once(self, http): + # A dead seed fails all three legs identically; compare() re-raises it + # once instead of burying three fake-empty "winners". + http.routes[SEED] = _dead() + with pytest.raises(UnreachableError): + compare(SEED, query="machine learning", max_pages=3, delay=0.0) + + def test_compare_surfaces_a_walled_seed_once(self, http): + http.routes[SEED] = _walled() + with pytest.raises(BlockedError): + compare(SEED, query="machine learning", max_pages=3, delay=0.0) + + +# --------------------------------------------------------------------------- # +# MCP tools: every tool raises the right seed error. +# --------------------------------------------------------------------------- # +pytest.importorskip("mcp") +from mcp.server.mcpserver.exceptions import ToolError, UnexpectedToolError # noqa: E402 + +from bytecrawl import mcp_server # noqa: E402 + + +@pytest.fixture(autouse=True) +def _no_delay(): + """The module-level scraper shouldn't sleep in tests.""" + mcp_server._scraper.delay = 0.0 + yield + + +# (tool callable, kwargs) for each of the six tools. Single-fetch tools and +# crawl tools reach the seed by different code paths (_open vs raise_for_seed); +# both must land on the same two exceptions. +MCP_TOOLS = [ + (mcp_server.fetch_markdown, {}), + (mcp_server.extract, {}), + (mcp_server.list_links, {}), + (mcp_server.fetch_json_api, {}), + (mcp_server.focused_crawl, {"query": "x", "max_pages": 3}), + (mcp_server.compare_strategies, {"query": "x", "max_pages": 3}), +] + + +# A seed error escaping a tool is re-raised as a ToolError so the SDK returns +# it to the agent as an is_error result carrying our message (a bare requests +# exception would be a crash, masked to "Error executing tool "). The +# typed seed error is preserved as __cause__, so the classification still holds. +@pytest.mark.parametrize("tool, kw", MCP_TOOLS) +def test_mcp_tool_reports_blocked_on_a_walled_seed(http, tool, kw): + http.routes[SEED] = _walled() + with pytest.raises(ToolError) as exc: + tool(SEED, **kw) + assert isinstance(exc.value.__cause__, BlockedError) + + +@pytest.mark.parametrize("tool, kw", MCP_TOOLS) +def test_mcp_tool_reports_unreachable_on_a_dead_seed(http, tool, kw): + http.routes[SEED] = _dead() + with pytest.raises(ToolError) as exc: + tool(SEED, **kw) + assert isinstance(exc.value.__cause__, UnreachableError) + + +@pytest.mark.parametrize("tool, kw", MCP_TOOLS) +def test_mcp_tool_reports_rate_limit_on_a_throttled_seed(http, tool, kw): + http.routes[SEED] = _throttled() + with pytest.raises(ToolError) as exc: + tool(SEED, **kw) + assert isinstance(exc.value.__cause__, RateLimitError) + assert "retry after ~30s" in str(exc.value) # the wait folds into the text + + +@pytest.mark.parametrize( + "route, needle", + [ + (_throttled(), "retry after ~30s"), + (_walled(), "Blocked"), + (_dead(), "Couldn't open"), + ], +) +def test_mcp_client_receives_the_seed_message(http, route, needle): + # The real SDK path: call_tool raises an *anticipated* ToolError (not the + # UnexpectedToolError a crash would raise), whose str carries our message. + # The RPC handler turns exactly this into an is_error result with the text + # intact — whereas a crash is masked to a bare "Error executing tool ". + import asyncio + + http.routes[SEED] = route + with pytest.raises(ToolError) as exc: + asyncio.run(mcp_server.server.call_tool("fetch_markdown", {"url": SEED})) + assert not isinstance(exc.value, UnexpectedToolError) # not a masked crash + assert needle in str(exc.value) # our message survives to the client + + +# --------------------------------------------------------------------------- # +# HTTP API: /api returns 403 {blocked} and 502 {unreachable} for every method. +# --------------------------------------------------------------------------- # +@pytest.fixture +def public_dns(monkeypatch): + """Let the SSRF guard resolve the fake seed host, offline. + + ``assert_public_url`` calls ``socket.getaddrinfo`` and requires a globally + routable address; a made-up ``.test`` host would fail to resolve (a real + network call, which the suite forbids) and 400 before the fetch we're + testing ever runs. Pin resolution to a fixed public IP so the guard passes + and control reaches the walled/dead response. + """ + import socket + + from bytecrawl import security + + def fake_getaddrinfo(host, *a, **kw): + return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", + ("93.184.216.34", 0))] + + monkeypatch.setattr(security.socket, "getaddrinfo", fake_getaddrinfo) + + +@pytest.fixture +def client(public_dns): + from landing.app import create_app + + app = create_app() + app.config.update(TESTING=True) + return app.test_client() + + +# Every /api method that reaches the seed. `select` gives extract a real +# selector so it takes the fetch path rather than the (fetch-first) discovery +# path — both fetch, but this keeps the call unambiguous. +API_METHODS = [ + {"method": "markdown"}, + {"method": "text"}, + {"method": "html"}, + {"method": "links"}, + {"method": "extract", "select": "h3::text"}, + {"method": "json"}, + {"method": "crawl", "query": "x", "strategy": "bfs"}, + {"method": "compare", "query": "x"}, +] + + +def _api_ids(params): + return params["method"] + + +@pytest.mark.parametrize("params", API_METHODS, ids=_api_ids) +def test_api_reports_a_walled_seed_as_403_blocked(client, http, params): + http.routes[SEED] = _walled() + resp = client.get("/api", query_string={"url": SEED, **params}) + assert resp.status_code == 403 + body = resp.get_json() + assert body["blocked"] is True + assert body["wall"] == "Cloudflare" + + +@pytest.mark.parametrize("params", API_METHODS, ids=_api_ids) +def test_api_reports_a_dead_seed_as_502_unreachable(client, http, params): + http.routes[SEED] = _dead() + resp = client.get("/api", query_string={"url": SEED, **params}) + assert resp.status_code == 502 + body = resp.get_json() + assert body["unreachable"] is True + + +@pytest.mark.parametrize("params", API_METHODS, ids=_api_ids) +def test_api_reports_a_throttled_seed_as_429_rate_limited(client, http, params): + http.routes[SEED] = _throttled() + resp = client.get("/api", query_string={"url": SEED, **params}) + assert resp.status_code == 429 + body = resp.get_json() + assert body["rate_limited"] is True + # The site's Retry-After is echoed in the body and the HTTP header so a + # client can back off by the amount the site actually asked for. + assert body["retry_after"] == 30 + assert resp.headers.get("Retry-After") == "30" + + +def test_api_walled_429_is_blocked_not_rate_limited(client, http): + """A 429 that carries a WAF signature is a wall, not a rate limit. + + Precedence in one place: the bot-wall answer (403 {blocked}) is more + specific than "slow down", so a walled 429 must not slip through as a 429 + {rate_limited}. + """ + http.routes[SEED] = _walled_429() + resp = client.get("/api", query_string={"url": SEED, "method": "markdown"}) + assert resp.status_code == 403 + body = resp.get_json() + assert body["blocked"] is True + assert body.get("rate_limited") is None + assert body["wall"] == "Cloudflare" + + +def test_api_walled_and_mcp_walled_agree(client, http): + """The contract in one place: the same walled seed, the tool and the + endpoint, the same named wall — one raised (as a ToolError wrapping the + typed BlockedError), one flagged in a 403 body.""" + http.routes[SEED] = _walled() + + with pytest.raises(ToolError) as exc: + mcp_server.fetch_markdown(SEED) + blocked = exc.value.__cause__ + assert isinstance(blocked, BlockedError) + + resp = client.get("/api", query_string={"url": SEED, "method": "markdown"}) + assert resp.status_code == 403 + assert resp.get_json()["wall"] == blocked.wall == "Cloudflare" + + +def test_auto_analyze_reports_a_dead_seed_as_unreachable(http): + """The 'auto' surface (/analyze) classifies a dead seed like the rest. + + Markdown in the playground runs /analyze, not /api?method=markdown. That + path used to swallow every fetch failure into a raw "Could not download the + page: " with no flag — so a mistyped domain showed red text there + while /api?method=links showed the friendly card on the same page. Now it + funnels the seed fetch through open_seed like every other surface, so it + raises UnreachableError and the response carries the `unreachable` flag and + the one shared wording the calm card renders from. + """ + http.routes[SEED] = _dead() + + from landing.app import create_app + + app = create_app() + app.config.update(TESTING=True) + resp = app.test_client().post("/analyze", json={"url": SEED, "lang": "en"}) + + body = resp.get_json() + assert body["ok"] is False + assert body["unreachable"] is True + assert body["error"] == bytecrawl.UNREACHABLE_MESSAGE + + +def test_auto_analyze_reports_a_walled_seed_as_blocked(http): + """And a bot wall on the 'auto' surface stays a named block, not unreachable.""" + http.routes[SEED] = _walled() + + from landing.app import create_app + + app = create_app() + app.config.update(TESTING=True) + resp = app.test_client().post("/analyze", json={"url": SEED, "lang": "en"}) + + body = resp.get_json() + assert body["ok"] is False + assert body["blocked"] is True + assert body["wall"] == "Cloudflare" + + +def test_auto_analyze_reports_a_throttled_seed_as_rate_limited(http): + """A 429 on the 'auto' surface (/analyze) reads as a rate limit, not a dead + seed — the same `rate_limited` flag, Retry-After and 429 status the rest give.""" + http.routes[SEED] = _throttled() + + from landing.app import create_app + + app = create_app() + app.config.update(TESTING=True) + resp = app.test_client().post("/analyze", json={"url": SEED, "lang": "en"}) + + assert resp.status_code == 429 + body = resp.get_json() + assert body["ok"] is False + assert body["rate_limited"] is True + assert body["retry_after"] == 30 + assert body["error"] == bytecrawl.RATE_LIMIT_MESSAGE + + +def test_api_unresolvable_host_is_502_unreachable_not_a_raw_400(monkeypatch): + """A host that doesn't resolve must get the friendly unreachable card too. + + The SSRF guard resolves DNS before the fetch. A mistyped domain fails that + resolution, and it used to escape as a raw ``400 {"error": "Cannot resolve + host: …"}`` — the one path that pre-checks, so the landing rendered red + error text while the playground (which never pre-checks) showed the card. + Now the guard reports an unresolvable host as UnreachableError, so /api + answers 502 {unreachable} with the one shared wording, matching every + other surface. This is the exact regression the landing showed. + """ + import socket + + from bytecrawl import UNREACHABLE_MESSAGE, security + + def boom(host, *a, **kw): + raise socket.gaierror("Name or service not known") + + monkeypatch.setattr(security.socket, "getaddrinfo", boom) + + from landing.app import create_app + + app = create_app() + app.config.update(TESTING=True) + resp = app.test_client().get( + "/api", query_string={"url": "https://en.wikipedi2a.org/wiki/x"}) + + assert resp.status_code == 502 + body = resp.get_json() + assert body["unreachable"] is True + # The friendly shared wording, not a raw "Cannot resolve host". + assert body["error"] == UNREACHABLE_MESSAGE + assert "Cannot resolve host" not in body["error"] diff --git a/web/landing/routes.py b/web/landing/routes.py index d7497c3..87006ba 100644 --- a/web/landing/routes.py +++ b/web/landing/routes.py @@ -158,10 +158,6 @@ def api(): url = services.clean_url(src.get("url") or "") if not url: return jsonify({"error": "the 'url' parameter is required"}), 400 - try: - services.assert_public_url(url) - except ValueError as e: - return jsonify({"error": str(e)}), 400 method = (src.get("method") or "markdown").lower() cache_key = "|".join([url, method, src.get("query", ""), src.get("select", ""), @@ -171,9 +167,31 @@ def api(): return jsonify(cached) try: + # The SSRF guard runs in the same funnel as the fetch so its failures + # get the same shaped answers: a non-public URL is a 400 ValueError, + # but a host that simply doesn't resolve is UnreachableError — the same + # 502 "couldn't open that page" a mistyped domain gets everywhere else, + # instead of a raw "Cannot resolve host" only on this pre-checked path. + services.assert_public_url(url) payload = services.run_api_method(method, url, src) except ValueError as e: return jsonify({"error": str(e)}), 400 + except bytecrawl.BlockedError as e: + return jsonify({"error": str(e), "blocked": True, "wall": e.wall}), 403 + except bytecrawl.RateLimitError as e: + # The seed is up but throttling us (429). An honest 429 back, with the + # site's Retry-After echoed both in the body and the header so a client + # can back off, instead of dressing a live-but-busy site as unreachable. + resp = jsonify({"error": str(e), "rate_limited": True, + "retry_after": e.retry_after}) + if e.retry_after is not None: + resp.headers["Retry-After"] = str(e.retry_after) + return resp, 429 + except bytecrawl.UnreachableError as e: + # The seed URL couldn't be opened at all (bad domain, refused, 404, + # two links pasted into one) — an honest 502 with a clear flag, the + # same answer every method and the MCP tools give. + return jsonify({"error": str(e), "unreachable": True}), 502 except Exception as e: return jsonify({"error": f"request failed: {e}"}), 502 diff --git a/web/landing/services.py b/web/landing/services.py index 4cdaac3..8ecadb4 100644 --- a/web/landing/services.py +++ b/web/landing/services.py @@ -14,7 +14,7 @@ import re import time -from bytecrawl import Scraper +from bytecrawl import BlockedError, RateLimitError, Scraper, UnreachableError, open_seed from bytecrawl.crawler import STRATEGIES as CRAWL_STRATEGIES from bytecrawl.crawler import compare, pagerank from bytecrawl.security import assert_public_url # re-exported for the routes @@ -80,7 +80,25 @@ def analyze_url(url: str, lang: str) -> tuple[dict, int, dict | None]: bot = Scraper(timeout=20) steps = [] try: - page = bot.static(url) + # open_seed is the shared classifier: a bot wall stays a BlockedError, + # and anything else that stopped the page loading (a domain that doesn't + # resolve, a refused connection, a 404) becomes UnreachableError. Without + # it, 'auto' was the one surface that dressed a dead seed as a raw + # requests traceback while /api?method=links gave the friendly card. + page = open_seed(lambda: bot.static(url)) + except BlockedError as e: + return {"ok": False, "error": str(e), "blocked": True, "wall": e.wall}, 403, None + except RateLimitError as e: + # Reachable but throttled (429): the same `rate_limited` flag and + # Retry-After every surface gives, so the frontend renders the calm + # "slow down and retry" card off the flag, not the raw error string. + return {"ok": False, "error": str(e), "rate_limited": True, + "retry_after": e.retry_after}, 429, None + except UnreachableError as e: + # The seed never opened — the same "couldn't open that page" answer, and + # the same `unreachable` flag, every other surface gives, so the frontend + # renders the calm card off the flag instead of the raw error string. + return {"ok": False, "error": str(e), "unreachable": True}, 200, None except Exception as e: return {"ok": False, "error": tr["download_err"].format(e=e)}, 200, None @@ -165,6 +183,22 @@ def run_crawl(url: str, strategy: str, query: str) -> tuple[dict, int, dict | No except Exception as e: return {"ok": False, "error": f"crawl failed: {e}"}, 200, None + if result.blocked is not None and not result.pages: + return {"ok": False, "error": str(result.blocked), + "blocked": True, "wall": result.blocked.wall}, 403, None + + if result.rate_limited is not None and not result.pages: + # Seed throttled: say "slow down and retry", carrying the wait, instead + # of a hollow "0 relevant". + return {"ok": False, "error": str(result.rate_limited), + "rate_limited": True, + "retry_after": result.rate_limited.retry_after}, 429, None + + if result.unreachable: + # Seed unreachable: say so plainly instead of a hollow "0 relevant". + return {"ok": False, "unreachable": True, + "error": "Couldn't open that page."}, 200, None + pr = pagerank(result.graph) props = {"strategy": strategy, **result.stats} payload = { @@ -181,31 +215,33 @@ def run_api_method(method: str, url: str, src) -> dict: """Dispatch one /api call to ByteCrawl and return its payload dict. ``src`` is the request's arg/form mapping (anything with ``.get``). Bad - input raises ValueError (the route maps it to 400); everything else - propagates for the route to turn into a 502. + input raises ValueError (the route maps it to 400); a walled seed raises + BlockedError (403) and an unopenable one UnreachableError (502), the same + across every method; anything else propagates for the route to turn into a + 502. """ bot = Scraper(timeout=15) if method in ("markdown", "md"): - page = bot.static(url) + page = open_seed(lambda: bot.static(url)) return {"url": url, "method": "markdown", "markdown": page.markdown(), "tokens": page.tokens(page.markdown())} if method == "text": - page = bot.static(url) + page = open_seed(lambda: bot.static(url)) return {"url": url, "method": "text", "text": page.soup.get_text(" ", strip=True)} if method == "html": - page = bot.static(url) + page = open_seed(lambda: bot.static(url)) return {"url": url, "method": "html", "html": page.html} if method == "links": - page = bot.static(url) + page = open_seed(lambda: bot.static(url)) return {"url": url, "method": "links", "links": page.links()} if method == "json": - page = bot.api(url) + page = open_seed(lambda: bot.api(url)) return {"url": url, "method": "json", "data": page.json()} if method == "extract": select = (src.get("select") or "").strip() - page = bot.static(url) + page = open_seed(lambda: bot.static(url)) if not select: # No selector is not an error, it is the question before it: # "what can I pull off this page?" Answering with the repeated @@ -225,6 +261,10 @@ def run_api_method(method: str, url: str, src) -> dict: pages = API_MAX_PAGES crawler = CRAWL_STRATEGIES[strategy](query=query, delay=0.0, timeout=8) result = crawler.crawl(url, max_pages=pages, max_depth=4) + # A walled seed raises the block (403); an unreachable one raises + # UnreachableError (502). Either way the route reports it plainly + # rather than letting an empty crawl read as "nothing relevant". + result.raise_for_seed() ranked = result.top(pages) if query else result.pages return {"url": url, "method": "crawl", "strategy": strategy, "query": query, "stats": result.stats, "pages": ranked} diff --git a/web/landing/static/i18n.js b/web/landing/static/i18n.js index 2b226b3..7da75d9 100644 --- a/web/landing/static/i18n.js +++ b/web/landing/static/i18n.js @@ -91,6 +91,10 @@ azr_title: "Analyze any URL", azr_sub: "Paste a page: ByteCrawl picks the strategy (static or browser) and explains what it did and why.", azr_hint: "Change the URL and try yours.", + unreach_t: "Couldn’t open that page", + unreach_d: "The site didn’t respond — check the URL is real or complete.", + ratelimit_t: "The site asked us to slow down", + ratelimit_d: "It’s up and reachable — just rate-limiting requests (HTTP 429). Wait a moment and try again.", analyze_btn: "Run", pm_markdown: "Markdown", pm_extract: "Extract", pm_json: "JSON API", pm_crawl: "Graph crawl", pm_links: "Links", @@ -103,6 +107,7 @@ badge_tokens: "Markdown saves {pct}% of LLM tokens", st_enter_url: "✗ Enter a URL", st_analyzing: "Analyzing the page...", st_strategy: "Strategy used:", st_three: "BFS, Shark-Search and OPIC on the same budget:", + pm_stop_confirm: "This run is still going. Stop it and switch?", pm_cmp_running: "running…", pm_cmp_head: "Same budget, three orderings — query:", pm_cmp_best: "Full ranking from the winner:", @@ -260,6 +265,10 @@ azr_title: "Analiza cualquier URL", azr_sub: "Pega una página: ByteCrawl elige la estrategia (estático o navegador) y te explica qué hizo y por qué.", azr_hint: "Cambia la URL y prueba la tuya.", + unreach_t: "No pude abrir esa página", + unreach_d: "El sitio no respondió — revisa que la URL exista o esté completa.", + ratelimit_t: "El sitio nos pidió bajar el ritmo", + ratelimit_d: "Está en línea y responde — solo está limitando las solicitudes (HTTP 429). Espera un momento y vuelve a intentar.", analyze_btn: "Ejecutar", pm_markdown: "Markdown", pm_extract: "Extract", pm_json: "API JSON", pm_crawl: "Crawl de grafo", pm_links: "Links", @@ -272,6 +281,7 @@ badge_tokens: "El Markdown ahorra {pct}% de tokens de LLM", st_enter_url: "✗ Escribe una URL", st_analyzing: "Analizando la página...", st_strategy: "Estrategia usada:", st_three: "BFS, Shark-Search y OPIC con el mismo presupuesto:", + pm_stop_confirm: "Este análisis sigue corriendo. ¿Detenerlo y cambiar?", pm_cmp_running: "corriendo…", pm_cmp_head: "Mismo presupuesto, tres órdenes — consulta:", pm_cmp_best: "Ranking completo del ganador:", @@ -429,6 +439,10 @@ azr_title: "Analise qualquer URL", azr_sub: "Cole uma página: o ByteCrawl escolhe a estratégia (estático ou navegador) e explica o que fez e por quê.", azr_hint: "Troque a URL e teste a sua.", + unreach_t: "Não consegui abrir essa página", + unreach_d: "O site não respondeu — confira se a URL existe ou está completa.", + ratelimit_t: "O site pediu para diminuirmos o ritmo", + ratelimit_d: "Está no ar e respondendo — só está limitando as requisições (HTTP 429). Aguarde um momento e tente de novo.", analyze_btn: "Executar", pm_markdown: "Markdown", pm_extract: "Extract", pm_json: "API JSON", pm_crawl: "Crawl de grafo", pm_links: "Links", @@ -441,6 +455,7 @@ badge_tokens: "O Markdown economiza {pct}% dos tokens de LLM", st_enter_url: "✗ Digite uma URL", st_analyzing: "Analisando a página...", st_strategy: "Estratégia usada:", st_three: "BFS, Shark-Search e OPIC com o mesmo orçamento:", + pm_stop_confirm: "Esta análise ainda está rodando. Parar e trocar?", pm_cmp_running: "rodando…", pm_cmp_head: "Mesmo orçamento, três ordens — consulta:", pm_cmp_best: "Ranking completo do vencedor:", diff --git a/web/landing/templates/landing.html b/web/landing/templates/landing.html index 8572c66..a3463fc 100644 --- a/web/landing/templates/landing.html +++ b/web/landing/templates/landing.html @@ -207,8 +207,13 @@ /* live results under the bar */ .hout { max-width: 620px; margin: 0 auto 22px; text-align: left; display: none; } - .hout.show { display: block; animation: ns-in .28s ease both; } - .hout-box { border: 1px solid var(--border); border-radius: 14px; background: var(--panel); overflow: hidden; } + /* The fade lives on the box, not the container: innerHTML swaps replace the + box with a fresh node, so the animation re-runs on every render. Put it on + .hout.show instead and it fires once (empty -> loading) but not on the + loading -> result swap, so the result hard-pops in — the flicker. */ + .hout.show { display: block; } + .hout-box { border: 1px solid var(--border); border-radius: 14px; background: var(--panel); overflow: hidden; + animation: ns-in .22s ease both; } .hout-head { display: flex; align-items: center; gap: 8px; padding: 10px 16px; border-bottom: 1px solid var(--border); font-size: 12px; color: var(--muted); background: var(--panel-2); } .hout-head .ok { color: var(--text); font-weight: 600; } @@ -233,7 +238,14 @@ .hout pre { padding: 14px 16px; margin: 0; white-space: pre-wrap; word-break: break-word; font-family: var(--mono); font-size: 12px; line-height: 1.6; color: var(--code); } .hout .err { padding: 14px 16px; color: #b91c1c; font-size: 13px; } - + /* bot-wall block: a calm notice, not an all-red shout */ + .hout .block { display: flex; gap: 13px; padding: 16px; align-items: flex-start; } + .hout .block .ic { flex: 0 0 auto; width: 30px; height: 30px; border-radius: 9px; + display: flex; align-items: center; justify-content: center; + background: #fef2f2; color: #dc2626; } + .hout .block .ic svg { width: 17px; height: 17px; } + .hout .block .title { font-weight: 600; font-size: 13.5px; color: var(--text); margin-bottom: 3px; } + .hout .block .detail { font-size: 12.5px; line-height: 1.55; color: var(--muted-2); } .install { display: inline-flex; align-items: center; gap: 12px; font-family: var(--mono); font-size: 14px; background: var(--panel); border: 1px solid var(--border); border-radius: 9px; padding: 11px 18px; @@ -769,6 +781,58 @@

    Build with ByteCrawl today

    hbarOut.classList.add('show'); } + // A block is one calm card on its own — no URL header bar above it. The page + // link lives inside the card as a quiet secondary action. + function hbarCard(body) { + hbarOut.innerHTML = '
    ' + body + '
    '; + hbarOut.classList.add('show'); + } + + // A bot-wall block reads as a calm notice — shield icon, a title, and the + // rest in muted grey — rather than a wall of red. The message arrives as + // "Blocked by X — detail…", so split on the dash into title + detail. + function blockCard(d) { + const shield = '' + + ''; + const parts = String(d.error || '').split(' — '); + const title = d.wall ? 'Blocked by ' + d.wall : (parts[0] || 'Blocked'); + const detail = parts.length > 1 ? parts.slice(1).join(' — ') : ''; + return '
    ' + shield + '
    ' + + '
    ' + esc(title) + '
    ' + + (detail ? '
    ' + esc(detail) + '
    ' : '') + + '
    '; + } + + // Seed URL we couldn't open (bad domain, 404, or two links pasted into one). + // Same calm card as a block, an alert-circle icon rather than the shield. + function unreachableCard() { + const alert = '' + + '' + + ''; + return '
    ' + alert + '
    ' + + '
    Couldn’t open that page
    ' + + '
    The site didn’t respond — check the URL is real ' + + 'or complete.
    ' + + '
    '; + } + + // Seed reachable but throttling us (HTTP 429). Same calm card, a clock icon — + // a "wait and retry", not a dead page or a wall; show the Retry-After when we + // got one. + function rateLimitCard(d) { + const clock = '' + + ''; + const wait = d && d.retry_after ? ' (~' + d.retry_after + 's)' : ''; + return '
    ' + clock + '
    ' + + '
    The site asked us to slow down
    ' + + '
    It’s up and reachable — just rate-limiting requests ' + + '(HTTP 429). Wait a moment and try again.' + esc(wait) + '
    ' + + '
    '; + } + // People paste URLs out of prose ("site.com · query: …"), so trim the // punctuation and stray words that ride along instead of failing on them. function cleanUrl(raw) { @@ -792,6 +856,9 @@

    Build with ByteCrawl today

    try { const r = await fetch('/api?' + params.toString()); const d = await r.json(); + if (d.blocked) { hbarCard(blockCard(d)); return; } + if (d.rate_limited) { hbarCard(rateLimitCard(d)); return; } + if (d.unreachable) { hbarCard(unreachableCard()); return; } if (d.error) { hbarBox('' + esc(url) + '', '
    ' + esc(d.error) + '
    '); return; } const link = (href, label) => diff --git a/web/landing/templates/try.html b/web/landing/templates/try.html index a0363bd..ed210a5 100644 --- a/web/landing/templates/try.html +++ b/web/landing/templates/try.html @@ -56,7 +56,13 @@ } footer { margin-top: auto; } a { color: inherit; text-decoration: none; } - .wrap { max-width: 1100px; margin: 0 auto; padding: 0 24px; } + /* width:100% matters for the analyzer's .wrap, which is a direct child of the + flex-column body: a flex item with `margin:0 auto` shrinks to its content + (max-width only caps it), so the card was narrow at idle and jumped wider + once the compare grid rendered. A definite width makes it fill the centred + 1100px column at every state. No-op for the nav/header/footer .wraps, which + are ordinary blocks and already fill their width. */ + .wrap { width: 100%; max-width: 1100px; margin: 0 auto; padding: 0 24px; } /* NAV */ nav { position: sticky; top: 0; z-index: 50; background: rgba(255,255,255,.75); backdrop-filter: blur(12px); border-bottom: 1px solid var(--border); } @@ -98,15 +104,27 @@ .btn:hover { background: #27272a; } .btn:disabled { opacity: .5; cursor: not-allowed; } /* gap, not a literal space: flex makes each text run its own item and eats - the whitespace between them, which is why the ✓ sat flush against the label. + the whitespace between them, so the spinner and label lean on the row gap. The heights here are deliberately integers: with line-height 1.5 the chip's content box was 19.5px, so centring it in the flex row put its text on a half-pixel and the digits rendered blurry. 20px row, 18px chip line, even padding — everything lands on whole pixels. */ .status { font-size: 14px; line-height: 20px; margin: 12px 0 0; min-height: 26px; display: flex; align-items: center; gap: 6px; } + /* An emptied status reserves nothing: at idle, and after a block clears the + line, the result (or the input) sits flush instead of under 38px of blank + row. Any real status text or spinner switches it straight back on. */ + .status:empty { min-height: 0; margin: 0; } .status.run { color: var(--muted); } .status.ok { color: var(--accent); font-weight: 600; } .status.err { color: var(--red); } + /* bot-wall block: a calm full-width notice, same shape as the home hero */ + .azr-block { display: flex; gap: 13px; padding: 16px; margin-top: 14px; + border: 1px solid var(--border); border-radius: 14px; background: var(--panel); } + .azr-block .ic { flex: 0 0 auto; width: 30px; height: 30px; border-radius: 9px; + display: flex; align-items: center; justify-content: center; background: #fef2f2; color: #dc2626; } + .azr-block .ic svg { width: 17px; height: 17px; } + .azr-block .title { font-weight: 600; font-size: 13.5px; color: var(--text); margin-bottom: 3px; } + .azr-block .detail { font-size: 12.5px; line-height: 1.55; color: var(--muted-2); } .time { margin-left: 2px; font-size: 13px; line-height: 18px; font-weight: 700; color: var(--accent); background: var(--panel-2); border: 1px solid var(--border); padding: 3px 9px; border-radius: 6px; font-family: var(--mono); font-variant-numeric: tabular-nums; } .result { margin-top: 14px; display: none; } .result.show { display: block; } @@ -199,7 +217,12 @@ .spinner { display: inline-block; width: 14px; height: 14px; border: 2px solid var(--border-2); border-top-color: var(--accent); border-radius: 50%; animation: spin .7s linear infinite; vertical-align: -2px; margin-right: 8px; } @keyframes spin { to { transform: rotate(360deg); } } .azr-out { margin-top: 18px; display: none; } - .azr-out.show { display: block; } + /* A spinner runs in the status row while loading and the output stays empty, + so the result comes in cold — fade it in instead of a hard pop, the same + gentle entrance the landing's output card uses. The load path clears the + 'show' class (out is emptied), so re-adding it on the result re-runs this. */ + .azr-out.show { display: block; animation: azr-in .22s ease both; } + @keyframes azr-in { from { opacity: 0; transform: translateY(4px); } to { opacity: 1; transform: none; } } .azr-steps { display: flex; flex-direction: column; gap: 10px; margin-bottom: 16px; } .azr-step { border: 1px solid var(--border); border-radius: 12px; padding: 14px 16px; background: var(--panel); } .azr-step .st { font-weight: 600; font-size: 14px; margin-bottom: 4px; color: var(--accent); } @@ -400,10 +423,19 @@

    Analyze any URL

    window.addEventListener('resize', pmFitHint); document.querySelectorAll('.azr-mode').forEach(b => { b.addEventListener('click', () => { + // A run is still in flight and you're leaving for another mode: ask, and if + // you confirm, abort it so it actually stops instead of finishing unseen. + if (pmAbort && b.dataset.mode !== pmMode()) { + if (!confirm(t('pm_stop_confirm'))) return; + pmAbort.abort(); + } document.querySelectorAll('.azr-mode').forEach(x => x.classList.remove('active')); b.classList.add('active'); pmSyncExtra(); - document.getElementById('azr-out').innerHTML = ''; + // Drop .show so the emptied box collapses instead of holding the reserved + // height as a blank gap after switching away from a result. + const o = document.getElementById('azr-out'); + o.className = 'azr-out'; o.innerHTML = ''; document.getElementById('azr-status').textContent = ''; }); }); @@ -494,16 +526,20 @@

    Analyze any URL

    // independent, so there is no reason to hold the first result hostage to the // slowest one — a card is painted as soon as its own strategy lands. function pmCard(r, seedHost, isBest) { + // A stable id per strategy so a card can be swapped in place as its own + // request lands, without re-rendering (and restarting the spinners of) the + // two cards beside it. + const id = `cmp-card-${r.strategy}`; const head = `
    ${r.strategy.toUpperCase()}` + `${isBest ? ' best' : ''}
    `; if (r.state === 'pending') { - return `
    ${head}` + + return `
    ${head}` + `
    ${pmEscape(t('pm_cmp_running'))}
    `; } if (r.state === 'err') { // One strategy failing is not a reason to blank the other two, so the error // stays inside its own card. - return `
    ${head}` + + return `
    ${head}` + `
    ✗ ${pmEscape(r.error || 'failed')}
    `; } // Every page listed is a page you can go read — the top 3 here are clickable @@ -513,43 +549,62 @@

    Analyze any URL

    `${pmLink(p.url, pmShort(p.url, seedHost))}` + `${(p.relevance || 0).toFixed(3)}`).join(''); const avg = (r.stats.avg_relevance ?? 0).toFixed(3); - return `
    ${head} + return `
    ${head}
    ${pmRelevant(r)} ${pmEscape(t('adv_relevant'))}${pmEscape(t('adv_avgrel'))} ${avg}${r.stats.requests} ${pmEscape(t('adv_pages'))}${r.stats.elapsed}s
    ${rows}
    `; } +// The grid is painted ONCE, all three cards pending, plus an empty tail slot. +// After this the box is never rebuilt wholesale: each card is swapped in place +// as it lands (pmUpdateCard) and the winner + full ranking are filled in at the +// end (pmSettleCompare). Rebuilding the whole innerHTML on every landing — the +// old behaviour — tore down all three cards each time, restarting the pending +// spinners and repainting the block; that full-grid repaint, three or four +// times over a ~1-minute crawl, was the flicker. function pmRenderCompare(runs, query, seedUrl) { const seedHost = pmHost(seedUrl); const out = document.getElementById('azr-out'); - const done = runs.filter(r => r.state === 'ok'); - // "best" is a claim about all three orderings, so it waits for all three. - // Awarding it while runs are still in flight would just mark whoever was - // fastest, which is the opposite of what the comparison is for. - const settled = runs.every(r => r.state !== 'pending'); - const best = settled && done.length - ? done.reduce((a, b) => (pmRelevant(b) > pmRelevant(a) ? b : a)) - : null; - const cards = runs.map(r => pmCard(r, seedHost, r === best)).join(''); - let tail = ''; - if (best) { - // The three cards only show each strategy's top 3. The winner's full ranked - // list is what you'd actually feed to an LLM, so it goes below in full, with - // clickable URLs. - const bestHits = (best.pages || []).filter(p => (p.relevance || 0) >= 0.1); - const bestList = (bestHits.length ? bestHits : (best.pages || [])) - .slice().sort((a, b) => (b.relevance || 0) - (a.relevance || 0)); - const bestRows = bestList.map(p => - `${pmLink(p.url, pmShort(p.url, seedHost))}` + - `${(p.relevance || 0).toFixed(3)}`).join(''); - tail = `
    ${pmEscape(t('pm_cmp_best'))} ${best.strategy.toUpperCase()} · ${bestHits.length}/${best.stats.requests} · ${best.stats.elapsed}s
    -
    ${bestRows}
    `; - } + const cards = runs.map(r => pmCard(r, seedHost, false)).join(''); out.innerHTML = `
    ${pmEscape(t('pm_cmp_head'))} ${pmEscape(query)}
    -
    ${cards}
    ${tail}`; +
    ${cards}
    `; out.className = 'azr-out show'; } +// Swap only the card whose request just landed. Its two siblings — and their +// still-spinning placeholders — are left in the DOM untouched, so nothing +// around the card repaints. +function pmUpdateCard(run, seedUrl) { + const el = document.getElementById(`cmp-card-${run.strategy}`); + if (el) el.outerHTML = pmCard(run, pmHost(seedUrl), false); +} + +// All three have landed. "best" is a claim about all three orderings, so it is +// awarded now, not to whoever was fastest — re-render just the winning card to +// mark it, and fill the tail slot with its full ranked list (what you'd feed an +// LLM). Touches only the winner card and the tail; the other two stay put. +function pmSettleCompare(runs, query, seedUrl) { + const seedHost = pmHost(seedUrl); + const done = runs.filter(r => r.state === 'ok'); + if (!done.length) return; + const best = done.reduce((a, b) => (pmRelevant(b) > pmRelevant(a) ? b : a)); + const winEl = document.getElementById(`cmp-card-${best.strategy}`); + if (winEl) winEl.outerHTML = pmCard(best, seedHost, true); + const bestHits = (best.pages || []).filter(p => (p.relevance || 0) >= 0.1); + const bestList = (bestHits.length ? bestHits : (best.pages || [])) + .slice().sort((a, b) => (b.relevance || 0) - (a.relevance || 0)); + const bestRows = bestList.map(p => + `${pmLink(p.url, pmShort(p.url, seedHost))}` + + `${(p.relevance || 0).toFixed(3)}`).join(''); + const tail = document.getElementById('cmp-tail'); + if (tail) tail.innerHTML = `
    ${pmEscape(t('pm_cmp_best'))} ${best.strategy.toUpperCase()} · ${bestHits.length}/${best.stats.requests} · ${best.stats.elapsed}s
    +
    ${bestRows}
    `; +} + +// Holds the current run's AbortController while a request is in flight (null +// otherwise), so switching modes mid-run can offer to stop and actually abort. +let pmAbort = null; + async function analyze() { const url = document.getElementById("azr-url").value.trim(); const mode = pmMode(); @@ -557,8 +612,55 @@

    Analyze any URL

    const btn = document.getElementById("azr-btn"); const status = document.getElementById("azr-status"); const out = document.getElementById("azr-out"); + // A bot-wall block reads as a calm full-width notice — shield, a title and + // the rest in grey — rather than a thin red status line. Backend sends + // "Blocked by X — detail…", so split on the dash into title + detail. + const showBlock = (d) => { + const shield = '' + + ''; + const parts = String(d.error || '').split(' — '); + const title = d.wall ? 'Blocked by ' + d.wall : (parts[0] || 'Blocked'); + const detail = parts.length > 1 ? parts.slice(1).join(' — ') : ''; + status.className = 'status'; status.innerHTML = ''; + out.className = 'azr-out show'; + out.innerHTML = '
    ' + shield + '
    ' + + '
    ' + pmEscape(title) + '
    ' + + (detail ? '
    ' + pmEscape(detail) + '
    ' : '') + + '
    '; + }; + // Seed URL we couldn't open (bad domain, 404, or two links pasted into one). + // Same calm card as a block, an alert-circle icon rather than the shield. + const showUnreachable = () => { + const alert = '' + + '' + + ''; + status.className = 'status'; status.innerHTML = ''; + out.className = 'azr-out show'; + out.innerHTML = '
    ' + alert + '
    ' + + '
    ' + pmEscape(t('unreach_t')) + '
    ' + + '
    ' + pmEscape(t('unreach_d')) + '
    ' + + '
    '; + }; + // Seed reachable but throttling us (HTTP 429). The same calm card, a clock + // icon — this is a "wait and retry", not a dead page or a wall. When the site + // sent a Retry-After we append the concrete wait to the shared wording. + const showRateLimit = (d) => { + const clock = '' + + ''; + const wait = d && d.retry_after ? ' (~' + d.retry_after + 's)' : ''; + status.className = 'status'; status.innerHTML = ''; + out.className = 'azr-out show'; + out.innerHTML = '
    ' + clock + '
    ' + + '
    ' + pmEscape(t('ratelimit_t')) + '
    ' + + '
    ' + pmEscape(t('ratelimit_d')) + pmEscape(wait) + '
    ' + + '
    '; + }; if (!url) { status.className = "status err"; status.textContent = t("st_enter_url"); return; } btn.disabled = true; + const ac = new AbortController(); pmAbort = ac; status.className = "status run"; status.innerHTML = '' + t("st_analyzing"); out.className = "azr-out"; out.innerHTML = ""; try { @@ -567,11 +669,15 @@

    Analyze any URL

    method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ url, lang: currentLang() }), + signal: ac.signal, }); const j = await r.json(); + if (j.blocked) { showBlock(j); return; } + if (j.rate_limited) { showRateLimit(j); return; } + if (j.unreachable) { showUnreachable(); return; } if (!j.ok) { status.className = "status err"; status.textContent = "✗ " + (j.error || "Error"); return; } status.className = "status ok"; - status.innerHTML = `✓ ${t("st_strategy")} ${j.method}`; + status.innerHTML = `${t("st_strategy")} ${j.method}`; renderAnalysis(j); document.getElementById("azr-hint").classList.add("show"); return; @@ -579,12 +685,23 @@

    Analyze any URL

    if (mode === 'crawl') { // same budget, three orderings — the only fair way to compare them const t0 = Date.now(); - // The grid is painted immediately, in a fixed order, with all three cards - // empty; each one fills in place when its own request lands. Shark - // typically comes back ~3s before the slowest run, and there is no reason - // to make you wait for the slowest to see it. + // The grid fills in place: each card is painted empty and swapped for its + // result when its own request lands — Shark typically returns ~3s before + // the slowest run, and there's no reason to wait for the slowest to see it. + // + // But it isn't painted *yet*. All three legs fetch the same seed first, so + // a seed-level failure (unreachable / walled) hits them identically — + // flashing three "running" spinners and then flipping the whole thing to + // "couldn't open that page" reads backwards. So the grid is revealed only + // once a leg gets *past* the seed; if every leg fails at it, the grid never + // appears and the failure card below shows on its own. const runs = PM_STRATS.map(strategy => ({ strategy, state: 'pending' })); - pmRenderCompare(runs, extra, url); + let gridPainted = false; + const revealGrid = () => { + if (gridPainted) return; + gridPainted = true; + pmRenderCompare(runs, extra, url); + }; let landed = 0; const tick = () => { status.className = 'status run'; @@ -596,6 +713,7 @@

    Analyze any URL

    const r = await fetch('/crawl', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ url, strategy: run.strategy, query: extra }), + signal: ac.signal, }); const d = await r.json(); Object.assign(run, d, { state: d.ok ? 'ok' : 'err' }); @@ -604,35 +722,61 @@

    Analyze any URL

    run.error = String(e); } landed++; + // Aborted mid-run (user switched modes and chose to stop): don't paint + // over the box the switch just cleared for the new mode. + if (ac.signal.aborted) return; tick(); - pmRenderCompare(runs, extra, url); + // A seed-level failure alone doesn't reveal the grid — every leg would + // fail identically and the failure card is the right answer. Anything + // that got past the seed reveals it; from then on every landing (a + // per-strategy error included) swaps its own card in place. + if (!(run.unreachable || run.blocked || run.rate_limited)) revealGrid(); + if (gridPainted) pmUpdateCard(run, url); })); + if (ac.signal.aborted) return; // stopped on a mode switch // Only a total wipeout is a page-level error; one strategy failing has // already been reported inside its own card. const okRuns = runs.filter(x => x.state === 'ok'); if (!okRuns.length) { + const blocked = runs.find(x => x.blocked); + if (blocked) { showBlock(blocked); return; } + const throttled = runs.find(x => x.rate_limited); + if (throttled) { showRateLimit(throttled); return; } + const dead = runs.find(x => x.unreachable); + if (dead) { showUnreachable(); return; } status.className = 'status err'; status.textContent = '✗ ' + ((runs.find(x => x.error) || {}).error || 'crawl failed'); return; } + // All three landed with at least one winner: award "best" and drop the + // full ranking below — in place, so the grid itself doesn't re-render. + pmSettleCompare(runs, extra, url); status.className = 'status ok'; // The .time chip is for a single datum (a method, a duration) — prose // inside it reads as a mislabelled value. The three runs are concurrent, // so the wall clock is the honest number, not the sum of the three. - status.innerHTML = `✓ ${t('st_three')} ${((Date.now() - t0) / 1000).toFixed(1)}s`; + status.innerHTML = `${t('st_three')} ${((Date.now() - t0) / 1000).toFixed(1)}s`; return; } const params = new URLSearchParams({ url, method: mode }); if (mode === 'extract') params.set('select', extra); - const r = await fetch('/api?' + params.toString()); + const r = await fetch('/api?' + params.toString(), { signal: ac.signal }); const d = await r.json(); + // Seed failures wear the same calm card here as in compare mode: a bot wall + // is the shield, a page that wouldn't open (bad domain, 404, two links + // pasted into one) is the alert-circle. Only a genuinely different error — + // a bad selector, a malformed request — falls through to the red line. + if (d.blocked) { showBlock(d); return; } + if (d.rate_limited) { showRateLimit(d); return; } + if (d.unreachable) { showUnreachable(); return; } if (d.error) { status.className = "status err"; status.textContent = "✗ " + d.error; return; } status.className = "status ok"; - status.innerHTML = `✓ /api?method=${mode}`; + status.innerHTML = `/api?method=${mode}`; pmRender(mode, d, url); } catch (e) { + if (ac.signal.aborted || e.name === 'AbortError') return; // user stopped it status.className = "status err"; status.textContent = "✗ " + e; - } finally { btn.disabled = false; } + } finally { btn.disabled = false; if (pmAbort === ac) pmAbort = null; } } function renderAnalysis(j) { From d01c7813814f3c16280e6b6eb7a4673797487f1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abraham=20P=C3=A9rez?= Date: Fri, 11 Sep 2026 14:11:44 -0600 Subject: [PATCH 4/6] test(seed): skip MCP-tools section cleanly on Python 3.9 The guard used pytest.importorskip("mcp") but the block below imports the mcp.server.mcpserver.exceptions submodule. On 3.9 CI (no mcp extra) a shallow mcp package can import while the submodule is absent, so the guard passed and collection then failed with ModuleNotFoundError. Point importorskip at the exact submodule these tests use, matching the sibling MCP test files, so the whole module skips instead of erroring. Co-Authored-By: Claude Opus 4.8 --- tests/test_seed_failures.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_seed_failures.py b/tests/test_seed_failures.py index 3d59276..95b58ea 100644 --- a/tests/test_seed_failures.py +++ b/tests/test_seed_failures.py @@ -240,7 +240,12 @@ def test_compare_surfaces_a_walled_seed_once(self, http): # --------------------------------------------------------------------------- # # MCP tools: every tool raises the right seed error. # --------------------------------------------------------------------------- # -pytest.importorskip("mcp") +# Guard on the exact submodule these tests import, not the top-level `mcp` +# package: on Python 3.9 (CI installs no mcp extra there) a shallow `mcp` can be +# importable without `mcp.server`, so importorskip("mcp") would pass and then the +# line below would ImportError at collection. Naming the submodule skips the +# module cleanly instead, the way the sibling MCP test files do. +pytest.importorskip("mcp.server.mcpserver.exceptions") from mcp.server.mcpserver.exceptions import ToolError, UnexpectedToolError # noqa: E402 from bytecrawl import mcp_server # noqa: E402 From a63352ad994dc2f1be5a3563ab2bd82371ae79c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abraham=20P=C3=A9rez?= Date: Fri, 11 Sep 2026 14:22:27 -0600 Subject: [PATCH 5/6] test(seed): run core and /api tests without the mcp extra; give /api CI its deps Two problems surfaced by the 3.9/3.12 split: - The MCP block used a module-level importorskip, so a run without the mcp extra (3.9 CI) aborted the whole module and silently skipped the core and /api tests too. Import the SDK softly and gate only the MCP tests with a skipif marker, resolving each tool by name so the parametrize list imports without the extra. Core and /api tests now run on 3.9 and 3.12; the MCP tests skip cleanly where the extra is absent. - The /api tests import the Flask landing app, which needs flask, python-dotenv and posthog. Those live only in web/landing/requirements.txt, so 3.12 failed with ModuleNotFoundError: No module named 'dotenv'. Add them to the dev extra (test-only, not runtime deps of the library) so CI exercises the /api seed-error surface, kept in sync with the web requirements. Co-Authored-By: Claude Opus 4.8 --- pyproject.toml | 5 ++- tests/test_seed_failures.py | 69 ++++++++++++++++++++++--------------- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 79f2862..2d5aa91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,10 @@ browser = ["playwright>=1.40"] llm = ["markdownify>=0.11", "trafilatura>=1.6"] mcp = ["mcp>=2; python_version >= '3.10'", "markdownify>=0.11", "trafilatura>=1.6"] all = ["playwright>=1.40", "markdownify>=0.11", "trafilatura>=1.6", "mcp>=2; python_version >= '3.10'"] -dev = ["pytest>=7", "pytest-cov", "ruff", "mypy", "bandit"] +# flask/python-dotenv/posthog are here (not runtime deps of the library) so the +# test suite can import the web landing app and exercise the /api seed-error +# surface. Kept in sync with web/landing/requirements.txt. +dev = ["pytest>=7", "pytest-cov", "ruff", "mypy", "bandit", "flask", "python-dotenv", "posthog"] [project.scripts] bytecrawl-mcp = "bytecrawl.mcp_server:main" diff --git a/tests/test_seed_failures.py b/tests/test_seed_failures.py index 95b58ea..60eac4e 100644 --- a/tests/test_seed_failures.py +++ b/tests/test_seed_failures.py @@ -240,34 +240,42 @@ def test_compare_surfaces_a_walled_seed_once(self, http): # --------------------------------------------------------------------------- # # MCP tools: every tool raises the right seed error. # --------------------------------------------------------------------------- # -# Guard on the exact submodule these tests import, not the top-level `mcp` -# package: on Python 3.9 (CI installs no mcp extra there) a shallow `mcp` can be -# importable without `mcp.server`, so importorskip("mcp") would pass and then the -# line below would ImportError at collection. Naming the submodule skips the -# module cleanly instead, the way the sibling MCP test files do. -pytest.importorskip("mcp.server.mcpserver.exceptions") -from mcp.server.mcpserver.exceptions import ToolError, UnexpectedToolError # noqa: E402 +# Only this block needs the `mcp` SDK (the mcp extra, Python 3.10+); the core +# tests above and the /api tests below need Flask, not mcp. Import the SDK +# softly and gate just the MCP tests with `requires_mcp`, so a run without the +# mcp extra (e.g. Python 3.9 CI) still collects and runs everything else instead +# of the whole module aborting on a module-level importorskip. +try: + from mcp.server.mcpserver.exceptions import ToolError, UnexpectedToolError -from bytecrawl import mcp_server # noqa: E402 + from bytecrawl import mcp_server + + _HAS_MCP = True +except ImportError: + _HAS_MCP = False + +requires_mcp = pytest.mark.skipif(not _HAS_MCP, reason="mcp extra not installed (Python 3.10+)") @pytest.fixture(autouse=True) def _no_delay(): - """The module-level scraper shouldn't sleep in tests.""" - mcp_server._scraper.delay = 0.0 + """The module-level MCP scraper shouldn't sleep in tests.""" + if _HAS_MCP: + mcp_server._scraper.delay = 0.0 yield -# (tool callable, kwargs) for each of the six tools. Single-fetch tools and -# crawl tools reach the seed by different code paths (_open vs raise_for_seed); -# both must land on the same two exceptions. +# (tool name, kwargs) for each of the six tools, resolved against mcp_server +# inside the test so this list imports without the mcp extra. Single-fetch tools +# and crawl tools reach the seed by different code paths (_open vs +# raise_for_seed); both must land on the same two exceptions. MCP_TOOLS = [ - (mcp_server.fetch_markdown, {}), - (mcp_server.extract, {}), - (mcp_server.list_links, {}), - (mcp_server.fetch_json_api, {}), - (mcp_server.focused_crawl, {"query": "x", "max_pages": 3}), - (mcp_server.compare_strategies, {"query": "x", "max_pages": 3}), + ("fetch_markdown", {}), + ("extract", {}), + ("list_links", {}), + ("fetch_json_api", {}), + ("focused_crawl", {"query": "x", "max_pages": 3}), + ("compare_strategies", {"query": "x", "max_pages": 3}), ] @@ -275,31 +283,35 @@ def _no_delay(): # it to the agent as an is_error result carrying our message (a bare requests # exception would be a crash, masked to "Error executing tool "). The # typed seed error is preserved as __cause__, so the classification still holds. -@pytest.mark.parametrize("tool, kw", MCP_TOOLS) -def test_mcp_tool_reports_blocked_on_a_walled_seed(http, tool, kw): +@requires_mcp +@pytest.mark.parametrize("tool_name, kw", MCP_TOOLS) +def test_mcp_tool_reports_blocked_on_a_walled_seed(http, tool_name, kw): http.routes[SEED] = _walled() with pytest.raises(ToolError) as exc: - tool(SEED, **kw) + getattr(mcp_server, tool_name)(SEED, **kw) assert isinstance(exc.value.__cause__, BlockedError) -@pytest.mark.parametrize("tool, kw", MCP_TOOLS) -def test_mcp_tool_reports_unreachable_on_a_dead_seed(http, tool, kw): +@requires_mcp +@pytest.mark.parametrize("tool_name, kw", MCP_TOOLS) +def test_mcp_tool_reports_unreachable_on_a_dead_seed(http, tool_name, kw): http.routes[SEED] = _dead() with pytest.raises(ToolError) as exc: - tool(SEED, **kw) + getattr(mcp_server, tool_name)(SEED, **kw) assert isinstance(exc.value.__cause__, UnreachableError) -@pytest.mark.parametrize("tool, kw", MCP_TOOLS) -def test_mcp_tool_reports_rate_limit_on_a_throttled_seed(http, tool, kw): +@requires_mcp +@pytest.mark.parametrize("tool_name, kw", MCP_TOOLS) +def test_mcp_tool_reports_rate_limit_on_a_throttled_seed(http, tool_name, kw): http.routes[SEED] = _throttled() with pytest.raises(ToolError) as exc: - tool(SEED, **kw) + getattr(mcp_server, tool_name)(SEED, **kw) assert isinstance(exc.value.__cause__, RateLimitError) assert "retry after ~30s" in str(exc.value) # the wait folds into the text +@requires_mcp @pytest.mark.parametrize( "route, needle", [ @@ -421,6 +433,7 @@ def test_api_walled_429_is_blocked_not_rate_limited(client, http): assert body["wall"] == "Cloudflare" +@requires_mcp def test_api_walled_and_mcp_walled_agree(client, http): """The contract in one place: the same walled seed, the tool and the endpoint, the same named wall — one raised (as a ToolError wrapping the From 3ad5184d436a230c6e14e131921e2cbf692fc6de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Abraham=20P=C3=A9rez?= Date: Fri, 11 Sep 2026 15:18:04 -0600 Subject: [PATCH 6/6] feat(landing): single-source demo examples for hero and playground MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a DEMO_EXAMPLES dict in content.py as the one source of truth for the worked examples, injected as JSON into both the "/" hero bar and the "/playground" so the two pages can never drift apart. Bring the hero to 5-mode parity (Crawl, Markdown, Links, JSON, Extract). Set the crawl demo to Wikipedia's Python page focused on "web framework": Shark-Search finds 5/6 relevant (top 0.679) while BFS and OPIC find 0 — the cleanest contrast of the candidates, on a clean seed URL with no parens. Co-Authored-By: Claude Opus 4.8 --- web/landing/content.py | 23 ++++++- web/landing/routes.py | 6 +- web/landing/templates/landing.html | 97 ++++++++++++++++++++++++++++-- web/landing/templates/try.html | 42 +++++++------ 4 files changed, 139 insertions(+), 29 deletions(-) diff --git a/web/landing/content.py b/web/landing/content.py index f4d8a05..2eb8717 100644 --- a/web/landing/content.py +++ b/web/landing/content.py @@ -35,7 +35,7 @@ "site": "quotes.toscrape.com", "description": "Hundreds of pages: walk a 'next' chain, or let BFS / Shark-Search / OPIC " "order a whole site under a request budget.", - "api": "?method=crawl&query=san+francisco", + "api": "?method=crawl&query=books", }, "login": { "title": "5 · API with login", @@ -54,6 +54,27 @@ } +# Worked examples behind the two live demos: the hero bar on "/" and the +# playground on "/playground". Both pages read from this one dict (injected as +# JSON into each template) so an example URL or query is edited in exactly one +# place and the two pages can never drift apart again. +# +# Keys are the playground's mode names. The hero bar only has Crawl and +# Markdown, so it uses "crawl" and "markdown"; the other three are +# playground-only. "query" is the focused-crawl topic (crawl) or the CSS +# selector (extract); "hint" is the i18n key for the hint under the playground's +# extra field. URLs carry their scheme; the hero strips it for display. +DEMO_EXAMPLES = { + "crawl": {"url": "https://en.wikipedia.org/wiki/Python_programming_language", + "query": "web framework", "hint": "pm_hint_crawl"}, + "markdown": {"url": "https://docs.python.org/3/tutorial/introduction.html"}, + "links": {"url": "https://news.ycombinator.com/"}, + "json": {"url": "https://data.sfgov.org/resource/wg3w-h783.json?$limit=5"}, + "extract": {"url": "https://books.toscrape.com/", + "query": "h3 a::attr(title)", "hint": "pm_hint_extract"}, +} + + ANALYZE_I18N = { "en": { "bad_url": "Enter a URL that starts with http:// or https://", diff --git a/web/landing/routes.py b/web/landing/routes.py index 87006ba..3b6ca80 100644 --- a/web/landing/routes.py +++ b/web/landing/routes.py @@ -21,7 +21,7 @@ import bytecrawl from landing import analytics, services -from landing.content import TECHNIQUES +from landing.content import DEMO_EXAMPLES, TECHNIQUES pages_bp = Blueprint("pages", __name__) api_bp = Blueprint("api", __name__) @@ -32,7 +32,7 @@ # --------------------------------------------------------------------------- # @pages_bp.route("/") def landing(): - return render_template("landing.html") + return render_template("landing.html", demos=DEMO_EXAMPLES) @pages_bp.route("/agent-onboarding/skill.json") @@ -86,7 +86,7 @@ def methods(): @pages_bp.route("/playground") def playground(): - return render_template("try.html", techniques=TECHNIQUES) + return render_template("try.html", techniques=TECHNIQUES, demos=DEMO_EXAMPLES) @pages_bp.route("/try") diff --git a/web/landing/templates/landing.html b/web/landing/templates/landing.html index a3463fc..5652509 100644 --- a/web/landing/templates/landing.html +++ b/web/landing/templates/landing.html @@ -459,16 +459,19 @@

    Crawling for AI
    - +
    - + (query to search)
    + + +
    @@ -623,6 +626,8 @@

    Build with ByteCrawl today