From ed1f85344edcce2e145eef8295fc6e1a5e045e8a Mon Sep 17 00:00:00 2001 From: Xi1495 Date: Sat, 7 Nov 2026 04:19:06 +0900 Subject: [PATCH 1/3] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E9=80=9A=E7=9F=A5?= =?UTF-8?q?=E5=88=97=E8=A1=A8=E8=A7=A3=E6=9E=90=E3=80=81=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E5=92=8C=E5=8E=86=E5=8F=B2=E7=BF=BB=E9=A1=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crawler/pyproject.toml | 2 + .../crawler/parse/fixtures/listing/README.md | 16 + .../parse/fixtures/listing/configured.html | 14 + .../parse/fixtures/listing/drupal.html | 58 ++++ .../parse/fixtures/listing/gpower.html | 23 ++ .../crawler/parse/fixtures/listing/trs.html | 16 + .../parse/fixtures/listing/vsb-calendar.html | 20 ++ .../crawler/parse/fixtures/listing/vsb.html | 22 ++ .../parse/fixtures/listing/webplus.html | 17 ++ .../parse/fixtures/listing/zju-http.html | 20 ++ crawler/src/crawler/parse/listing.py | 285 ++++++++++++++++++ crawler/src/crawler/parse/listing_test.py | 285 ++++++++++++++++++ crawler/uv.lock | 145 +++++++++ docs/development.md | 25 ++ 14 files changed, 948 insertions(+) create mode 100644 crawler/src/crawler/parse/fixtures/listing/README.md create mode 100644 crawler/src/crawler/parse/fixtures/listing/configured.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/drupal.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/gpower.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/trs.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/vsb-calendar.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/vsb.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/webplus.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/zju-http.html create mode 100644 crawler/src/crawler/parse/listing.py create mode 100644 crawler/src/crawler/parse/listing_test.py diff --git a/crawler/pyproject.toml b/crawler/pyproject.toml index 76ad4c5..c640430 100644 --- a/crawler/pyproject.toml +++ b/crawler/pyproject.toml @@ -5,6 +5,7 @@ description = "鼠钟的抓取节点：抓取、初筛和抽取" requires-python = ">=3.14" dependencies = [ "httpx>=0.28", + "lxml>=6.1.3", "pydantic>=2.14.1", ] @@ -14,6 +15,7 @@ dev = [ "pyright>=1.1.400", "pytest>=9.1", "ruff>=0.17", + "types-lxml>=2026.2.16", ] [build-system] diff --git a/crawler/src/crawler/parse/fixtures/listing/README.md b/crawler/src/crawler/parse/fixtures/listing/README.md new file mode 100644 index 0000000..b608c5b --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/README.md @@ -0,0 +1,16 @@ +# 列表结构样例 + +2026-11-07 通过原版 `Fetcher` 各取一页，只截取列表及翻页结构。标题、摘要、日期、通知链接和图片地址全部替换为自编内容；没有通知正文、快照、附件或学生信息。样例中日期均为 2026-09-10、2026-09-09。 + +| 样例 | 系统及结构依据 | 结构来源 | +| --- | --- | --- | +| `vsb.html` | Visual SiteBuilder，原页有 `Announced by Visual SiteBuilder 9`；日期为日、年月分栏，标题与摘要分开 | [厦大信息学院招生信息](https://informatics.xmu.edu.cn/zsxx.htm) | +| `vsb-calendar.html` | 同一建站系统的另一种日期布局，链接有完整 `title` 属性 | [西电电子工程学院研究生列表](https://see.xidian.edu.cn/yjs/158.htm) | +| `webplus.html` | WebPlus，原页的 `webplus-list` 及 Sudy 资源；`news_list`、`news_title`、`news_meta` | [东南大学网安硕士生招生](https://cyber.seu.edu.cn/18210/list.htm) | +| `trs.html` | TRS 类静态发布，`YYYYMM/tYYYYMMDD_编号.html`，日期在独立 `pull-right` 中；包含一条跳往研招系统的置顶链接 | [计算所硕士招生](https://www.ict.ac.cn/yjsjy/zsxx/sszs/) | +| `gpower.html` | GPower 页面，原页的 `gpowersoft.js` 和 `/cms/web/`；`infoBox` 内标题、摘要、日期分开 | [北科大人工智能学院通知](https://ai.ustb.edu.cn/xwgg/tzgg/) | +| `drupal.html` | 原页明确声明 Drupal 10，Views 容器、`list-page`、`time/@datetime`；只有页码的翻页器 | [中大计算机学院硕士招生](https://cse.sysu.edu.cn/admission/master) | +| `configured.html` | 独立招生发布系统，作为需要配置的页面测试，不计入前述五种通用系统 | [国科大硕士招生](https://admission.ucas.ac.cn/showarticle/newslist/1beb211c-ea0d-4c97-b5ce-61efa7addfcd) | +| `zju-http.html` | WebPlus 的另一种布局，使用开发约定中的 XPath 配置和显式 HTTP，列出 `jzlb`、`btt3`、`fbsj4` | [浙大计算机招生信息](http://www.cs.zju.edu.cn/csen/27006/list.htm) | + +Drupal 样例用于补充不同系统的解析覆盖，不改动 `data/sources.csv`。分页边界、未知日期、重复置顶、重定向、HTTP、非法配置和失败场景使用测试里自编的页面及 `MockTransport`，不访问外网。 diff --git a/crawler/src/crawler/parse/fixtures/listing/configured.html b/crawler/src/crawler/parse/fixtures/listing/configured.html new file mode 100644 index 0000000..a1ba74d --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/configured.html @@ -0,0 +1,14 @@ + + + +
+
+ + diff --git a/crawler/src/crawler/parse/fixtures/listing/drupal.html b/crawler/src/crawler/parse/fixtures/listing/drupal.html new file mode 100644 index 0000000..7af2ee9 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/drupal.html @@ -0,0 +1,58 @@ + + + + + + +
+ + +
+ + + + + + + + +
+ + diff --git a/crawler/src/crawler/parse/fixtures/listing/gpower.html b/crawler/src/crawler/parse/fixtures/listing/gpower.html new file mode 100644 index 0000000..7fd6809 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/gpower.html @@ -0,0 +1,23 @@ + + + +
+ 下一页 +
+ + diff --git a/crawler/src/crawler/parse/fixtures/listing/trs.html b/crawler/src/crawler/parse/fixtures/listing/trs.html new file mode 100644 index 0000000..29e97a4 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/trs.html @@ -0,0 +1,16 @@ + + + +
+ +
+ + diff --git a/crawler/src/crawler/parse/fixtures/listing/vsb-calendar.html b/crawler/src/crawler/parse/fixtures/listing/vsb-calendar.html new file mode 100644 index 0000000..57ce3d0 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/vsb-calendar.html @@ -0,0 +1,20 @@ + + + +
+ + + 下一页
+ + diff --git a/crawler/src/crawler/parse/fixtures/listing/vsb.html b/crawler/src/crawler/parse/fixtures/listing/vsb.html new file mode 100644 index 0000000..e335115 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/vsb.html @@ -0,0 +1,22 @@ + + + +
+ 下一页
+ + diff --git a/crawler/src/crawler/parse/fixtures/listing/webplus.html b/crawler/src/crawler/parse/fixtures/listing/webplus.html new file mode 100644 index 0000000..72c7603 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/webplus.html @@ -0,0 +1,17 @@ + + + +
+ + +
+ + diff --git a/crawler/src/crawler/parse/fixtures/listing/zju-http.html b/crawler/src/crawler/parse/fixtures/listing/zju-http.html new file mode 100644 index 0000000..5b23bbe --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/zju-http.html @@ -0,0 +1,20 @@ + + + +
+ + + +
+
示例招生通知甲
+
2026-09-10
+
+
+
示例招生通知乙
+
2026-09-09
+
+
+ +
+ + diff --git a/crawler/src/crawler/parse/listing.py b/crawler/src/crawler/parse/listing.py new file mode 100644 index 0000000..923d24b --- /dev/null +++ b/crawler/src/crawler/parse/listing.py @@ -0,0 +1,285 @@ +import codecs +import re +from calendar import monthrange +from dataclasses import dataclass +from datetime import date, datetime, timedelta, timezone +from urllib.parse import urldefrag, urljoin, urlsplit, urlunsplit + +from lxml import etree, html +from lxml.html import HtmlElement +from pydantic import ( + BaseModel, + ConfigDict, + InstanceOf, + TypeAdapter, + ValidationError, + field_validator, +) + +from crawler.crawl.contract import ListedItem, WorkSource +from crawler.crawl.fetch import Fetcher +from crawler.crawl.rounds import StepError + + +class Selectors(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + items: str + title: str + link: str + date: str | None = None + next: str | None = None + + @field_validator("items", "title", "link", "date", "next") + @classmethod + def valid_xpath(cls, value: str | None) -> str | None: + if value is not None: + if not value.strip(): + raise ValueError("XPath cannot be empty") + try: + etree.XPath(value, regexp=False) + except etree.XPathError as error: + raise ValueError("invalid XPath") from error + return value + + +class ListConfig(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + selectors: Selectors | None = None + force_http: bool = False + + +_ELEMENTS = TypeAdapter(list[InstanceOf[HtmlElement]]) +_DATE = re.compile(r"(? list[HtmlElement]: + result: object = node.xpath(expression, smart_strings=False) + try: + return _ELEMENTS.validate_python(result) + except ValidationError as error: + raise StepError("error", "配置不对: items 必须选中条目元素") from error + + +def _value(node: HtmlElement, expression: str | None, separator: str = "") -> str: + if expression is None: + return "" + result: object = node.xpath(expression, smart_strings=False) + if isinstance(result, list): + try: + values = TypeAdapter(list[InstanceOf[HtmlElement] | str]).validate_python(result) + except ValidationError as error: + raise StepError("error", "配置不对: 字段 XPath 必须返回文字或元素") from error + if not values: + return "" + value = values[0] + elif isinstance(result, str): + value = result + else: + raise StepError("error", "配置不对: 字段 XPath 必须返回文字或元素") + text = separator.join(value.itertext()) if isinstance(value, HtmlElement) else value + return " ".join(text.split()) + + +def _date(text: str) -> date | None: + match = _DATE.search(text) + if match: + year, month, day = map(int, match.groups()) + elif match := _SPLIT_DATE.fullmatch(text): + day, year, month = map(int, match.groups()) + else: + return None + try: + return date(year, month, day) + except ValueError: + return None + + +def _url(base: str, href: str) -> str | None: + if not href or href.startswith("#"): + return None + absolute, _ = urldefrag(urljoin(base, href)) + parts = urlsplit(absolute) + return absolute if parts.scheme in {"http", "https"} and parts.netloc else None + + +def _next(root: HtmlElement, url: str) -> str | None: + for anchor in root.iter("a"): + if anchor.get("aria-disabled") == "true" or "disabled" in anchor.get("class", ""): + continue + label = "".join("".join(anchor.itertext()).split()) + title = "".join(anchor.get("title", "").split()) + is_next = "next" in anchor.get("rel", "").split() or any( + re.fullmatch(r"(?:下一?页|Next)[>\u00bb\u203a]*", value, re.IGNORECASE) + for value in (label, title) + ) + if is_next and (target := _url(url, anchor.get("href", ""))): + return target + # Some Drupal pagers show only numbered pages, with no separate Next link. + for anchor in root.iter("a"): + if anchor.get("aria-current") != "page": + continue + row = anchor.getparent() + current = re.search(r"(\d+)\s*$", "".join(anchor.itertext())) + if row is None or "pager" not in row.get("class", "") or current is None: + continue + following = row.getnext() + if following is None: + continue + link = next(following.iter("a"), None) + if link is None: + continue + number = re.search(r"(\d+)\s*$", "".join(link.itertext())) + if number is not None and int(number[1]) == int(current[1]) + 1: + return _url(url, link.get("href", "")) + return None + + +def parse_listing( + body: bytes, + url: str, + selectors: Selectors | None = None, + content_type: str | None = None, +) -> ListPage: + if not body.strip(): + return ListPage([], None) + declaration = (content_type or "") + " " + body[:4096].decode("ascii", errors="ignore") + match = re.search(r"charset\s*=\s*['\"]?([\w.-]+)", declaration, re.IGNORECASE) + try: + encoding = codecs.lookup(match[1] if match else "utf-8").name + root = html.document_fromstring( + body, parser=html.HTMLParser(encoding=encoding, no_network=True) + ) + except (LookupError, etree.ParserError) as error: + raise StepError("error", "列表页无法解析") from error + try: + configured = selectors is not None + candidates = (selectors,) if selectors is not None else _TEMPLATES + rows: list[HtmlElement] = [] + for candidate in candidates: + rows = _elements(root, candidate.items) + if rows: + selectors = candidate + break + if selectors is None or not rows: + return ListPage([], None) + items: dict[str, ListedItem] = {} + for row in rows: + title = _value(row, selectors.title) + target = _url(url, _value(row, selectors.link)) + if title and target: + item = ListedItem( + url=target, + title=title, + published_on=_date(_value(row, selectors.date, " ")), + ) + items.setdefault(target, item) + following = _url(url, _value(root, selectors.next)) if configured else _next(root, url) + return ListPage(list(items.values()), following) + except etree.XPathError as error: + raise StepError("error", "配置不对: XPath 无法计算") from error + except ValidationError as error: + raise StepError("error", "列表条目或配置字段不符合结构") from error + + +async def list_items( + fetcher: Fetcher, source: WorkSource, *, today: date | None = None +) -> list[ListedItem]: + try: + config = ListConfig.model_validate(source.config or {}) + except ValidationError as error: + location = ".".join(str(value) for value in error.errors(include_input=False)[0]["loc"]) + raise StepError("error", f"配置不对: {location}") from error + + def request_url(url: str) -> str: + parts = urlsplit(url) + if config.force_http and parts.netloc == urlsplit(source.url).netloc: + return urlunsplit(parts._replace(scheme="http")) + return url + + now = today or datetime.now(timezone(timedelta(hours=8))).date() + year = now.year - 2 + cutoff = now.replace(year=year, day=min(now.day, monthrange(year, now.month)[1])) + current = request_url(source.url) + visited: set[str] = set() + items: dict[str, ListedItem] = {} + while True: + if current in visited: + raise StepError("error", "分页链接循环") + visited.add(current) + response = await fetcher.get(current) + visited.add(response.url) + if not 200 <= response.status < 300: + raise StepError("http_status", f"HTTP {response.status}") + page = parse_listing(response.body, response.url, config.selectors, response.content_type) + if not page.items: + return [] + fresh = [item for item in page.items if item.url not in items] + if ( + source.backfill + and fresh + and all(item.published_on is not None and item.published_on < cutoff for item in fresh) + ): + break + for item in page.items: + if not source.backfill or item.published_on is None or item.published_on >= cutoff: + items.setdefault(item.url, item) + if not source.backfill or page.next_url is None: + break + current = request_url(page.next_url) + return list(items.values()) diff --git a/crawler/src/crawler/parse/listing_test.py b/crawler/src/crawler/parse/listing_test.py new file mode 100644 index 0000000..8f7ceed --- /dev/null +++ b/crawler/src/crawler/parse/listing_test.py @@ -0,0 +1,285 @@ +from datetime import date +from pathlib import Path + +import httpx +import pytest +from pydantic import JsonValue + +from crawler.crawl.contract import WorkSource +from crawler.crawl.fetch import Fetcher +from crawler.crawl.fetch_test import FakeSites +from crawler.crawl.rounds import NoticePage, StepError, Steps, crawl_source +from crawler.crawl.rounds_test import FakeWebsite, extract, screen +from crawler.crawl.website import Website +from crawler.parse.listing import Selectors, list_items, parse_listing + +FIXTURES = Path(__file__).parent / "fixtures" / "listing" +BASE = "https://example.edu.cn/notices.htm" +TODAY = date(2026, 11, 7) + + +def source(*, backfill: bool = False, config: dict[str, JsonValue] | None = None) -> WorkSource: + return WorkSource( + id=7, url=BASE, section="招生信息", browser=False, config=config, backfill=backfill + ) + + +def page(rows: list[tuple[str, str | None]], following: str | None = None) -> bytes: + items = "".join( + f'
  • 自编通知{number}' + f'{published or ""}
    • ' + for number, published in rows + ) + next_link = f'' if following else "" + return f'{next_link}'.encode() + + +@pytest.mark.parametrize( + ("name", "urls", "next_url"), + [ + ("vsb", ["/info/1001/101.htm", "/info/1001/102.htm"], "https://example.edu.cn/page2.htm"), + ( + "vsb-calendar", + ["/info/1001/101.htm", "/info/1001/102.htm"], + "https://example.edu.cn/page2.htm", + ), + ( + "webplus", + ["/2026/0910/c100a101/page.htm", "/2026/0910/c100a102/page.htm"], + "https://example.edu.cn/page2.htm", + ), + ("trs", ["https://admission.example.edu.cn/notice/1", "/202609/t20260909_102.html"], None), + ( + "gpower", + ["/" + "1" * 32 + ".htm", "/" + "2" * 32 + ".htm"], + "https://example.edu.cn/page2.htm", + ), + ("drupal", ["/article/101", "/article/102"], "https://example.edu.cn/notices.htm?page=1"), + ], +) +def test_real_cms_structures_keep_titles_dates_and_links( + name: str, urls: list[str], next_url: str | None +) -> None: + result = parse_listing((FIXTURES / f"{name}.html").read_bytes(), BASE) + assert [item.title for item in result.items] == ["示例招生通知甲", "示例招生通知乙"] + assert [item.published_on for item in result.items] == [date(2026, 9, 10), date(2026, 9, 9)] + expected = [ + value if value.startswith("https:") else "https://example.edu.cn" + value for value in urls + ] + assert [item.url for item in result.items] == expected + assert result.next_url == next_url + + +def test_configured_structure_and_legacy_encoding() -> None: + selectors = Selectors( + items="//ul[@class='b-list']/li", + title="normalize-space(.//a)", + link=".//a/@href", + date=".//span[@class='m-date']", + ) + text = (FIXTURES / "configured.html").read_text(encoding="utf-8") + result = parse_listing(text.encode("gb18030"), BASE, selectors, "text/html; charset=gb18030") + assert [(item.title, item.published_on) for item in result.items] == [ + ("示例招生通知甲", date(2026, 9, 10)), + ("示例招生通知乙", date(2026, 9, 9)), + ] + assert result.items[0].url == "https://example.edu.cn/showarticle/Article/example/101" + + +def test_documented_http_configuration_reads_the_real_page_structure() -> None: + config: dict[str, JsonValue] = { + "selectors": { + "items": "//div[contains(concat(' ',normalize-space(@class),' '),' jzlb ')]", + "title": "./div[@class='btt3']/a", + "link": "./div[@class='btt3']/a/@href", + "date": "./div[@class='fbsj4']", + "next": "//a[@class='next']/@href", + }, + "force_http": True, + } + sites = FakeSites( + { + "http://example.edu.cn/notices.htm": httpx.Response( + 200, content=(FIXTURES / "zju-http.html").read_bytes() + ) + } + ) + result = sites.run(lambda f: list_items(f, source(config=config), today=TODAY)) + assert [(item.title, item.published_on) for item in result] == [ + ("示例招生通知甲", date(2026, 9, 10)), + ("示例招生通知乙", date(2026, 9, 9)), + ] + assert result[0].url == "http://example.edu.cn/csen/2026/0910/c27006a101/page.htm" + + +@pytest.mark.parametrize("backfill", [False, True]) +def test_backfill_keeps_boundary_and_unknown_dates_and_ignores_repeated_pinned_rows( + backfill: bool, +) -> None: + sites = FakeSites( + { + BASE: httpx.Response( + 200, + content=page([("1", "2026-09-10"), ("2", "2026-09-09"), ("9", "2020-01-01")], "p2"), + ), + "https://example.edu.cn/p2": httpx.Response( + 200, + content=page( + [("1", "2026-09-10"), ("3", "2024-11-07"), ("4", None), ("5", "2024-11-06")], + "p3", + ), + ), + "https://example.edu.cn/p3": httpx.Response( + 200, + content=page([("1", "2026-09-10"), ("6", "2024-10-01")], "must-not-fetch"), + ), + } + ) + result = sites.run(lambda f: list_items(f, source(backfill=backfill), today=TODAY)) + assert [item.title for item in result] == ( + ["自编通知1", "自编通知2", "自编通知3", "自编通知4"] + if backfill + else ["自编通知1", "自编通知2", "自编通知9"] + ) + assert len(sites.requests) == (3 if backfill else 1) + + +def test_leap_day_backfill_uses_calendar_years() -> None: + sites = FakeSites( + { + BASE: httpx.Response(200, content=page([("1", "2022-02-28")], "p2")), + "https://example.edu.cn/p2": httpx.Response( + 200, content=page([("2", "2022-02-27")], "must-not-fetch") + ), + } + ) + result = sites.run(lambda f: list_items(f, source(backfill=True), today=date(2024, 2, 29))) + assert [item.published_on for item in result] == [date(2022, 2, 28)] + assert len(sites.requests) == 2 + + +def test_numbered_pager_advances_one_page_and_stops_at_last() -> None: + pager = ( + '' + ) + sites = FakeSites( + { + BASE: httpx.Response( + 200, + content=page([("1", "2026-09-10")]) + + pager.format(first='aria-current="page"', second="").encode(), + ), + BASE + "?page=1": httpx.Response( + 200, + content=page([("2", "2026-09-09")]) + + pager.format(first="", second='aria-current="page"').encode(), + ), + } + ) + result = sites.run(lambda f: list_items(f, source(backfill=True), today=TODAY)) + assert [item.title for item in result] == ["自编通知1", "自编通知2"] + assert len(sites.requests) == 2 + + +@pytest.mark.parametrize("configured", [False, True]) +def test_explicit_http_config_and_redirect_base_are_used_for_pagination_and_links( + configured: bool, +) -> None: + sites = FakeSites( + { + "http://example.edu.cn/notices.htm": httpx.Response( + 302, headers={"location": "http://example.edu.cn/board/index.htm"} + ), + "http://example.edu.cn/board/index.htm": httpx.Response( + 200, + content=page([("1", "2026-09-10")], "https://example.edu.cn/board/p2").replace( + b"/info/1001/1.htm", b"./info/1001/1.htm#section" + ), + ), + "http://example.edu.cn/board/p2": httpx.Response( + 200, content=page([("2", "2023-01-01")]) + ), + } + ) + selectors: dict[str, JsonValue] = { + "items": "//li", + "title": ".//a", + "link": ".//a/@href", + "date": ".//span", + "next": "//a[@rel='next']/@href", + } + config: dict[str, JsonValue] = {"force_http": True} + if configured: + config["selectors"] = selectors + result = sites.run(lambda f: list_items(f, source(backfill=True, config=config), today=TODAY)) + assert [item.url for item in result] == ["http://example.edu.cn/board/info/1001/1.htm"] + assert [url for _, url in sites.requests] == list(sites.pages) + + +@pytest.mark.parametrize( + "config", + [ + {"unknown": True}, + {"force_http": "true"}, + {"selectors": {"items": "[", "title": ".//a", "link": ".//a/@href"}}, + {"selectors": {"items": "string(//li)", "title": ".//a", "link": ".//a/@href"}}, + {"selectors": {"items": "//li", "title": ".//missing:title", "link": ".//a/@href"}}, + ], +) +def test_bad_configuration_is_recorded_by_the_round(config: dict[str, JsonValue]) -> None: + sites = FakeSites({BASE: httpx.Response(200, content=page([("1", "2026-09-10")]))}) + website = FakeWebsite([], frozenset()) + + async def read(_fetcher: Fetcher, _source: WorkSource, _url: str) -> NoticePage: + raise AssertionError("a failed list must not read notices") + + async def run(fetcher: Fetcher) -> None: + async with Website( + "http://site", "local-test", httpx.MockTransport(website.handle) + ) as client: + await crawl_source( + source(config=config), fetcher, client, Steps(list_items, screen, read, extract) + ) + + sites.run(run) + [record] = website.called("POST /runs") + assert record["failure"] == "error" + assert record["detail"].startswith("配置不对") + assert website.called("POST /sources/7/listing") == [] + + +@pytest.mark.parametrize("status", [403, 500]) +def test_historical_http_failure_does_not_return_partial_backfill(status: int) -> None: + sites = FakeSites( + { + BASE: httpx.Response(200, content=page([("1", "2026-09-10")], "p2")), + "https://example.edu.cn/p2": httpx.Response(status), + } + ) + with pytest.raises(StepError, match=f"HTTP {status}") as caught: + sites.run(lambda f: list_items(f, source(backfill=True), today=TODAY)) + assert caught.value.failure == "http_status" + + +def test_unknown_and_invalid_dates_do_not_invent_publication_dates() -> None: + body = page([("1", "2026-02-30"), ("2", "9月1日")]) + result = parse_listing(body, BASE) + assert [item.published_on for item in result.items] == [None, None] + assert parse_listing(b"", BASE).items == [] + assert parse_listing(b"", BASE).items == [] + + +def test_cyclic_pagination_is_an_error_instead_of_a_partial_success() -> None: + sites = FakeSites( + { + BASE: httpx.Response(200, content=page([("1", "2026-09-10")], "p2")), + "https://example.edu.cn/p2": httpx.Response( + 200, content=page([("2", "2026-09-09")], BASE) + ), + } + ) + with pytest.raises(StepError, match="分页链接循环"): + sites.run(lambda f: list_items(f, source(backfill=True), today=TODAY)) + assert len(sites.requests) == 2 diff --git a/crawler/uv.lock b/crawler/uv.lock index f1660c4..fbf61b9 100644 --- a/crawler/uv.lock +++ b/crawler/uv.lock @@ -43,6 +43,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/bd/551ee6af426af84ca33e02622be722925c196608e9127d731ef17c47f06e/argcomplete-3.7.2-py3-none-any.whl", hash = "sha256:6029205678bdd9c1c728a155f5f9ecf5812393f969eef58807641a2bc2aa5b19", size = 43294, upload-time = "2026-08-06T04:53:20.246Z" }, ] +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + [[package]] name = "black" version = "26.10.1" @@ -98,6 +111,7 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "httpx" }, + { name = "lxml" }, { name = "pydantic" }, ] @@ -107,11 +121,13 @@ dev = [ { name = "pyright" }, { name = "pytest" }, { name = "ruff" }, + { name = "types-lxml" }, ] [package.metadata] requires-dist = [ { name = "httpx", specifier = ">=0.28" }, + { name = "lxml", specifier = ">=6.1.3" }, { name = "pydantic", specifier = ">=2.14.1" }, ] @@ -121,6 +137,16 @@ dev = [ { name = "pyright", specifier = ">=1.1.400" }, { name = "pytest", specifier = ">=9.1" }, { name = "ruff", specifier = ">=0.17" }, + { name = "types-lxml", specifier = ">=2026.2.16" }, +] + +[[package]] +name = "cssselect" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/8b/dc32df939ab541fca6ee8964d26aa231dbe231cdc2b2713228161441ba9c/cssselect-1.6.0.tar.gz", hash = "sha256:8c83a7139e97b93aa5ebdc0f46e785f7056a08a8bf201e597a6a2629d7eb11db", size = 51743, upload-time = "2026-10-09T20:05:09.484Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/ae/f24b3aac56ba91a29c9d3a31c07a9ad4e9eb500e5d212742bb6d348edaef/cssselect-1.6.0-py3-none-any.whl", hash = "sha256:6df6eab9b264c0f2092a6e386b33610e1684a25e27925ecebe25e3d97cbf3525", size = 22244, upload-time = "2026-10-09T20:05:08.215Z" }, ] [[package]] @@ -254,6 +280,80 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "lxml" +version = "6.1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/ad/28ecd7cb894d172f3c9c80a075eeeb2017ac62e3632cee05a5f9493547eb/lxml-6.1.3.tar.gz", hash = "sha256:45222d94ddd511536f3b2f7d9deae3b2339b4ce0f075f1ca25703b07cad9dd21", size = 4211198, upload-time = "2026-09-02T14:48:02.287Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/15/fc75a70b0af6021d0ea16811f1fc71cc42cd06ce90fe10f007a69b2eed84/lxml-6.1.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2bec13085dc8ef48a3fe62f7dfcacfeda2c785cdf19cc8eeda2bb9ed081da165", size = 8609725, upload-time = "2026-09-02T14:49:00.156Z" }, + { url = "https://files.pythonhosted.org/packages/84/ef/398fcf9018f881ec9aeaafae1ddd6586dfb13314a35d35e899de373dcae0/lxml-6.1.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4f4db7c7e954d289d71878938348b3d91b904a3e8210a11939359fb758a58e7d", size = 4639629, upload-time = "2026-09-02T14:49:02.81Z" }, + { url = "https://files.pythonhosted.org/packages/a7/2d/49b6a6ad7ce8f64b07b9fe852ff0c6d3fcbb26db61bee4f63d4120180a1c/lxml-6.1.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2cae5d5c90a62d9139c512a0cb1aad1d182b022b5740daea2617eb5bf7fc658e", size = 4965074, upload-time = "2026-09-02T14:49:05.133Z" }, + { url = "https://files.pythonhosted.org/packages/66/bc/6230cf80e4331c33383b0b6b73dc31a393dd76edd4cb73d761de5123034d/lxml-6.1.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c6c0c13128a32eb04a51357e56a094e13aa8e6d3d1884de2e9ae923f6915e1a8", size = 5099355, upload-time = "2026-09-02T14:49:07.343Z" }, + { url = "https://files.pythonhosted.org/packages/ac/cf/d1143d9b7717e07a82f158a1fc9ce6e581fdad1226734950af869e3ffde4/lxml-6.1.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2221e88679d1351e9a40aaee54bc65679b9795bbd0160bc3d5e36b163344eb75", size = 5036795, upload-time = "2026-09-02T14:49:09.65Z" }, + { url = "https://files.pythonhosted.org/packages/31/6f/194bb00ffb89712c30f5a7e1b8e685590e140fad6c8261fec172c09a3dc0/lxml-6.1.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cfb398886a7eb4c719161c3efcff2a1248febc53a4d8e5072d2d8a87fed84ac9", size = 5658740, upload-time = "2026-09-02T14:49:11.9Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/27e3cee3dcdb3b7bc09727b642bdbfcd098490ea77df04611db9060d7722/lxml-6.1.3-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7eb78ba28b187e1e9203a55c60fcf70df2d22cb205fe6d51b9383d6097419f0", size = 5245991, upload-time = "2026-09-02T14:49:14.154Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e9/8312560579fc980bbd2233a8a673cc46f7d613d3633f2bf08a21e8f4ad13/lxml-6.1.3-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:ea6b1e9105b4b24a34c722432d9fb578f9ed83af21fa1abda639011e0f22bbb6", size = 5354136, upload-time = "2026-09-02T14:49:16.459Z" }, + { url = "https://files.pythonhosted.org/packages/74/d8/eda60f4f73a9c780b5d6e1175484f66e6c81a2c93346e2906a1fec9c7a02/lxml-6.1.3-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:e8b17e23df3e827a69d25af70990ca2420e92668aaffaeeb3cd2351d7916a023", size = 4704379, upload-time = "2026-09-02T14:49:19.032Z" }, + { url = "https://files.pythonhosted.org/packages/ba/c8/c9cc60057be78ac34bd2b842e45e6e88edbfe5e532e82c3b82381b7aab49/lxml-6.1.3-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b7c37339d7e75cab9a123a04248e243cefefb302ad6db566ea0c77cbcde421e", size = 5258676, upload-time = "2026-09-02T14:49:21.306Z" }, + { url = "https://files.pythonhosted.org/packages/41/7b/66894008fee8d1785b8db129747ae963fd427b68f456918df7f2f24a8b98/lxml-6.1.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:83e3a51e7933db700a0da0db31849db3a24022d9970da9bb73001e1d0326fd92", size = 5090069, upload-time = "2026-09-02T14:49:23.562Z" }, + { url = "https://files.pythonhosted.org/packages/8b/31/c1b60404859f4c3cd1f41f29c65a24e25cea78fde822d9574a21f66810be/lxml-6.1.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9bde9ae026a55b9a192078dfa6e27dd0ca4a050171ab6272e92f97b757dfdf48", size = 4741958, upload-time = "2026-09-02T14:49:26.037Z" }, + { url = "https://files.pythonhosted.org/packages/23/b8/6285f0cf546f14da2554cabdeaf7c2c2ff3190c74807f0de2e8810a786f9/lxml-6.1.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1a635e837b50a1819bebfedaac5916498ea024120969da8790500148fb0a894d", size = 5683245, upload-time = "2026-09-02T14:49:28.438Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f6/2168cab44336dcb15fed0f0b78577225b83297cdf0dee349c95420c3dcb0/lxml-6.1.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d0c5c362bc94f1929dc7e96e715bbe7bd17037f802e6d8f0d1545df9133c0559", size = 5246087, upload-time = "2026-09-02T14:49:30.955Z" }, + { url = "https://files.pythonhosted.org/packages/f5/89/32f5de69a0a31f30e6164981851f87b37ecb2c4ee838e504b88d49d4818e/lxml-6.1.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c59e4265608da6a041f54646ecc0c9ecdbb19aaf14c4c684bb6c2114998cc415", size = 5269352, upload-time = "2026-09-02T14:49:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/a2/a1/741d952ed3a7ef7a50055c6415aec3f067015e97f72f4389ce77b09657ba/lxml-6.1.3-cp314-cp314-win32.whl", hash = "sha256:2e62c569ec7531b679b184cbfe335c501c1d13c4b363560013019962eb630e6d", size = 3662783, upload-time = "2026-09-02T14:50:23.751Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bc/5811cc73cac05e324e05ba9b0924e1a163a317a167ede8a9c748b11db30a/lxml-6.1.3-cp314-cp314-win_amd64.whl", hash = "sha256:66299564c046bc7e0cc5de5106601eae907e9fa5904cd68a323380a8502f7861", size = 4073951, upload-time = "2026-09-02T14:50:26.348Z" }, + { url = "https://files.pythonhosted.org/packages/92/18/3768c8b01ac3a9bed1914715e6011711b00e2a11628ffa6f7fa37f8e0269/lxml-6.1.3-cp314-cp314-win_arm64.whl", hash = "sha256:ebd054ad1737a68fb7c5c073d405cef2b88bb824e294de3b4a4e995b47f0e376", size = 3749279, upload-time = "2026-09-02T14:50:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/72/38/84684784738d9451db2b330de2483f496690c3a5c642071df24135739b37/lxml-6.1.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5a143e6207579de8baeded4eaac9134413200359f1969d636f0bfb98ee8c3c8f", size = 8860296, upload-time = "2026-09-02T14:49:36.346Z" }, + { url = "https://files.pythonhosted.org/packages/24/b7/fc4c50bb1b38e864010ea396046cabe85129bf9e65b11edcfbc37d356241/lxml-6.1.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a1cec0f99b9b914d39176347a93b7610dc09324491aee1cbc57cd291a41a1d55", size = 4755190, upload-time = "2026-09-02T14:49:39.872Z" }, + { url = "https://files.pythonhosted.org/packages/94/e2/ee9aa6ed2b666b2db1f6f7fd48964ff9da39ebe827ef5eac0ab881f639d9/lxml-6.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f6b9d2aad499c769ee8287609ab0e6de99d8bcea99c6e6c2e64945259fd52fb2", size = 4979517, upload-time = "2026-09-02T14:49:42.153Z" }, + { url = "https://files.pythonhosted.org/packages/29/e3/e7763d1661b283ddd4fa36f91b9a497db6b8d2aff55028b16c7f642e0755/lxml-6.1.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a23fefdb345b2d4d0ff2860571b5ff9a89a28b6a120f720e8fb0324d346626", size = 5115270, upload-time = "2026-09-02T14:49:44.493Z" }, + { url = "https://files.pythonhosted.org/packages/2d/cd/22205d5b4d177e3f4156f780412426ee7c7f8107809f119f0dcc40fa51e3/lxml-6.1.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:545ccc14fb05485f48b4439ec35beb16d5b5280eb6c81c658bd4707a2a119414", size = 5032449, upload-time = "2026-09-02T14:49:46.841Z" }, + { url = "https://files.pythonhosted.org/packages/da/43/06a4626c3bb79ef8c501b674afab8100d64e798665bb2a97d1c960636a49/lxml-6.1.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:93476b6514b373fc6ca67d26c442784f7807c86f00635bfe79f935c3eab2af17", size = 5603325, upload-time = "2026-09-02T14:49:49.664Z" }, + { url = "https://files.pythonhosted.org/packages/d0/9c/733682a0c2de9f5779ba207bbb3f3f6be8c6bda863fc01739b186b38783a/lxml-6.1.3-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8db38ff3fb7aee7d6a82ae4da2eef1178656fe1216841fbd24870062a9d60473", size = 5229023, upload-time = "2026-09-02T14:49:52.447Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8a/e69cdaca3fd33a647942925664f01b20908d41a6968c182305be9c38fb11/lxml-6.1.3-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:25f4118c438f96bb466e83108506d03d5c31b1bd2387e83e5b070bda6ded9c37", size = 5317811, upload-time = "2026-09-02T14:49:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b2/0c397588174403c2ab68fc464abf97e03e7324f9c6cb6a99023104707195/lxml-6.1.3-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:1beb0f9909b26cee938df9ba56b15252a84429b1fc30ce6fca161390b9789a70", size = 4646516, upload-time = "2026-09-02T14:49:57.761Z" }, + { url = "https://files.pythonhosted.org/packages/56/7e/cfea25afafbe49db8b225764f7f74bb37c2a7f5e717d917d3d4a5e098ed4/lxml-6.1.3-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3a27ac6c780c8b8a1cd231b58407634cafc1c4cc28cd6c7141362df0f36351e7", size = 5240626, upload-time = "2026-09-02T14:50:00.279Z" }, + { url = "https://files.pythonhosted.org/packages/a1/75/7a587771bb52ebb0e2c57b6dbe9fd96a70fbb54d72ddd97d54c5f8ec18d5/lxml-6.1.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a1932d7ce78a561367512c594fe66eac2b2ec9b9264cfd9b5f950622f4a116e2", size = 5086619, upload-time = "2026-09-02T14:50:03.245Z" }, + { url = "https://files.pythonhosted.org/packages/1e/01/94c0ebe6d831861542d251e038052e52bf6d33f1d18f1cfffdc82851065a/lxml-6.1.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:7d0f5976aa2701996f759b30172925829867547bb073af0ae67d1307a0f0262c", size = 4758828, upload-time = "2026-09-02T14:50:05.873Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/938d67bd0e5b1fdfa52be28aefdffbad57e1f6b8e921c2aab88542c75f40/lxml-6.1.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e7ce578aa8a80910a72a8ca0bbea3baae10100827249001999726a788456d8", size = 5627083, upload-time = "2026-09-02T14:50:08.555Z" }, + { url = "https://files.pythonhosted.org/packages/d8/65/4e51522f6c214650db0abb7b16ccd11b1238b8a05a8d59aa4ebed59c9f67/lxml-6.1.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d97c5227621af74b111882a290b10f371780a38eef9d9e730408fba2259b52fb", size = 5235170, upload-time = "2026-09-02T14:50:11.255Z" }, + { url = "https://files.pythonhosted.org/packages/92/c2/e73d19365665f6b16ef84df21199befc3b06e4c539046ad2d9595f6fb9ea/lxml-6.1.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:da707f14ea3c35ee463d50acd596d6488e4b2b4ae7cf77a5bf93f55c023d63e8", size = 5252273, upload-time = "2026-09-02T14:50:13.782Z" }, + { url = "https://files.pythonhosted.org/packages/48/a9/7f386c84c9fe2854e1ca6e231c285e1c8f392971ac353c6865e6ec49faff/lxml-6.1.3-cp314-cp314t-win32.whl", hash = "sha256:9efe56a68179f3adc4de41861c9358931db03837c48dd5e1c78077b84dd07f3a", size = 3902712, upload-time = "2026-09-02T14:50:16.171Z" }, + { url = "https://files.pythonhosted.org/packages/82/a6/8a3eb793f7900ef01c7f99e6f5fcbcfbdff35251cfaef66b32a4c16352d6/lxml-6.1.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c9389b3784b56c58d933b5e0aecdf28f901b073ff385358d8a7d40907f6e14b2", size = 4400979, upload-time = "2026-09-02T14:50:18.621Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c4/3807bea283b4fe9e9d9f5dde46a73df91178472b335d2778e10b2a37aa22/lxml-6.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32a409be3190b088f960ac92bfedfbef2f86c49ff940765e1548177592d20026", size = 3823401, upload-time = "2026-09-02T14:50:21.119Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8e/4614fcd65496054cfb7172662f3576a59200278739506433b8c241ea422a/lxml-6.1.3-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:6ea2f13dce778ca072ccee598bca46a092ce192e8fd907b6c1f0e52c800529a0", size = 8609378, upload-time = "2026-09-02T14:50:31.772Z" }, + { url = "https://files.pythonhosted.org/packages/f2/51/2cdce3c65fa99a6195dd8fbd512d33407c1000ad99f63e0a285b63d7a8eb/lxml-6.1.3-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:c581b1d68b3845fb86c6b2983e755b29bf001461c59fa411d2c26a911b6559a9", size = 4640022, upload-time = "2026-09-02T14:50:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/52/09/0b30084e9eb1c546a4be3d9c56df70058d116b1a320400a59b0f7da87bf0/lxml-6.1.3-cp315-cp315-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e01125896585139453cab8cb235893644d8815d7509520da95ae3ee8d1c1f79", size = 5037928, upload-time = "2026-09-02T14:50:37.007Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0e/5c37275a3e361f6138dc06db748ea565c1fe8a5f4ee5e2ddd80047c81a89/lxml-6.1.3-cp315-cp315-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:290f66b97ede0e552e1cb44a0fd8a74f9753ee635b50830a0b122fb72788d015", size = 5661932, upload-time = "2026-09-02T14:50:39.777Z" }, + { url = "https://files.pythonhosted.org/packages/70/c5/b71ffb289b15e2642e2a3cf6d468c44da39ea119061a99e5b05e3d10f217/lxml-6.1.3-cp315-cp315-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73fc05988ed20809450474ba760a87c8ad4e455fc09783c02195e56ec634b41a", size = 5249209, upload-time = "2026-09-02T14:50:42.141Z" }, + { url = "https://files.pythonhosted.org/packages/81/ea/9910da149a23932f9301652e57661cd9e42b0df18f12be21159b7255f92b/lxml-6.1.3-cp315-cp315-manylinux_2_31_armv7l.whl", hash = "sha256:dc3a44689eea43eab836e5c98a8ab015dc2419987d1ea6eafc7c590cdff86bed", size = 4704543, upload-time = "2026-09-02T14:50:44.634Z" }, + { url = "https://files.pythonhosted.org/packages/76/07/9290329cd188c62e22021f79df04ee0cc33d9a93b0d38bd65ccd452ad9d0/lxml-6.1.3-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:209c3ccbfe35a04ac6d24f0611f9d1cbf8025d49991b14acd935236234d6c156", size = 5261298, upload-time = "2026-09-02T14:50:47.301Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0c/aba78bd3401cd99b73a0aed8e2b9b43e14be94fab3603d4bbc8a62365f2a/lxml-6.1.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2f5b2a2b9811b853b39bfa41367c6d78747b8e3e80e07fc5a24aae295c1a4d7d", size = 5090453, upload-time = "2026-09-02T14:50:49.952Z" }, + { url = "https://files.pythonhosted.org/packages/8d/dc/fa4426c3355aa0216cbeb3911495b5f65a26e0df85859a89928fe28f0396/lxml-6.1.3-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:6a406d0b3cb207b0fa460ed4dc93e866f44f105da0169361cb18ff998a44c7f0", size = 4744709, upload-time = "2026-09-02T14:50:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/be/2b/224fe7918658ab7c532ac2412f3c1eb28f71e6364fb07566262d0cc6a7b6/lxml-6.1.3-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:53258656846f5c48996b882fb4b135885e088a3ad3d96b4bc0530f95124d1f69", size = 5685802, upload-time = "2026-09-02T14:50:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/21/44/7d480819b9adcae5f84dd8ac529132c6b7a578544398225cd20321adcd91/lxml-6.1.3-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:aa633613ff907ea91b9b0489a1f0da1b8725d8c6ccec6b77e8a1c9c235044bb0", size = 5249019, upload-time = "2026-09-02T14:50:57.985Z" }, + { url = "https://files.pythonhosted.org/packages/72/83/385a267ea1b6b283f2249dd827ef360a295e9db14e13ef4665a120c60d64/lxml-6.1.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:90f709b9accab6b2e4d14f5c8718203877a0486bcb3afd74d8b539ecd1e961d4", size = 5271886, upload-time = "2026-09-02T14:51:01.667Z" }, + { url = "https://files.pythonhosted.org/packages/d8/0d/f967b0eb172ae876855a402d6d9b11fa86e3e0c89ca9bbfeadf7ffbfa719/lxml-6.1.3-cp315-cp315-win32.whl", hash = "sha256:b4fc6b03b9d9d90557274f571ab30e7fbbfc527955536935d96f98b6817a86e4", size = 3662894, upload-time = "2026-09-02T14:51:45.173Z" }, + { url = "https://files.pythonhosted.org/packages/f4/48/d8a8c4160a29e663109ad520bac2deb37fcd014756d024561e8bc3e611ec/lxml-6.1.3-cp315-cp315-win_amd64.whl", hash = "sha256:33cadd956b667997e4de1635fce9541f2e8ede2038fcde8cf55aa14d571d1bad", size = 4074626, upload-time = "2026-09-02T14:51:47.77Z" }, + { url = "https://files.pythonhosted.org/packages/25/20/3e1395d34d19f9254625d0b567b81cf70d37d3417be074f4d63b94a2be3c/lxml-6.1.3-cp315-cp315-win_arm64.whl", hash = "sha256:8a330c0ee5fa318c7b5cbbaad882baeca3f570357e7eb25ab34bf31008150758", size = 3749495, upload-time = "2026-09-02T14:51:50.663Z" }, + { url = "https://files.pythonhosted.org/packages/8f/c6/7465ffd9c43883526a382df6fa4846c9d8d419214f7effbf65270e795471/lxml-6.1.3-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:0bf5a3e397df2ec4258eb5eea4c1ac6cf013ca1abd04a176903bff20a70021fe", size = 8857677, upload-time = "2026-09-02T14:51:05.109Z" }, + { url = "https://files.pythonhosted.org/packages/ed/eb/1f3a917e299df43c8162c3e6f64fc2cea3bcf277910f35bff5b8e5d39901/lxml-6.1.3-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:13d22c0d57355366b393936acf6b98a5e0edeadddd3fccbc6a846c50a76b8741", size = 4754522, upload-time = "2026-09-02T14:51:08.137Z" }, + { url = "https://files.pythonhosted.org/packages/d7/f9/f81b4bdb6efb7a596be29603d8758154d00a5f545db9f3cef9d9041c8f64/lxml-6.1.3-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cad7617727a96d189bd6f979d0fadf765198c7934e85f4edaba9bf3ad919a300", size = 5033744, upload-time = "2026-09-02T14:51:10.633Z" }, + { url = "https://files.pythonhosted.org/packages/c8/0f/26d9bfaacb319c86e0eca8a1a0bf1130d36a7afbd318883e23caea63763d/lxml-6.1.3-cp315-cp315t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cae82b5ca24b0c2beedb269f6e2a96f466acd926879ab00ae19f1a65cbf9ffb0", size = 5615269, upload-time = "2026-09-02T14:51:13.357Z" }, + { url = "https://files.pythonhosted.org/packages/5d/90/73675f3f4141350ed65d6fec533b107d4e802c5caa340cf111771edd86e0/lxml-6.1.3-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:69cafd61aea04ebb3502c93c2aaa568b12931ca0802231e0b5de76bf8b6e74bd", size = 5236280, upload-time = "2026-09-02T14:51:16.051Z" }, + { url = "https://files.pythonhosted.org/packages/fd/be/ed260767e7977de463a0f91f3f4fffcab85c0a2a024a21ffe1fa442c2c79/lxml-6.1.3-cp315-cp315t-manylinux_2_31_armv7l.whl", hash = "sha256:dc205732d593118cf701d986f40e9de7801bb2e371cb189ddbda9b7348f4d97e", size = 4650718, upload-time = "2026-09-02T14:51:19.102Z" }, + { url = "https://files.pythonhosted.org/packages/d0/fd/e9839d03b1e767f2725cf7d7d81b80d5f3f9fdc10ad8827e2479311b046e/lxml-6.1.3-cp315-cp315t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88e719b9437f148f7e1465df845c758dd1598618cbea3a2fd1e61a715542f2b2", size = 5243376, upload-time = "2026-09-02T14:51:21.606Z" }, + { url = "https://files.pythonhosted.org/packages/34/a5/4606e347e2788c301f677004aa83e28d24da9fe663a24380122af57be6fc/lxml-6.1.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:40983eabefd13da003e68170928c7acc011f0d095eefce5871a3c71c9385fb9a", size = 5092340, upload-time = "2026-09-02T14:51:24.21Z" }, + { url = "https://files.pythonhosted.org/packages/ea/99/3314a8661cdf30f493c55a87db283961dfaae08451976a2ca418958e1804/lxml-6.1.3-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:fad67b12ffe0f71e02b4932b04883cbc76a9072bbd30731409d3523cf058b011", size = 4758768, upload-time = "2026-09-02T14:51:26.813Z" }, + { url = "https://files.pythonhosted.org/packages/30/58/3bdc577f78ea8b7d72d39a84506f7001d5b28728f43e5b84891e3b7d9a4a/lxml-6.1.3-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6cd11e7550d89e551a87dcec30f04b1fca32e86b68708aa01a4daa455d8605e5", size = 5649546, upload-time = "2026-09-02T14:51:29.453Z" }, + { url = "https://files.pythonhosted.org/packages/6a/e4/652633de1a2395949ebb7a8fc7d089aba12a2b45f0fefbc9d29e3e3ab3cf/lxml-6.1.3-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ca0ec532ad2f5ba1e5ec120ac157769c57f01855b3d8bf37213f5d88abd9ba0a", size = 5234874, upload-time = "2026-09-02T14:51:32.262Z" }, + { url = "https://files.pythonhosted.org/packages/65/a6/c4581d171de30449304b4859bbd3607e9b40da13c0f88b68e6097c8d785e/lxml-6.1.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e99e09ab7741f1281e2677f4c0058c7f5267d182530b09c87e4f6aa26adf3887", size = 5260043, upload-time = "2026-09-02T14:51:34.841Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/ed6ee6186a89e69ca4ea9658b2a278f46a5efe8b5d4db56c7197f18653fe/lxml-6.1.3-cp315-cp315t-win32.whl", hash = "sha256:ace1d2c83b2bd24db5940600541140e87a325e119cb32d5fa9ad720d7e76648e", size = 3901093, upload-time = "2026-09-02T14:51:37.234Z" }, + { url = "https://files.pythonhosted.org/packages/67/9d/11d10257a4a048d04195d638bb61f0246ce2448eb05f682bcbab25a257a8/lxml-6.1.3-cp315-cp315t-win_amd64.whl", hash = "sha256:b49638355ea3bebba70da783ccbc630fd72afa16bc46c54474bfa1f9a915bbc6", size = 4395446, upload-time = "2026-09-02T14:51:39.884Z" }, + { url = "https://files.pythonhosted.org/packages/f8/b7/44edd7de434181c582892e68d1ffe6775ca403ce14aea07cb5a218a936cf/lxml-6.1.3-cp315-cp315t-win_arm64.whl", hash = "sha256:5a721a98c649855963811b59b55755b30566e7f7fc40bdc9803d66dee9f811cf", size = 3822836, upload-time = "2026-09-02T14:51:42.471Z" }, +] + [[package]] name = "markupsafe" version = "3.0.4" @@ -590,6 +690,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/60/a3604c79cb02777b6cb84da894104021986f06c4bc41375102cc95049a32/ruff-0.17.2-py3-none-win_arm64.whl", hash = "sha256:195ad918027f817c119310ca89a42d347887dfc28934aaa869325e9c5dc9d962", size = 9827036, upload-time = "2026-10-22T18:37:58.031Z" }, ] +[[package]] +name = "soupsieve" +version = "4.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/7c/a64eedae0b7776cb9602d2da9cb8f8caae8e1b4c1da5e8e87bf1e68121ec/soupsieve-4.0.1.tar.gz", hash = "sha256:7621e3c5f6b371ed2c5eabaac5de300ac3d66d05e6d7f34783e31b63c4f90c75", size = 153302, upload-time = "2026-11-04T16:36:27.749Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/5c/4f68370847b561a389dd9a2d7eb3d19fd8cbb8acbae543a0c6b8d0d9700b/soupsieve-4.0.1-py3-none-any.whl", hash = "sha256:2e68cf7dd5a0f29d81f9d485275ccf294e6475f615e211895d91cb81f8614f04", size = 50688, upload-time = "2026-11-04T16:36:26.158Z" }, +] + [[package]] name = "typeguard" version = "4.6.0" @@ -602,6 +711,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/eb/461d5f167b6f5c7d97696f397c82f82e3480e003fce3f0a1cd1dd26e2eb2/typeguard-4.6.0-py3-none-any.whl", hash = "sha256:79878165bb86f2cf5d41d159a0ff1792a796cf496882d2fe1b1c6c7049b9cdd7", size = 36884, upload-time = "2026-07-26T08:40:21.868Z" }, ] +[[package]] +name = "types-html5lib" +version = "1.1.11.20260518" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "types-webencodings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b8/5a/0c708d1b0d35ad48b6a223c77c4a882fd016b40c25becb082a92e02a9c00/types_html5lib-1.1.11.20260518.tar.gz", hash = "sha256:4f33c087cb1119d65c4c80eca4323c2b501f9eaf8af9616b8b732ed4d8eae8fa", size = 18420, upload-time = "2026-05-18T06:07:23.662Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/d0/b088b9f11eb69637d6826843f06caaff60247156735a25512922d3dc2c13/types_html5lib-1.1.11.20260518-py3-none-any.whl", hash = "sha256:9baa7912224ebb37027c5ccb7e3768e43ea47b1dfdd977e7ddc4b0a4a550584d", size = 24339, upload-time = "2026-05-18T06:07:22.876Z" }, +] + +[[package]] +name = "types-lxml" +version = "2026.2.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "cssselect" }, + { name = "types-html5lib" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dd/ad/c70ac8cbdc28eb58a17301c69b4925af54b614e47f9b2ebc9de5cc10f786/types_lxml-2026.2.16.tar.gz", hash = "sha256:b3a1340cc06db98d541c785732f6f68bea438daff4e2b7809ef748d545d01406", size = 161204, upload-time = "2026-02-17T02:34:50.855Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/5c/03ec9befbf4bb5309bfd576c6a5ac1c75633f78f6b64cf1f594e97cd3d23/types_lxml-2026.2.16-py3-none-any.whl", hash = "sha256:5dd81ffa54830e5f361988737c5f1d6a0ae48b2742790637ec560df790ea0401", size = 97040, upload-time = "2026-02-17T02:34:49.286Z" }, +] + +[[package]] +name = "types-webencodings" +version = "0.6.0.20261104" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/63/b8/722e153333019ac643dfac24be7c8d2fa21cbea791c94c39a1c58f413211/types_webencodings-0.6.0.20261104.tar.gz", hash = "sha256:3021d7ea7011a4a51a43b2dc4c0e4856c8d9b1bb1a1a3d2500ab177224225b08", size = 7818, upload-time = "2026-11-04T08:13:58.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/3b/93a20758bf4b0a388a97672c7bd8212856bf744f6894fdaa8d2f877815cd/types_webencodings-0.6.0.20261104-py3-none-any.whl", hash = "sha256:a719903a309f35d43a99461e801df94af2def074dec2694423c7a2601ddcc45d", size = 8577, upload-time = "2026-11-04T08:13:57.726Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" diff --git a/docs/development.md b/docs/development.md index 2cbc755..c4d4bb4 100644 --- a/docs/development.md +++ b/docs/development.md @@ -64,6 +64,31 @@ crawler/ 抓取节点 - 判别模型和 LLM 的提示词放在调用它的代码旁边。输出都按固定的结构解析，解析不了的当失败处理。 - 每一轮抓取都交一份抓取记录，失败也交，写明原因；不在一轮里反复重试。各步抓不成时抛 `FetchError`（`Fetcher` 抛的）或 `StepError`（写明失败类别，比如 `http_status` 和「HTTP 403」），一轮的流程把它记进抓取记录；别的异常是 bug，照常抛出。 +### 2.1 列表配置 + +`parse/listing.py` 的 `list_items(fetcher, source)` 可作为 `Steps.list_items`。不填配置时按常见建站系统的列表结构取；特殊页面用 `ListConfig` 校验 `WorkSource.config`。例如浙大计算机学院的 `http://www.cs.zju.edu.cn/csen/27006/list.htm`： + +```json +{ + "selectors": { + "items": "//div[contains(concat(' ',normalize-space(@class),' '),' jzlb ')]", + "title": "./div[@class='btt3']/a", + "link": "./div[@class='btt3']/a/@href", + "date": "./div[@class='fbsj4']", + "next": "//a[@class='next']/@href" + }, + "force_http": true +} +``` + +- `selectors` 可省略，表示通用解析；填写时 `items`、`title`、`link` 必填。全部选择器都是 XPath 1.0：`items` 相对于整页，必须选中条目元素；`title`、`link`、`date` 相对于每个条目；`next` 相对于整页。字段取第一个匹配的字符串或元素的文字，空白归一化。日期和翻页选择器可省略，分别表示日期未知、没有下一页。 +- `force_http` 默认 `false`。只有明确只能用 HTTP 的来源才设为 `true`（可单独配置 `{"force_http": true}`）：列表请求及同站翻页使用 HTTP；不自动在 HTTPS 失败后降级，不改写外站链接。 +- 链接按请求跳转后的实际地址解析为绝对网址，去掉片段；只接受 HTTP(S) 链接，不执行脚本。日期只认完整年月日，也支持同一个日期容器里分开的日和年月；缺失或无效时为 `null`，不从标题或当前年份猜。 +- 通用翻页优先读取「下一页」或 `rel="next"`；只有页码时，从 `aria-current="page"` 标明的当前页取紧邻的下一页码，不把「尾页」当下一页，不执行拼页码的脚本。 +- 普通轮次只读第一页；`backfill` 为真时保留近两年及日期未知的条目，按北京时间的今天计算两年前的同月同日，闰日取该月最后一天。已读网址去重；一页中此前未见的条目都有日期且都早于范围时停止，重复的置顶通知不阻碍截止判断；日期未知的条目不能作为停止依据。没有下一页也停止，翻页形成循环则记为错误。 +- 未识别出列表时返回空列表，由一轮流程记成 `no_items`；任一历史页读不到条目也不交部分回补结果。非 2xx 状态抛 `StepError("http_status", "HTTP …")`；非法配置（含 XPath 语法、返回类型错误）转成 `StepError("error", "配置不对：…")`，不让校验异常造成租约反复重来。 +- 模板及替换了通知内容的页面样例和测试放在 `parse/`，按页面结构补通用能力，不按学校域名写特判。配置供来源管理及 #46 的试抓生成共用；浏览器页面的取得不在本步骤实现。 + ## 3. 时间 - 时刻存成带时区的时间戳；通知里写的"某天几点"一律按北京时间理解，时间值和换算的规则见[总体设计 §8.2](design.md#82-抽取)。 From eb1fa218acc36d194dc8c34dbf5bd8c9ef80da97 Mon Sep 17 00:00:00 2001 From: Xi1495 Date: Sat, 7 Nov 2026 22:05:04 +0900 Subject: [PATCH 2/3] =?UTF-8?q?=E6=8C=89=E9=87=8D=E5=A4=8D=E7=BB=93?= =?UTF-8?q?=E6=9E=84=E8=AF=86=E5=88=AB=E5=88=97=E8=A1=A8=E5=B9=B6=E5=88=86?= =?UTF-8?q?=E7=A6=BB=E7=BF=BB=E9=A1=B5=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crawler/src/crawler/crawl/listing.py | 65 +++++ .../crawler/parse/fixtures/listing/README.md | 8 + .../parse/fixtures/listing/date-in-link.html | 17 ++ .../parse/fixtures/listing/empty-link.html | 29 ++ .../fixtures/listing/month-day-year.html | 53 ++++ .../parse/fixtures/listing/opaque-links.html | 9 + .../parse/fixtures/listing/plain-span.html | 13 + .../parse/fixtures/listing/right-span.html | 5 + .../parse/fixtures/listing/table-rows.html | 41 +++ .../fixtures/listing/year-month-day.html | 37 +++ crawler/src/crawler/parse/listing.py | 272 ++++++++---------- crawler/src/crawler/parse/listing_test.py | 118 +++++++- docs/development.md | 17 +- 13 files changed, 505 insertions(+), 179 deletions(-) create mode 100644 crawler/src/crawler/crawl/listing.py create mode 100644 crawler/src/crawler/parse/fixtures/listing/date-in-link.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/empty-link.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/month-day-year.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/opaque-links.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/plain-span.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/right-span.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/table-rows.html create mode 100644 crawler/src/crawler/parse/fixtures/listing/year-month-day.html diff --git a/crawler/src/crawler/crawl/listing.py b/crawler/src/crawler/crawl/listing.py new file mode 100644 index 0000000..32202b1 --- /dev/null +++ b/crawler/src/crawler/crawl/listing.py @@ -0,0 +1,65 @@ +import time +from calendar import monthrange +from collections.abc import Callable +from datetime import datetime, timedelta, timezone +from urllib.parse import urlsplit, urlunsplit + +from pydantic import ValidationError + +from crawler.crawl.contract import ListedItem, WorkSource +from crawler.crawl.fetch import Fetcher +from crawler.crawl.rounds import StepError +from crawler.parse.listing import ListConfig, parse_listing + + +async def list_items( + fetcher: Fetcher, source: WorkSource, clock: Callable[[], float] = time.time +) -> list[ListedItem]: + try: + config = ListConfig.model_validate(source.config or {}) + except ValidationError as error: + location = ".".join(str(value) for value in error.errors(include_input=False)[0]["loc"]) + raise StepError("error", f"配置不对: {location}") from error + + def request_url(url: str) -> str: + parts = urlsplit(url) + if config.force_http and parts.netloc == urlsplit(source.url).netloc: + return urlunsplit(parts._replace(scheme="http")) + return url + + now = datetime.fromtimestamp(clock(), timezone(timedelta(hours=8))).date() + year = now.year - 2 + cutoff = now.replace(year=year, day=min(now.day, monthrange(year, now.month)[1])) + current = request_url(source.url) + visited: set[str] = set() + items: dict[str, ListedItem] = {} + while True: + if current in visited: + raise StepError("error", "分页链接循环") + visited.add(current) + response = await fetcher.get(current) + visited.add(response.url) + if not 200 <= response.status < 300: + raise StepError("http_status", f"HTTP {response.status}") + try: + page = parse_listing( + response.body, response.url, config.selectors, response.content_type + ) + except ValueError as error: + raise StepError("error", f"配置不对: {error}") from error + if not page.items: + break + fresh = [item for item in page.items if item.url not in items] + if ( + source.backfill + and fresh + and all(item.published_on is not None and item.published_on < cutoff for item in fresh) + ): + break + for item in page.items: + if not source.backfill or item.published_on is None or item.published_on >= cutoff: + items.setdefault(item.url, item) + if not source.backfill or page.next_url is None: + break + current = request_url(page.next_url) + return list(items.values()) diff --git a/crawler/src/crawler/parse/fixtures/listing/README.md b/crawler/src/crawler/parse/fixtures/listing/README.md index b608c5b..fe55827 100644 --- a/crawler/src/crawler/parse/fixtures/listing/README.md +++ b/crawler/src/crawler/parse/fixtures/listing/README.md @@ -12,5 +12,13 @@ | `drupal.html` | 原页明确声明 Drupal 10，Views 容器、`list-page`、`time/@datetime`；只有页码的翻页器 | [中大计算机学院硕士招生](https://cse.sysu.edu.cn/admission/master) | | `configured.html` | 独立招生发布系统，作为需要配置的页面测试，不计入前述五种通用系统 | [国科大硕士招生](https://admission.ucas.ac.cn/showarticle/newslist/1beb211c-ea0d-4c97-b5ce-61efa7addfcd) | | `zju-http.html` | WebPlus 的另一种布局，使用开发约定中的 XPath 配置和显式 HTTP，列出 `jzlb`、`btt3`、`fbsj4` | [浙大计算机招生信息](http://www.cs.zju.edu.cn/csen/27006/list.htm) | +| `opaque-links.html` | 通知链接为不带 `info/` 的字母数字文件名 | [北科大研究生院](https://yzxc.ustb.edu.cn/xlytm/hdap/index.htm) | +| `month-day-year.html` | 月日与年份分栏，标题和摘要在同一链接内 | [北科大计通学院](https://scce.ustb.edu.cn/xydt/tzgg.htm) | +| `year-month-day.html` | 年份与月日分栏 | [电子科大信通学院](https://www.sice.uestc.edu.cn/index/tzgg/yjsk.htm) | +| `table-rows.html` | 每条通知使用嵌套表格的行排版 | [东南大学研究生院](https://yzb.seu.edu.cn/6676/list.htm) | +| `date-in-link.html` | 日期在标题链接内部 | [西电人工智能学院](https://sai.xidian.edu.cn/tzgg.htm) | +| `plain-span.html` | 日期在无 class 的 `span` 内 | [西电研究生院](https://gr.xidian.edu.cn/yjsy/yjszs.htm) | +| `empty-link.html` | 通知链接前有一个空链接 | [西电网信学院](https://ce.xidian.edu.cn/sy/tzgg.htm) | +| `right-span.html` | 日期在 `right` 元素内 | [沈自](https://sia.cas.cn/zpjy/yjsjy/zs/zsgg/) | Drupal 样例用于补充不同系统的解析覆盖，不改动 `data/sources.csv`。分页边界、未知日期、重复置顶、重定向、HTTP、非法配置和失败场景使用测试里自编的页面及 `MockTransport`，不访问外网。 diff --git a/crawler/src/crawler/parse/fixtures/listing/date-in-link.html b/crawler/src/crawler/parse/fixtures/listing/date-in-link.html new file mode 100644 index 0000000..8725c90 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/date-in-link.html @@ -0,0 +1,17 @@ + diff --git a/crawler/src/crawler/parse/fixtures/listing/empty-link.html b/crawler/src/crawler/parse/fixtures/listing/empty-link.html new file mode 100644 index 0000000..f2bf067 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/empty-link.html @@ -0,0 +1,29 @@ + diff --git a/crawler/src/crawler/parse/fixtures/listing/month-day-year.html b/crawler/src/crawler/parse/fixtures/listing/month-day-year.html new file mode 100644 index 0000000..64ef01f --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/month-day-year.html @@ -0,0 +1,53 @@ + diff --git a/crawler/src/crawler/parse/fixtures/listing/opaque-links.html b/crawler/src/crawler/parse/fixtures/listing/opaque-links.html new file mode 100644 index 0000000..624e71a --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/opaque-links.html @@ -0,0 +1,9 @@ + diff --git a/crawler/src/crawler/parse/fixtures/listing/plain-span.html b/crawler/src/crawler/parse/fixtures/listing/plain-span.html new file mode 100644 index 0000000..4c2ffcc --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/plain-span.html @@ -0,0 +1,13 @@ + diff --git a/crawler/src/crawler/parse/fixtures/listing/right-span.html b/crawler/src/crawler/parse/fixtures/listing/right-span.html new file mode 100644 index 0000000..f60a9b3 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/right-span.html @@ -0,0 +1,5 @@ + diff --git a/crawler/src/crawler/parse/fixtures/listing/table-rows.html b/crawler/src/crawler/parse/fixtures/listing/table-rows.html new file mode 100644 index 0000000..7ff4dae --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/table-rows.html @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + +
    自编招生通知12026-09-10
    + + + + + + + + + + + + + + + + + + + +
    自编招生通知22026-09-9
    + + + + diff --git a/crawler/src/crawler/parse/fixtures/listing/year-month-day.html b/crawler/src/crawler/parse/fixtures/listing/year-month-day.html new file mode 100644 index 0000000..91f5f18 --- /dev/null +++ b/crawler/src/crawler/parse/fixtures/listing/year-month-day.html @@ -0,0 +1,37 @@ + diff --git a/crawler/src/crawler/parse/listing.py b/crawler/src/crawler/parse/listing.py index 923d24b..a5a43da 100644 --- a/crawler/src/crawler/parse/listing.py +++ b/crawler/src/crawler/parse/listing.py @@ -1,24 +1,14 @@ import codecs import re -from calendar import monthrange from dataclasses import dataclass -from datetime import date, datetime, timedelta, timezone -from urllib.parse import urldefrag, urljoin, urlsplit, urlunsplit +from datetime import date +from urllib.parse import urldefrag, urljoin, urlsplit from lxml import etree, html from lxml.html import HtmlElement -from pydantic import ( - BaseModel, - ConfigDict, - InstanceOf, - TypeAdapter, - ValidationError, - field_validator, -) +from pydantic import BaseModel, ConfigDict, field_validator -from crawler.crawl.contract import ListedItem, WorkSource -from crawler.crawl.fetch import Fetcher -from crawler.crawl.rounds import StepError +from crawler.crawl.contract import ListedItem class Selectors(BaseModel): @@ -48,58 +38,10 @@ class ListConfig(BaseModel): force_http: bool = False -_ELEMENTS = TypeAdapter(list[InstanceOf[HtmlElement]]) -_DATE = re.compile(r"(? list[HtmlElement]: - result: object = node.xpath(expression, smart_strings=False) - try: - return _ELEMENTS.validate_python(result) - except ValidationError as error: - raise StepError("error", "配置不对: items 必须选中条目元素") from error - - -def _value(node: HtmlElement, expression: str | None, separator: str = "") -> str: + result: list[object] | str | float | bool = node.xpath(expression, smart_strings=False) + if not isinstance(result, list): + raise ValueError("items 必须选中条目元素") + values: list[object] = result + elements: list[HtmlElement] = [] + for value in values: + if not isinstance(value, HtmlElement): + raise ValueError("items 必须选中条目元素") + elements.append(value) + return elements + + +def _value(node: HtmlElement, expression: str | None, *, href: bool = False) -> str: if expression is None: return "" - result: object = node.xpath(expression, smart_strings=False) + result: list[object] | str | float | bool = node.xpath(expression, smart_strings=False) if isinstance(result, list): - try: - values = TypeAdapter(list[InstanceOf[HtmlElement] | str]).validate_python(result) - except ValidationError as error: - raise StepError("error", "配置不对: 字段 XPath 必须返回文字或元素") from error - if not values: - return "" - value = values[0] - elif isinstance(result, str): + values: list[object] = result + if not all(isinstance(v, (HtmlElement, str)) for v in values): + raise ValueError("字段 XPath 必须返回文字或元素") + value = values[0] if values else "" + else: value = result + if isinstance(value, HtmlElement): + text = value.get("href", "") if href else " ".join(value.itertext()) + elif isinstance(value, str): + text = value else: - raise StepError("error", "配置不对: 字段 XPath 必须返回文字或元素") - text = separator.join(value.itertext()) if isinstance(value, HtmlElement) else value + raise ValueError("字段 XPath 必须返回文字或元素") return " ".join(text.split()) def _date(text: str) -> date | None: - match = _DATE.search(text) + match = _SPLIT_DATE.search(text) if match: - year, month, day = map(int, match.groups()) - elif match := _SPLIT_DATE.fullmatch(text): day, year, month = map(int, match.groups()) + elif match := _YEAR_LAST_DATE.search(text): + month, day, year = map(int, match.groups()) + elif (match := _YEAR_FIRST_DATE.search(text)) or (match := _DATE.search(text)): + year, month, day = map(int, match.groups()) else: return None try: @@ -203,83 +152,86 @@ def parse_listing( try: encoding = codecs.lookup(match[1] if match else "utf-8").name root = html.document_fromstring( - body, parser=html.HTMLParser(encoding=encoding, no_network=True) + body, parser=html.HTMLParser(encoding=encoding, no_network=True, remove_comments=True) ) except (LookupError, etree.ParserError) as error: - raise StepError("error", "列表页无法解析") from error + raise ValueError("列表页无法解析") from error try: - configured = selectors is not None - candidates = (selectors,) if selectors is not None else _TEMPLATES - rows: list[HtmlElement] = [] - for candidate in candidates: - rows = _elements(root, candidate.items) - if rows: - selectors = candidate - break - if selectors is None or not rows: - return ListPage([], None) - items: dict[str, ListedItem] = {} - for row in rows: - title = _value(row, selectors.title) - target = _url(url, _value(row, selectors.link)) - if title and target: - item = ListedItem( - url=target, - title=title, - published_on=_date(_value(row, selectors.date, " ")), - ) - items.setdefault(target, item) - following = _url(url, _value(root, selectors.next)) if configured else _next(root, url) - return ListPage(list(items.values()), following) + if selectors is not None: + items: dict[str, ListedItem] = {} + for row in _elements(root, selectors.items): + title = _value(row, selectors.title) + target = _url(url, _value(row, selectors.link, href=True)) + if title and target: + items.setdefault( + target, + ListedItem( + url=target, title=title, published_on=_date(_value(row, selectors.date)) + ), + ) + return ListPage( + list(items.values()), _url(url, _value(root, selectors.next, href=True)) + ) except etree.XPathError as error: - raise StepError("error", "配置不对: XPath 无法计算") from error - except ValidationError as error: - raise StepError("error", "列表条目或配置字段不符合结构") from error + raise ValueError("XPath 无法计算") from error + return ListPage(_generic_items(root, url), _next(root, url)) -async def list_items( - fetcher: Fetcher, source: WorkSource, *, today: date | None = None -) -> list[ListedItem]: - try: - config = ListConfig.model_validate(source.config or {}) - except ValidationError as error: - location = ".".join(str(value) for value in error.errors(include_input=False)[0]["loc"]) - raise StepError("error", f"配置不对: {location}") from error +def _row_date(row: HtmlElement) -> tuple[bool, date | None]: + for node in row.iter(): + value = " ".join(node.itertext()) + if any( + pattern.fullmatch(value.strip()) + for pattern in (_DATE, _SPLIT_DATE, _YEAR_LAST_DATE, _YEAR_FIRST_DATE) + ): + return True, _date(value) + text = " ".join(row.itertext()) + if _DATE.search(text) or _SPLIT_DATE.search(text): + return True, _date(text) + for node in row.iter(): + value = node.get("datetime", "") + if _DATE.search(value): + return True, _date(value) + return False, None + + +def _title(anchor: HtmlElement) -> str: + text = anchor.get("title", "").strip() + if not text: + for expression in ( + ".//h1 | .//h2 | .//h3 | " + ".//*[contains(concat(' ',normalize-space(@class),' '),' title ')]", + ".//*[contains(concat(' ',normalize-space(@class),' '),' text ')]", + "./p[1]", + ): + candidates = _elements(anchor, expression) + if candidates: + text = "".join(candidates[0].itertext()) + break + if not text: + text = " ".join(anchor.itertext()) + return " ".join(_DATE.sub("", text).split()).strip(" []【】()\uff08\uff09") - def request_url(url: str) -> str: - parts = urlsplit(url) - if config.force_http and parts.netloc == urlsplit(source.url).netloc: - return urlunsplit(parts._replace(scheme="http")) - return url - now = today or datetime.now(timezone(timedelta(hours=8))).date() - year = now.year - 2 - cutoff = now.replace(year=year, day=min(now.day, monthrange(year, now.month)[1])) - current = request_url(source.url) - visited: set[str] = set() - items: dict[str, ListedItem] = {} - while True: - if current in visited: - raise StepError("error", "分页链接循环") - visited.add(current) - response = await fetcher.get(current) - visited.add(response.url) - if not 200 <= response.status < 300: - raise StepError("http_status", f"HTTP {response.status}") - page = parse_listing(response.body, response.url, config.selectors, response.content_type) - if not page.items: - return [] - fresh = [item for item in page.items if item.url not in items] - if ( - source.backfill - and fresh - and all(item.published_on is not None and item.published_on < cutoff for item in fresh) - ): - break - for item in page.items: - if not source.backfill or item.published_on is None or item.published_on >= cutoff: - items.setdefault(item.url, item) - if not source.backfill or page.next_url is None: - break - current = request_url(page.next_url) - return list(items.values()) +def _generic_items(root: HtmlElement, url: str) -> list[ListedItem]: + best: list[ListedItem] = [] + best_count = 0 + for parent in root.iter(): + items: dict[str, ListedItem] = {} + dated = 0 + for row in parent: + links: dict[str, str] = {} + for anchor in row.iter("a"): + target = _url(url, anchor.get("href", "")) + title = _title(anchor) + if target and title: + links.setdefault(target, title) + if len(links) != 1: + continue + target, title = next(iter(links.items())) + has_date, published = _row_date(row) + dated += has_date + items.setdefault(target, ListedItem(url=target, title=title, published_on=published)) + if dated and dated >= best_count: + best_count, best = dated, list(items.values()) + return best diff --git a/crawler/src/crawler/parse/listing_test.py b/crawler/src/crawler/parse/listing_test.py index 8f7ceed..df1e9b2 100644 --- a/crawler/src/crawler/parse/listing_test.py +++ b/crawler/src/crawler/parse/listing_test.py @@ -1,4 +1,4 @@ -from datetime import date +from datetime import date, datetime, timedelta, timezone from pathlib import Path import httpx @@ -8,10 +8,11 @@ from crawler.crawl.contract import WorkSource from crawler.crawl.fetch import Fetcher from crawler.crawl.fetch_test import FakeSites +from crawler.crawl.listing import list_items from crawler.crawl.rounds import NoticePage, StepError, Steps, crawl_source from crawler.crawl.rounds_test import FakeWebsite, extract, screen from crawler.crawl.website import Website -from crawler.parse.listing import Selectors, list_items, parse_listing +from crawler.parse.listing import Selectors, parse_listing FIXTURES = Path(__file__).parent / "fixtures" / "listing" BASE = "https://example.edu.cn/notices.htm" @@ -104,7 +105,13 @@ def test_documented_http_configuration_reads_the_real_page_structure() -> None: ) } ) - result = sites.run(lambda f: list_items(f, source(config=config), today=TODAY)) + result = sites.run( + lambda f: list_items( + f, + source(config=config), + clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), + ) + ) assert [(item.title, item.published_on) for item in result] == [ ("示例招生通知甲", date(2026, 9, 10)), ("示例招生通知乙", date(2026, 9, 9)), @@ -135,7 +142,13 @@ def test_backfill_keeps_boundary_and_unknown_dates_and_ignores_repeated_pinned_r ), } ) - result = sites.run(lambda f: list_items(f, source(backfill=backfill), today=TODAY)) + result = sites.run( + lambda f: list_items( + f, + source(backfill=backfill), + clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), + ) + ) assert [item.title for item in result] == ( ["自编通知1", "自编通知2", "自编通知3", "自编通知4"] if backfill @@ -153,7 +166,13 @@ def test_leap_day_backfill_uses_calendar_years() -> None: ), } ) - result = sites.run(lambda f: list_items(f, source(backfill=True), today=date(2024, 2, 29))) + result = sites.run( + lambda f: list_items( + f, + source(backfill=True), + clock=lambda: datetime(2024, 2, 29, tzinfo=timezone(timedelta(hours=8))).timestamp(), + ) + ) assert [item.published_on for item in result] == [date(2022, 2, 28)] assert len(sites.requests) == 2 @@ -178,7 +197,13 @@ def test_numbered_pager_advances_one_page_and_stops_at_last() -> None: ), } ) - result = sites.run(lambda f: list_items(f, source(backfill=True), today=TODAY)) + result = sites.run( + lambda f: list_items( + f, + source(backfill=True), + clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), + ) + ) assert [item.title for item in result] == ["自编通知1", "自编通知2"] assert len(sites.requests) == 2 @@ -213,7 +238,13 @@ def test_explicit_http_config_and_redirect_base_are_used_for_pagination_and_link config: dict[str, JsonValue] = {"force_http": True} if configured: config["selectors"] = selectors - result = sites.run(lambda f: list_items(f, source(backfill=True, config=config), today=TODAY)) + result = sites.run( + lambda f: list_items( + f, + source(backfill=True, config=config), + clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), + ) + ) assert [item.url for item in result] == ["http://example.edu.cn/board/info/1001/1.htm"] assert [url for _, url in sites.requests] == list(sites.pages) @@ -259,7 +290,15 @@ def test_historical_http_failure_does_not_return_partial_backfill(status: int) - } ) with pytest.raises(StepError, match=f"HTTP {status}") as caught: - sites.run(lambda f: list_items(f, source(backfill=True), today=TODAY)) + sites.run( + lambda f: list_items( + f, + source(backfill=True), + clock=lambda: datetime( + 2026, 11, 7, tzinfo=timezone(timedelta(hours=8)) + ).timestamp(), + ) + ) assert caught.value.failure == "http_status" @@ -281,5 +320,66 @@ def test_cyclic_pagination_is_an_error_instead_of_a_partial_success() -> None: } ) with pytest.raises(StepError, match="分页链接循环"): - sites.run(lambda f: list_items(f, source(backfill=True), today=TODAY)) + sites.run( + lambda f: list_items( + f, + source(backfill=True), + clock=lambda: datetime( + 2026, 11, 7, tzinfo=timezone(timedelta(hours=8)) + ).timestamp(), + ) + ) + assert len(sites.requests) == 2 + + +@pytest.mark.parametrize( + "name", + [ + "opaque-links", + "month-day-year", + "year-month-day", + "table-rows", + "date-in-link", + "plain-span", + "empty-link", + "right-span", + ], +) +def test_repeated_structure_finds_dates_without_url_or_class_templates(name: str) -> None: + result = parse_listing((FIXTURES / f"{name}.html").read_bytes(), BASE) + assert [(item.title, item.published_on, item.url) for item in result.items] == [ + ("自编招生通知1", date(2026, 9, 10), "https://example.edu.cn/article/101.htm"), + ("自编招生通知2", date(2026, 9, 9), "https://example.edu.cn/article/102.htm"), + ] + + +def test_empty_historical_page_finishes_backfill_without_discarding_items() -> None: + sites = FakeSites( + { + BASE: httpx.Response(200, content=page([("1", "2026-09-10")], "p2")), + "https://example.edu.cn/p2": httpx.Response(200, text=""), + } + ) + result = sites.run( + lambda f: list_items( + f, + source(backfill=True), + clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), + ) + ) + assert [item.title for item in result] == ["自编通知1"] assert len(sites.requests) == 2 + + +def test_configured_links_and_next_elements_use_href_instead_of_text() -> None: + selectors = Selectors( + items="//li", title=".//a", link=".//a", date=".//span", next="//a[@rel='next']" + ) + result = parse_listing(page([("1", "2026-09-10")], "p2"), BASE, selectors) + assert result.items[0].url == "https://example.edu.cn/info/1001/1.htm" + assert result.next_url == "https://example.edu.cn/p2" + + +def test_html_comments_are_ignored_during_structure_detection() -> None: + result = parse_listing(page([("1", "2026-09-10")]) + b"", BASE) + assert len(result.items) == 1 diff --git a/docs/development.md b/docs/development.md index c4d4bb4..f6c2d26 100644 --- a/docs/development.md +++ b/docs/development.md @@ -66,28 +66,25 @@ crawler/ 抓取节点 ### 2.1 列表配置 -`parse/listing.py` 的 `list_items(fetcher, source)` 可作为 `Steps.list_items`。不填配置时按常见建站系统的列表结构取；特殊页面用 `ListConfig` 校验 `WorkSource.config`。例如浙大计算机学院的 `http://www.cs.zju.edu.cn/csen/27006/list.htm`： +特殊列表通过 `ListConfig` 校验 `WorkSource.config`，管理员与 #46 的试抓生成使用同一格式。例如只能用 HTTP 的浙大计算机学院列表： ```json { "selectors": { "items": "//div[contains(concat(' ',normalize-space(@class),' '),' jzlb ')]", "title": "./div[@class='btt3']/a", - "link": "./div[@class='btt3']/a/@href", + "link": "./div[@class='btt3']/a", "date": "./div[@class='fbsj4']", - "next": "//a[@class='next']/@href" + "next": "//a[@class='next']" }, "force_http": true } ``` -- `selectors` 可省略，表示通用解析；填写时 `items`、`title`、`link` 必填。全部选择器都是 XPath 1.0：`items` 相对于整页，必须选中条目元素；`title`、`link`、`date` 相对于每个条目；`next` 相对于整页。字段取第一个匹配的字符串或元素的文字，空白归一化。日期和翻页选择器可省略，分别表示日期未知、没有下一页。 -- `force_http` 默认 `false`。只有明确只能用 HTTP 的来源才设为 `true`（可单独配置 `{"force_http": true}`）：列表请求及同站翻页使用 HTTP；不自动在 HTTPS 失败后降级，不改写外站链接。 -- 链接按请求跳转后的实际地址解析为绝对网址，去掉片段；只接受 HTTP(S) 链接，不执行脚本。日期只认完整年月日，也支持同一个日期容器里分开的日和年月；缺失或无效时为 `null`，不从标题或当前年份猜。 -- 通用翻页优先读取「下一页」或 `rel="next"`；只有页码时，从 `aria-current="page"` 标明的当前页取紧邻的下一页码，不把「尾页」当下一页，不执行拼页码的脚本。 -- 普通轮次只读第一页；`backfill` 为真时保留近两年及日期未知的条目，按北京时间的今天计算两年前的同月同日，闰日取该月最后一天。已读网址去重；一页中此前未见的条目都有日期且都早于范围时停止，重复的置顶通知不阻碍截止判断；日期未知的条目不能作为停止依据。没有下一页也停止，翻页形成循环则记为错误。 -- 未识别出列表时返回空列表，由一轮流程记成 `no_items`；任一历史页读不到条目也不交部分回补结果。非 2xx 状态抛 `StepError("http_status", "HTTP …")`；非法配置（含 XPath 语法、返回类型错误）转成 `StepError("error", "配置不对：…")`，不让校验异常造成租约反复重来。 -- 模板及替换了通知内容的页面样例和测试放在 `parse/`，按页面结构补通用能力，不按学校域名写特判。配置供来源管理及 #46 的试抓生成共用；浏览器页面的取得不在本步骤实现。 +- `selectors` 可省略，表示按重复条目结构及完整日期通用解析；填写时 `items`、`title`、`link` 必填。 +- 选择器都是 XPath 1.0。`items` 相对于整页，选中条目元素；`title`、`link`、`date` 相对于条目；`next` 相对于整页。各字段取第一个匹配值；`title`、`date` 取字符串或元素文字，`link`、`next` 取字符串或元素的 `href`，按跳转后的页面地址解析成绝对网址。只接受 HTTP(S)，不执行脚本。 +- `date`、`next` 可省略，分别表示日期未知、没有下一页。日期只认完整年月日，也支持日与年月分开写，不猜年份。 +- `force_http` 默认 `false`，只有明确只能用 HTTP 的来源才设为 `true`；可单独配置 `{"force_http": true}`，对列表及同站翻页生效，不自动从 HTTPS 降级。 ## 3. 时间 From a7aa7995fa5b136f5fd1aa566ed1722ffb47374b Mon Sep 17 00:00:00 2001 From: Xi1495 Date: Sun, 8 Nov 2026 02:11:27 +0900 Subject: [PATCH 3/3] =?UTF-8?q?=E4=BF=9D=E7=95=99=E5=88=97=E8=A1=A8?= =?UTF-8?q?=E6=A0=87=E9=A2=98=E5=B9=B6=E6=AD=A3=E7=A1=AE=E7=BB=93=E6=9D=9F?= =?UTF-8?q?=E5=BE=AA=E7=8E=AF=E7=BF=BB=E9=A1=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crawler/src/crawler/crawl/listing.py | 4 +- crawler/src/crawler/parse/listing.py | 120 +++++++++++++++++----- crawler/src/crawler/parse/listing_test.py | 119 +++++++++++++++++---- 3 files changed, 198 insertions(+), 45 deletions(-) diff --git a/crawler/src/crawler/crawl/listing.py b/crawler/src/crawler/crawl/listing.py index 32202b1..4eb1bdb 100644 --- a/crawler/src/crawler/crawl/listing.py +++ b/crawler/src/crawler/crawl/listing.py @@ -35,7 +35,7 @@ def request_url(url: str) -> str: items: dict[str, ListedItem] = {} while True: if current in visited: - raise StepError("error", "分页链接循环") + break visited.add(current) response = await fetcher.get(current) visited.add(response.url) @@ -46,7 +46,7 @@ def request_url(url: str) -> str: response.body, response.url, config.selectors, response.content_type ) except ValueError as error: - raise StepError("error", f"配置不对: {error}") from error + raise StepError("error", str(error)) from error if not page.items: break fresh = [item for item in page.items if item.url not in items] diff --git a/crawler/src/crawler/parse/listing.py b/crawler/src/crawler/parse/listing.py index a5a43da..7696e7b 100644 --- a/crawler/src/crawler/parse/listing.py +++ b/crawler/src/crawler/parse/listing.py @@ -177,27 +177,72 @@ def parse_listing( return ListPage(_generic_items(root, url), _next(root, url)) -def _row_date(row: HtmlElement) -> tuple[bool, date | None]: - for node in row.iter(): - value = " ".join(node.itertext()) - if any( - pattern.fullmatch(value.strip()) - for pattern in (_DATE, _SPLIT_DATE, _YEAR_LAST_DATE, _YEAR_FIRST_DATE) - ): - return True, _date(value) - text = " ".join(row.itertext()) - if _DATE.search(text) or _SPLIT_DATE.search(text): - return True, _date(text) +@dataclass(frozen=True) +class Publication: + value: date | None + start: int + end: int + node: HtmlElement | None = None + + +def _row_text(row: HtmlElement) -> tuple[str, dict[HtmlElement, tuple[int, int]]]: + parts: list[str] = [] + ranges: dict[HtmlElement, tuple[int, int]] = {} + length = 0 + + def append(value: str | None) -> None: + nonlocal length + if value is not None: + parts.append(value + " ") + length += len(value) + 1 + + def visit(node: HtmlElement) -> None: + start = length + append(node.text) + for child in node: + visit(child) + append(child.tail) + ranges[node] = (start, length) + + visit(row) + return "".join(parts), ranges + + +def _row_date( + row: HtmlElement, text: str, ranges: dict[HtmlElement, tuple[int, int]] +) -> Publication | None: + patterns = (_SPLIT_DATE, _YEAR_LAST_DATE, _YEAR_FIRST_DATE, _DATE) + nodes = sorted( + row.iter(), + key=lambda node: node.tag == "a" or next(node.iterancestors("a"), None) is not None, + ) + for node in nodes: + start, end = ranges[node] + value = text[start:end] + for pattern in patterns: + if pattern.fullmatch(value.strip()) and (match := pattern.search(value)): + return Publication( + _date(match[0]), start + match.start(), start + match.end(), node + ) + for pattern in patterns: + if match := pattern.search(text): + return Publication(_date(match[0]), match.start(), match.end()) for node in row.iter(): value = node.get("datetime", "") if _DATE.search(value): - return True, _date(value) - return False, None + return Publication(_date(value), 0, 0, node) + return None -def _title(anchor: HtmlElement) -> str: +def _title( + anchor: HtmlElement, + row_text: str, + ranges: dict[HtmlElement, tuple[int, int]], + published: Publication | None, +) -> str: text = anchor.get("title", "").strip() if not text: + title_node = anchor for expression in ( ".//h1 | .//h2 | .//h3 | " ".//*[contains(concat(' ',normalize-space(@class),' '),' title ')]", @@ -206,32 +251,57 @@ def _title(anchor: HtmlElement) -> str: ): candidates = _elements(anchor, expression) if candidates: - text = "".join(candidates[0].itertext()) + title_node = candidates[0] break - if not text: - text = " ".join(anchor.itertext()) - return " ".join(_DATE.sub("", text).split()).strip(" []【】()\uff08\uff09") + start, end = ranges[title_node] + text = row_text[start:end] + if published and start <= published.start < published.end <= end: + first, last = published.start - start, published.end - start + left, right = text[:first].rstrip(), text[last:].lstrip() + if ( + left + and right + and {"[": "]", "【": "】", "(": ")", "\uff08": "\uff09"}.get(left[-1]) == right[0] + ): + first, last = len(left) - 1, len(text) - len(right) + 1 + text = text[:first] + text[last:] + return " ".join(text.split()) def _generic_items(root: HtmlElement, url: str) -> list[ListedItem]: best: list[ListedItem] = [] - best_count = 0 + best_score = (0, 0) for parent in root.iter(): items: dict[str, ListedItem] = {} dated = 0 + separate_dates = 0 for row in parent: + text, ranges = _row_text(row) + published = _row_date(row, text, ranges) links: dict[str, str] = {} for anchor in row.iter("a"): target = _url(url, anchor.get("href", "")) - title = _title(anchor) + title = _title(anchor, text, ranges, published) if target and title: links.setdefault(target, title) if len(links) != 1: continue target, title = next(iter(links.items())) - has_date, published = _row_date(row) - dated += has_date - items.setdefault(target, ListedItem(url=target, title=title, published_on=published)) - if dated and dated >= best_count: - best_count, best = dated, list(items.values()) + dated += published is not None + separate_dates += ( + published is not None + and published.node is not None + and published.node.tag != "a" + and next(published.node.iterancestors("a"), None) is None + ) + items.setdefault( + target, + ListedItem( + url=target, title=title, published_on=published.value if published else None + ), + ) + # A date outside the link identifies publication better than a date inside a title. + score = (dated, separate_dates) + if dated and score >= best_score: + best_score, best = score, list(items.values()) return best diff --git a/crawler/src/crawler/parse/listing_test.py b/crawler/src/crawler/parse/listing_test.py index df1e9b2..ba5da48 100644 --- a/crawler/src/crawler/parse/listing_test.py +++ b/crawler/src/crawler/parse/listing_test.py @@ -250,16 +250,27 @@ def test_explicit_http_config_and_redirect_base_are_used_for_pagination_and_link @pytest.mark.parametrize( - "config", + ("config", "detail"), [ - {"unknown": True}, - {"force_http": "true"}, - {"selectors": {"items": "[", "title": ".//a", "link": ".//a/@href"}}, - {"selectors": {"items": "string(//li)", "title": ".//a", "link": ".//a/@href"}}, - {"selectors": {"items": "//li", "title": ".//missing:title", "link": ".//a/@href"}}, + ({"unknown": True}, "配置不对: unknown"), + ({"force_http": "true"}, "配置不对: force_http"), + ( + {"selectors": {"items": "[", "title": ".//a", "link": ".//a/@href"}}, + "配置不对: selectors.items", + ), + ( + {"selectors": {"items": "string(//li)", "title": ".//a", "link": ".//a/@href"}}, + "items 必须选中条目元素", + ), + ( + {"selectors": {"items": "//li", "title": ".//missing:title", "link": ".//a/@href"}}, + "XPath 无法计算", + ), ], ) -def test_bad_configuration_is_recorded_by_the_round(config: dict[str, JsonValue]) -> None: +def test_bad_configuration_is_recorded_by_the_round( + config: dict[str, JsonValue], detail: str +) -> None: sites = FakeSites({BASE: httpx.Response(200, content=page([("1", "2026-09-10")]))}) website = FakeWebsite([], frozenset()) @@ -277,7 +288,7 @@ async def run(fetcher: Fetcher) -> None: sites.run(run) [record] = website.called("POST /runs") assert record["failure"] == "error" - assert record["detail"].startswith("配置不对") + assert record["detail"] == detail assert website.called("POST /sources/7/listing") == [] @@ -310,7 +321,7 @@ def test_unknown_and_invalid_dates_do_not_invent_publication_dates() -> None: assert parse_listing(b"", BASE).items == [] -def test_cyclic_pagination_is_an_error_instead_of_a_partial_success() -> None: +def test_cyclic_pagination_finishes_backfill_without_discarding_items() -> None: sites = FakeSites( { BASE: httpx.Response(200, content=page([("1", "2026-09-10")], "p2")), @@ -319,16 +330,14 @@ def test_cyclic_pagination_is_an_error_instead_of_a_partial_success() -> None: ), } ) - with pytest.raises(StepError, match="分页链接循环"): - sites.run( - lambda f: list_items( - f, - source(backfill=True), - clock=lambda: datetime( - 2026, 11, 7, tzinfo=timezone(timedelta(hours=8)) - ).timestamp(), - ) + result = sites.run( + lambda f: list_items( + f, + source(backfill=True), + clock=lambda: datetime(2026, 11, 7, tzinfo=timezone(timedelta(hours=8))).timestamp(), ) + ) + assert [item.title for item in result] == ["自编通知1", "自编通知2"] assert len(sites.requests) == 2 @@ -383,3 +392,77 @@ def test_configured_links_and_next_elements_use_href_instead_of_text() -> None: def test_html_comments_are_ignored_during_structure_detection() -> None: result = parse_listing(page([("1", "2026-09-10")]) + b"", BASE) assert len(result.items) == 1 + + +@pytest.mark.parametrize( + ("row", "title"), + [ + ( + '【博士】自编招生通知\uff08第二批\uff092026-09-10', + "【博士】自编招生通知\uff08第二批\uff09", + ), + ( + '关于2026年9月10日开展夏令营的通知2026-09-09', + "关于2026年9月10日开展夏令营的通知", + ), + ( + '关于2026年9月10日开展夏令营的通知' + "2026-09-09", + "关于 2026年9月10日 开展夏令营的通知", + ), + ( + '通知' + "2026-09-10", + "关于2026年9月10日开展夏令营的通知", + ), + ( + '09.10 2026【博士】通知甲\uff08第二批\uff09', + "【博士】通知甲\uff08第二批\uff09", + ), + ('102026-09通知甲', "通知甲"), + ( + '[2026-09-10]【博士】通知甲\uff08第二批\uff09', + "【博士】通知甲\uff08第二批\uff09", + ), + ( + '[2026-09-10]关于2026年9月11日开展夏令营的通知', + "关于2026年9月11日开展夏令营的通知", + ), + ('【2026-09-10】通知甲\uff08第二批\uff09', "通知甲\uff08第二批\uff09"), + ], +) +def test_title_removes_only_the_publication_date_and_its_enclosing_brackets( + row: str, title: str +) -> None: + result = parse_listing(f'
    • {row}
    '.encode(), BASE) + [item] = result.items + assert item.title == title + assert item.published_on == (date(2026, 9, 9) if "2026-09-09" in row else date(2026, 9, 10)) + + +def test_page_parse_failure_is_not_reported_as_a_configuration_error() -> None: + sites = FakeSites( + { + BASE: httpx.Response( + 200, + content=page([("1", "2026-09-10")]), + headers={"content-type": "text/html; charset=unknown-charset"}, + ) + } + ) + with pytest.raises(StepError) as caught: + sites.run(lambda f: list_items(f, source())) + assert caught.value.detail == "列表页无法解析" + + +def test_repeated_links_can_be_the_list_rows() -> None: + body = ( + '
    ' + '2026-09-10通知甲' + '2026-09-09通知乙
    ' + ) + result = parse_listing(body.encode(), BASE) + assert [(item.title, item.published_on) for item in result.items] == [ + ("通知甲", date(2026, 9, 10)), + ("通知乙", date(2026, 9, 9)), + ]