diff --git a/.github/workflows/pgdg-cve-scraper.yml b/.github/workflows/pgdg-cve-scraper.yml new file mode 100644 index 0000000..97b64bb --- /dev/null +++ b/.github/workflows/pgdg-cve-scraper.yml @@ -0,0 +1,110 @@ +name: pgdg-cve-scraper + +# PGDG ships security updates on Tuesdays (every ~4 weeks). This runs +# weekly Monday evenings UTC and opens a PR with newly-disclosed CVEs +# so a human can review + merge. Manual trigger via workflow_dispatch. + +on: + schedule: + # Mondays 22:00 UTC. Picked so we land after PGDG's Tuesday-batch of + # fixes (i.e., the *next* Monday after a PGDG release). + - cron: '0 22 * * 1' + workflow_dispatch: + # Note: deliberately no `pull_request:` trigger. The default GITHUB_TOKEN + # on PR events is read-only, so this workflow's "open a new PR" step + # would fail with "GitHub Actions is not permitted to create or approve + # pull requests." If you want a PR-validation pass, add a separate + # workflow that just runs the scraper in --check mode. + +concurrency: + group: pgdg-cve-scraper + cancel-in-progress: false + +permissions: {} + +jobs: + scrape: + permissions: + contents: write + pull-requests: write + name: Scrape PGDG security index + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: "3.11" + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Sync deps + run: uv sync --quiet + + - name: Run scraper (CI mode) + # --check exit codes: + # 0 = no proposed additions + # 1 = proposed additions, written to stdout (this is what we want + # to open a PR for) + # 2+ = hard failure (parse error, network error, missing + # sentinel, etc.). Propagate as a workflow failure WITHOUT + # writing the (likely empty or partial) stdout, so the + # resulting commit doesn't carry a misleading file. + id: scrape + run: | + set +e + PROPOSED=$(uv run python tools/scrape_pgdg.py --check 2>/dev/null) + rc=$? + set -e + echo "scraper exit code: $rc" + if [ $rc -eq 1 ]; then + echo "needs_pr=true" >> "$GITHUB_OUTPUT" + echo "$PROPOSED" > proposed-cves.json + elif [ $rc -eq 0 ]; then + echo "needs_pr=false" >> "$GITHUB_OUTPUT" + else + echo "scraper: hard failure (exit code $rc); not opening a PR" >&2 + exit "$rc" + fi + + - name: Open PR with proposed additions + if: steps.scrape.outputs.needs_pr == 'true' + uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0 + with: + # Explicit base is required when the push event delivers a merge + # commit (detached HEAD), e.g. after merging main into the source + # branch on a feature PR. Default would otherwise fail with + # "the 'base' input must be supplied." + base: ${{ github.event.repository.default_branch }} + branch: chore/pgdg-cve-scrape + title: "chore: add newly disclosed PostgreSQL CVEs" + body: | + Automated PGDG security-index scrape. + + The scraper (`tools/scrape_pgdg.py`) ran against + https://www.postgresql.org/support/security/?cve=title and + found CVE entries newer than what's in `data/cves.json`. The + proposed additions are in `proposed-cves.json` below. + + **Action items for a human:** + - [ ] Eyeball each entry: confirm it actually affects + PostgreSQL 15-18. + - [ ] Sanity-check summaries (one-line, technical). + - [ ] Confirm CVSS thresholds (tool defaults to >= 7.0). + - [ ] Merge the JSON into `data/cves.json`: + ```bash + uv run python tools/scrape_pgdg.py --write --yes + uv run python tools/generate_cve_sql.py + git add data/cves.json pgFirstAid.sql view_pgFirstAid.sql view_pgFirstAid_managed.sql + git commit -m "chore: refresh CVE catalog" + ``` + - [ ] Delete `proposed-cves.json` before merge. + + scraper exit code: ${{ steps.scrape.outputs.needs_pr }} (non-zero = additions pending) + add-paths: proposed-cves.json + commit-message: "chore: scrape PGDG for new CVEs" + delete-branch: true + labels: automated,security diff --git a/.github/workflows/release-notes-scout.yml b/.github/workflows/release-notes-scout.yml new file mode 100644 index 0000000..71d5c11 --- /dev/null +++ b/.github/workflows/release-notes-scout.yml @@ -0,0 +1,117 @@ +name: release-notes-scout + +# Weekly Monday evening UTC scout of PostgreSQL release notes. +# Proposes high/medium severity bug fixes that match data/known_bugs.json +# entries missing from the curated list. A PR is opened for human review; +# never auto-merges. + +on: + schedule: + - cron: '0 22 * * 1' # Mondays 22:00 UTC, after PGDG's Tuesday batch. + workflow_dispatch: + # Note: deliberately no `pull_request:` trigger. The default GITHUB_TOKEN + # on PR events is read-only, so this workflow's "open a new PR" step + # would fail with "GitHub Actions is not permitted to create or approve + # pull requests." If you want a PR-validation pass, add a separate + # workflow that just runs the scout in --check mode. + +concurrency: + group: release-notes-scout + cancel-in-progress: false + +permissions: {} + +jobs: + scout: + permissions: + contents: write + pull-requests: write + name: Scout release-notes for high-severity bug fixes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 + with: + python-version: "3.11" + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + - name: Sync deps + run: uv sync --quiet + + - name: Run scout (CI mode) + # --check exit codes: + # 0 = no proposed additions + # 1 = proposed additions, written to stdout (this is what we want + # to open a PR for) + # 2+ = hard failure (parse error, network error, missing + # sentinel, etc.). Propagate as a workflow failure WITHOUT + # writing the (likely empty or partial) stdout, so the + # resulting commit doesn't carry a misleading file. + id: scout + run: | + set +e + PROPOSED=$(uv run python tools/scrape_release_notes.py --check 2>/dev/null) + rc=$? + set -e + echo "scout exit code: $rc" + if [ $rc -eq 1 ]; then + echo "needs_pr=true" >> "$GITHUB_OUTPUT" + echo "$PROPOSED" > proposed-bugs.json + elif [ $rc -eq 0 ]; then + echo "needs_pr=false" >> "$GITHUB_OUTPUT" + else + echo "scout: hard failure (exit code $rc); not opening a PR" >&2 + exit "$rc" + fi + + - name: Open PR with proposed bug entries + if: steps.scout.outputs.needs_pr == 'true' + uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6.1.0 + with: + # Explicit base is required when the push event delivers a merge + # commit (detached HEAD), e.g. after merging main into the source + # branch on a feature PR. Default would otherwise fail with + # "the 'base' input must be supplied." + base: ${{ github.event.repository.default_branch }} + branch: chore/release-notes-scout + title: "chore: add notable PostgreSQL bug fixes" + body: | + Automated PostgreSQL release-notes scout. + + `tools/scrape_release_notes.py` fetched the latest ~2 minor + release notes per major in {15,16,17,18}, filtered the + `
  • ` entries by severity keywords, deduped + against `data/known_bugs.json`, and surfaced the top 5 per + major for review. The candidates are in `proposed-bugs.json`. + + **Action items for a human:** + - [ ] Decide which entries earn a permanent seat. The list + is biased toward data-integrity / crash / replication + bugs; doc-only or trivial entries should be dropped. + - [ ] Rename curator IDs if needed (e.g. `PG17-a1b2c3d4e` + → `PG17-MERGE-RACE-CONDITION-01`) before merging. + - [ ] Merge selected entries into `data/known_bugs.json`: + ```bash + uv run python tools/scrape_release_notes.py --write --yes + uv run python tools/generate_cve_sql.py + git add data/known_bugs.json pgFirstAid.sql view_pgFirstAid.sql view_pgFirstAid_managed.sql + git commit -m "chore: refresh known-bugs catalog" + ``` + - [ ] Delete `proposed-bugs.json` before merge. + + Notes on the keyword filter (`HIGH`/`MEDIUM` lists live in + `tools/scrape_release_notes.py`): the curated list is small, + intent is curated quality over recall, and silent false + negatives are preferred over noisy false positives. Tune the + keyword lists if the proposals get noisy or thin. + + scout exit code: ${{ steps.scout.outputs.needs_pr }} (non-zero = proposals pending) + add-paths: proposed-bugs.json + commit-message: "chore: scout release notes for new bug fixes" + delete-branch: true + labels: automated diff --git a/tools/README.md b/tools/README.md index 16e4095..df69366 100644 --- a/tools/README.md +++ b/tools/README.md @@ -4,23 +4,34 @@ Source-of-truth data + generator for the inline CVE and known-bug VALUES rows th ## What's here -``` +```text data/ cves.json curated CVE entries (one source of truth) known_bugs.json curated non-CVE release-note bugs tools/ generate_cve_sql.py reads JSON → writes VALUES rows into the SQL files + scrape_pgdg.py fetches PGDG security index, proposes new CVEs + scrape_release_notes.py fetches PGDG release notes, proposes notable bugs tests/ test_generate_cve_sql.py unit tests for the generator + test_scrape_pgdg.py PGDG scraper + filter tests with fixtures + test_scrape_release_notes.py release-notes scraper + filter tests with fixtures + fixtures/ + pgdg_index_synthetic.html hand-written two-CVE fixture + pgdg_index_realistic.html snapshot of the live PGDG security index + release_notes_synthetic.html hand-written six-listitem fixture + release_index_realistic.html snapshot of the live release-notes index .github/workflows/ cve-data-sync.yml CI drift check: fails if SQL files are out of sync with JSON + pgdg-cve-scraper.yml Weekly PGDG security scrape → opens a PR + release-notes-scout.yml Weekly PGDG release-notes scout → opens a PR ``` ## How to add a CVE Edit `data/cves.json` and add an entry: -```json +```text { "cve_id": "CVE-2027-NNNNN", "cvss": 7.5, @@ -61,7 +72,7 @@ Each bug entry covers exactly one major (`issue_id` starts with `PG`). Ru ## Manual flags -``` +```bash python tools/generate_cve_sql.py # write fresh rows into all SQL files python tools/generate_cve_sql.py --check # exit non-zero if any file is stale (CI mode) ``` diff --git a/tools/scrape_pgdg.py b/tools/scrape_pgdg.py new file mode 100644 index 0000000..58290c2 --- /dev/null +++ b/tools/scrape_pgdg.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +"""Scrape the PostgreSQL security index and propose new CVE entries for data/cves.json. + +Source of truth for the live CVE catalog: + https://www.postgresql.org/support/security/?cve=title + +Workflow: + Default --propose Fetch the index, parse out CVEs, print the proposed + additions vs. data/cves.json as a JSON snippet. No + files are modified. + --check CI mode: exit non-zero when any additions are + pending (the GitHub Action uses this to decide + whether to open a PR). + --write --yes Patch data/cves.json with the proposed additions. + Use --yes to skip the confirmation prompt. + +Inclusion rule (project policy): + - CVE must affect at least one of PG 15, 16, 17, 18 (--majors 15,16,17,18). + - CVSS v3 base score must be >= 7.0 (--min-cvss 7.0). + - Skip CVEs already in data/cves.json. + +No external deps: uses urllib.request + stdlib re. The HTML is small and +table-uniform, so we regex over the rendered page rather than depend on +BeautifulSoup. The HTML structure can shift if PGDG redesigns: when that +happens, the unit-test fixtures will catch it and we'll tighten the regex. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import urllib.request +from pathlib import Path + +PGDG_URL = "https://www.postgresql.org/support/security/?cve=title" +PGDG_BASE = "https://www.postgresql.org" + +REPO_ROOT = Path(__file__).resolve().parent.parent +CVES_JSON = REPO_ROOT / "data" / "cves.json" + +DEFAULT_MIN_CVSS = 7.0 +DEFAULT_MAJORS = ["15", "16", "17", "18"] +DEFAULT_USER_AGENT = "pgFirstAid-scraper/0.1 (+https://github.com/bringnow/pgFirstAid)" + + +# ---------- HTML helper ---------- + +def _strip_html(s: str) -> str: + """Drop tags, decode common entities, collapse whitespace.""" + s = re.sub(r"<[^>]+>", "", s) + s = (s.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace(""", '"') + .replace("'", "'") + .replace(" ", " ")) + return re.sub(r"\s+", " ", s).strip() + + +# ---------- Parsing ---------- + +# PGDG index page renders the CVE catalog as the first with a / pair. We capture the table body and +# pull each ; each row has exactly 5 (.*?)", re.DOTALL) +_TD_RE = re.compile(r"]*>(.*?)", re.DOTALL) + +_CVE_LINK_RE = re.compile(r'href="(/support/security/(CVE-\d{4}-\d+)/)"') +_CVSS_RE = re.compile( + r'href="https://nvd\.nist\.gov/vuln-metrics[^"]*"[^>]*>([0-9.]+)' +) +_VERSION_RE = re.compile(r"^(\d+)\.(\d+)$") + + +def parse_pgdg_table(html_text: str, *, majors: set[str] | None = None) -> list[dict]: + """Parse the PGDG security index and return one dict per CVE entry. + + If `majors` is provided, only entries whose `fixed_in` map contains at + least one of those majors are kept. Without it, the function preserves + every entry whose `fixed_in` has any major. The caller is expected to + apply any severity / major-scope filter separately. + + Returned dict shape: + { + "cve_id": "CVE-YYYY-NNNN", + "cvss": float, + "summary": str, # one-line description from the index + "doc_link": "https://www.postgresql.org/support/security/CVE-YYYY-NNNN/", + "fixed_in": {"15": 19, "16": 15, "17": 11, "18": 5}, # subset for our majors + "affected": ["15","16","17","18"], # raw major list (debug) + "fixed_raw": ["15.19","16.15",...], # raw version list (debug) + } + """ + m = _TABLE_BODY_RE.search(html_text) + if not m: + raise RuntimeError("PGDG index table not found; markup may have changed") + body = m.group(1) + + entries: list[dict] = [] + for row_match in _ROW_RE.finditer(body): + cells = _TD_RE.findall(row_match.group(1)) + if len(cells) != 5: + continue # skip thead/sub-header rows / malformed rows + + # ----- Cell 0: Reference ----- + cve_m = _CVE_LINK_RE.search(cells[0]) + if not cve_m: + continue + cve_id = cve_m.group(2) + doc_link = PGDG_BASE + cve_m.group(1) + + # ----- Cell 1: Affected majors ----- + affected_str = _strip_html(cells[1]) + affected_majors = [m.strip() for m in affected_str.split(",") if m.strip()] + if not affected_majors: + continue + + # ----- Cell 2: Fixed versions (positional to Affected) ----- + fixed_str = _strip_html(cells[2]) + fixed_versions = [m.strip() for m in fixed_str.split(",") if m.strip()] + if len(fixed_versions) != len(affected_majors): + # Out-of-sync table; bail on this row rather than mis-map. + continue + + fixed_in: dict[str, int] = {} + for major, version in zip(affected_majors, fixed_versions): + if majors is not None and major not in majors: + continue # skip majors outside the caller's scope + vm = _VERSION_RE.match(version) + if not vm or int(vm.group(1)) != int(major): + continue # malformed "X.Y" or major/version mismatch in upstream table + fixed_in[major] = int(vm.group(2)) + + if not fixed_in: + continue # no impact on PG 15-18 + + # ----- Cell 3: Component + CVSS score ----- + cvss_m = _CVSS_RE.search(cells[3]) + if not cvss_m: + continue + try: + cvss = float(cvss_m.group(1)) + except ValueError: + continue + + # ----- Cell 4: Description ----- + summary = _strip_html(cells[4]) + summary = re.sub(r"\s*more details\s*$", "", summary, flags=re.IGNORECASE) + if not summary: + continue + + entries.append({ + "cve_id": cve_id, + "cvss": cvss, + "summary": summary, + "doc_link": doc_link, + "fixed_in": fixed_in, + # Internal fields for downstream tooling (the writer strips these). + "_affected": affected_majors, + "_fixed_raw": fixed_versions, + }) + return entries + + +# ---------- Filtering ---------- + +def filter_proposed( + entries: list[dict], + existing: list[dict], + *, + min_cvss: float, + majors: list[str], +) -> list[dict]: + """Return entries that should be ADDED to data/cves.json. + + Rule: must affect at least one of `majors`, must be at least `min_cvss`, + must not already be present by cve_id. + """ + majors_set = set(majors) + existing_ids = {row["cve_id"] for row in existing} + additions: list[dict] = [] + for entry in entries: + if entry["cve_id"] in existing_ids: + continue # already curated + if entry["cvss"] < min_cvss: + continue # below severity threshold + if not (set(entry["fixed_in"]) & majors_set): + continue # doesn't affect any of our majors + additions.append(entry) + # Highest CVSS first; deterministic tiebreaker on id. + additions.sort(key=lambda e: (-e["cvss"], e["cve_id"])) + return additions + + +def to_json_format(proposed: list[dict]) -> list[dict]: + """Strip internal fields; return only the keys the JSON schema uses.""" + return [ + { + "cve_id": p["cve_id"], + "cvss": p["cvss"], + "summary": p["summary"], + "doc_link": p["doc_link"], + "fixed_in": p["fixed_in"], + } + for p in proposed + ] + + +# ---------- File IO ---------- + +def load_existing() -> list[dict]: + if not CVES_JSON.exists(): + return [] + return json.loads(CVES_JSON.read_text()).get("cves", []) + + +def merge_into_data(proposed: list[dict]) -> None: + """Patch data/cves.json in place with the proposed additions.""" + doc = json.loads(CVES_JSON.read_text()) + doc.setdefault("cves", []) + by_id = {row["cve_id"]: row for row in doc["cves"]} + for row in to_json_format(proposed): + if row["cve_id"] in by_id: + continue + doc["cves"].append(row) + # Bump last_reviewed. + from datetime import date + doc["last_reviewed"] = date.today().isoformat() + # Re-sort: by CVSS desc, then cve_id, for stable diffs. + doc["cves"].sort(key=lambda r: (-float(r["cvss"]), r["cve_id"])) + CVES_JSON.write_text(json.dumps(doc, indent=2, ensure_ascii=False) + "\n") + + +# ---------- HTTP ---------- + +def _resolve_cafile() -> str | None: + """Find a usable CA bundle, or None to fall back to system default. + + Some Python builds (notably Python 3.14 on Nix) ship with ssl.get_default_verify_paths() + pointing at /etc/ssl/cert.pem that doesn't exist, which makes urllib + fail with CERTIFICATE_VERIFY_FAILED. We try a few well-known paths. + """ + import ssl + import os + candidates = [ + os.environ.get("SSL_CERT_FILE"), + os.environ.get("CURL_CA_BUNDLE"), + "/etc/ssl/certs/ca-certificates.crt", # Debian/Ubuntu/GitHub Actions + "/etc/ssl/certs/ca-bundle.crt", # RHEL/CentOS + "/etc/pki/tls/certs/ca-bundle.crt", # older RHEL + "/etc/ssl/cert.pem", # macOS / some Python distros + ] + for path in candidates: + if path and os.path.exists(path): + return path + # Fall back to whatever Python's ssl layer thinks is correct. + default = ssl.get_default_verify_paths() + return default.cafile or (default.capath if os.path.isdir(default.capath or "") else None) + + +def fetch_pgdg_index(url: str = PGDG_URL, user_agent: str = DEFAULT_USER_AGENT, timeout: float = 30.0) -> str: + import ssl + ctx = ssl.create_default_context() + cafile = _resolve_cafile() + if cafile and os.path.isfile(cafile): + ctx.load_verify_locations(cafile) + elif cafile and os.path.isdir(cafile): + ctx.load_verify_locations(capath=cafile) + req = urllib.request.Request( + url, + headers={"User-Agent": user_agent, "Accept": "text/html,application/xhtml+xml"}, + ) + with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp: + return resp.read().decode("utf-8", "replace") + + +# ---------- CLI ---------- + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + mode = p.add_mutually_exclusive_group() + mode.add_argument("--propose", action="store_true", default=True, + help="print proposed additions as JSON (default)") + mode.add_argument("--check", action="store_true", + help="CI mode: exit non-zero when additions are pending") + mode.add_argument("--write", action="store_true", + help="patch data/cves.json with proposed additions (use with --yes)") + p.add_argument("--yes", action="store_true", + help="skip confirmation prompt for --write") + p.add_argument("--min-cvss", type=float, default=DEFAULT_MIN_CVSS, + help=f"include CVEs at or above this CVSS v3 score (default {DEFAULT_MIN_CVSS})") + p.add_argument("--majors", default=",".join(DEFAULT_MAJORS), + help=f"comma-separated PG major versions to consider (default {','.join(DEFAULT_MAJORS)})") + p.add_argument("--url", default=PGDG_URL, help="PGDG URL (override for testing)") + p.add_argument("--user-agent", default=DEFAULT_USER_AGENT, + help="User-Agent header for the HTTP request") + args = p.parse_args(argv) + + majors = [m.strip() for m in args.majors.split(",") if m.strip()] + + # ---- Fetch + parse ---- + try: + html = fetch_pgdg_index(url=args.url, user_agent=args.user_agent) + except Exception as exc: + print(f"scraper: HTTP fetch failed: {exc}", file=sys.stderr) + return 2 + + try: + entries = parse_pgdg_table(html, majors=set(majors)) + except RuntimeError as exc: + print(f"scraper: parse failed: {exc}", file=sys.stderr) + return 2 + + # ---- Filter + propose ---- + existing = load_existing() + proposed = filter_proposed(entries, existing, min_cvss=args.min_cvss, majors=majors) + cleaned = to_json_format(proposed) + + # ---- Mode dispatch ---- + if args.check: + if cleaned: + print(f"FAIL: {len(cleaned)} new CVE(s) pending for review " + f"(see data/cves.json and propose output)", file=sys.stderr) + print(json.dumps(cleaned, indent=2, ensure_ascii=False)) + return 1 + print("ok: no new CVEs pending") + return 0 + + if args.write: + if not cleaned: + print("scraper: --write invoked but no new CVEs to add", file=sys.stderr) + return 0 + if not args.yes: + print("scraper: --write requires --yes to confirm overwrite of data/cves.json", file=sys.stderr) + return 2 + merge_into_data(proposed) + print(f"scraper: wrote {len(cleaned)} entry/entries into {CVES_JSON.relative_to(REPO_ROOT)}") + return 0 + + # Default: print proposed as JSON + print(json.dumps(cleaned, indent=2, ensure_ascii=False)) + if cleaned: + print(f"\n# ({len(cleaned)} proposed addition(s); threshold CVSS >= {args.min_cvss}, majors {majors})", + file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/scrape_release_notes.py b/tools/scrape_release_notes.py new file mode 100644 index 0000000..e63efa1 --- /dev/null +++ b/tools/scrape_release_notes.py @@ -0,0 +1,482 @@ +#!/usr/bin/env python3 +"""Scrape PostgreSQL release notes and propose notable bug fixes. + +Source of truth for the live catalog: + https://www.postgresql.org/docs/release/ (release index, lists all minors) + https://www.postgresql.org/docs/release/X.Y/ (per-minor change list) + +Workflow: + Default --propose Fetch latest releases, extract bug-fix listitems, classify + severity, dedupe against data/known_bugs.json, print the + top-N proposals as JSON. No files are modified. + --check CI mode: exit non-zero when proposals are non-empty. + --write --yes Patch data/known_bugs.json with the proposals. + +Inclusion rule (project policy, matches the prior conversation): + - Limit to the last `revisions-back` releases per major in {15,16,17,18}. + - Each proposal needs a stable identifier. We use the upstream commit hash + when available (`https://postgr.es/c/`); otherwise a content hash. + - Dedupe against data/known_bugs.json by the proposed issue_id. + - Output is sorted by severity (highest first), then by major, then by id. + +Caveats: the release-notes catalog is fundamentally noisier than the CVE +catalog: each minor has ~50-200 listitems. We rely on keyword severity +filtering plus a per-major top-N cap to keep the proposal list reviewable. + +No external deps: urllib.request + stdlib re. Same SSL-CA resolution as +tools/scrape_pgdg.py. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +import urllib.request +from pathlib import Path +from typing import Any + +PGDG_RELEASE_INDEX = "https://www.postgresql.org/docs/release/" +PGDG_DOCS_BASE = "https://www.postgresql.org/docs" + +REPO_ROOT = Path(__file__).resolve().parent.parent +BUGS_JSON = REPO_ROOT / "data" / "known_bugs.json" + +DEFAULT_MAJORS = ["15", "16", "17", "18"] +DEFAULT_REVISIONS_BACK = 2 # how many recent minors per major to scan +DEFAULT_TOP_PER_MAJOR = 5 # output cap per major +DEFAULT_USER_AGENT = "pgFirstAid-scout/0.1 (+https://github.com/bringnow/pgFirstAid)" + + +# ---------- Severity keywords ---------- + +# Each listitem's first

    summarizes the fix. These keyword lists classify. +# "high" = data integrity / crash / wrong results. +# "medium" = replication / vacuum / partition / pl/* / catalog / planner. +# "low" = everything else. (Default omits low unless --include-low.) + +HIGH_KEYWORDS = ( + "data corruption", + "data loss", + "wrong results", + "incorrect results", + "crash", + "panicked", + "panic", + "server crash", + "stack overflow", + "memory corruption", + "memory-safety", + "use-after-free", + "buffer overrun", + "buffer overflow", + "integer overflow", + "integer underflow", + "diverg", + "divergence", + "silent", + "leaks data", + "data leak", + "discloses", + "leaks memory", + "memory leak", + "could result in", +) + +MEDIUM_KEYWORDS = ( + "replication", + "replica", + "standby", + "failover", + "subtransact", + "subtransaction", + "partition", + "vacuum", + "analyze", + "autovacuum", + "trigger", + "foreign key", + "foreign keys", + "alter table", + "catalog", + "pl/pgsql", + "pl/perl", + "pl/python", + "pl/tcl", + "extension", + "index", + "btree", + "gin", + "gist", + "brin", + "hash index", + "sequence", + "serial", + "identity column", + "subquery", + "join order", + "planner", + "statistics", + "pg_upgrade", + "pg_dump", + "pg_restore", + "wal", + "checkpoint", + "logical replication", + "subscription", + "publication", + "deadlock", + "lock", + "parallel", + "cte", + "recursive", + "upsert", + "merge", + "json", + "jsonb", + "tsvector", + "tsquery", +) + + +# ---------- SSL ---------- (mirrors scrape_pgdg.py) + +def _resolve_cafile() -> str | None: + """Best-effort CA bundle discovery. Returns None if nothing usable is found.""" + candidates = [ + os.environ.get("SSL_CERT_FILE"), + os.environ.get("CURL_CA_BUNDLE"), + "/etc/ssl/certs/ca-certificates.crt", + "/etc/ssl/certs/ca-bundle.crt", + "/etc/pki/tls/certs/ca-bundle.crt", + "/etc/ssl/cert.pem", + ] + for path in candidates: + if path and os.path.exists(path): + return path + return None + + +def _fetch(url: str, timeout: float = 30.0, user_agent: str = DEFAULT_USER_AGENT) -> str | None: + """Fetch a URL. Returns None on 404/410; any other HTTPError re-raises. + + PGDG occasionally has indexing gaps (a release is announced and indexed + but the per-minor page hasn't been uploaded yet). We treat that as + "skip this revision and continue" rather than a hard error. Anything + else (5xx, network error, etc.) propagates so the caller can decide. + """ + import ssl + ctx = ssl.create_default_context() + cafile = _resolve_cafile() + if cafile: + ctx.load_verify_locations(cafile) + req = urllib.request.Request( + url, headers={"User-Agent": user_agent, "Accept": "text/html,*/*"} + ) + try: + with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp: + return resp.read().decode("utf-8", "replace") + except urllib.error.HTTPError as exc: + if exc.code in (404, 410): + return None + raise + + +# ---------- Release index ---------- + +_VERSION_LINK_RE = re.compile(r']*href="(/docs/release/(\d+)\.(\d+)/?)"[^>]*>') + + +def list_recent_releases( + html: str, *, + majors: list[str], + revisions_back: int, +) -> dict[str, list[str]]: + """Return {major: [minor1, minor2, ...]} for the most recent `revisions_back`. + + Each major's list is sorted ascending so callers can slice off the top + `revisions_back` newest entries (the rightmost end of the list). + """ + found: dict[str, list[str]] = {m: [] for m in majors} + majors_set = set(majors) + for path, major, minor in _VERSION_LINK_RE.findall(html): + if major not in majors_set: + continue + if minor not in found[major]: + found[major].append(minor) + for entries in found.values(): + entries.sort(key=lambda m: int(m)) + return found + + +# ---------- Per-release page parsing ---------- + +_LISTITEM_RE = re.compile( + r'

  • (.*?)
    • ', + re.DOTALL, +) +_FIRST_P_RE = re.compile(r"

    (.*?)

    ", re.DOTALL) +_COMMIT_LINK_RE = re.compile(r'href="https://postgr\.es/c/([0-9a-f]{6,})"') +_TAG_STRIP_RE = re.compile(r"<[^>]+>") +_WHITESPACE_RE = re.compile(r"\s+") + + +def _strip_tags(html_text: str) -> str: + text = _TAG_STRIP_RE.sub("", html_text) + return _WHITESPACE_RE.sub(" ", text).strip() + + +def _classify(text: str) -> str: + lower = text.lower() + for kw in HIGH_KEYWORDS: + if kw in lower: + return "high" + for kw in MEDIUM_KEYWORDS: + if kw in lower: + return "medium" + return "low" + + +def parse_release_page(html: str, *, major: str, minor: str, doc_link: str) -> list[dict]: + """Return listitems from a release-notes page as proposed-addition dicts.""" + entries: list[dict] = [] + for li_m in _LISTITEM_RE.finditer(html): + body = li_m.group(1) + # Use the FIRST

    as the summary title; usually "Fix ... (Author Name)". + first_p = _FIRST_P_RE.search(body) + if not first_p: + continue + summary = _strip_tags(first_p.group(1)) + if not summary: + continue + # Trim "(Author Name)" off the end so the proposed summary reads cleanly. + summary = re.sub(r"\s*\([A-Z][\w .'-]*(?:,\s*[A-Z][\w .'-]*)*\)\s*$", "", summary) + if not summary: + continue + + # Stable identifier: commit hash if any, else summary hash. + commit_match = _COMMIT_LINK_RE.search(body) + if commit_match: + issue_id = f"PG{major}-{commit_match.group(1)[:9]}" + else: + digest = hashlib.sha1(summary.encode("utf-8")).hexdigest()[:8] + issue_id = f"PG{major}-FIX-{digest}" + + severity = _classify(summary) + + entries.append({ + "issue_id": issue_id, + "summary": summary, + "doc_link": doc_link, + "fixed_in_minor": int(minor), + # Internal fields: stripped by to_json_format. + "_severity": severity, + "_commit": commit_match.group(1) if commit_match else None, + }) + return entries + + +# ---------- Filtering ---------- + +def filter_proposed( + entries: list[dict], + existing: list[dict], + *, + majors: list[str], + min_severity: str = "medium", + top_per_major: int = DEFAULT_TOP_PER_MAJOR, +) -> list[dict]: + """Reduce scraped listitems to a human-reviewable proposal set. + + - Skip existing entries (matched by issue_id). + - Keep only entries whose severity is >= min_severity. + - Limit to N entries per major, highest severity first. + """ + sev_rank = {"low": 0, "medium": 1, "high": 2} + threshold = sev_rank[min_severity] + + existing_ids = {row["issue_id"] for row in existing} + + # Filter + score. + qualifying = [] + for entry in entries: + major = doc_link_major(entry["doc_link"]) + if major not in majors: + continue + if entry["issue_id"] in existing_ids: + continue + sev = entry["_severity"] + if sev_rank[sev] < threshold: + continue + qualifying.append(entry) + + # Group by major; sort each group by severity desc, then by minor desc, then by id. + by_major: dict[str, list[dict]] = {m: [] for m in majors} + for entry in qualifying: + major = doc_link_major(entry["doc_link"]) + by_major.setdefault(major, []).append(entry) + result: list[dict] = [] + for major in majors: + items = by_major.get(major, []) + items.sort(key=lambda e: (-sev_rank[e["_severity"]], -e["fixed_in_minor"], e["issue_id"])) + result.extend(items[:top_per_major]) + return result + + +_DOC_LINK_MAJOR_RE = re.compile(r"/release/(\d+)\.(\d+)/?") + + +def doc_link_major(doc_link: str) -> str: + m = _DOC_LINK_MAJOR_RE.search(doc_link) + return m.group(1) if m else "" + + +def to_json_format(proposed: list[dict]) -> list[dict]: + """Strip internal fields; return only the keys the JSON schema uses.""" + return [ + { + "issue_id": p["issue_id"], + "summary": p["summary"], + "doc_link": p["doc_link"], + "fixed_in_minor": p["fixed_in_minor"], + } + for p in proposed + ] + + +# ---------- File IO ---------- + +def load_existing() -> list[dict]: + if not BUGS_JSON.exists(): + return [] + return json.loads(BUGS_JSON.read_text()).get("bugs", []) + + +def merge_into_data(proposed: list[dict]) -> None: + """Patch data/known_bugs.json in place with the proposed additions.""" + doc = json.loads(BUGS_JSON.read_text()) + doc.setdefault("bugs", []) + by_id = {row["issue_id"]: row for row in doc["bugs"]} + for row in to_json_format(proposed): + if row["issue_id"] in by_id: + continue + doc["bugs"].append(row) + from datetime import date + doc["last_reviewed"] = date.today().isoformat() + # Stable sort: by major, then minor desc, then id. + doc["bugs"].sort(key=lambda r: (r["issue_id"].split("-")[0], -int(r["fixed_in_minor"]), r["issue_id"])) + BUGS_JSON.write_text(json.dumps(doc, indent=2, ensure_ascii=False) + "\n") + + +# ---------- CLI ---------- + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + mode = p.add_mutually_exclusive_group() + mode.add_argument("--propose", action="store_true", default=True) + mode.add_argument("--check", action="store_true") + mode.add_argument("--write", action="store_true") + p.add_argument("--yes", action="store_true") + p.add_argument("--majors", default=",".join(DEFAULT_MAJORS)) + p.add_argument("--revisions-back", type=int, default=DEFAULT_REVISIONS_BACK) + p.add_argument("--top-per-major", type=int, default=DEFAULT_TOP_PER_MAJOR) + p.add_argument( + "--min-severity", choices=["low", "medium", "high"], default="medium", + help="drop entries below this severity band (default medium)", + ) + p.add_argument( + "--index-url", default=PGDG_RELEASE_INDEX, + help="release-notes index URL (override for testing)", + ) + p.add_argument("--user-agent", default=DEFAULT_USER_AGENT) + args = p.parse_args(argv) + + majors = [m.strip() for m in args.majors.split(",") if m.strip()] + + # ---- Fetch release index ---- + try: + index_html = _fetch(args.index_url, user_agent=args.user_agent) + except Exception as exc: + print(f"scout: index fetch failed: {exc}", file=sys.stderr) + return 2 + + if index_html is None: + print("scout: index returned 404/410", file=sys.stderr) + return 2 + + recent = list_recent_releases(index_html, majors=majors, revisions_back=args.revisions_back) + if not any(recent.values()): + print("scout: index returned no recent releases for our majors", file=sys.stderr) + return 2 + + # ---- Fetch each candidate release page ---- + all_entries: list[dict] = [] + skipped: list[str] = [] + for major in majors: + minors = recent.get(major, []) + # Take the most recent N releases per major. + for minor in minors[-args.revisions_back:]: + url = f"{PGDG_DOCS_BASE}/release/{major}.{minor}/" + try: + html = _fetch(url, user_agent=args.user_agent) + except Exception as exc: + print(f"scout: page fetch failed {major}.{minor}: {exc}", file=sys.stderr) + continue + if html is None: + # 404 / 410: PGDG indexing gap. Skip and continue. + skipped.append(f"{major}.{minor}") + continue + all_entries.extend(parse_release_page( + html, major=major, minor=minor, doc_link=url, + )) + + if skipped and args.check: + for s in skipped: + print(f"scout: skipped {s} (404/410 from PGDG)", file=sys.stderr) + + # ---- Filter + propose ---- + existing = load_existing() + proposed = filter_proposed( + all_entries, existing, + majors=majors, + min_severity=args.min_severity, + top_per_major=args.top_per_major, + ) + cleaned = to_json_format(proposed) + + # ---- Mode dispatch ---- + if args.check: + if cleaned: + print(f"FAIL: {len(cleaned)} new bug-fix proposal(s) pending review", file=sys.stderr) + print(json.dumps(cleaned, indent=2, ensure_ascii=False)) + return 1 + print("ok: no new bug-fix proposals pending") + return 0 + + if args.write: + if not cleaned: + print("scout: --write invoked but no new proposals", file=sys.stderr) + return 0 + if not args.yes: + print("scout: --write requires --yes to confirm overwrite of data/known_bugs.json", file=sys.stderr) + return 2 + merge_into_data(proposed) + print(f"scout: wrote {len(cleaned)} entry/entries into {BUGS_JSON.relative_to(REPO_ROOT)}") + return 0 + + print(json.dumps(cleaned, indent=2, ensure_ascii=False)) + if cleaned: + majors_repr = ",".join(majors) + print( + f"\n# ({len(cleaned)} proposal(s); min severity {args.min_severity}; majors {majors_repr}; " + f"top {args.top_per_major}/major)", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/tests/fixtures/pgdg_index_realistic.html b/tools/tests/fixtures/pgdg_index_realistic.html new file mode 100644 index 0000000..1dbec06 --- /dev/null +++ b/tools/tests/fixtures/pgdg_index_realistic.html @@ -0,0 +1,924 @@ +

    cells. +_TABLE_BODY_RE = re.compile( + r'\s*.*?\s*(.*?)\s*
    ', + re.DOTALL, +) +_ROW_RE = re.compile(r"
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ReferenceAffectedFixedComponent & CVSS v3 Base ScoreDescription
    + CVE-2026-19385
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24client
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL pg_dump heap buffer overflow executes arbitrary code

    more details
    + CVE-2026-18408
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24client
    + 8.8
    AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H +     		PostgreSQL psql \unrestrict lets superuser of pg_dump origin server execute arbitrary code in psql client

    more details
    + CVE-2026-18024
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 4.3
    AV:N/AC:L/PR:L/UI:N/S:U/C:L/I:N/A:N +     		PostgreSQL ascii() function reads past end of buffer

    more details
    + CVE-2026-16241
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24client
    + 3.8
    AV:N/AC:L/PR:H/UI:N/S:U/C:N/I:L/A:L +     		PostgreSQL ECPG integer underflow can crash the client

    more details
    + CVE-2026-16239
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL type confusion in cursor CLOSE + DECLARE executes arbitrary code

    more details
    + CVE-2026-16238
    + Announcement
    +     		1818.5core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL type confusion in pg_restore_attribute_stats() executes arbitrary code

    more details
    + CVE-2026-15742
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24contrib module
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL fuzzystrmatch writes effectively-arbitrary addresses, via integer wraparound

    more details
    + CVE-2026-15741
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 8.8
    AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H +     		PostgreSQL expression deparse allows SQL injection via EXTRACT argument

    more details
    + CVE-2026-14681
    + Announcement
    +     		18, 1718.5, 17.11core server
    + 4.2
    AV:N/AC:H/PR:L/UI:N/S:U/C:L/I:L/A:N +     		PostgreSQL improper enforcement of GSSAPI encryption when coupled with SSL

    more details
    + CVE-2026-14680
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL type confusion via "internal" arguments

    more details
    + CVE-2026-14679
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 8.2
    AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:L/A:H +     		PostgreSQL stack buffer overflow in argument match writes 0x0 and 0x1 to server memory

    more details
    + CVE-2026-14678
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24contrib module
    + 4.3
    AV:N/AC:L/PR:L/UI:N/S:U/C:L/I:N/A:N +     		PostgreSQL pg_trgm picksplit reads past end of buffer

    more details
    + CVE-2026-14677
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL 32-bit pltcl and plperl undersize allocations, via integer wraparound

    more details
    + CVE-2026-14676
    + Announcement
    +     		1818.5contrib module
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL pg_stat_statements heap buffer overflow executes arbitrary code

    more details
    + CVE-2026-14673
    + Announcement
    +     		18, 16, 15, 1418.5, 16.15, 15.19, 14.24contrib module
    + 3.8
    AV:N/AC:L/PR:H/UI:N/S:U/C:L/I:L/A:N +     		PostgreSQL amcheck does not clear untrusted search path

    more details
    + CVE-2026-14672
    + Announcement
    +     		18, 17, 1618.5, 17.11, 16.15core server
    + 5.3
    AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:N/A:N +     		PostgreSQL observable response discrepancy with non-default scram_iterations provides user existence oracle

    more details
    + CVE-2026-14671
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24contrib module
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL refint plan cache type confusion executes arbitrary code

    more details
    + CVE-2026-14670
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL plperl tied object heap buffer overflow executes arbitrary code

    more details
    + CVE-2026-14669
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL to_char heap buffer overflow executes arbitrary code

    more details
    + CVE-2026-14668
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 8.1
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:H +     		PostgreSQL ctid type confusion in selectivity estimator discloses derivative of arbitrary read

    more details
    + CVE-2026-14666
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 4.2
    AV:N/AC:H/PR:L/UI:N/S:U/C:L/I:L/A:N +     		PostgreSQL row security caching disregards role modifications

    more details
    + CVE-2026-14664
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL regexp heap buffer overflow executes arbitrary code

    more details
    + CVE-2026-14663
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24contrib module
    + 6.5
    AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:N +     		PostgreSQL pgcrypto, for OpenSSL-disabled ciphers, silently encrypts to and decrypts from cleartext

    more details
    + CVE-2026-14662
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL tsvector and tsquery undersize allocations, via integer wraparound

    more details
    + CVE-2026-6638
    + Announcement
    +     		18, 17, 1618.4, 17.10, 16.14core server
    + 3.7
    AV:N/AC:H/PR:L/UI:R/S:U/C:L/I:L/A:N +     		PostgreSQL REFRESH PUBLICATION allows SQL injection via table name

    more details
    + CVE-2026-6637
    + Announcement
    +     		18, 17, 16, 15, 1418.4, 17.10, 16.14, 15.18, 14.23contrib module
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL refint allows stack buffer overflow and SQL injection

    more details
    + CVE-2026-6575
    + Announcement
    +     		1818.4core server
    + 4.3
    AV:N/AC:L/PR:L/UI:N/S:U/C:L/I:N/A:N +     		PostgreSQL pg_restore_attribute_stats accepts values that cause query planning to read past end of stats array

    more details
    + CVE-2026-6479
    + Announcement
    +     		18, 17, 16, 15, 1418.4, 17.10, 16.14, 15.18, 14.23core server
    + 7.5
    AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H +     		PostgreSQL SSL/GSS init causes denial of service, via uncontrolled recursion

    more details
    + CVE-2026-6478
    + Announcement
    +     		18, 17, 16, 15, 1418.4, 17.10, 16.14, 15.18, 14.23core server
    + 6.5
    AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:N +     		PostgreSQL discloses MD5-hashed passwords via covert timing channel

    more details
    + CVE-2026-6477
    + Announcement
    +     		18, 17, 16, 15, 1418.4, 17.10, 16.14, 15.18, 14.23client
    + 8.8
    AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H +     		PostgreSQL libpq lo_* functions let server superuser overwrite client stack memory

    more details
    + CVE-2026-6476
    + Announcement
    +     		18, 1718.4, 17.10client
    + 7.2
    AV:N/AC:L/PR:H/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL pg_createsubscriber allows SQL injection via subscription name

    more details
    + CVE-2026-6475
    + Announcement
    +     		18, 17, 16, 15, 1418.4, 17.10, 16.14, 15.18, 14.23client
    + 8.8
    AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H +     		PostgreSQL pg_basebackup and pg_rewind can overwrite unrelated files of origin superuser choice

    more details
    + CVE-2026-6474
    + Announcement
    +     		18, 17, 16, 15, 1418.4, 17.10, 16.14, 15.18, 14.23core server
    + 4.3
    AV:N/AC:L/PR:L/UI:N/S:U/C:L/I:N/A:N +     		PostgreSQL timeofday() can disclose portions of server memory

    more details
    + CVE-2026-6473
    + Announcement
    +     		18, 17, 16, 15, 1418.4, 17.10, 16.14, 15.18, 14.23core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL server undersizes allocations, via integer wraparound

    more details
    + CVE-2026-6472
    + Announcement
    +     		18, 17, 16, 15, 1418.4, 17.10, 16.14, 15.18, 14.23core server
    + 5.4
    AV:N/AC:L/PR:L/UI:N/S:U/C:L/I:L/A:N +     		PostgreSQL CREATE TYPE does not check multirange schema CREATE privilege

    more details
    + CVE-2026-6471
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 7.2
    AV:N/AC:L/PR:H/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL logical decoding can dlopen arbitrary file

    more details
    + CVE-2026-6470
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 4.3
    AV:N/AC:L/PR:L/UI:N/S:U/C:N/I:N/A:L +     		PostgreSQL fails to check type USAGE privilege

    more details
    + CVE-2026-6469
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24core server
    + 3.8
    AV:N/AC:L/PR:H/UI:N/S:U/C:N/I:L/A:L +     		PostgreSQL ALTER TABLE ALTER TYPE resets extended statistics ownership

    more details
    + CVE-2026-6464
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24client
    + 8.1
    AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL psql COPY FROM STDIN early failure processes data lines as psql commands

    more details
    + CVE-2026-2007
    + Announcement
    +     		1818.2contrib module
    + 8.2
    AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:L/A:H +     		PostgreSQL pg_trgm heap buffer overflow writes pattern onto server memory

    more details
    + CVE-2026-2006
    + Announcement
    +     		18, 17, 16, 15, 1418.2, 17.8, 16.12, 15.16, 14.21core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL missing validation of multibyte character length executes arbitrary code

    more details
    + CVE-2026-2005
    + Announcement
    +     		18, 17, 16, 15, 1418.2, 17.8, 16.12, 15.16, 14.21contrib module
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL pgcrypto heap buffer overflow executes arbitrary code

    more details
    + CVE-2026-2004
    + Announcement
    +     		18, 17, 16, 15, 1418.2, 17.8, 16.12, 15.16, 14.21contrib module
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL intarray missing validation of type of input to selectivity estimator executes arbitrary code

    more details
    + CVE-2026-2003
    + Announcement
    +     		18, 17, 16, 15, 1418.2, 17.8, 16.12, 15.16, 14.21core server
    + 4.3
    AV:N/AC:L/PR:L/UI:N/S:U/C:L/I:N/A:N +     		PostgreSQL oidvector discloses a few bytes of memory

    more details
    + CVE-2025-12818
    + Announcement
    +     		18, 17, 16, 15, 1418.1, 17.7, 16.11, 15.15, 14.20core server
    + 5.9
    AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H +     		PostgreSQL libpq undersizes allocations, via integer wraparound

    more details
    + CVE-2025-12817
    + Announcement
    +     		18, 17, 16, 15, 1418.1, 17.7, 16.11, 15.15, 14.20core server
    + 3.1
    AV:N/AC:H/PR:L/UI:N/S:U/C:N/I:N/A:L +     		PostgreSQL CREATE STATISTICS does not check for schema CREATE privilege

    more details
    + CVE-2025-8715
    + Announcement
    +     		17, 16, 15, 1417.6, 16.10, 15.14, 14.19core server
    + 8.8
    AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H +     		PostgreSQL pg_dump newline in object name executes arbitrary code in psql client and in restore target server

    more details
    + CVE-2025-8714
    + Announcement
    +     		17, 16, 15, 1417.6, 16.10, 15.14, 14.19core server
    + 8.8
    AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H +     		PostgreSQL pg_dump lets superuser of origin server execute arbitrary code in psql client

    more details
    + CVE-2025-8713
    + Announcement
    +     		17, 16, 15, 1417.6, 16.10, 15.14, 14.19core server
    + 3.1
    AV:N/AC:H/PR:L/UI:N/S:U/C:L/I:N/A:N +     		PostgreSQL optimizer statistics can expose sampled data within a view, partition, or child table

    more details
    + CVE-2025-4207
    + Announcement
    +     		17, 16, 15, 1417.5, 16.9, 15.13, 14.18core server
    + 5.9
    AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:N/A:H +     		PostgreSQL GB18030 encoding validation can read one byte past end of allocation for text that fails validation

    more details
    + CVE-2025-1094
    + Announcement
    +     		17, 16, 15, 1417.3, 16.7, 15.11, 14.16core server
    + 8.1
    AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL quoting APIs miss neutralizing quoting syntax in text that fails encoding validation

    more details
    + CVE-2024-10979
    + Announcement
    +     		17, 16, 15, 1417.1, 16.5, 15.9, 14.14core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL PL/Perl environment variable changes execute arbitrary code

    more details
    + CVE-2024-10978
    + Announcement
    +     		17, 16, 15, 1417.1, 16.5, 15.9, 14.14core server
    + 4.2
    AV:N/AC:H/PR:L/UI:N/S:U/C:L/I:L/A:N +     		PostgreSQL SET ROLE, SET SESSION AUTHORIZATION reset to wrong user ID

    more details
    + CVE-2024-10977
    + Announcement
    +     		17, 16, 15, 1417.1, 16.5, 15.9, 14.14client
    + 3.1
    AV:N/AC:H/PR:N/UI:R/S:U/C:N/I:L/A:N +     		PostgreSQL libpq retains an error message from man-in-the-middle

    more details
    + CVE-2024-10976
    + Announcement
    +     		17, 16, 15, 1417.1, 16.5, 15.9, 14.14core server
    + 4.2
    AV:N/AC:H/PR:L/UI:N/S:U/C:L/I:L/A:N +     		PostgreSQL row security below e.g. subqueries disregards user ID changes

    more details
    + CVE-2024-7348
    + Announcement
    +     		16, 15, 1416.4, 15.8, 14.13core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL relation replacement during pg_dump executes arbitrary SQL

    more details
    + CVE-2024-4317
    + Announcement
    +     		16, 15, 1416.3, 15.7, 14.12core server
    + 3.1
    AV:N/AC:H/PR:L/UI:N/S:U/C:L/I:N/A:N +     		Restrict visibility of "pg_stats_ext" and "pg_stats_ext_exprs" entries to the table owner

    more details
    + CVE-2024-0985
    + Announcement
    +     		16, 15, 1416.2, 15.6, 14.11core server
    + 8.0
    AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H +     		PostgreSQL non-owner REFRESH MATERIALIZED VIEW CONCURRENTLY executes arbitrary SQL

    more details
    + CVE-2023-39418
    + Announcement
    +     		1515.4core server
    + 3.1
    AV:N/AC:H/PR:L/UI:N/S:U/C:N/I:L/A:N +     		MERGE fails to enforce UPDATE or SELECT row security policies

    more details
    + CVE-2023-39417
    + Announcement
    +     		15, 1415.4, 14.9core server
    + 7.5
    AV:N/AC:H/PR:L/UI:N/S:U/C:H/I:H/A:H +     		Extension script @substitutions@ within quoting allow SQL injection

    more details
    + CVE-2023-5870
    + Announcement
    +     		16, 15, 1416.1, 15.5, 14.10core server
    + 2.2
    AV:N/AC:H/PR:H/UI:N/S:U/C:N/I:N/A:L +     		Role "pg_signal_backend" can signal certain superuser processes

    more details
    + CVE-2023-5869
    + Announcement
    +     		16, 15, 1416.1, 15.5, 14.10core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		Buffer overrun from integer overflow in array modification

    more details
    + CVE-2023-5868
    + Announcement
    +     		16, 15, 1416.1, 15.5, 14.10core server
    + 4.3
    AV:N/AC:L/PR:L/UI:N/S:U/C:L/I:N/A:N +     		Memory disclosure in aggregate function calls

    more details
    + CVE-2023-2455
    + Announcement
    +     		15, 1415.3, 14.8core server
    + 4.2
    AV:N/AC:H/PR:L/UI:N/S:U/C:L/I:L/A:N +     		Row security policies disregard user ID changes after inlining

    more details
    + CVE-2023-2454
    + Announcement
    +     		15, 1415.3, 14.8core server
    + 7.2
    AV:N/AC:L/PR:H/UI:N/S:U/C:H/I:H/A:H +     		CREATE SCHEMA ... schema_element defeats protective search_path changes

    more details
    + CVE-2022-41862
    + Announcement
    +     		15, 1415.2, 14.7client
    + 3.7
    AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:N/A:N +     		Client memory disclosure when connecting, with Kerberos, to modified server

    more details
    + CVE-2022-2625
    + Announcement
    +     		1414.5core server
    + 7.1
    AV:N/AC:H/PR:L/UI:R/S:U/C:H/I:H/A:H +     		Extension scripts replace objects not belonging to the extension

    more details
    + CVE-2022-1552
    + Announcement
    +     		1414.3core server
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		Autovacuum, REINDEX, and others omit "security restricted operation" sandbox

    more details
    + CVE-2021-23222
    + Announcement
    +     		1414.1client
    + 3.7
    AV:N/AC:H/PR:N/UI:N/S:U/C:L/I:N/A:N +     		libpq processes unencrypted bytes from man-in-the-middle

    more details
    + CVE-2021-23214
    + Announcement
    +     		1414.1core server
    + 8.1
    AV:N/AC:H/PR:N/UI:N/S:U/C:H/I:H/A:H +     		Server processes unencrypted bytes from man-in-the-middle

    more details
    \ No newline at end of file diff --git a/tools/tests/fixtures/pgdg_index_synthetic.html b/tools/tests/fixtures/pgdg_index_synthetic.html new file mode 100644 index 0000000..2267fce --- /dev/null +++ b/tools/tests/fixtures/pgdg_index_synthetic.html @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + +
    ReferenceAffectedFixedComponent & CVSS v3 Base ScoreDescription
    CVE-2026-19385
    + Announcement
    +     		18, 17, 16, 15, 1418.5, 17.11, 16.15, 15.19, 14.24client
    + 8.8
    AV:N/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:H +     		PostgreSQL pg_dump heap buffer overflow executes arbitrary code

    more details
    CVE-2024-0985
    + Announcement
    +     		16, 15, 1416.2, 15.6, 14.11core server
    + 8.0
    AV:N/AC:L/PR:L/UI:R/S:U/C:H/I:H/A:H +     		Non-owner REFRESH MATERIALIZED VIEW CONCURRENTLY executes arbitrary SQL

    more details
    + diff --git a/tools/tests/fixtures/release_index_realistic.html b/tools/tests/fixtures/release_index_realistic.html new file mode 100644 index 0000000..fe13648 --- /dev/null +++ b/tools/tests/fixtures/release_index_realistic.html @@ -0,0 +1,3861 @@ + + + + PostgreSQL: Release Notes + + + + + + + + + + + + + + + + +
    +
    +
    + + +
    +
    +
    +
    August 13, 2026: PostgreSQL 18.6, 17.11, 16.15, 15.19, 14.24 and 19 Beta 3 Released! + +
    +
    +
    + +
    +
    +
    +
    + +
    +

    Quick Links

    + + + +
    + +
    +
    +
    +
    + + + +
    +
    +
    +

    Release Notes

    +
    +

    Below is the complete archive of release notes for every version of PostgreSQL.

    + + +
    + PostgreSQL 18 + +
    + + + +
    + PostgreSQL 17 + +
    + + + +
    + PostgreSQL 16 + +
    + + + +
    + PostgreSQL 15 + +
    + + + +
    + PostgreSQL 14 + +
    + + + +
    + PostgreSQL 13 + +
    + + + +
    + PostgreSQL 12 + +
    + + + +
    + PostgreSQL 11 + +
    + + + +
    + PostgreSQL 10 + +
    + + + +
    + PostgreSQL 9.6 + +
    + + + +
    + PostgreSQL 9.5 + +
    + + + +
    + PostgreSQL 9.4 + +
    + + + +
    + PostgreSQL 9.3 + +
    + + + +
    + PostgreSQL 9.2 + +
    + + + +
    + PostgreSQL 9.1 + +
    + + + +
    + PostgreSQL 9.0 + +
    + + + +
    + PostgreSQL 8.4 + +
    + + + +
    + PostgreSQL 8.3 + +
    + + + +
    + PostgreSQL 8.2 + +
    + + + +
    + PostgreSQL 8.1 + +
    + + + +
    + PostgreSQL 8.0 + +
    + + + +
    + PostgreSQL 7.4 + +
    + + + +
    + PostgreSQL 7.3 + +
    + + + +
    + PostgreSQL 7.2 + +
    + + + +
    + PostgreSQL 7.1 + +
    + + + +
    + PostgreSQL 7.0 + +
    + + + +
    + PostgreSQL 6.5 + +
    + + + +
    + PostgreSQL 6.4 + +
    + + + +
    + PostgreSQL 6.3 + +
    + + + +
    + PostgreSQL 6.2 + +
    + + + +
    + PostgreSQL 6.1 + +
    + + + +
    + PostgreSQL 6.0 + +
    + + + +
    + PostgreSQL 1 + +
    + + + +
    + Postgres95 + +
    + + +
    +
    + + +
    +
    +
    +
    + + + + + + + diff --git a/tools/tests/fixtures/release_notes_synthetic.html b/tools/tests/fixtures/release_notes_synthetic.html new file mode 100644 index 0000000..5932b66 --- /dev/null +++ b/tools/tests/fixtures/release_notes_synthetic.html @@ -0,0 +1,32 @@ + + + +PostgreSQL: Release Notes + +

    Release Notes

    +

    E.X. Release 17.11

    +

    E.X.1. Migration to Version 17.11

    +

    E.X.2. Changes

    + + + diff --git a/tools/tests/test_scrape_pgdg.py b/tools/tests/test_scrape_pgdg.py new file mode 100644 index 0000000..5b45092 --- /dev/null +++ b/tools/tests/test_scrape_pgdg.py @@ -0,0 +1,273 @@ +"""Tests for tools/scrape_pgdg.py. + +Parses canned HTML fixtures and asserts the PGDG scraper produces the +expected CVE dicts. Avoids network so tests are deterministic. +""" + +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from pathlib import Path +from typing import Any + +TOOLS_DIR = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(TOOLS_DIR)) + +import scrape_pgdg as scraper # noqa: E402 + +FIXTURES_DIR = TOOLS_DIR / "tests" / "fixtures" + + +def _load(name: str) -> str: + return (FIXTURES_DIR / name).read_text() + + +class ParseSyntheticTests(unittest.TestCase): + """Hand-written fixture exercising the parser's core behavior.""" + + @classmethod + def setUpClass(cls) -> None: + cls.html = _load("pgdg_index_synthetic.html") + + def test_parses_two_cves(self) -> None: + entries = scraper.parse_pgdg_table(self.html) + self.assertEqual(len(entries), 2) + + def test_first_cve_ids_and_fixed_map(self) -> None: + entries = scraper.parse_pgdg_table(self.html) + first = entries[0] + self.assertEqual(first["cve_id"], "CVE-2026-19385") + self.assertEqual(first["cvss"], 8.8) + self.assertEqual( + first["doc_link"], + "https://www.postgresql.org/support/security/CVE-2026-19385/", + ) + # Only majors 15-18 are kept; 14 is dropped. + self.assertEqual( + first["fixed_in"], {"15": 19, "16": 15, "17": 11, "18": 5}, + ) + + def test_second_cve_partial_major_set(self) -> None: + entries = scraper.parse_pgdg_table(self.html) + second = entries[1] + self.assertEqual(second["cve_id"], "CVE-2024-0985") + # Source listed only 16, 15, 14: only 15 and 16 should appear. + self.assertEqual(second["fixed_in"], {"15": 6, "16": 2}) + + def test_summary_strips_more_details_link(self) -> None: + entries = scraper.parse_pgdg_table(self.html) + for entry in entries: + self.assertNotIn("more details", entry["summary"]) + # No HTML tags leaked through. + self.assertNotIn("<", entry["summary"]) + self.assertNotIn(">", entry["summary"]) + + +class ParseRealisticFixtureTests(unittest.TestCase): + """The real PGDG index page, captured as a fixture on 2026-08-23. + + Acts as a sanity floor: when PGDG redesigns the table, this test fails + loudly so we tighten the parser. + """ + + @classmethod + def setUpClass(cls) -> None: + cls.html = _load("pgdg_index_realistic.html") + + def test_parses_at_least_60_cves(self) -> None: + # PGDG publishes ~30 CVEs per major x 5 majors = ~150 historically; + # 60 is a loose floor: if we drop below this, suspect a parse regression. + entries = scraper.parse_pgdg_table(self.html) + self.assertGreater(len(entries), 60, f"only {len(entries)} entries parsed") + + def test_every_entry_has_required_fields(self) -> None: + entries = scraper.parse_pgdg_table(self.html) + for entry in entries: + self.assertRegex(entry["cve_id"], r"^CVE-\d{4}-\d+$") + self.assertGreaterEqual(entry["cvss"], 0.0) + self.assertLessEqual(entry["cvss"], 10.0) + self.assertTrue(entry["summary"]) + self.assertTrue(entry["doc_link"].startswith("https://www.postgresql.org")) + self.assertTrue( + set(entry["fixed_in"]).issubset({"15", "16", "17", "18"}), + f"unexpected majors in fixed_in: {entry['fixed_in']!r}", + ) + for major, minor in entry["fixed_in"].items(): + # Repo's version_num convention: major*10000 + minor. + self.assertGreaterEqual(minor, 0) + self.assertLess(minor, 100) + + def test_fixed_versions_align_with_affected_majors(self) -> None: + entries = scraper.parse_pgdg_table(self.html) + for entry in entries: + for major, minor in entry["fixed_in"].items(): + # PG 15.x -> 15 + x*100ish. Confirm the major portion matches. + # Recreate: major * 10000 + minor. + reconstructed = int(major) * 10000 + minor + # Just sanity-check the major part matches; minor is sane. + self.assertEqual( + reconstructed // 10000, int(major), + f"major mismatch in {entry['cve_id']}: {entry['fixed_in']!r}", + ) + + +class FilterProposedTests(unittest.TestCase): + """Filter logic that decides what makes it into the proposed-additions set.""" + + ENTRY_HIGH = { + "cve_id": "CVE-2026-19385", + "cvss": 8.8, + "summary": "pg_dump heap overflow", + "doc_link": "https://www.postgresql.org/support/security/CVE-2026-19385/", + "fixed_in": {"15": 19, "16": 15, "17": 11, "18": 5}, + "_affected": ["18", "17", "16", "15", "14"], + "_fixed_raw": ["18.5", "17.11", "16.15", "15.19", "14.24"], + } + ENTRY_MEDIUM = dict(ENTRY_HIGH, cve_id="CVE-2024-0985", cvss=4.2, + fixed_in={"15": 6, "16": 2}) + ENTRY_OLDER = dict(ENTRY_HIGH, cve_id="CVE-2026-99999", + fixed_in={"10": 5, "9": 6}) # doesn't affect 15-18 + EXISTING = [ + {"cve_id": "CVE-2026-19385", "cvss": 8.8, "summary": "x", + "doc_link": "x", "fixed_in": {"15": 19}}, # already curated + ] + + def test_filter_drops_already_curated(self) -> None: + out = scraper.filter_proposed( + [self.ENTRY_HIGH], self.EXISTING, min_cvss=7.0, majors=["15", "16", "17", "18"] + ) + self.assertEqual(out, []) + + def test_filter_drops_low_cvss(self) -> None: + out = scraper.filter_proposed( + [self.ENTRY_MEDIUM], [], min_cvss=7.0, majors=["15", "16", "17", "18"] + ) + self.assertEqual(out, []) + + def test_filter_drops_out_of_scope_majors(self) -> None: + out = scraper.filter_proposed( + [self.ENTRY_OLDER], [], min_cvss=7.0, majors=["15", "16", "17", "18"] + ) + self.assertEqual(out, []) + + def test_filter_keeps_qualifying_new_cves(self) -> None: + out = scraper.filter_proposed( + [self.ENTRY_HIGH], [], min_cvss=7.0, majors=["15", "16", "17", "18"] + ) + self.assertEqual(len(out), 1) + self.assertEqual(out[0]["cve_id"], "CVE-2026-19385") + + def test_filter_sorted_highest_cvss_first(self) -> None: + a = dict(self.ENTRY_HIGH, cve_id="CVE-A", cvss=9.9) + b = dict(self.ENTRY_HIGH, cve_id="CVE-B", cvss=7.1) + out = scraper.filter_proposed( + [b, a], [], min_cvss=7.0, majors=["15", "16", "17", "18"] + ) + self.assertEqual([e["cve_id"] for e in out], ["CVE-A", "CVE-B"]) + + +class ToJsonFormatTests(unittest.TestCase): + def test_internal_fields_stripped(self) -> None: + cleaned = scraper.to_json_format([ + {"cve_id": "CVE-X", "cvss": 7.5, "summary": "x", + "doc_link": "y", "fixed_in": {"15": 1}, "_affected": ["15"], "_fixed_raw": ["15.1"]} + ]) + self.assertEqual(len(cleaned), 1) + self.assertNotIn("_affected", cleaned[0]) + self.assertNotIn("_fixed_raw", cleaned[0]) + self.assertEqual(cleaned[0]["cve_id"], "CVE-X") + + +class MergeIntoDataTests(unittest.TestCase): + def test_appends_only_new_entries_no_duplicates(self) -> None: + with tempfile.TemporaryDirectory() as d: + data_path = Path(d) / "cves.json" + data_path.write_text(json.dumps({ + "version": 1, + "cves": [ + {"cve_id": "CVE-KEEP", "cvss": 8.0, "summary": "kept", + "doc_link": "https://x", "fixed_in": {"15": 1}}, + ], + })) + # Point the scraper's CVES_JSON at the temp path by patching the + # module attribute, then revert after the test. + original_path = scraper.CVES_JSON + scraper.CVES_JSON = data_path + try: + scraper.merge_into_data([ + {"cve_id": "CVE-KEEP", "cvss": 8.0, "summary": "kept", + "doc_link": "https://x", "fixed_in": {"15": 1}, + "_affected": [], "_fixed_raw": []}, + {"cve_id": "CVE-NEW", "cvss": 9.9, "summary": "new", + "doc_link": "https://y", "fixed_in": {"16": 4}, + "_affected": ["16"], "_fixed_raw": ["16.4"]}, + ]) + finally: + scraper.CVES_JSON = original_path + + doc = json.loads(data_path.read_text()) + cve_ids = [c["cve_id"] for c in doc["cves"]] + # CVE-KEEP not duplicated; CVE-NEW appended. + self.assertEqual(sorted(cve_ids), ["CVE-KEEP", "CVE-NEW"]) + # Sorted highest CVSS first. + self.assertEqual(doc["cves"][0]["cve_id"], "CVE-NEW") + # last_reviewed was bumped. + self.assertTrue(doc["last_reviewed"]) + + +class StrigHtmlTests(unittest.TestCase): + def test_decodes_entities_and_drops_tags(self) -> None: + out = scraper._strip_html( + "

    PostgreSQL "pg_dump" leaks files & arbitrary code

    " + ) + self.assertEqual(out, 'PostgreSQL "pg_dump" leaks files & arbitrary code') + + +class FetchIndexUrlTests(unittest.TestCase): + """fetch_pgdg_index passes its `url` argument to urllib.request.Request. + + Without this, the CLI's --url flag is decorative — the request always + hits the hard-coded PGDG_URL. These tests verify the wiring. + """ + + def test_request_uses_supplied_url(self) -> None: + import unittest.mock as mock + + captured: dict = {} + def fake_urlopen(req, timeout, context): + captured["url"] = req.full_url + captured["headers"] = dict(req.header_items()) + return mock.MagicMock( + __enter__=lambda s: s, + __exit__=lambda s, *a: False, + read=lambda: b"placeholder", + ) + + with mock.patch("urllib.request.urlopen", side_effect=fake_urlopen): + scraper.fetch_pgdg_index(url="https://example.com/security") + + self.assertEqual(captured["url"], "https://example.com/security") + + def test_default_url_unchanged(self) -> None: + import unittest.mock as mock + + captured: dict = {} + def fake_urlopen(req, timeout, context): + captured["url"] = req.full_url + return mock.MagicMock( + __enter__=lambda s: s, + __exit__=lambda s, *a: False, + read=lambda: b"x", + ) + + with mock.patch("urllib.request.urlopen", side_effect=fake_urlopen): + scraper.fetch_pgdg_index() + + self.assertEqual(captured["url"], scraper.PGDG_URL) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tools/tests/test_scrape_release_notes.py b/tools/tests/test_scrape_release_notes.py new file mode 100644 index 0000000..f318972 --- /dev/null +++ b/tools/tests/test_scrape_release_notes.py @@ -0,0 +1,254 @@ +"""Tests for tools/scrape_release_notes.py. + +Frozen HTML fixtures + targeted unit tests for the parser, severity +classifier, and proposal-filter pipeline. +""" + +from __future__ import annotations + +import json +import sys +import tempfile +import unittest +from pathlib import Path + +TOOLS_DIR = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(TOOLS_DIR)) + +import scrape_release_notes as scout # noqa: E402 + +FIXTURES_DIR = TOOLS_DIR / "tests" / "fixtures" + + +def _load(name: str) -> str: + return (FIXTURES_DIR / name).read_text() + + +# ---------- Page parser ---------- + +class ParseReleasePageTests(unittest.TestCase): + + @classmethod + def setUpClass(cls) -> None: + cls.html = _load("release_notes_synthetic.html") + + def test_parses_all_listitems(self) -> None: + entries = scout.parse_release_page( + self.html, major="17", minor="11", + doc_link="https://www.postgresql.org/docs/release/17.11/", + ) + self.assertEqual(len(entries), 6) + + def test_severity_classifier_flags_data_integrity(self) -> None: + entries = scout.parse_release_page( + self.html, major="17", minor="11", + doc_link="x/", + ) + # The memory-safety bugs entry + mem_safety = next(e for e in entries if "memory-safety" in e["summary"]) + self.assertEqual(mem_safety["_severity"], "high") + + def test_severity_classifier_flags_crash(self) -> None: + entries = scout.parse_release_page(self.html, major="17", minor="11", doc_link="x/") + crash = next(e for e in entries if "crash" in e["summary"]) + self.assertEqual(crash["_severity"], "high") + + def test_severity_classifier_flags_replication_medium(self) -> None: + entries = scout.parse_release_page(self.html, major="17", minor="11", doc_link="x/") + repl = next(e for e in entries if "logical replication" in e["summary"]) + self.assertEqual(repl["_severity"], "medium") + + def test_severity_classifier_low_for_documentation(self) -> None: + entries = scout.parse_release_page(self.html, major="17", minor="11", doc_link="x/") + doc = next(e for e in entries if "documentation" in e["summary"]) + self.assertEqual(doc["_severity"], "low") + + def test_strips_trailing_author_attribution(self) -> None: + entries = scout.parse_release_page(self.html, major="17", minor="11", doc_link="x/") + for entry in entries: + # "(Author Name)" should be trimmed from the summary. + self.assertNotRegex(entry["summary"], r"\([A-Z][\w .'-]*(?:,.*?)*\)\s*$", + f"author not stripped: {entry['summary']!r}") + + def test_strips_trailing_section_marker(self) -> None: + entries = scout.parse_release_page(self.html, major="17", minor="11", doc_link="x/") + for entry in entries: + # The section anchor at the end (from postgr.es/c/) is HTML, + # but if our _strip_tags missed it the text would contain it. + self.assertNotIn("sec.", entry["summary"]) + + def test_stable_id_uses_commit_hash(self) -> None: + entries = scout.parse_release_page(self.html, major="17", minor="11", doc_link="x/") + mem_safety = next(e for e in entries if "memory-safety" in e["summary"]) + # Commit hash from URL is "a1b2c3d4e" → issue_id uses first 9 chars with major prefix. + self.assertEqual(mem_safety["issue_id"], "PG17-a1b2c3d4e") + self.assertEqual(mem_safety["_commit"], "a1b2c3d4e") + + def test_stable_id_falls_back_to_summary_hash(self) -> None: + entries = scout.parse_release_page(self.html, major="17", minor="11", doc_link="x/") + no_commit = next(e for e in entries if "no commit URL" in e["summary"]) + # digest is sha1("...")[:8] + self.assertTrue(no_commit["issue_id"].startswith("PG17-FIX-")) + self.assertEqual(len(no_commit["issue_id"].split("-")[-1]), 8) + self.assertIsNone(no_commit["_commit"]) + + +# ---------- Release-index parser ---------- + +class ListRecentReleasesTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + # The realistic fixture is committed to the repo. If it's + # missing (e.g. first-time contributor check-out), skip the + # network-bootstrap fallback: tests should be deterministic and + # not silently start hitting PGDG on every CI run. + realistic = FIXTURES_DIR / "release_index_realistic.html" + if not realistic.exists(): + raise unittest.SkipTest( + f"missing fixture: {realistic}. Add the file before running this test." + ) + cls.index_html = _load("release_index_realistic.html") + + def test_finds_per_major_minors(self) -> None: + recent = scout.list_recent_releases( + self.index_html, majors=["15", "16", "17", "18"], + revisions_back=2, + ) + for major in ("15", "16", "17", "18"): + self.assertIn(major, recent) + self.assertGreaterEqual(len(recent[major]), 2) + # Each major's list is in ascending numeric order. + for entries in recent.values(): + self.assertEqual(entries, sorted(entries, key=int)) + + +# ---------- Filter pipeline ---------- + +class FilterProposedTests(unittest.TestCase): + def _entries(self, ids_and_severities): + return [ + {"issue_id": i, "_severity": s, "doc_link": "https://x/release/15.1/", + "summary": i, "fixed_in_minor": 1, "_commit": None} + for i, s in ids_and_severities + ] + + def test_keeps_high_and_medium_by_default(self) -> None: + all_entries = self._entries([("X", "high"), ("Y", "medium"), ("Z", "low")]) + out = scout.filter_proposed( + all_entries, [], + majors=["15"], min_severity="medium", top_per_major=10, + ) + ids = {e["issue_id"] for e in out} + self.assertEqual(ids, {"X", "Y"}) + + def test_drop_threshold_low_only_returns_high(self) -> None: + all_entries = self._entries([("X", "high"), ("Y", "medium"), ("Z", "low")]) + out = scout.filter_proposed( + all_entries, [], majors=["15"], min_severity="high", top_per_major=10, + ) + ids = {e["issue_id"] for e in out} + self.assertEqual(ids, {"X"}) + + def test_drop_threshold_low_returns_everything(self) -> None: + all_entries = self._entries([("X", "high"), ("Y", "low")]) + out = scout.filter_proposed( + all_entries, [], majors=["15"], min_severity="low", top_per_major=10, + ) + ids = {e["issue_id"] for e in out} + self.assertEqual(ids, {"X", "Y"}) + + def test_dedupes_against_existing_by_id(self) -> None: + all_entries = self._entries([("X", "high"), ("Y", "high")]) + existing = [{"issue_id": "X"}] + out = scout.filter_proposed( + all_entries, existing, + majors=["15"], min_severity="medium", top_per_major=10, + ) + ids = {e["issue_id"] for e in out} + self.assertEqual(ids, {"Y"}) + + def test_caps_per_major(self) -> None: + all_entries = self._entries( + [(f"PG15-X{i:02}", "high") for i in range(20)] + ) + out = scout.filter_proposed( + all_entries, [], majors=["15"], min_severity="high", top_per_major=5, + ) + self.assertEqual(len(out), 5) + + def test_filters_by_major_scope(self) -> None: + # Mix of PG 15 and PG 17 entries; with majors=[17] only the 17 ones come through. + all_entries = ( + self._entries([("PG15-X", "high"), ("PG15-Y", "high")]) + + [ + {"issue_id": "PG17-Z", "_severity": "high", + "doc_link": "https://x/release/17.5/", "summary": "x", + "fixed_in_minor": 5, "_commit": None} + ] + ) + out = scout.filter_proposed( + all_entries, [], majors=["17"], min_severity="medium", top_per_major=10, + ) + ids = {e["issue_id"] for e in out} + self.assertEqual(ids, {"PG17-Z"}) + + def test_output_sorted_highest_severity_first(self) -> None: + all_entries = self._entries( + [("PG15-A", "low"), ("PG15-B", "high"), ("PG15-C", "medium")] + ) + out = scout.filter_proposed( + all_entries, [], majors=["15"], min_severity="low", top_per_major=10, + ) + self.assertEqual([e["issue_id"] for e in out], ["PG15-B", "PG15-C", "PG15-A"]) + + +class ToJsonFormatTests(unittest.TestCase): + def test_internal_fields_stripped(self) -> None: + cleaned = scout.to_json_format([ + {"issue_id": "PG15-X", "summary": "s", "doc_link": "u", + "fixed_in_minor": 1, "_severity": "high", "_commit": "abcdefg"} + ]) + self.assertEqual(len(cleaned), 1) + self.assertNotIn("_severity", cleaned[0]) + self.assertNotIn("_commit", cleaned[0]) + self.assertEqual(cleaned[0]["issue_id"], "PG15-X") + + +class MergeIntoDataTests(unittest.TestCase): + def test_appends_only_new_entries(self) -> None: + with tempfile.TemporaryDirectory() as d: + data_path = Path(d) / "known_bugs.json" + data_path.write_text(json.dumps({ + "version": 1, "last_reviewed": "2026-01-01", + "bugs": [{"issue_id": "PG15-KEEP", "summary": "kept", + "doc_link": "https://x", "fixed_in_minor": 1}], + })) + original_path = scout.BUGS_JSON + scout.BUGS_JSON = data_path + try: + scout.merge_into_data([ + {"issue_id": "PG15-KEEP", "summary": "kept", + "doc_link": "https://x", "fixed_in_minor": 1, + "_severity": "high", "_commit": None}, + {"issue_id": "PG15-NEW", "summary": "new", + "doc_link": "https://y", "fixed_in_minor": 4, + "_severity": "high", "_commit": None}, + ]) + finally: + scout.BUGS_JSON = original_path + + doc = json.loads(data_path.read_text()) + ids = sorted(b["issue_id"] for b in doc["bugs"]) + self.assertEqual(ids, ["PG15-KEEP", "PG15-NEW"]) + self.assertTrue(doc["last_reviewed"] >= "2026-08-01") + + +class DocLinkMajorTests(unittest.TestCase): + def test_extracts_major_from_doc_link(self) -> None: + self.assertEqual(scout.doc_link_major("https://x/release/17.11/"), "17") + self.assertEqual(scout.doc_link_major("https://x/release/15.19/"), "15") + self.assertEqual(scout.doc_link_major(""), "") + + +if __name__ == "__main__": + unittest.main(verbosity=2)