From 3af36cb6dd94d1934b21a290cd9871dee92f1437 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 07:14:30 +0200 Subject: [PATCH 01/46] Apply ruff format Mechanical formatting only, so later diffs (and the PostToolUse format hook) don't mix whitespace churn into behavior changes. Co-Authored-By: Claude Opus 5.5 --- configure_transmission.py | 56 +- fetch_torrents.py | 596 ++++++++---- tests/test_configure_transmission.py | 179 ++-- tests/test_fetch_torrents.py | 1305 ++++++++++++++++++-------- tests/test_torrent_scrape.py | 499 ++++++---- torrent_scrape.py | 126 ++- 6 files changed, 1833 insertions(+), 928 deletions(-) diff --git a/configure_transmission.py b/configure_transmission.py index 264f2e5..9eba1f6 100644 --- a/configure_transmission.py +++ b/configure_transmission.py @@ -9,6 +9,7 @@ build_port_forwarding_overrides()'s first-run default - see its docstring - which only ever applies before there's anything on disk to disturb. """ + import json import os import sys @@ -17,25 +18,25 @@ def build_rpc_overrides(env): overrides = {} - whitelist = env.get('TRANSMISSION_RPC_WHITELIST', '').strip() + whitelist = env.get("TRANSMISSION_RPC_WHITELIST", "").strip() if whitelist: - overrides['rpc-whitelist-enabled'] = True - overrides['rpc-whitelist'] = whitelist + overrides["rpc-whitelist-enabled"] = True + overrides["rpc-whitelist"] = whitelist - username = env.get('TRANSMISSION_RPC_USERNAME', '').strip() - password = env.get('TRANSMISSION_RPC_PASSWORD', '').strip() + username = env.get("TRANSMISSION_RPC_USERNAME", "").strip() + password = env.get("TRANSMISSION_RPC_PASSWORD", "").strip() if username and password: - overrides['rpc-authentication-required'] = True - overrides['rpc-username'] = username + overrides["rpc-authentication-required"] = True + overrides["rpc-username"] = username # Plaintext is intentional: transmission-daemon hashes rpc-password # in place on its next start, same as if set by hand. - overrides['rpc-password'] = password + overrides["rpc-password"] = password return overrides def _parse_positive_int(value): - value = (value or '').strip() + value = (value or "").strip() if not value: return None try: @@ -54,21 +55,23 @@ def build_peer_limit_overrides(env): honor (or that blocks all peers, in the case of 0).""" overrides = {} - global_limit = _parse_positive_int(env.get('TRANSMISSION_PEER_LIMIT_GLOBAL', '')) + global_limit = _parse_positive_int(env.get("TRANSMISSION_PEER_LIMIT_GLOBAL", "")) if global_limit is not None: - overrides['peer-limit-global'] = global_limit + overrides["peer-limit-global"] = global_limit - per_torrent_limit = _parse_positive_int(env.get('TRANSMISSION_PEER_LIMIT_PER_TORRENT', '')) + per_torrent_limit = _parse_positive_int( + env.get("TRANSMISSION_PEER_LIMIT_PER_TORRENT", "") + ) if per_torrent_limit is not None: - overrides['peer-limit-per-torrent'] = per_torrent_limit + overrides["peer-limit-per-torrent"] = per_torrent_limit return overrides def _parse_bool(value): - value = (value or '').strip().lower() - if value in ('true', 'false'): - return value == 'true' + value = (value or "").strip().lower() + if value in ("true", "false"): + return value == "true" return None @@ -82,11 +85,11 @@ def build_port_forwarding_overrides(env, settings_file_exists): Transmission itself would otherwise pick its own default. Once the file exists, this is untouched unless TRANSMISSION_PORT_FORWARDING says otherwise, same as every other setting here.""" - explicit = _parse_bool(env.get('TRANSMISSION_PORT_FORWARDING', '')) + explicit = _parse_bool(env.get("TRANSMISSION_PORT_FORWARDING", "")) if explicit is not None: - return {'port-forwarding-enabled': explicit} + return {"port-forwarding-enabled": explicit} if not settings_file_exists: - return {'port-forwarding-enabled': False} + return {"port-forwarding-enabled": False} return {} @@ -95,7 +98,9 @@ def warn_if_open_without_auth(overrides): anyone who can reach the RPC port has full control (add/remove/delete data) - Transmission has no read-only RPC mode. Returns a warning message to log, or None if there's nothing to warn about.""" - if overrides.get('rpc-whitelist-enabled') and not overrides.get('rpc-authentication-required'): + if overrides.get("rpc-whitelist-enabled") and not overrides.get( + "rpc-authentication-required" + ): return ( "TRANSMISSION_RPC_WHITELIST is set but TRANSMISSION_RPC_USERNAME/" "TRANSMISSION_RPC_PASSWORD are not - the web UI/RPC endpoint is " @@ -114,22 +119,25 @@ def load_settings(path): if not os.path.exists(path): return {} try: - with open(path, 'r', encoding='utf-8') as f: + with open(path, "r", encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError) as exc: - print(f"configure_transmission: could not read {path} ({exc}) - starting fresh.", file=sys.stderr) + print( + f"configure_transmission: could not read {path} ({exc}) - starting fresh.", + file=sys.stderr, + ) return {} def save_settings(path, settings): tmp_path = f"{path}.tmp" - with open(tmp_path, 'w', encoding='utf-8') as f: + with open(tmp_path, "w", encoding="utf-8") as f: json.dump(settings, f, indent=4, sort_keys=True) os.replace(tmp_path, path) def main(): - settings_path = os.getenv('TRANSMISSION_SETTINGS_FILE', '/config/settings.json') + settings_path = os.getenv("TRANSMISSION_SETTINGS_FILE", "/config/settings.json") settings_file_exists = os.path.exists(settings_path) overrides = { **build_rpc_overrides(os.environ), diff --git a/fetch_torrents.py b/fetch_torrents.py index 085528a..78fb3df 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -16,15 +16,18 @@ import torrent_scrape # Configure logging -log_dir = os.getenv('FETCH_TORRENTS_LOG_DIR', '/logs') +log_dir = os.getenv("FETCH_TORRENTS_LOG_DIR", "/logs") log_file = os.path.join(log_dir, "fetch_torrents.log") ratio_log_file = os.path.join(log_dir, "fetch_torrents_ratios.log") ratio_history_file = os.path.join(log_dir, "fetch_torrents_ratio_history.json") -old_release_check_state_file = os.path.join(log_dir, "fetch_torrents_old_release_check_state.json") +old_release_check_state_file = os.path.join( + log_dir, "fetch_torrents_old_release_check_state.json" +) removed_history_file = os.path.join(log_dir, "fetch_torrents_removed_history.json") + def parse_log_level(env_var: str, default: int = logging.INFO) -> int: - value = os.getenv(env_var, '').strip() + value = os.getenv(env_var, "").strip() if not value: return default if value.isdigit(): @@ -37,14 +40,14 @@ def parse_log_level(env_var: str, default: int = logging.INFO) -> int: def parse_bool(env_var: str, default: bool = False) -> bool: - value = os.getenv(env_var, '').strip().lower() + value = os.getenv(env_var, "").strip().lower() if not value: return default - return value in ('1', 'true', 'yes', 'on') + return value in ("1", "true", "yes", "on") def parse_int(env_var: str, default: int) -> int: - value = os.getenv(env_var, '').strip() + value = os.getenv(env_var, "").strip() if not value: return default try: @@ -52,7 +55,10 @@ def parse_int(env_var: str, default: int) -> int: except ValueError: return default -log_level = parse_log_level('FETCH_TORRENTS_LOG_LEVEL', parse_log_level('LOG_LEVEL', logging.INFO)) + +log_level = parse_log_level( + "FETCH_TORRENTS_LOG_LEVEL", parse_log_level("LOG_LEVEL", logging.INFO) +) formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s") stream_handler = logging.StreamHandler() @@ -62,21 +68,26 @@ def parse_int(env_var: str, default: int) -> int: # grows without bound for the lifetime of a "deploy and forget" container - # see FETCH_TORRENTS_LOG_MAX_BYTES/FETCH_TORRENTS_LOG_BACKUP_COUNT in the # README. 5MB x 3 backups keeps a generous amount of history bounded to ~20MB. -log_max_bytes = parse_int('FETCH_TORRENTS_LOG_MAX_BYTES', 5 * 1024 * 1024) -log_backup_count = parse_int('FETCH_TORRENTS_LOG_BACKUP_COUNT', 3) +log_max_bytes = parse_int("FETCH_TORRENTS_LOG_MAX_BYTES", 5 * 1024 * 1024) +log_backup_count = parse_int("FETCH_TORRENTS_LOG_BACKUP_COUNT", 3) file_handler = logging.handlers.RotatingFileHandler( - log_file, maxBytes=log_max_bytes, backupCount=log_backup_count, + log_file, + maxBytes=log_max_bytes, + backupCount=log_backup_count, ) file_handler.setFormatter(formatter) + class RatioOnlyFilter(logging.Filter): def filter(self, record: logging.LogRecord) -> bool: return record.getMessage().startswith("[ratio]") + class NonRatioFilter(logging.Filter): def filter(self, record: logging.LogRecord) -> bool: return not record.getMessage().startswith("[ratio]") + class ImportantMessageFilter(logging.Filter): def __init__(self, level: int, important_prefixes=None): super().__init__() @@ -104,7 +115,8 @@ def get_always_log_prefixes(): "Old-release check complete", ] -always_log_enabled = parse_bool('FETCH_TORRENTS_ALWAYS_LOG', True) + +always_log_enabled = parse_bool("FETCH_TORRENTS_ALWAYS_LOG", True) important_prefixes = get_always_log_prefixes() if always_log_enabled else [] stream_handler.setLevel(logging.DEBUG) @@ -121,12 +133,14 @@ def get_always_log_prefixes(): else: file_handler.setLevel(log_level) -ratio_handler = logging.FileHandler(ratio_log_file, mode='w', delay=True, encoding='utf-8') +ratio_handler = logging.FileHandler( + ratio_log_file, mode="w", delay=True, encoding="utf-8" +) ratio_handler.setLevel(logging.INFO) ratio_handler.addFilter(RatioOnlyFilter()) ratio_handler.setFormatter(formatter) -logger = logging.getLogger('fetch_torrents') +logger = logging.getLogger("fetch_torrents") logger.setLevel(logging.DEBUG) logger.addHandler(stream_handler) logger.addHandler(file_handler) @@ -144,12 +158,12 @@ def get_rpc_credentials(): TRANSMISSION_RPC_WHITELIST is widened beyond localhost) makes every one of this script's own localhost RPC calls fail with 401 - silently, since callers only log the failure - permanently disabling cleanup.""" - username = os.getenv('TRANSMISSION_RPC_USERNAME', '').strip() - password = os.getenv('TRANSMISSION_RPC_PASSWORD', '').strip() + username = os.getenv("TRANSMISSION_RPC_USERNAME", "").strip() + password = os.getenv("TRANSMISSION_RPC_PASSWORD", "").strip() return username or None, password or None -def wait_for_transmission_rpc(host='localhost', port=9091, timeout=30): +def wait_for_transmission_rpc(host="localhost", port=9091, timeout=30): start = time.time() while time.time() - start < timeout: try: @@ -164,12 +178,12 @@ def get_previous_ratios(log_file): if not os.path.exists(log_file): return {} ratios = {} - with open(log_file, 'r', encoding='utf-8') as f: + with open(log_file, "r", encoding="utf-8") as f: lines = f.readlines() # Only parse the latest ratio segment between the last start and end markers. - start_marker = '[ratio] RATIOS START' - end_marker = '[ratio] RATIOS END' + start_marker = "[ratio] RATIOS START" + end_marker = "[ratio] RATIOS END" start_index = None end_index = None @@ -182,25 +196,26 @@ def get_previous_ratios(log_file): if start_index is not None: if end_index is not None and end_index > start_index: - lines = lines[start_index + 1:end_index] + lines = lines[start_index + 1 : end_index] else: - lines = lines[start_index + 1:] + lines = lines[start_index + 1 :] for line in lines: - match = re.search(r'\[ratio\]\s+(.+?)\s+→\s+(\d+\.\d+)', line) + match = re.search(r"\[ratio\]\s+(.+?)\s+→\s+(\d+\.\d+)", line) if match: name = match.group(1).strip() ratio = float(match.group(2)) ratios[name] = ratio return ratios + distro_patterns = { - 'ubuntu': re.compile(r'^ubuntu-|^lbuntu-|^xbuntu-'), - 'debian': re.compile(r'^debian-'), - 'kali': re.compile(r'^kali-linux-'), - 'arch': re.compile(r'^archlinux-'), - 'mint': re.compile(r'^linuxmint-'), - 'fedora': re.compile(r'^Fedora-Workstation-Live-'), + "ubuntu": re.compile(r"^ubuntu-|^lbuntu-|^xbuntu-"), + "debian": re.compile(r"^debian-"), + "kali": re.compile(r"^kali-linux-"), + "arch": re.compile(r"^archlinux-"), + "mint": re.compile(r"^linuxmint-"), + "fedora": re.compile(r"^Fedora-Workstation-Live-"), # devuan has no entry here deliberately: it publishes a single # multi-file torrent per release (see fetch_devuan_latest()), so its # name (e.g. "devuan_excalibur") carries no dotted version string for @@ -215,12 +230,12 @@ def get_previous_ratios(log_file): # instead of distro_patterns above wherever Devuan needs identifying without # a dotted version to parse: should_fetch_torrent()'s ratio gate and # _old_version_candidates()'s cleanup grouping. -DEVUAN_NAME_PATTERN = re.compile(r'^devuan_') +DEVUAN_NAME_PATTERN = re.compile(r"^devuan_") # Every distro fetch_torrents.py knows how to fetch, for FETCH_TORRENTS_DISTROS # validation - deliberately not the same set as distro_patterns above (see # the devuan comment there). -ALL_DISTROS = tuple(distro_patterns.keys()) + ('devuan',) +ALL_DISTROS = tuple(distro_patterns.keys()) + ("devuan",) # Distros fetched when FETCH_TORRENTS_DISTROS is unset. Arch and Devuan are # opt-in only - see tasks/01-demand-checks-for-default-distros.md: @@ -230,15 +245,16 @@ def get_previous_ratios(log_file): # - Devuan bundles every edition/arch into one multi-file torrent under a # single info-hash, so any demand check only ever reads the swarm as a # whole, never which of the bundled files anyone actually wants. -OPT_IN_DISTROS = ('arch', 'devuan') +OPT_IN_DISTROS = ("arch", "devuan") DEFAULT_DISTROS = tuple(d for d in ALL_DISTROS if d not in OPT_IN_DISTROS) -def parse_supported_distros(env_var='FETCH_TORRENTS_DISTROS'): - value = os.getenv(env_var, '').strip() + +def parse_supported_distros(env_var="FETCH_TORRENTS_DISTROS"): + value = os.getenv(env_var, "").strip() if not value: return list(DEFAULT_DISTROS) - requested = [entry.strip().lower() for entry in value.split(',') if entry.strip()] + requested = [entry.strip().lower() for entry in value.split(",") if entry.strip()] valid = [d for d in ALL_DISTROS if d in requested] invalid = [entry for entry in requested if entry not in ALL_DISTROS] @@ -282,12 +298,14 @@ def parse_supported_distros(env_var='FETCH_TORRENTS_DISTROS'): # # FETCH_TORRENTS_INCLUDE_LOW_DEMAND opts back into all of the above. LOW_DEMAND_PATTERN = re.compile( - r'(cloud-genericcloud|^kali-linux-\d+\.\d+-installer-netinst-)', re.IGNORECASE + r"(cloud-genericcloud|^kali-linux-\d+\.\d+-installer-netinst-)", re.IGNORECASE ) + def is_low_demand_variant(name): return bool(LOW_DEMAND_PATTERN.search(name)) + def filter_low_demand(torrents, include_low_demand): if include_low_demand: return dict(torrents) @@ -296,58 +314,63 @@ def filter_low_demand(torrents, include_low_demand): if is_low_demand_variant(name): logger.info( "Skipping %s – low-demand image family (set " - "FETCH_TORRENTS_INCLUDE_LOW_DEMAND=true to include).", name, + "FETCH_TORRENTS_INCLUDE_LOW_DEMAND=true to include).", + name, ) else: kept[name] = url return kept + def get_distro(name): for distro, pattern in distro_patterns.items(): if pattern.match(name): return distro return None + def version_to_tuple(v): try: - return tuple(map(int, v.split('.'))) + return tuple(map(int, v.split("."))) except ValueError as exc: raise ValueError(f"Invalid version string '{v}'") from exc + def parse_version_type(name, distro): - if distro == 'ubuntu': - parts = name.split('-') + if distro == "ubuntu": + parts = name.split("-") prefix = parts[0] version = parts[1] - suffix = '-'.join(parts[2:]) + suffix = "-".join(parts[2:]) type_ = f"{prefix}-{suffix}" - elif distro == 'debian': - parts = name.split('-') + elif distro == "debian": + parts = name.split("-") version = parts[1] arch = parts[2] - type_suffix = '-'.join(parts[3:]) + type_suffix = "-".join(parts[3:]) type_ = f"{arch}-{type_suffix}" - elif distro == 'kali': - parts = name.split('-') + elif distro == "kali": + parts = name.split("-") version = parts[2] - type_ = '-'.join(parts[3:]) - elif distro == 'arch': - parts = name.split('-') + type_ = "-".join(parts[3:]) + elif distro == "arch": + parts = name.split("-") version = parts[1] - type_ = '' - elif distro == 'mint': - parts = name.split('-') + type_ = "" + elif distro == "mint": + parts = name.split("-") version = parts[1] - type_ = '-'.join(parts[2:]) - elif distro == 'fedora': - parts = name.split('-') + type_ = "-".join(parts[2:]) + elif distro == "fedora": + parts = name.split("-") version = parts[-1] type_ = parts[-2] else: - version = '' - type_ = '' + version = "" + type_ = "" return version, type_ + def _should_fetch_devuan(name, ratios, removed_history): """Devuan bundles every edition/arch into one multi-file torrent per release (see fetch_devuan_latest()), so there's no per-name version to @@ -361,11 +384,14 @@ def _should_fetch_devuan(name, ratios, removed_history): if name in ratios: return True # already holding this exact release - prior_ratios = [r for n, r in ratios.items() if n != name and DEVUAN_NAME_PATTERN.match(n)] + prior_ratios = [ + r for n, r in ratios.items() if n != name and DEVUAN_NAME_PATTERN.match(n) + ] if not prior_ratios: prior_ratios = [ - record['ratio'] for n, record in removed_history.items() - if n != name and DEVUAN_NAME_PATTERN.match(n) and 'ratio' in record + record["ratio"] + for n, record in removed_history.items() + if n != name and DEVUAN_NAME_PATTERN.match(n) and "ratio" in record ] if not prior_ratios: @@ -383,7 +409,7 @@ def should_fetch_torrent(name, ratios, removed_history=None): below - bypassing the ratio gate for exactly the versions cleanup already gave up on. Live ratios still take priority when a name somehow appears in both.""" - if os.getenv('SKIP_RATIO_CHECK', 'false').lower() == 'true': + if os.getenv("SKIP_RATIO_CHECK", "false").lower() == "true": return True distro = get_distro(name) if not distro: @@ -410,19 +436,25 @@ def should_fetch_torrent(name, ratios, removed_history=None): if t == type_: type_ratios[version_to_tuple(v_str)] = r except Exception as exc: - logger.warning("Skipping stored ratio entry %s due to parse error: %s", n, exc) + logger.warning( + "Skipping stored ratio entry %s due to parse error: %s", n, exc + ) for n, record in removed_history.items(): - if n in ratios or 'ratio' not in record: + if n in ratios or "ratio" not in record: continue d = get_distro(n) if d == distro: try: v_str, t = parse_version_type(n, d) if t == type_: - type_ratios.setdefault(version_to_tuple(v_str), record['ratio']) + type_ratios.setdefault(version_to_tuple(v_str), record["ratio"]) except Exception as exc: - logger.warning("Skipping removed-history ratio entry %s due to parse error: %s", n, exc) + logger.warning( + "Skipping removed-history ratio entry %s due to parse error: %s", + n, + exc, + ) if not type_ratios: return True # no previous, fetch @@ -433,8 +465,16 @@ def should_fetch_torrent(name, ratios, removed_history=None): prev_max = max(prev_versions) return type_ratios[prev_max] >= 1.0 -def format_run_summary(elapsed, success_count, existing_count, not_found_count, failure_count, - old_added=None, old_skipped=None): + +def format_run_summary( + elapsed, + success_count, + existing_count, + not_found_count, + failure_count, + old_added=None, + old_skipped=None, +): """The single end-of-run log line. old_added/old_skipped are only non-None when FETCH_TORRENTS_CHECK_OLD_RELEASES ran, so the count of torrents that flag introduced is visible at a glance rather than only @@ -456,7 +496,7 @@ def _safe_watch_path(name): the check here means a future looser fetch_*() regex (parsing scraped HTML/JSON from upstream) can't turn a crafted name into a write/delete outside watch_dir.""" - if not name or name in ('.', '..') or '/' in name or '\\' in name: + if not name or name in (".", "..") or "/" in name or "\\" in name: return None return os.path.join(watch_dir, f"{name}.torrent") @@ -489,17 +529,20 @@ def download_torrent(name, url): logger.error(f"Failed to download {url}: {e}") return "failed" + def fetch_ubuntu_lts(): url = "https://releases.ubuntu.com/" try: - text = requests.get("https://changelogs.ubuntu.com/meta-release-lts", timeout=30).text - blocks = [b for b in text.strip().split("\n\n") if "Supported: 1" in b] + text = requests.get( + "https://changelogs.ubuntu.com/meta-release-lts", timeout=30 + ).text + blocks = [b for b in text.strip().split("\n\n") if "Supported: 1" in b] results = {} # newest first (optional – remove reversed() if order is irrelevant) for block in reversed(blocks): - version = re.search(r"Version:\s*([\d.]+)", block).group(1) - codename = re.search(r"Dist:\s*(\w+)", block).group(1) + version = re.search(r"Version:\s*([\d.]+)", block).group(1) + codename = re.search(r"Dist:\s*(\w+)", block).group(1) torrent_urls = [ f"https://releases.ubuntu.com/{codename}/ubuntu-{version}-desktop-amd64.iso.torrent", @@ -520,13 +563,14 @@ def fetch_ubuntu_lts(): logger.error(f"Ubuntu fetch error: {e}") return False + def fetch_debian_stable(): urls = [ - "https://cdimage.debian.org/debian-cd/current/amd64/bt-dvd/", - "https://cdimage.debian.org/debian-cd/current/arm64/bt-dvd/", - "https://cdimage.debian.org/debian-cd/current/amd64/bt-cd/", - "https://cdimage.debian.org/debian-cd/current/arm64/bt-cd/" - ] + "https://cdimage.debian.org/debian-cd/current/amd64/bt-dvd/", + "https://cdimage.debian.org/debian-cd/current/arm64/bt-dvd/", + "https://cdimage.debian.org/debian-cd/current/amd64/bt-cd/", + "https://cdimage.debian.org/debian-cd/current/arm64/bt-cd/", + ] results = {} for url in urls: @@ -534,9 +578,9 @@ def fetch_debian_stable(): r = requests.get(url, timeout=30) r.raise_for_status() soup = BeautifulSoup(r.text, "html.parser") - - for link in soup.find_all('a', href=True): - href = link['href'] + + for link in soup.find_all("a", href=True): + href = link["href"] if ".iso.torrent" in href: torrent_url = url + href # Keep the .iso extension so this matches what Transmission @@ -545,15 +589,16 @@ def fetch_debian_stable(): results[name] = torrent_url break else: - logger.warning("No Debian DVD-1 torrent found.") - + logger.warning("No Debian DVD-1 torrent found.") + except Exception as e: logger.error(f"Debian fetch error: {e}") return results + def _kali_torrent_urls(ver): - base_cd = f"https://cdimage.kali.org/kali-{ver}/kali-linux-{ver}-installer" + base_cd = f"https://cdimage.kali.org/kali-{ver}/kali-linux-{ver}-installer" base_arm = f"https://kali.download/arm-images/kali-{ver}/kali-linux-{ver}" base_cloud = f"https://kali.download/cloud-images/kali-{ver}/kali-linux-{ver}-cloud-genericcloud" return [ @@ -563,11 +608,9 @@ def _kali_torrent_urls(ver): f"{base_cd}-arm64.iso.torrent", f"{base_cd}-netinst-arm64.iso.torrent", f"{base_cd}-purple-amd64.iso.torrent", - f"{base_arm}-raspberry-pi-armhf.img.xz.torrent", f"{base_arm}-raspberry-pi-zero-2-w-armhf.img.xz.torrent", f"{base_arm}-raspberry-pi-zero-w-armel.img.xz.torrent", - f"{base_cloud}-amd64.tar.xz.torrent", f"{base_cloud}-arm64.tar.xz.torrent", ] @@ -614,8 +657,8 @@ def fetch_kali_old_versions(): soup = BeautifulSoup(html, "html.parser") versions = [] - for link in soup.find_all('a', href=True): - match = re.match(r'^kali-(\d+\.\d+)/$', link['href']) + for link in soup.find_all("a", href=True): + match = re.match(r"^kali-(\d+\.\d+)/$", link["href"]) if match: versions.append(match.group(1)) @@ -636,6 +679,7 @@ def fetch_kali_old_versions(): logger.error("Kali old-release fetch error: %s", exc) return {} + def fetch_arch_latest(): """Opt-in (FETCH_TORRENTS_DISTROS=arch,...): Arch's official torrent publishes no tracker at all - only DHT and ~400 HTTP webseeds - so real @@ -662,7 +706,7 @@ def fetch_arch_latest(): link = row.find("a", href=re.compile(torrent_url_pattern)) if not link: continue - href = link['href'] + href = link["href"] version = re.sub(torrent_url_pattern, "\\1", href) candidates.append((version, href)) @@ -704,6 +748,7 @@ def fetch_devuan_latest(): logger.error(f"Devuan fetch error: {e}") return False + def fetch_linuxmint_cinnamon(): url = "https://www.linuxmint.com/download.php" try: @@ -724,6 +769,7 @@ def fetch_linuxmint_cinnamon(): logger.error(f"Linux Mint fetch error: {e}") return False + def fetch_fedora_workstation(): url = "https://torrent.fedoraproject.org/torrents/" try: @@ -731,15 +777,17 @@ def fetch_fedora_workstation(): r.raise_for_status() soup = BeautifulSoup(r.text, "html.parser") - pattern = re.compile(r"^Fedora-Workstation-Live-([A-Za-z0-9_]+)-(\d+)\.torrent$") + pattern = re.compile( + r"^Fedora-Workstation-Live-([A-Za-z0-9_]+)-(\d+)\.torrent$" + ) latest = {} - for link in soup.find_all('a', href=True): - match = pattern.match(link['href']) + for link in soup.find_all("a", href=True): + match = pattern.match(link["href"]) if not match: continue arch, version = match.group(1), int(match.group(2)) if arch not in latest or version > latest[arch][0]: - latest[arch] = (version, link['href']) + latest[arch] = (version, link["href"]) if not latest: logger.warning("No Fedora Workstation torrents found on %s", url) @@ -770,19 +818,23 @@ def fetch_fedora_workstation_old_versions(): r.raise_for_status() soup = BeautifulSoup(r.text, "html.parser") - pattern = re.compile(r"^Fedora-Workstation-Live-([A-Za-z0-9_]+)-(\d+)\.torrent$") + pattern = re.compile( + r"^Fedora-Workstation-Live-([A-Za-z0-9_]+)-(\d+)\.torrent$" + ) by_arch = {} - for link in soup.find_all('a', href=True): - match = pattern.match(link['href']) + for link in soup.find_all("a", href=True): + match = pattern.match(link["href"]) if not match: continue arch, version = match.group(1), int(match.group(2)) - by_arch.setdefault(arch, []).append((version, link['href'])) + by_arch.setdefault(arch, []).append((version, link["href"])) results = {} for versions in by_arch.values(): versions.sort(key=lambda entry: entry[0], reverse=True) - for _version, href in versions[1:]: # newest is fetch_fedora_workstation()'s job + for _version, href in versions[ + 1: + ]: # newest is fetch_fedora_workstation()'s job name = href.replace(".torrent", "") results[name] = url + href return results @@ -792,7 +844,9 @@ def fetch_fedora_workstation_old_versions(): return {} -def has_unmet_demand(scrape_result, min_leechers, min_leecher_ratio, allow_zero_seeders): +def has_unmet_demand( + scrape_result, min_leechers, min_leecher_ratio, allow_zero_seeders +): """True if a live tracker scrape shows real, currently-unserved demand: at least min_leechers leechers in absolute terms (so a single leecher never justifies a download on its own), AND at least min_leecher_ratio @@ -816,16 +870,18 @@ def has_unmet_demand(scrape_result, min_leechers, min_leecher_ratio, allow_zero_ scraping failed.""" if not scrape_result: return False - leechers = scrape_result.get('leechers', 0) + leechers = scrape_result.get("leechers", 0) if leechers < min_leechers: return False - seeders = scrape_result.get('seeders', 0) + seeders = scrape_result.get("seeders", 0) if seeders == 0: return allow_zero_seeders return leechers >= min_leecher_ratio * seeders -def evaluate_old_release_demand(name, url, min_leechers, min_leecher_ratio, allow_zero_seeders, timeout=15): +def evaluate_old_release_demand( + name, url, min_leechers, min_leecher_ratio, allow_zero_seeders, timeout=15 +): """Downloads name's .torrent metadata and scrapes its trackers for live demand, without adding it to Transmission. Returns (keep, torrent_bytes, scrape_result): keep is True once has_unmet_demand() passes; @@ -853,7 +909,9 @@ def evaluate_old_release_demand(name, url, min_leechers, min_leecher_ratio, allo finally: os.remove(tmp.name) - keep = has_unmet_demand(scrape_result, min_leechers, min_leecher_ratio, allow_zero_seeders) + keep = has_unmet_demand( + scrape_result, min_leechers, min_leecher_ratio, allow_zero_seeders + ) return keep, torrent_bytes, scrape_result @@ -861,16 +919,20 @@ def load_old_release_check_state(path): if not os.path.exists(path): return {} try: - with open(path, 'r', encoding='utf-8') as f: + with open(path, "r", encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError) as exc: - logger.warning("Could not read old-release check state at %s (%s) - starting fresh.", path, exc) + logger.warning( + "Could not read old-release check state at %s (%s) - starting fresh.", + path, + exc, + ) return {} def save_old_release_check_state(path, state): tmp_path = f"{path}.tmp" - with open(tmp_path, 'w', encoding='utf-8') as f: + with open(tmp_path, "w", encoding="utf-8") as f: json.dump(state, f, indent=2, sort_keys=True) os.replace(tmp_path, path) @@ -883,7 +945,7 @@ def should_recheck_old_release(name, check_state, today, recheck_interval_days): record = check_state.get(name) if record is None: return True - last_checked = date.fromisoformat(record['last_checked']) + last_checked = date.fromisoformat(record["last_checked"]) return (today - last_checked).days >= recheck_interval_days @@ -891,16 +953,20 @@ def load_removed_history(path): if not os.path.exists(path): return {} try: - with open(path, 'r', encoding='utf-8') as f: + with open(path, "r", encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError) as exc: - logger.warning("Could not read removed-torrent history at %s (%s) - starting fresh.", path, exc) + logger.warning( + "Could not read removed-torrent history at %s (%s) - starting fresh.", + path, + exc, + ) return {} def save_removed_history(path, history): tmp_path = f"{path}.tmp" - with open(tmp_path, 'w', encoding='utf-8') as f: + with open(tmp_path, "w", encoding="utf-8") as f: json.dump(history, f, indent=2, sort_keys=True) os.replace(tmp_path, path) @@ -919,7 +985,11 @@ def record_removal(history, name, today, reason, ratio): Transmission stops reporting this (now-deleted) torrent - see its removed_history fallback.""" updated = dict(history) - updated[name] = {'removed_date': today.isoformat(), 'reason': reason, 'ratio': round(float(ratio), 3)} + updated[name] = { + "removed_date": today.isoformat(), + "reason": reason, + "ratio": round(float(ratio), 3), + } return updated @@ -931,14 +1001,22 @@ def record_removal(history, name, today, reason, ratio): # check - no separate discovery needed. Debian and Mint don't expose a # similarly easy back-catalog with the current fetch approach. OLD_RELEASE_DISCOVERY = { - 'kali': fetch_kali_old_versions, - 'fedora': fetch_fedora_workstation_old_versions, + "kali": fetch_kali_old_versions, + "fedora": fetch_fedora_workstation_old_versions, } -def check_old_releases_for_demand(selected_distros, include_low_demand, min_leechers, min_leecher_ratio, - allow_zero_seeders, check_state=None, removed_history=None, today=None, - recheck_interval_days=7): +def check_old_releases_for_demand( + selected_distros, + include_low_demand, + min_leechers, + min_leecher_ratio, + allow_zero_seeders, + check_state=None, + removed_history=None, + today=None, + recheck_interval_days=7, +): """FETCH_TORRENTS_CHECK_OLD_RELEASES: for distros with old-release discovery, scrape each not-yet-seeded candidate's trackers and add it to the watch dir only if there's live unmet demand - no need to leech it @@ -984,22 +1062,32 @@ def check_old_releases_for_demand(selected_distros, include_low_demand, min_leec if name in removed_history: logger.info( "Skipping old release %s - previously removed on %s, not re-adding automatically.", - name, removed_history[name]['removed_date'], + name, + removed_history[name]["removed_date"], ) skipped += 1 continue - if not should_recheck_old_release(name, check_state, today, recheck_interval_days): - logger.debug("Skipping old release %s - checked recently, not due for recheck.", name) + if not should_recheck_old_release( + name, check_state, today, recheck_interval_days + ): + logger.debug( + "Skipping old release %s - checked recently, not due for recheck.", + name, + ) skipped += 1 continue keep, torrent_bytes, scrape_result = evaluate_old_release_demand( - name, url, min_leechers, min_leecher_ratio, allow_zero_seeders, + name, + url, + min_leechers, + min_leecher_ratio, + allow_zero_seeders, ) check_state[name] = { - 'last_checked': today.isoformat(), - 'leechers': scrape_result['leechers'] if scrape_result else None, + "last_checked": today.isoformat(), + "leechers": scrape_result["leechers"] if scrape_result else None, } if keep: @@ -1007,33 +1095,48 @@ def check_old_releases_for_demand(selected_distros, include_low_demand, min_leec f.write(torrent_bytes) logger.info( "Found unmet demand on old release %s (%d leechers via %s) - added.", - name, scrape_result['leechers'], scrape_result['tracker'], + name, + scrape_result["leechers"], + scrape_result["tracker"], ) added += 1 else: - if scrape_result is not None and scrape_result.get('seeders', 0) == 0 and not allow_zero_seeders: + if ( + scrape_result is not None + and scrape_result.get("seeders", 0) == 0 + and not allow_zero_seeders + ): logger.info( "Skipping old release %s - %d leechers but 0 seeders via %s (no verified-complete copy " "in the swarm; set FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS=true to gamble on it).", - name, scrape_result['leechers'], scrape_result['tracker'], + name, + scrape_result["leechers"], + scrape_result["tracker"], ) elif scrape_result is not None: logger.info( "Skipping old release %s - %d leechers / %d seeders via %s " "(need >= %d leechers and >= %.1fx as many leechers as seeders).", - name, scrape_result['leechers'], scrape_result.get('seeders', 0), - scrape_result['tracker'], min_leechers, min_leecher_ratio, + name, + scrape_result["leechers"], + scrape_result.get("seeders", 0), + scrape_result["tracker"], + min_leechers, + min_leecher_ratio, ) elif torrent_bytes is not None: logger.info( - "Skipping old release %s - could not determine live demand from any tracker.", name, + "Skipping old release %s - could not determine live demand from any tracker.", + name, ) skipped += 1 return added, skipped, check_state -def log_seed_ratios_via_http(rpc_url="http://localhost:9091/transmission/rpc", auth: tuple | None = None): +def log_seed_ratios_via_http( + rpc_url="http://localhost:9091/transmission/rpc", auth: tuple | None = None +): if auth is None: username, password = get_rpc_credentials() auth = (username, password) if username and password else None @@ -1043,7 +1146,7 @@ def log_seed_ratios_via_http(rpc_url="http://localhost:9091/transmission/rpc", a headers = {"X-Transmission-Session-Id": r.headers["X-Transmission-Session-Id"]} payload = { "method": "torrent-get", - "arguments": {"fields": ["name", "uploadRatio"]} + "arguments": {"fields": ["name", "uploadRatio"]}, } r = requests.post(rpc_url, json=payload, headers=headers, auth=auth, timeout=15) r.raise_for_status() @@ -1063,16 +1166,17 @@ def log_seed_ratios_via_http(rpc_url="http://localhost:9091/transmission/rpc", a logger.info("[ratio] RATIOS END") logger.info("") + def _devuan_sort_key(torrent): """Devuan releases carry no dotted version in their name to compare (see DEVUAN_NAME_PATTERN), so _old_version_candidates() orders them by when Transmission added them instead - the most-recently-added one is treated as the current release, the same role the highest parsed version plays for every other distro.""" - added = getattr(torrent, 'added_date', None) + added = getattr(torrent, "added_date", None) if added is None: return 0.0 - if hasattr(added, 'timestamp'): + if hasattr(added, "timestamp"): return added.timestamp() return float(added) @@ -1093,7 +1197,9 @@ def _old_version_candidates(torrents): groups = {} for torrent in torrents: if DEVUAN_NAME_PATTERN.match(torrent.name): - groups.setdefault(('devuan', ''), []).append((_devuan_sort_key(torrent), torrent)) + groups.setdefault(("devuan", ""), []).append( + (_devuan_sort_key(torrent), torrent) + ) continue distro = get_distro(torrent.name) if not distro: @@ -1102,7 +1208,9 @@ def _old_version_candidates(torrents): version_str, type_ = parse_version_type(torrent.name, distro) version = version_to_tuple(version_str) except Exception as exc: - logger.warning("Skipping %s during cleanup due to parse error: %s", torrent.name, exc) + logger.warning( + "Skipping %s during cleanup due to parse error: %s", torrent.name, exc + ) continue groups.setdefault((distro, type_), []).append((version, torrent)) @@ -1115,6 +1223,7 @@ def _old_version_candidates(torrents): candidates.extend(torrent for _version, torrent in entries[1:]) return candidates + def _remove_watch_file(name): """Best-effort: once cleanup deletes a torrent's downloaded data, also remove its leftover .torrent file from /watch - otherwise these @@ -1136,7 +1245,7 @@ def plan_cleanup(torrents, skip_ratio_check=False, min_ratio=1.0): to_remove = [] to_keep_low_ratio = [] for torrent in _old_version_candidates(torrents): - ratio = float(getattr(torrent, 'ratio', 0.0) or 0.0) + ratio = float(getattr(torrent, "ratio", 0.0) or 0.0) if not skip_ratio_check and ratio < min_ratio: to_keep_low_ratio.append((torrent, ratio)) else: @@ -1144,23 +1253,26 @@ def plan_cleanup(torrents, skip_ratio_check=False, min_ratio=1.0): return to_remove, to_keep_low_ratio + def cleanup_old_versions(): """Space-constrained option (CLEANUP_KEEP_ONLY_LATEST_VERSION=true): remove superseded versions as soon as they clear the ratio floor, regardless of whether anyone's still downloading them.""" username, password = get_rpc_credentials() - tc = Client(host='localhost', port=9091, username=username, password=password) + tc = Client(host="localhost", port=9091, username=username, password=password) torrents = tc.get_torrents() - skip_ratio_check = parse_bool('CLEANUP_SKIP_RATIO_CHECK', False) - min_ratio = float(os.getenv('CLEANUP_MIN_RATIO', '1.0')) + skip_ratio_check = parse_bool("CLEANUP_SKIP_RATIO_CHECK", False) + min_ratio = float(os.getenv("CLEANUP_MIN_RATIO", "1.0")) to_remove, to_keep_low_ratio = plan_cleanup(torrents, skip_ratio_check, min_ratio) for torrent, ratio in to_keep_low_ratio: logger.info( "Keeping old version %s – ratio %.3f is below cleanup threshold %.3f.", - torrent.name, ratio, min_ratio, + torrent.name, + ratio, + min_ratio, ) removed_history = load_removed_history(removed_history_file) @@ -1170,8 +1282,11 @@ def cleanup_old_versions(): tc.remove_torrent(torrent.id, delete_data=True) _remove_watch_file(torrent.name) removed_history = record_removal( - removed_history, torrent.name, today, 'keep_only_latest', - float(getattr(torrent, 'ratio', 0.0) or 0.0), + removed_history, + torrent.name, + today, + "keep_only_latest", + float(getattr(torrent, "ratio", 0.0) or 0.0), ) save_removed_history(removed_history_file, removed_history) @@ -1189,7 +1304,7 @@ def plan_disk_pressure_cleanup(torrents, used_percent, threshold_percent): if threshold_percent <= 0 or used_percent < threshold_percent: return [] candidates = _old_version_candidates(torrents) - candidates.sort(key=lambda t: float(getattr(t, 'ratio', 0.0) or 0.0)) + candidates.sort(key=lambda t: float(getattr(t, "ratio", 0.0) or 0.0)) return candidates @@ -1206,14 +1321,16 @@ def enforce_disk_usage_limit(threshold_percent, download_path="/downloads"): return username, password = get_rpc_credentials() - tc = Client(host='localhost', port=9091, username=username, password=password) + tc = Client(host="localhost", port=9091, username=username, password=password) torrents = tc.get_torrents() candidates = plan_disk_pressure_cleanup(torrents, used_percent, threshold_percent) if not candidates: logger.warning( "Downloads folder usage is %.1f%% (>= %.1f%% threshold) but no superseded " - "versions are available to free space.", used_percent, threshold_percent, + "versions are available to free space.", + used_percent, + threshold_percent, ) return @@ -1224,14 +1341,19 @@ def enforce_disk_usage_limit(threshold_percent, download_path="/downloads"): used_percent = (used / total * 100) if total else 0.0 if used_percent < threshold_percent: break - ratio = float(getattr(torrent, 'ratio', 0.0) or 0.0) + ratio = float(getattr(torrent, "ratio", 0.0) or 0.0) logger.warning( "Downloads folder usage %.1f%% >= %.1f%% threshold - removing %s (ratio %.3f) to free space.", - used_percent, threshold_percent, torrent.name, ratio, + used_percent, + threshold_percent, + torrent.name, + ratio, ) tc.remove_torrent(torrent.id, delete_data=True) _remove_watch_file(torrent.name) - removed_history = record_removal(removed_history, torrent.name, today, 'disk_pressure', ratio) + removed_history = record_removal( + removed_history, torrent.name, today, "disk_pressure", ratio + ) save_removed_history(removed_history_file, removed_history) @@ -1242,16 +1364,18 @@ def load_ratio_history(path): if not os.path.exists(path): return {} try: - with open(path, 'r', encoding='utf-8') as f: + with open(path, "r", encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError) as exc: - logger.warning("Could not read ratio history at %s (%s) – starting fresh.", path, exc) + logger.warning( + "Could not read ratio history at %s (%s) – starting fresh.", path, exc + ) return {} def save_ratio_history(path, history): tmp_path = f"{path}.tmp" - with open(tmp_path, 'w', encoding='utf-8') as f: + with open(tmp_path, "w", encoding="utf-8") as f: json.dump(history, f, indent=2, sort_keys=True) os.replace(tmp_path, path) @@ -1262,16 +1386,27 @@ def save_ratio_history(path, history): # taken every RATIO_HISTORY_SAMPLE_INTERVAL_DAYS, pruned once older than # window_days + the interval. That's a fixed handful of small numbers per # torrent forever, not one entry per run. -def update_ratio_history(history, current_ratios, today, sample_interval_days=RATIO_HISTORY_SAMPLE_INTERVAL_DAYS, window_days=30): +def update_ratio_history( + history, + current_ratios, + today, + sample_interval_days=RATIO_HISTORY_SAMPLE_INTERVAL_DAYS, + window_days=30, +): max_age_days = window_days + sample_interval_days updated = {} for name, ratio in current_ratios.items(): samples = list(history.get(name, [])) - if not samples or (today - date.fromisoformat(samples[-1]['date'])).days >= sample_interval_days: - samples.append({'date': today.isoformat(), 'ratio': ratio}) + if ( + not samples + or (today - date.fromisoformat(samples[-1]["date"])).days + >= sample_interval_days + ): + samples.append({"date": today.isoformat(), "ratio": ratio}) updated[name] = [ - s for s in samples - if (today - date.fromisoformat(s['date'])).days <= max_age_days + s + for s in samples + if (today - date.fromisoformat(s["date"])).days <= max_age_days ] return updated @@ -1280,13 +1415,19 @@ def _stagnation_anchor(samples, today, window_days): """Ratio of the oldest recorded sample that's at least window_days old, or None if we don't have one yet - i.e. this torrent is too new to judge either way.""" - old_enough = [s for s in samples if (today - date.fromisoformat(s['date'])).days >= window_days] + old_enough = [ + s + for s in samples + if (today - date.fromisoformat(s["date"])).days >= window_days + ] if not old_enough: return None - return min(old_enough, key=lambda s: s['date'])['ratio'] + return min(old_enough, key=lambda s: s["date"])["ratio"] -def plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_delta=0.02): +def plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 +): """Default cleanup path: among superseded (non-latest) versions per (distro, type) group, remove ones whose ratio has grown less than min_ratio_delta over the last window_days - nobody's downloading them @@ -1296,7 +1437,7 @@ def plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_ to_remove = [] to_keep = [] for torrent in _old_version_candidates(torrents): - ratio = float(getattr(torrent, 'ratio', 0.0) or 0.0) + ratio = float(getattr(torrent, "ratio", 0.0) or 0.0) anchor = _stagnation_anchor(history.get(torrent.name, []), today, window_days) if anchor is None: to_keep.append((torrent, None)) @@ -1311,43 +1452,58 @@ def plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_ def cleanup_stagnant_torrents(): username, password = get_rpc_credentials() - tc = Client(host='localhost', port=9091, username=username, password=password) + tc = Client(host="localhost", port=9091, username=username, password=password) torrents = tc.get_torrents() - window_days = int(os.getenv('CLEANUP_STAGNATION_WINDOW_DAYS', '30')) - min_ratio_delta = float(os.getenv('CLEANUP_STAGNATION_MIN_RATIO_DELTA', '0.02')) + window_days = int(os.getenv("CLEANUP_STAGNATION_WINDOW_DAYS", "30")) + min_ratio_delta = float(os.getenv("CLEANUP_STAGNATION_MIN_RATIO_DELTA", "0.02")) today = date.today() history = load_ratio_history(ratio_history_file) - current_ratios = {t.name: float(getattr(t, 'ratio', 0.0) or 0.0) for t in torrents} - history = update_ratio_history(history, current_ratios, today, window_days=window_days) + current_ratios = {t.name: float(getattr(t, "ratio", 0.0) or 0.0) for t in torrents} + history = update_ratio_history( + history, current_ratios, today, window_days=window_days + ) - to_remove, to_keep = plan_stagnation_cleanup(torrents, history, today, window_days, min_ratio_delta) + to_remove, to_keep = plan_stagnation_cleanup( + torrents, history, today, window_days, min_ratio_delta + ) for torrent, delta in to_keep: if delta is None: logger.info( "Keeping old version %s – not enough ratio history yet (need %d days).", - torrent.name, window_days, + torrent.name, + window_days, ) else: logger.info( "Keeping old version %s – ratio grew by %.3f over the last %d days.", - torrent.name, delta, window_days, + torrent.name, + delta, + window_days, ) removed_history = load_removed_history(removed_history_file) for torrent in to_remove: logger.info( "Removing stagnant torrent: %s (no meaningful ratio growth in %d days)", - torrent.name, window_days, + torrent.name, + window_days, ) tc.remove_torrent(torrent.id, delete_data=True) _remove_watch_file(torrent.name) - removed_history = record_removal(removed_history, torrent.name, today, 'stagnant', current_ratios[torrent.name]) + removed_history = record_removal( + removed_history, + torrent.name, + today, + "stagnant", + current_ratios[torrent.name], + ) save_removed_history(removed_history_file, removed_history) save_ratio_history(ratio_history_file, history) + if __name__ == "__main__": start_time = time.time() logger.info("Starting torrent fetch run.") @@ -1361,24 +1517,29 @@ def cleanup_stagnant_torrents(): not_found_count = 0 distro_funcs = [ - ('ubuntu', fetch_ubuntu_lts), - ('debian', fetch_debian_stable), - ('kali', fetch_kali_latest), - ('arch', fetch_arch_latest), - ('mint', fetch_linuxmint_cinnamon), - ('fedora', fetch_fedora_workstation), - ('devuan', fetch_devuan_latest), + ("ubuntu", fetch_ubuntu_lts), + ("debian", fetch_debian_stable), + ("kali", fetch_kali_latest), + ("arch", fetch_arch_latest), + ("mint", fetch_linuxmint_cinnamon), + ("fedora", fetch_fedora_workstation), + ("devuan", fetch_devuan_latest), ] selected_distros = parse_supported_distros() logger.info("Selected distros for this run: %s", ", ".join(selected_distros)) - include_low_demand = parse_bool('FETCH_TORRENTS_INCLUDE_LOW_DEMAND', False) - logger.info("Including low-demand image families (arm64/cloud/etc.): %s", include_low_demand) + include_low_demand = parse_bool("FETCH_TORRENTS_INCLUDE_LOW_DEMAND", False) + logger.info( + "Including low-demand image families (arm64/cloud/etc.): %s", include_low_demand + ) for distro, func in distro_funcs: if distro not in selected_distros: - logger.info("Skipping distro %s because it is not enabled by FETCH_TORRENTS_DISTROS.", distro) + logger.info( + "Skipping distro %s because it is not enabled by FETCH_TORRENTS_DISTROS.", + distro, + ) continue logger.info(f"Fetching latest {distro} torrents...") @@ -1397,34 +1558,61 @@ def cleanup_stagnant_torrents(): else: failure_count += 1 else: - logger.info(f"Skipping {name} due to low ratio on previous version.") + logger.info( + f"Skipping {name} due to low ratio on previous version." + ) else: failure_count += 1 old_added = old_skipped = None - check_old_releases = parse_bool('FETCH_TORRENTS_CHECK_OLD_RELEASES', False) + check_old_releases = parse_bool("FETCH_TORRENTS_CHECK_OLD_RELEASES", False) if check_old_releases: - old_release_min_leechers = int(os.getenv('FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS', '10')) - old_release_min_leecher_ratio = float(os.getenv('FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO', '0.1')) - old_release_allow_zero_seeders = parse_bool('FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS', False) - old_release_recheck_days = int(os.getenv('FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS', '7')) - skip_previously_removed = parse_bool('FETCH_TORRENTS_OLD_RELEASE_SKIP_REMOVED', True) + old_release_min_leechers = int( + os.getenv("FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS", "10") + ) + old_release_min_leecher_ratio = float( + os.getenv("FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO", "0.1") + ) + old_release_allow_zero_seeders = parse_bool( + "FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS", False + ) + old_release_recheck_days = int( + os.getenv("FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS", "7") + ) + skip_previously_removed = parse_bool( + "FETCH_TORRENTS_OLD_RELEASE_SKIP_REMOVED", True + ) check_state = load_old_release_check_state(old_release_check_state_file) - removed_history = load_removed_history(removed_history_file) if skip_previously_removed else {} + removed_history = ( + load_removed_history(removed_history_file) + if skip_previously_removed + else {} + ) logger.info( "Checking old releases for unmet demand (min leechers: %d, min leecher/seeder ratio: %.1fx, " "allow zero-seeder swarms: %s, recheck interval: %d days)...", - old_release_min_leechers, old_release_min_leecher_ratio, old_release_allow_zero_seeders, + old_release_min_leechers, + old_release_min_leecher_ratio, + old_release_allow_zero_seeders, old_release_recheck_days, ) old_added, old_skipped, check_state = check_old_releases_for_demand( - selected_distros, include_low_demand, old_release_min_leechers, old_release_min_leecher_ratio, - old_release_allow_zero_seeders, check_state, removed_history, date.today(), old_release_recheck_days, + selected_distros, + include_low_demand, + old_release_min_leechers, + old_release_min_leecher_ratio, + old_release_allow_zero_seeders, + check_state, + removed_history, + date.today(), + old_release_recheck_days, ) save_old_release_check_state(old_release_check_state_file, check_state) - logger.info("Old-release check complete: %d added, %d skipped.", old_added, old_skipped) + logger.info( + "Old-release check complete: %d added, %d skipped.", old_added, old_skipped + ) if wait_for_transmission_rpc(): try: @@ -1432,10 +1620,12 @@ def cleanup_stagnant_torrents(): except Exception as exc: logger.error("Could not query Transmission: %s", exc) else: - logger.error("Transmission RPC not available on localhost:9091; skipping ratio logs.") - + logger.error( + "Transmission RPC not available on localhost:9091; skipping ratio logs." + ) + try: - if parse_bool('CLEANUP_KEEP_ONLY_LATEST_VERSION', False): + if parse_bool("CLEANUP_KEEP_ONLY_LATEST_VERSION", False): cleanup_old_versions() else: cleanup_stagnant_torrents() @@ -1443,15 +1633,27 @@ def cleanup_stagnant_torrents(): logger.error("Could not clean up old versions: %s", exc) try: - disk_usage_threshold = float(os.getenv('CLEANUP_DISK_USAGE_THRESHOLD_PERCENT', '95')) + disk_usage_threshold = float( + os.getenv("CLEANUP_DISK_USAGE_THRESHOLD_PERCENT", "95") + ) enforce_disk_usage_limit(disk_usage_threshold) except Exception as exc: logger.error("Could not enforce disk usage limit: %s", exc) total, used, free = shutil.disk_usage("/downloads") - logger.info(f"Downloads folder usage: {used // (2**30)} GB used / {total // (2**30)} GB total") + logger.info( + f"Downloads folder usage: {used // (2**30)} GB used / {total // (2**30)} GB total" + ) elapsed = time.time() - start_time - logger.info(format_run_summary( - elapsed, success_count, existing_count, not_found_count, failure_count, old_added, old_skipped, - )) + logger.info( + format_run_summary( + elapsed, + success_count, + existing_count, + not_found_count, + failure_count, + old_added, + old_skipped, + ) + ) diff --git a/tests/test_configure_transmission.py b/tests/test_configure_transmission.py index 9292a45..f01cd8a 100644 --- a/tests/test_configure_transmission.py +++ b/tests/test_configure_transmission.py @@ -2,6 +2,7 @@ Run with: python -m unittest discover -s tests """ + import os import sys import tempfile @@ -19,56 +20,62 @@ def test_no_relevant_env_vars_yields_no_overrides(self): self.assertEqual(ct.build_rpc_overrides({}), {}) def test_whitelist_env_var_broadens_and_keeps_whitelist_enabled(self): - overrides = ct.build_rpc_overrides({'TRANSMISSION_RPC_WHITELIST': '10.0.0.*'}) + overrides = ct.build_rpc_overrides({"TRANSMISSION_RPC_WHITELIST": "10.0.0.*"}) self.assertEqual( overrides, - {'rpc-whitelist-enabled': True, 'rpc-whitelist': '10.0.0.*'}, + {"rpc-whitelist-enabled": True, "rpc-whitelist": "10.0.0.*"}, ) def test_blank_whitelist_env_var_is_ignored(self): - self.assertEqual(ct.build_rpc_overrides({'TRANSMISSION_RPC_WHITELIST': ' '}), {}) + self.assertEqual( + ct.build_rpc_overrides({"TRANSMISSION_RPC_WHITELIST": " "}), {} + ) def test_username_and_password_enable_authentication(self): - overrides = ct.build_rpc_overrides({ - 'TRANSMISSION_RPC_USERNAME': 'alice', - 'TRANSMISSION_RPC_PASSWORD': 'hunter2', - }) + overrides = ct.build_rpc_overrides( + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + } + ) self.assertEqual( overrides, { - 'rpc-authentication-required': True, - 'rpc-username': 'alice', - 'rpc-password': 'hunter2', + "rpc-authentication-required": True, + "rpc-username": "alice", + "rpc-password": "hunter2", }, ) def test_username_without_password_does_not_enable_authentication(self): - overrides = ct.build_rpc_overrides({'TRANSMISSION_RPC_USERNAME': 'alice'}) + overrides = ct.build_rpc_overrides({"TRANSMISSION_RPC_USERNAME": "alice"}) self.assertEqual(overrides, {}) def test_password_without_username_does_not_enable_authentication(self): - overrides = ct.build_rpc_overrides({'TRANSMISSION_RPC_PASSWORD': 'hunter2'}) + overrides = ct.build_rpc_overrides({"TRANSMISSION_RPC_PASSWORD": "hunter2"}) self.assertEqual(overrides, {}) def test_whitelist_and_auth_can_combine(self): - overrides = ct.build_rpc_overrides({ - 'TRANSMISSION_RPC_WHITELIST': '*', - 'TRANSMISSION_RPC_USERNAME': 'alice', - 'TRANSMISSION_RPC_PASSWORD': 'hunter2', - }) + overrides = ct.build_rpc_overrides( + { + "TRANSMISSION_RPC_WHITELIST": "*", + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + } + ) self.assertEqual( overrides, { - 'rpc-whitelist-enabled': True, - 'rpc-whitelist': '*', - 'rpc-authentication-required': True, - 'rpc-username': 'alice', - 'rpc-password': 'hunter2', + "rpc-whitelist-enabled": True, + "rpc-whitelist": "*", + "rpc-authentication-required": True, + "rpc-username": "alice", + "rpc-password": "hunter2", }, ) @@ -78,44 +85,58 @@ def test_no_relevant_env_vars_yields_no_overrides(self): self.assertEqual(ct.build_peer_limit_overrides({}), {}) def test_global_limit_env_var_sets_override(self): - overrides = ct.build_peer_limit_overrides({'TRANSMISSION_PEER_LIMIT_GLOBAL': '1000'}) + overrides = ct.build_peer_limit_overrides( + {"TRANSMISSION_PEER_LIMIT_GLOBAL": "1000"} + ) - self.assertEqual(overrides, {'peer-limit-global': 1000}) + self.assertEqual(overrides, {"peer-limit-global": 1000}) def test_per_torrent_limit_env_var_sets_override(self): - overrides = ct.build_peer_limit_overrides({'TRANSMISSION_PEER_LIMIT_PER_TORRENT': '300'}) + overrides = ct.build_peer_limit_overrides( + {"TRANSMISSION_PEER_LIMIT_PER_TORRENT": "300"} + ) - self.assertEqual(overrides, {'peer-limit-per-torrent': 300}) + self.assertEqual(overrides, {"peer-limit-per-torrent": 300}) def test_both_limits_can_combine(self): - overrides = ct.build_peer_limit_overrides({ - 'TRANSMISSION_PEER_LIMIT_GLOBAL': '1000', - 'TRANSMISSION_PEER_LIMIT_PER_TORRENT': '300', - }) + overrides = ct.build_peer_limit_overrides( + { + "TRANSMISSION_PEER_LIMIT_GLOBAL": "1000", + "TRANSMISSION_PEER_LIMIT_PER_TORRENT": "300", + } + ) - self.assertEqual(overrides, {'peer-limit-global': 1000, 'peer-limit-per-torrent': 300}) + self.assertEqual( + overrides, {"peer-limit-global": 1000, "peer-limit-per-torrent": 300} + ) def test_blank_env_vars_are_ignored(self): - overrides = ct.build_peer_limit_overrides({ - 'TRANSMISSION_PEER_LIMIT_GLOBAL': ' ', - 'TRANSMISSION_PEER_LIMIT_PER_TORRENT': '', - }) + overrides = ct.build_peer_limit_overrides( + { + "TRANSMISSION_PEER_LIMIT_GLOBAL": " ", + "TRANSMISSION_PEER_LIMIT_PER_TORRENT": "", + } + ) self.assertEqual(overrides, {}) def test_non_numeric_env_vars_are_ignored(self): - overrides = ct.build_peer_limit_overrides({ - 'TRANSMISSION_PEER_LIMIT_GLOBAL': 'unlimited', - 'TRANSMISSION_PEER_LIMIT_PER_TORRENT': 'lots', - }) + overrides = ct.build_peer_limit_overrides( + { + "TRANSMISSION_PEER_LIMIT_GLOBAL": "unlimited", + "TRANSMISSION_PEER_LIMIT_PER_TORRENT": "lots", + } + ) self.assertEqual(overrides, {}) def test_zero_and_negative_values_are_ignored(self): - overrides = ct.build_peer_limit_overrides({ - 'TRANSMISSION_PEER_LIMIT_GLOBAL': '0', - 'TRANSMISSION_PEER_LIMIT_PER_TORRENT': '-5', - }) + overrides = ct.build_peer_limit_overrides( + { + "TRANSMISSION_PEER_LIMIT_GLOBAL": "0", + "TRANSMISSION_PEER_LIMIT_PER_TORRENT": "-5", + } + ) self.assertEqual(overrides, {}) @@ -124,7 +145,7 @@ class BuildPortForwardingOverridesTests(unittest.TestCase): def test_no_env_var_on_first_run_defaults_to_disabled(self): overrides = ct.build_port_forwarding_overrides({}, settings_file_exists=False) - self.assertEqual(overrides, {'port-forwarding-enabled': False}) + self.assertEqual(overrides, {"port-forwarding-enabled": False}) def test_no_env_var_on_later_run_yields_no_override(self): overrides = ct.build_port_forwarding_overrides({}, settings_file_exists=True) @@ -133,28 +154,32 @@ def test_no_env_var_on_later_run_yields_no_override(self): def test_explicit_true_enables_it_even_on_first_run(self): overrides = ct.build_port_forwarding_overrides( - {'TRANSMISSION_PORT_FORWARDING': 'true'}, settings_file_exists=False, + {"TRANSMISSION_PORT_FORWARDING": "true"}, + settings_file_exists=False, ) - self.assertEqual(overrides, {'port-forwarding-enabled': True}) + self.assertEqual(overrides, {"port-forwarding-enabled": True}) def test_explicit_false_disables_it_on_a_later_run(self): overrides = ct.build_port_forwarding_overrides( - {'TRANSMISSION_PORT_FORWARDING': 'false'}, settings_file_exists=True, + {"TRANSMISSION_PORT_FORWARDING": "false"}, + settings_file_exists=True, ) - self.assertEqual(overrides, {'port-forwarding-enabled': False}) + self.assertEqual(overrides, {"port-forwarding-enabled": False}) def test_blank_env_var_on_first_run_falls_back_to_default(self): overrides = ct.build_port_forwarding_overrides( - {'TRANSMISSION_PORT_FORWARDING': ' '}, settings_file_exists=False, + {"TRANSMISSION_PORT_FORWARDING": " "}, + settings_file_exists=False, ) - self.assertEqual(overrides, {'port-forwarding-enabled': False}) + self.assertEqual(overrides, {"port-forwarding-enabled": False}) def test_invalid_env_var_on_later_run_yields_no_override(self): overrides = ct.build_port_forwarding_overrides( - {'TRANSMISSION_PORT_FORWARDING': 'sometimes'}, settings_file_exists=True, + {"TRANSMISSION_PORT_FORWARDING": "sometimes"}, + settings_file_exists=True, ) self.assertEqual(overrides, {}) @@ -162,18 +187,22 @@ def test_invalid_env_var_on_later_run_yields_no_override(self): class WarnIfOpenWithoutAuthTests(unittest.TestCase): def test_warns_when_whitelist_broadened_without_auth(self): - warning = ct.warn_if_open_without_auth({'rpc-whitelist-enabled': True, 'rpc-whitelist': '*'}) + warning = ct.warn_if_open_without_auth( + {"rpc-whitelist-enabled": True, "rpc-whitelist": "*"} + ) self.assertIsNotNone(warning) def test_no_warning_when_auth_also_required(self): - warning = ct.warn_if_open_without_auth({ - 'rpc-whitelist-enabled': True, - 'rpc-whitelist': '*', - 'rpc-authentication-required': True, - 'rpc-username': 'alice', - 'rpc-password': 'hunter2', - }) + warning = ct.warn_if_open_without_auth( + { + "rpc-whitelist-enabled": True, + "rpc-whitelist": "*", + "rpc-authentication-required": True, + "rpc-username": "alice", + "rpc-password": "hunter2", + } + ) self.assertIsNone(warning) @@ -183,21 +212,25 @@ def test_no_warning_when_whitelist_untouched(self): class ApplyOverridesTests(unittest.TestCase): def test_overrides_take_precedence_but_other_keys_survive(self): - settings = {'peer-port': 51413, 'rpc-whitelist-enabled': True, 'rpc-whitelist': '127.0.0.1'} - overrides = {'rpc-whitelist': '10.0.0.*'} + settings = { + "peer-port": 51413, + "rpc-whitelist-enabled": True, + "rpc-whitelist": "127.0.0.1", + } + overrides = {"rpc-whitelist": "10.0.0.*"} merged = ct.apply_overrides(settings, overrides) - self.assertEqual(merged['peer-port'], 51413) - self.assertEqual(merged['rpc-whitelist'], '10.0.0.*') - self.assertTrue(merged['rpc-whitelist-enabled']) + self.assertEqual(merged["peer-port"], 51413) + self.assertEqual(merged["rpc-whitelist"], "10.0.0.*") + self.assertTrue(merged["rpc-whitelist-enabled"]) class SettingsFilePersistenceTests(unittest.TestCase): def test_round_trips_through_disk(self): - tmp_dir = tempfile.mkdtemp(prefix='configure_transmission_') - path = os.path.join(tmp_dir, 'settings.json') - settings = {'rpc-whitelist': '10.0.0.*'} + tmp_dir = tempfile.mkdtemp(prefix="configure_transmission_") + path = os.path.join(tmp_dir, "settings.json") + settings = {"rpc-whitelist": "10.0.0.*"} ct.save_settings(path, settings) loaded = ct.load_settings(path) @@ -205,16 +238,16 @@ def test_round_trips_through_disk(self): self.assertEqual(loaded, settings) def test_missing_file_returns_empty_settings(self): - tmp_dir = tempfile.mkdtemp(prefix='configure_transmission_') - path = os.path.join(tmp_dir, 'does-not-exist.json') + tmp_dir = tempfile.mkdtemp(prefix="configure_transmission_") + path = os.path.join(tmp_dir, "does-not-exist.json") self.assertEqual(ct.load_settings(path), {}) def test_corrupt_file_returns_empty_settings_instead_of_raising(self): - tmp_dir = tempfile.mkdtemp(prefix='configure_transmission_') - path = os.path.join(tmp_dir, 'settings.json') - with open(path, 'w', encoding='utf-8') as f: - f.write('{not valid json') + tmp_dir = tempfile.mkdtemp(prefix="configure_transmission_") + path = os.path.join(tmp_dir, "settings.json") + with open(path, "w", encoding="utf-8") as f: + f.write("{not valid json") self.assertEqual(ct.load_settings(path), {}) diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index 5318978..22848ef 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -8,6 +8,7 @@ Run with: python -m unittest discover -s tests """ + import json import os import subprocess @@ -19,12 +20,14 @@ from datetime import date, timedelta from unittest.mock import MagicMock -if 'transmission_rpc' not in sys.modules: - stub = types.ModuleType('transmission_rpc') +if "transmission_rpc" not in sys.modules: + stub = types.ModuleType("transmission_rpc") stub.Client = MagicMock() - sys.modules['transmission_rpc'] = stub + sys.modules["transmission_rpc"] = stub -os.environ.setdefault('FETCH_TORRENTS_LOG_DIR', tempfile.mkdtemp(prefix='fetch_torrents_test_logs_')) +os.environ.setdefault( + "FETCH_TORRENTS_LOG_DIR", tempfile.mkdtemp(prefix="fetch_torrents_test_logs_") +) REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if REPO_ROOT not in sys.path: @@ -245,43 +248,63 @@ class ShouldFetchTorrentRatioKeyTests(unittest.TestCase): def test_finds_previous_version_and_honors_low_ratio_ubuntu(self): ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.5} - self.assertFalse(ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios)) + self.assertFalse( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) def test_finds_previous_version_and_honors_low_ratio_debian(self): ratios = {"debian-12.4.0-amd64-DVD-1.iso": 0.5} - self.assertFalse(ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios)) + self.assertFalse( + ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios) + ) def test_finds_previous_version_and_honors_low_ratio_kali(self): ratios = {"kali-linux-2023.4-installer-amd64.iso": 0.3} - self.assertFalse(ft.should_fetch_torrent("kali-linux-2024.1-installer-amd64.iso", ratios)) + self.assertFalse( + ft.should_fetch_torrent("kali-linux-2024.1-installer-amd64.iso", ratios) + ) def test_finds_previous_version_and_honors_low_ratio_arch(self): ratios = {"archlinux-2024.04.01-x86_64.iso": 0.2} - self.assertFalse(ft.should_fetch_torrent("archlinux-2024.05.01-x86_64.iso", ratios)) + self.assertFalse( + ft.should_fetch_torrent("archlinux-2024.05.01-x86_64.iso", ratios) + ) def test_finds_previous_version_and_allows_high_ratio_ubuntu(self): ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.5} - self.assertTrue(ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios)) + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) def test_finds_previous_version_and_allows_high_ratio_debian(self): ratios = {"debian-12.4.0-amd64-DVD-1.iso": 1.5} - self.assertTrue(ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios)) + self.assertTrue( + ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios) + ) def test_finds_previous_version_and_honors_low_ratio_mint(self): ratios = {"linuxmint-22.2-cinnamon-64bit.iso": 0.4} - self.assertFalse(ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios)) + self.assertFalse( + ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios) + ) def test_finds_previous_version_and_allows_high_ratio_mint(self): ratios = {"linuxmint-22.2-cinnamon-64bit.iso": 1.5} - self.assertTrue(ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios)) + self.assertTrue( + ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios) + ) def test_finds_previous_version_and_honors_low_ratio_fedora(self): ratios = {"Fedora-Workstation-Live-x86_64-43": 0.4} - self.assertFalse(ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios)) + self.assertFalse( + ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios) + ) def test_finds_previous_version_and_allows_high_ratio_fedora(self): ratios = {"Fedora-Workstation-Live-x86_64-43": 1.5} - self.assertTrue(ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios)) + self.assertTrue( + ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios) + ) class ShouldFetchTorrentRemovedHistoryFallbackTests(unittest.TestCase): @@ -294,22 +317,34 @@ class ShouldFetchTorrentRemovedHistoryFallbackTests(unittest.TestCase): def test_falls_back_to_removed_history_ratio_when_not_live(self): removed_history = { - 'ubuntu-23.10-desktop-amd64.iso': { - 'removed_date': '2026-01-01', 'reason': 'stagnant', 'ratio': 0.4, + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.4, }, } self.assertFalse( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios={}, removed_history=removed_history) + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) ) def test_removed_history_ratio_above_floor_allows_fetch(self): removed_history = { - 'ubuntu-23.10-desktop-amd64.iso': { - 'removed_date': '2026-01-01', 'reason': 'keep_only_latest', 'ratio': 1.5, + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "keep_only_latest", + "ratio": 1.5, }, } self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios={}, removed_history=removed_history) + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) ) def test_live_ratio_takes_priority_over_removed_history(self): @@ -317,31 +352,52 @@ def test_live_ratio_takes_priority_over_removed_history(self): # still reports hasn't been removed), but live data winning if it ever # happens is the safer default. removed_history = { - 'ubuntu-23.10-desktop-amd64.iso': { - 'removed_date': '2026-01-01', 'reason': 'stagnant', 'ratio': 0.1, + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, }, } - ratios = {'ubuntu-23.10-desktop-amd64.iso': 1.5} + ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.5} self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios, removed_history=removed_history) + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios, + removed_history=removed_history, + ) ) def test_removed_history_entry_without_ratio_is_ignored(self): # Legacy entries recorded before this field existed have no 'ratio' # key - must not crash, and must not count as a usable prior version. removed_history = { - 'ubuntu-23.10-desktop-amd64.iso': {'removed_date': '2026-01-01', 'reason': 'stagnant'}, + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + }, } self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios={}, removed_history=removed_history) + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) ) def test_removed_history_from_a_different_distro_is_not_consulted(self): removed_history = { - 'debian-12.4.0-amd64-DVD-1.iso': {'removed_date': '2026-01-01', 'reason': 'stagnant', 'ratio': 0.1}, + "debian-12.4.0-amd64-DVD-1.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + }, } self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios={}, removed_history=removed_history) + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) ) @@ -370,18 +426,30 @@ def test_already_holding_this_exact_release_is_always_fetched(self): def test_falls_back_to_removed_history_when_no_live_ratio(self): removed_history = { - "devuan_daedalus": {"removed_date": "2026-01-01", "reason": "stagnant", "ratio": 0.4}, + "devuan_daedalus": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.4, + }, } self.assertFalse( - ft.should_fetch_torrent("devuan_excalibur", ratios={}, removed_history=removed_history) + ft.should_fetch_torrent( + "devuan_excalibur", ratios={}, removed_history=removed_history + ) ) def test_other_distros_removed_history_not_consulted_for_devuan(self): removed_history = { - "debian-12.4.0-amd64-DVD-1.iso": {"removed_date": "2026-01-01", "reason": "stagnant", "ratio": 0.1}, + "debian-12.4.0-amd64-DVD-1.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + }, } self.assertTrue( - ft.should_fetch_torrent("devuan_excalibur", ratios={}, removed_history=removed_history) + ft.should_fetch_torrent( + "devuan_excalibur", ratios={}, removed_history=removed_history + ) ) @@ -394,9 +462,9 @@ class RatioLogSurvivesRestartTests(unittest.TestCase): about what happens between import and the first get_previous_ratios() call.""" def test_previous_ratios_are_readable_after_a_fresh_start(self): - log_dir = tempfile.mkdtemp(prefix='fetch_torrents_ratio_restart_') - ratio_log_path = os.path.join(log_dir, 'fetch_torrents_ratios.log') - with open(ratio_log_path, 'w', encoding='utf-8') as f: + log_dir = tempfile.mkdtemp(prefix="fetch_torrents_ratio_restart_") + ratio_log_path = os.path.join(log_dir, "fetch_torrents_ratios.log") + with open(ratio_log_path, "w", encoding="utf-8") as f: f.write( "2026-08-01 00:00:00,000 INFO: [ratio] RATIOS START\n" "2026-08-01 00:00:00,000 INFO: [ratio] archlinux-2026.04.01-x86_64.iso → 0.664\n" @@ -415,13 +483,17 @@ def test_previous_ratios_are_readable_after_a_fresh_start(self): ) env = dict(os.environ, FETCH_TORRENTS_LOG_DIR=log_dir) result = subprocess.run( - [sys.executable, '-c', script], - env=env, capture_output=True, text=True, timeout=30, + [sys.executable, "-c", script], + env=env, + capture_output=True, + text=True, + timeout=30, ) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn( - "archlinux-2026.04.01-x86_64.iso", result.stdout, + "archlinux-2026.04.01-x86_64.iso", + result.stdout, "get_previous_ratios() found nothing from the prior run's ratio " "log - it was likely wiped by the ratio FileHandler's mode='w' " "open at import time, before ever being read. That silently " @@ -438,7 +510,7 @@ class LogFileRotationTests(unittest.TestCase): was in effect during test collection).""" def _run(self, extra_env=None): - log_dir = tempfile.mkdtemp(prefix='fetch_torrents_log_rotation_') + log_dir = tempfile.mkdtemp(prefix="fetch_torrents_log_rotation_") script = ( "import os, sys, types\n" "from unittest.mock import MagicMock\n" @@ -453,8 +525,11 @@ def _run(self, extra_env=None): ) env = dict(os.environ, FETCH_TORRENTS_LOG_DIR=log_dir, **(extra_env or {})) result = subprocess.run( - [sys.executable, '-c', script], - env=env, capture_output=True, text=True, timeout=30, + [sys.executable, "-c", script], + env=env, + capture_output=True, + text=True, + timeout=30, ) self.assertEqual(result.returncode, 0, result.stderr) lines = result.stdout.splitlines() @@ -462,21 +537,27 @@ def _run(self, extra_env=None): def test_defaults_to_a_bounded_rotating_handler(self): handler_type, max_bytes, backup_count = self._run() - self.assertEqual(handler_type, 'RotatingFileHandler') + self.assertEqual(handler_type, "RotatingFileHandler") self.assertEqual(max_bytes, 5 * 1024 * 1024) self.assertEqual(backup_count, 3) def test_env_vars_override_rotation_settings(self): - _handler_type, max_bytes, backup_count = self._run({ - 'FETCH_TORRENTS_LOG_MAX_BYTES': '1000', 'FETCH_TORRENTS_LOG_BACKUP_COUNT': '1', - }) + _handler_type, max_bytes, backup_count = self._run( + { + "FETCH_TORRENTS_LOG_MAX_BYTES": "1000", + "FETCH_TORRENTS_LOG_BACKUP_COUNT": "1", + } + ) self.assertEqual(max_bytes, 1000) self.assertEqual(backup_count, 1) def test_invalid_env_vars_fall_back_to_defaults(self): - _handler_type, max_bytes, backup_count = self._run({ - 'FETCH_TORRENTS_LOG_MAX_BYTES': 'not-a-number', 'FETCH_TORRENTS_LOG_BACKUP_COUNT': 'also-bad', - }) + _handler_type, max_bytes, backup_count = self._run( + { + "FETCH_TORRENTS_LOG_MAX_BYTES": "not-a-number", + "FETCH_TORRENTS_LOG_BACKUP_COUNT": "also-bad", + } + ) self.assertEqual(max_bytes, 5 * 1024 * 1024) self.assertEqual(backup_count, 3) @@ -489,16 +570,14 @@ class FetchUbuntuLtsNamingTests(unittest.TestCase): tasks/02-ratio-key-mismatch-ubuntu-debian.md.""" def test_candidate_names_include_arch_and_iso_suffix(self): - meta_release = ( - "Dist: noble\n" - "Version: 24.04\n" - "Supported: 1\n" - ) + meta_release = "Dist: noble\nVersion: 24.04\nSupported: 1\n" class FakeResponse: text = meta_release - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_ubuntu_lts() self.assertTrue(results) @@ -519,7 +598,9 @@ class FakeResponse: def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_debian_stable() self.assertIn("debian-12.5.0-amd64-DVD-1.iso", results) @@ -537,7 +618,9 @@ def test_candidate_name_matches_torrent_internal_name(self): class FakeResponse: text = html - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_linuxmint_cinnamon() self.assertIn("linuxmint-22.3-cinnamon-64bit.iso", results) @@ -570,16 +653,16 @@ class FakeResponse: def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_fedora_workstation() self.assertEqual( results, { - "Fedora-Workstation-Live-x86_64-44": - "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-x86_64-44.torrent", - "Fedora-Workstation-Live-aarch64-44": - "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-aarch64-44.torrent", + "Fedora-Workstation-Live-x86_64-44": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-x86_64-44.torrent", + "Fedora-Workstation-Live-aarch64-44": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-aarch64-44.torrent", }, ) @@ -606,12 +689,16 @@ class FakeResponse: def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_arch_latest() self.assertEqual( results, - {"archlinux-2026.09.01": "https://archlinux.org/releng/releases/2026.09.01/torrent/"}, + { + "archlinux-2026.09.01": "https://archlinux.org/releng/releases/2026.09.01/torrent/" + }, ) def test_no_available_releases_returns_false(self): @@ -623,7 +710,9 @@ class FakeResponse: def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): result = ft.fetch_arch_latest() self.assertFalse(result) @@ -640,13 +729,15 @@ def test_extracts_current_release_torrent(self): html = ( '' + "" ) class FakeResponse: text = html - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_devuan_latest() self.assertEqual( @@ -658,7 +749,9 @@ def test_returns_false_when_no_torrent_link_found(self): class FakeResponse: text = "nothing here" - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): result = ft.fetch_devuan_latest() self.assertFalse(result) @@ -674,34 +767,42 @@ class ParseSupportedDistrosTests(unittest.TestCase): def test_default_excludes_opt_in_distros(self): with unittest.mock.patch.dict(os.environ, {}, clear=False): - os.environ.pop('FETCH_TORRENTS_DISTROS', None) + os.environ.pop("FETCH_TORRENTS_DISTROS", None) result = ft.parse_supported_distros() - self.assertNotIn('arch', result) - self.assertNotIn('devuan', result) - for expected in ('ubuntu', 'debian', 'kali', 'mint', 'fedora'): + self.assertNotIn("arch", result) + self.assertNotIn("devuan", result) + for expected in ("ubuntu", "debian", "kali", "mint", "fedora"): self.assertIn(expected, result) def test_explicit_opt_in_distros_are_honored(self): - with unittest.mock.patch.dict(os.environ, {'FETCH_TORRENTS_DISTROS': 'arch,devuan'}): + with unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "arch,devuan"} + ): result = ft.parse_supported_distros() - self.assertEqual(sorted(result), ['arch', 'devuan']) + self.assertEqual(sorted(result), ["arch", "devuan"]) def test_can_mix_default_and_opt_in_distros(self): - with unittest.mock.patch.dict(os.environ, {'FETCH_TORRENTS_DISTROS': 'ubuntu,arch'}): + with unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "ubuntu,arch"} + ): result = ft.parse_supported_distros() - self.assertEqual(sorted(result), ['arch', 'ubuntu']) + self.assertEqual(sorted(result), ["arch", "ubuntu"]) def test_unknown_entries_are_dropped_but_valid_ones_kept(self): - with unittest.mock.patch.dict(os.environ, {'FETCH_TORRENTS_DISTROS': 'ubuntu,nonsense'}): + with unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "ubuntu,nonsense"} + ): result = ft.parse_supported_distros() - self.assertEqual(result, ['ubuntu']) + self.assertEqual(result, ["ubuntu"]) def test_all_invalid_falls_back_to_default_not_all_distros(self): - with unittest.mock.patch.dict(os.environ, {'FETCH_TORRENTS_DISTROS': 'nonsense'}): + with unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "nonsense"} + ): result = ft.parse_supported_distros() self.assertEqual(sorted(result), sorted(ft.DEFAULT_DISTROS)) - self.assertNotIn('arch', result) - self.assertNotIn('devuan', result) + self.assertNotIn("arch", result) + self.assertNotIn("devuan", result) class LowDemandVariantTests(unittest.TestCase): @@ -791,8 +892,8 @@ class DownloadTorrent404Tests(unittest.TestCase): same way as a genuine download failure.""" def setUp(self): - self.tmp_watch_dir = tempfile.mkdtemp(prefix='fetch_torrents_test_watch_') - patcher = unittest.mock.patch.object(ft, 'watch_dir', self.tmp_watch_dir) + self.tmp_watch_dir = tempfile.mkdtemp(prefix="fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) @@ -800,13 +901,18 @@ def test_404_response_is_not_found_not_failed(self): class FakeResponse: status_code = 404 - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): status = ft.download_torrent( - 'ubuntu-14.04.6-live-server-amd64.iso', 'https://example.invalid/x.torrent' + "ubuntu-14.04.6-live-server-amd64.iso", + "https://example.invalid/x.torrent", ) - self.assertEqual(status, 'not_found') - dest = os.path.join(self.tmp_watch_dir, 'ubuntu-14.04.6-live-server-amd64.iso.torrent') + self.assertEqual(status, "not_found") + dest = os.path.join( + self.tmp_watch_dir, "ubuntu-14.04.6-live-server-amd64.iso.torrent" + ) self.assertFalse(os.path.exists(dest)) def test_other_http_error_is_still_failed(self): @@ -816,30 +922,36 @@ class FakeResponse: def raise_for_status(self): raise Exception("500 Server Error") - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): status = ft.download_torrent( - 'ubuntu-24.04-desktop-amd64.iso', 'https://example.invalid/x.torrent' + "ubuntu-24.04-desktop-amd64.iso", "https://example.invalid/x.torrent" ) - self.assertEqual(status, 'failed') + self.assertEqual(status, "failed") def test_successful_download_is_still_added(self): class FakeResponse: status_code = 200 - content = b'fake torrent bytes' + content = b"fake torrent bytes" def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): status = ft.download_torrent( - 'ubuntu-24.04-desktop-amd64.iso', 'https://example.invalid/x.torrent' + "ubuntu-24.04-desktop-amd64.iso", "https://example.invalid/x.torrent" ) - self.assertEqual(status, 'added') - dest = os.path.join(self.tmp_watch_dir, 'ubuntu-24.04-desktop-amd64.iso.torrent') - with open(dest, 'rb') as f: - self.assertEqual(f.read(), b'fake torrent bytes') + self.assertEqual(status, "added") + dest = os.path.join( + self.tmp_watch_dir, "ubuntu-24.04-desktop-amd64.iso.torrent" + ) + with open(dest, "rb") as f: + self.assertEqual(f.read(), b"fake torrent bytes") class SafeWatchPathTests(unittest.TestCase): @@ -850,46 +962,46 @@ class SafeWatchPathTests(unittest.TestCase): upstream content into a path escape.""" def setUp(self): - patcher = unittest.mock.patch.object(ft, 'watch_dir', '/watch') + patcher = unittest.mock.patch.object(ft, "watch_dir", "/watch") patcher.start() self.addCleanup(patcher.stop) def test_normal_name_resolves_under_watch_dir(self): self.assertEqual( - ft._safe_watch_path('ubuntu-24.04-desktop-amd64.iso'), - os.path.join('/watch', 'ubuntu-24.04-desktop-amd64.iso.torrent'), + ft._safe_watch_path("ubuntu-24.04-desktop-amd64.iso"), + os.path.join("/watch", "ubuntu-24.04-desktop-amd64.iso.torrent"), ) def test_rejects_path_separators(self): - self.assertIsNone(ft._safe_watch_path('archlinux-../../../etc/cron.d/evil')) - self.assertIsNone(ft._safe_watch_path('foo/bar')) - self.assertIsNone(ft._safe_watch_path('foo\\bar')) + self.assertIsNone(ft._safe_watch_path("archlinux-../../../etc/cron.d/evil")) + self.assertIsNone(ft._safe_watch_path("foo/bar")) + self.assertIsNone(ft._safe_watch_path("foo\\bar")) def test_rejects_empty_or_dot_names(self): - self.assertIsNone(ft._safe_watch_path('')) + self.assertIsNone(ft._safe_watch_path("")) self.assertIsNone(ft._safe_watch_path(None)) - self.assertIsNone(ft._safe_watch_path('.')) - self.assertIsNone(ft._safe_watch_path('..')) + self.assertIsNone(ft._safe_watch_path(".")) + self.assertIsNone(ft._safe_watch_path("..")) class DownloadTorrentUnsafeNameTests(unittest.TestCase): def setUp(self): - self.tmp_watch_dir = tempfile.mkdtemp(prefix='fetch_torrents_test_watch_') - patcher = unittest.mock.patch.object(ft, 'watch_dir', self.tmp_watch_dir) + self.tmp_watch_dir = tempfile.mkdtemp(prefix="fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) def test_download_torrent_refuses_unsafe_name_without_any_request(self): - with unittest.mock.patch.object(ft.requests, 'get') as mock_get: - status = ft.download_torrent('../evil', 'https://example.invalid/x.torrent') + with unittest.mock.patch.object(ft.requests, "get") as mock_get: + status = ft.download_torrent("../evil", "https://example.invalid/x.torrent") mock_get.assert_not_called() - self.assertEqual(status, 'failed') + self.assertEqual(status, "failed") self.assertEqual(os.listdir(self.tmp_watch_dir), []) def test_remove_watch_file_refuses_unsafe_name_without_touching_disk(self): - with unittest.mock.patch.object(ft.os, 'remove') as mock_remove: - ft._remove_watch_file('../evil') + with unittest.mock.patch.object(ft.os, "remove") as mock_remove: + ft._remove_watch_file("../evil") mock_remove.assert_not_called() @@ -903,7 +1015,9 @@ class UpdateRatioHistoryTests(unittest.TestCase): def test_new_torrent_gets_a_first_sample(self): today = date(2026, 8, 29) - history = ft.update_ratio_history({}, {"archlinux-2026.08.01-x86_64.iso": 1.2}, today) + history = ft.update_ratio_history( + {}, {"archlinux-2026.08.01-x86_64.iso": 1.2}, today + ) self.assertEqual( history, @@ -916,7 +1030,9 @@ def test_recent_sample_is_not_duplicated(self): "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-25", "ratio": 1.0}], } - updated = ft.update_ratio_history(history, {"archlinux-2026.08.01-x86_64.iso": 1.1}, today) + updated = ft.update_ratio_history( + history, {"archlinux-2026.08.01-x86_64.iso": 1.1}, today + ) # Only 4 days since the last sample (< the 7-day interval) - no new # sample yet, and the ratio value on the existing sample is untouched. @@ -931,7 +1047,9 @@ def test_new_sample_added_once_interval_elapses(self): "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-20", "ratio": 1.0}], } - updated = ft.update_ratio_history(history, {"archlinux-2026.08.01-x86_64.iso": 1.3}, today) + updated = ft.update_ratio_history( + history, {"archlinux-2026.08.01-x86_64.iso": 1.3}, today + ) self.assertEqual( updated, @@ -987,7 +1105,9 @@ def test_latest_or_only_version_is_never_removed_even_if_flat(self): "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-06-30", "ratio": 1.0}], } - to_remove, to_keep = ft.plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_delta=0.02) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) self.assertEqual(to_remove, []) self.assertEqual(to_keep, []) @@ -1002,7 +1122,9 @@ def test_old_version_with_no_ratio_growth_is_removed(self): "ubuntu-23.10-desktop-amd64.iso": [{"date": "2026-07-20", "ratio": 1.495}], } - to_remove, to_keep = ft.plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_delta=0.02) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) self.assertEqual(names_of(to_remove), ["ubuntu-23.10-desktop-amd64.iso"]) self.assertEqual(to_keep, []) @@ -1017,7 +1139,9 @@ def test_old_version_still_growing_is_kept(self): "ubuntu-23.10-desktop-amd64.iso": [{"date": "2026-07-20", "ratio": 1.2}], } - to_remove, to_keep = ft.plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_delta=0.02) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) self.assertEqual(to_remove, []) self.assertEqual(len(to_keep), 1) @@ -1036,7 +1160,9 @@ def test_old_version_without_enough_history_is_kept(self): "ubuntu-23.10-desktop-amd64.iso": [{"date": "2026-08-24", "ratio": 0.1}], } - to_remove, to_keep = ft.plan_stagnation_cleanup(torrents, history, today, window_days=30, min_ratio_delta=0.02) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) self.assertEqual(to_remove, []) self.assertEqual(len(to_keep), 1) @@ -1051,7 +1177,9 @@ def test_old_version_with_no_history_at_all_is_kept(self): FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), ] - to_remove, to_keep = ft.plan_stagnation_cleanup(torrents, {}, today, window_days=30, min_ratio_delta=0.02) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, {}, today, window_days=30, min_ratio_delta=0.02 + ) self.assertEqual(to_remove, []) self.assertEqual(len(to_keep), 1) @@ -1060,7 +1188,9 @@ def test_unrelated_single_torrent_groups_are_ignored(self): today = date(2026, 8, 29) torrents = [FakeTorrent("some-random-linux-distro-1.0.iso", 1, ratio=5.0)] - to_remove, to_keep = ft.plan_stagnation_cleanup(torrents, {}, today, window_days=30, min_ratio_delta=0.02) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, {}, today, window_days=30, min_ratio_delta=0.02 + ) self.assertEqual(to_remove, []) self.assertEqual(to_keep, []) @@ -1071,9 +1201,11 @@ class RatioHistoryPersistenceTests(unittest.TestCase): tolerate a missing or corrupt file rather than crashing a run.""" def test_round_trips_through_disk(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_ratio_history_') - path = os.path.join(tmp_dir, 'fetch_torrents_ratio_history.json') - history = {"archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-29", "ratio": 1.2}]} + tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_ratio_history_") + path = os.path.join(tmp_dir, "fetch_torrents_ratio_history.json") + history = { + "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-29", "ratio": 1.2}] + } ft.save_ratio_history(path, history) loaded = ft.load_ratio_history(path) @@ -1081,16 +1213,16 @@ def test_round_trips_through_disk(self): self.assertEqual(loaded, history) def test_missing_file_returns_empty_history(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_ratio_history_') - path = os.path.join(tmp_dir, 'does-not-exist.json') + tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_ratio_history_") + path = os.path.join(tmp_dir, "does-not-exist.json") self.assertEqual(ft.load_ratio_history(path), {}) def test_corrupt_file_returns_empty_history_instead_of_raising(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_ratio_history_') - path = os.path.join(tmp_dir, 'fetch_torrents_ratio_history.json') - with open(path, 'w', encoding='utf-8') as f: - f.write('{not valid json') + tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_ratio_history_") + path = os.path.join(tmp_dir, "fetch_torrents_ratio_history.json") + with open(path, "w", encoding="utf-8") as f: + f.write("{not valid json") self.assertEqual(ft.load_ratio_history(path), {}) @@ -1113,7 +1245,9 @@ def test_returns_only_the_non_newest_versions(self): class FakeResponse: text = html - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_kali_old_versions() self.assertTrue(results) @@ -1126,7 +1260,9 @@ def test_single_version_folder_returns_nothing(self): class FakeResponse: text = html - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_kali_old_versions() self.assertEqual(results, {}) @@ -1152,16 +1288,16 @@ class FakeResponse: def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_fedora_workstation_old_versions() self.assertEqual( results, { - "Fedora-Workstation-Live-x86_64-43": - "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-x86_64-43.torrent", - "Fedora-Workstation-Live-aarch64-43": - "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-aarch64-43.torrent", + "Fedora-Workstation-Live-x86_64-43": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-x86_64-43.torrent", + "Fedora-Workstation-Live-aarch64-43": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-aarch64-43.torrent", }, ) @@ -1174,7 +1310,9 @@ class FakeResponse: def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()): + with unittest.mock.patch.object( + ft.requests, "get", return_value=FakeResponse() + ): results = ft.fetch_fedora_workstation_old_versions() self.assertEqual(results, {}) @@ -1191,20 +1329,28 @@ class HasUnmetDemandTests(unittest.TestCase): def test_none_scrape_result_is_no_demand(self): self.assertFalse( - ft.has_unmet_demand(None, min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False) + ft.has_unmet_demand( + None, min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False + ) ) def test_below_absolute_threshold_is_no_demand(self): self.assertFalse( ft.has_unmet_demand( - {'leechers': 0, 'seeders': 0}, min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False + {"leechers": 0, "seeders": 0}, + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) ) def test_single_leecher_on_huge_swarm_is_no_demand(self): self.assertFalse( ft.has_unmet_demand( - {'leechers': 1, 'seeders': 1000}, min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False + {"leechers": 1, "seeders": 1000}, + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) ) @@ -1212,21 +1358,30 @@ def test_meets_absolute_but_not_ratio_is_no_demand(self): # 11 leechers clears a min_leechers=10 floor, but 11 < 1.2x the 10 seeders. self.assertFalse( ft.has_unmet_demand( - {'leechers': 11, 'seeders': 10}, min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=False + {"leechers": 11, "seeders": 10}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) ) def test_meets_both_absolute_and_ratio_thresholds_is_demand(self): self.assertTrue( ft.has_unmet_demand( - {'leechers': 25, 'seeders': 10}, min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=False + {"leechers": 25, "seeders": 10}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) ) def test_exactly_at_ratio_threshold_is_demand(self): self.assertTrue( ft.has_unmet_demand( - {'leechers': 12, 'seeders': 10}, min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=False + {"leechers": 12, "seeders": 10}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) ) @@ -1238,25 +1393,36 @@ def test_zero_seeders_is_excluded_by_default(self): # opts into the gamble via allow_zero_seeders. self.assertFalse( ft.has_unmet_demand( - {'leechers': 10, 'seeders': 0}, min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=False + {"leechers": 10, "seeders": 0}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) ) def test_zero_seeders_counts_as_demand_when_explicitly_allowed(self): self.assertTrue( ft.has_unmet_demand( - {'leechers': 10, 'seeders': 0}, min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=True + {"leechers": 10, "seeders": 0}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=True, ) ) def test_zero_seeders_still_needs_the_absolute_floor_when_allowed(self): self.assertFalse( ft.has_unmet_demand( - {'leechers': 5, 'seeders': 0}, min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=True + {"leechers": 5, "seeders": 0}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=True, ) ) - def test_lopsided_but_substantial_swarms_still_count_as_demand_at_default_ratio(self): + def test_lopsided_but_substantial_swarms_still_count_as_demand_at_default_ratio( + self, + ): # FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO defaults to 0.1: we # have no way of knowing whether existing seeders have spare upload # capacity or are bandwidth-limited, so leechers don't need to @@ -1264,12 +1430,18 @@ def test_lopsided_but_substantial_swarms_still_count_as_demand_at_default_ratio( # and 100:1000 both clearly indicate room for another seeder. self.assertTrue( ft.has_unmet_demand( - {'leechers': 10, 'seeders': 100}, min_leechers=1, min_leecher_ratio=0.1, allow_zero_seeders=False + {"leechers": 10, "seeders": 100}, + min_leechers=1, + min_leecher_ratio=0.1, + allow_zero_seeders=False, ) ) self.assertTrue( ft.has_unmet_demand( - {'leechers': 100, 'seeders': 1000}, min_leechers=1, min_leecher_ratio=0.1, allow_zero_seeders=False + {"leechers": 100, "seeders": 1000}, + min_leechers=1, + min_leecher_ratio=0.1, + allow_zero_seeders=False, ) ) @@ -1279,7 +1451,10 @@ def test_extreme_oversupply_still_fails_at_default_ratio(self): # permissive 0.1 default - it needs at least 67.3 leechers. self.assertFalse( ft.has_unmet_demand( - {'leechers': 3, 'seeders': 673}, min_leechers=1, min_leecher_ratio=0.1, allow_zero_seeders=False + {"leechers": 3, "seeders": 673}, + min_leechers=1, + min_leecher_ratio=0.1, + allow_zero_seeders=False, ) ) @@ -1289,11 +1464,18 @@ def test_404_is_not_kept_and_not_scraped(self): class FakeResponse: status_code = 404 - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()), \ - unittest.mock.patch.object(ft.torrent_scrape, 'scrape_torrent') as mock_scrape: + with ( + unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()), + unittest.mock.patch.object( + ft.torrent_scrape, "scrape_torrent" + ) as mock_scrape, + ): keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - 'ubuntu-14.04.6-desktop-amd64.iso', 'https://example.invalid/x.torrent', - min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False, + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) mock_scrape.assert_not_called() @@ -1302,10 +1484,15 @@ class FakeResponse: self.assertIsNone(scrape_result) def test_download_failure_is_not_kept(self): - with unittest.mock.patch.object(ft.requests, 'get', side_effect=ConnectionError("nope")): + with unittest.mock.patch.object( + ft.requests, "get", side_effect=ConnectionError("nope") + ): keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - 'ubuntu-14.04.6-desktop-amd64.iso', 'https://example.invalid/x.torrent', - min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False, + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) self.assertFalse(keep) @@ -1315,75 +1502,108 @@ def test_download_failure_is_not_kept(self): def test_scrape_below_threshold_is_not_kept(self): class FakeResponse: status_code = 200 - content = b'fake torrent bytes' + content = b"fake torrent bytes" def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()), \ - unittest.mock.patch.object( - ft.torrent_scrape, 'scrape_torrent', - return_value={'tracker': 'http://t.example/scrape', 'seeders': 10, 'leechers': 0, 'completed': 0}, - ): + with ( + unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()), + unittest.mock.patch.object( + ft.torrent_scrape, + "scrape_torrent", + return_value={ + "tracker": "http://t.example/scrape", + "seeders": 10, + "leechers": 0, + "completed": 0, + }, + ), + ): keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - 'ubuntu-14.04.6-desktop-amd64.iso', 'https://example.invalid/x.torrent', - min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False, + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) self.assertFalse(keep) - self.assertEqual(torrent_bytes, b'fake torrent bytes') - self.assertEqual(scrape_result['leechers'], 0) + self.assertEqual(torrent_bytes, b"fake torrent bytes") + self.assertEqual(scrape_result["leechers"], 0) def test_scrape_meeting_threshold_is_kept(self): class FakeResponse: status_code = 200 - content = b'fake torrent bytes' + content = b"fake torrent bytes" def raise_for_status(self): pass - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()), \ - unittest.mock.patch.object( - ft.torrent_scrape, 'scrape_torrent', - return_value={'tracker': 'http://t.example/scrape', 'seeders': 10, 'leechers': 30, 'completed': 0}, - ): + with ( + unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()), + unittest.mock.patch.object( + ft.torrent_scrape, + "scrape_torrent", + return_value={ + "tracker": "http://t.example/scrape", + "seeders": 10, + "leechers": 30, + "completed": 0, + }, + ), + ): keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - 'ubuntu-14.04.6-desktop-amd64.iso', 'https://example.invalid/x.torrent', - min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False, + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) self.assertTrue(keep) - self.assertEqual(torrent_bytes, b'fake torrent bytes') - self.assertEqual(scrape_result['leechers'], 30) + self.assertEqual(torrent_bytes, b"fake torrent bytes") + self.assertEqual(scrape_result["leechers"], 30) def test_meets_absolute_floor_but_already_well_seeded_is_not_kept(self): class FakeResponse: status_code = 200 - content = b'fake torrent bytes' + content = b"fake torrent bytes" def raise_for_status(self): pass # 1 leecher clears a min_leechers=1 floor, but 1000 seeders already # cover it - not unmet demand. - with unittest.mock.patch.object(ft.requests, 'get', return_value=FakeResponse()), \ - unittest.mock.patch.object( - ft.torrent_scrape, 'scrape_torrent', - return_value={'tracker': 'http://t.example/scrape', 'seeders': 1000, 'leechers': 1, 'completed': 0}, - ): + with ( + unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()), + unittest.mock.patch.object( + ft.torrent_scrape, + "scrape_torrent", + return_value={ + "tracker": "http://t.example/scrape", + "seeders": 1000, + "leechers": 1, + "completed": 0, + }, + ), + ): keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - 'ubuntu-14.04.6-desktop-amd64.iso', 'https://example.invalid/x.torrent', - min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False, + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, ) self.assertFalse(keep) - self.assertEqual(scrape_result['leechers'], 1) + self.assertEqual(scrape_result["leechers"], 1) class CheckOldReleasesForDemandTests(unittest.TestCase): def setUp(self): - self.tmp_watch_dir = tempfile.mkdtemp(prefix='fetch_torrents_test_watch_') - patcher = unittest.mock.patch.object(ft, 'watch_dir', self.tmp_watch_dir) + self.tmp_watch_dir = tempfile.mkdtemp(prefix="fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) @@ -1391,71 +1611,136 @@ def test_adds_only_torrents_with_unmet_demand(self): # Neither name matches LOW_DEMAND_PATTERN - that filtering is # covered separately by test_low_demand_variant_is_filtered_before_evaluation. candidates = { - 'kali-linux-2026.1-installer-purple-amd64.iso': 'https://example.invalid/dead.torrent', - 'kali-linux-2026.1-installer-amd64.iso': 'https://example.invalid/alive.torrent', + "kali-linux-2026.1-installer-purple-amd64.iso": "https://example.invalid/dead.torrent", + "kali-linux-2026.1-installer-amd64.iso": "https://example.invalid/alive.torrent", } - def fake_evaluate(name, url, min_leechers, min_leecher_ratio, allow_zero_seeders, timeout=15): - if 'purple' in name: - return False, b'dead bytes', {'tracker': 't', 'seeders': 5, 'leechers': 0, 'completed': 0} - return True, b'alive bytes', {'tracker': 't', 'seeders': 5, 'leechers': 30, 'completed': 0} + def fake_evaluate( + name, url, min_leechers, min_leecher_ratio, allow_zero_seeders, timeout=15 + ): + if "purple" in name: + return ( + False, + b"dead bytes", + {"tracker": "t", "seeders": 5, "leechers": 0, "completed": 0}, + ) + return ( + True, + b"alive bytes", + {"tracker": "t", "seeders": 5, "leechers": 30, "completed": 0}, + ) - with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {'kali': lambda: candidates}, clear=True), \ - unittest.mock.patch.object(ft, 'evaluate_old_release_demand', side_effect=fake_evaluate): - added, skipped, _ = ft.check_old_releases_for_demand(['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False) + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand", side_effect=fake_evaluate + ), + ): + added, skipped, _ = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) self.assertEqual(added, 1) self.assertEqual(skipped, 1) - alive_path = os.path.join(self.tmp_watch_dir, 'kali-linux-2026.1-installer-amd64.iso.torrent') - dead_path = os.path.join(self.tmp_watch_dir, 'kali-linux-2026.1-installer-purple-amd64.iso.torrent') - with open(alive_path, 'rb') as f: - self.assertEqual(f.read(), b'alive bytes') + alive_path = os.path.join( + self.tmp_watch_dir, "kali-linux-2026.1-installer-amd64.iso.torrent" + ) + dead_path = os.path.join( + self.tmp_watch_dir, "kali-linux-2026.1-installer-purple-amd64.iso.torrent" + ) + with open(alive_path, "rb") as f: + self.assertEqual(f.read(), b"alive bytes") self.assertFalse(os.path.exists(dead_path)) def test_skips_distros_without_old_release_discovery(self): with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {}, clear=True): - added, skipped, _ = ft.check_old_releases_for_demand(['ubuntu'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False) + added, skipped, _ = ft.check_old_releases_for_demand( + ["ubuntu"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) self.assertEqual((added, skipped), (0, 0)) def test_already_present_torrent_is_not_re_evaluated(self): - name = 'kali-linux-2026.1-installer-amd64.iso' - existing_path = os.path.join(self.tmp_watch_dir, f'{name}.torrent') - with open(existing_path, 'wb') as f: - f.write(b'already here') - - with unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, - {'kali': lambda: {name: 'https://example.invalid/x.torrent'}}, clear=True, - ), \ - unittest.mock.patch.object(ft, 'evaluate_old_release_demand') as mock_evaluate: - added, skipped, _ = ft.check_old_releases_for_demand(['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False) + name = "kali-linux-2026.1-installer-amd64.iso" + existing_path = os.path.join(self.tmp_watch_dir, f"{name}.torrent") + with open(existing_path, "wb") as f: + f.write(b"already here") + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, + {"kali": lambda: {name: "https://example.invalid/x.torrent"}}, + clear=True, + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): + added, skipped, _ = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) mock_evaluate.assert_not_called() self.assertEqual((added, skipped), (0, 0)) def test_unsafe_candidate_name_is_skipped_without_evaluation(self): - name = 'kali-linux-2026.1-installer-../../etc/cron.d/evil' - - with unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, - {'kali': lambda: {name: 'https://example.invalid/x.torrent'}}, clear=True, - ), \ - unittest.mock.patch.object(ft, 'evaluate_old_release_demand') as mock_evaluate: - added, skipped, _ = ft.check_old_releases_for_demand(['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False) + name = "kali-linux-2026.1-installer-../../etc/cron.d/evil" + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, + {"kali": lambda: {name: "https://example.invalid/x.torrent"}}, + clear=True, + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): + added, skipped, _ = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) mock_evaluate.assert_not_called() self.assertEqual((added, skipped), (0, 1)) def test_low_demand_variant_is_filtered_before_evaluation(self): - name = 'kali-linux-2026.1-installer-netinst-amd64.iso' - - with unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, - {'kali': lambda: {name: 'https://example.invalid/x.torrent'}}, clear=True, - ), \ - unittest.mock.patch.object(ft, 'evaluate_old_release_demand') as mock_evaluate: - added, skipped, _ = ft.check_old_releases_for_demand(['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False) + name = "kali-linux-2026.1-installer-netinst-amd64.iso" + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, + {"kali": lambda: {name: "https://example.invalid/x.torrent"}}, + clear=True, + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): + added, skipped, _ = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) mock_evaluate.assert_not_called() self.assertEqual((added, skipped), (0, 0)) @@ -1468,7 +1753,11 @@ class FormatRunSummaryTests(unittest.TestCase): def test_without_old_release_check(self): summary = ft.format_run_summary( - elapsed=12.5, success_count=3, existing_count=1, not_found_count=0, failure_count=0, + elapsed=12.5, + success_count=3, + existing_count=1, + not_found_count=0, + failure_count=0, ) self.assertEqual( summary, @@ -1477,8 +1766,13 @@ def test_without_old_release_check(self): def test_with_old_release_check_appends_counts(self): summary = ft.format_run_summary( - elapsed=12.5, success_count=3, existing_count=1, not_found_count=0, failure_count=0, - old_added=2, old_skipped=6, + elapsed=12.5, + success_count=3, + existing_count=1, + not_found_count=0, + failure_count=0, + old_added=2, + old_skipped=6, ) self.assertEqual( summary, @@ -1488,8 +1782,13 @@ def test_with_old_release_check_appends_counts(self): def test_still_starts_with_run_complete_for_always_log_filter(self): summary = ft.format_run_summary( - elapsed=1.0, success_count=0, existing_count=0, not_found_count=0, failure_count=0, - old_added=0, old_skipped=0, + elapsed=1.0, + success_count=0, + existing_count=0, + not_found_count=0, + failure_count=0, + old_added=0, + old_skipped=0, ) self.assertTrue(summary.startswith("Run complete in")) @@ -1501,26 +1800,42 @@ class ShouldRecheckOldReleaseTests(unittest.TestCase): a while instead.""" def test_never_checked_is_due(self): - self.assertTrue(ft.should_recheck_old_release('x', {}, date(2026, 1, 8), recheck_interval_days=7)) + self.assertTrue( + ft.should_recheck_old_release( + "x", {}, date(2026, 1, 8), recheck_interval_days=7 + ) + ) def test_checked_recently_is_not_due(self): - state = {'x': {'last_checked': '2026-01-05', 'leechers': 0}} - self.assertFalse(ft.should_recheck_old_release('x', state, date(2026, 1, 8), recheck_interval_days=7)) + state = {"x": {"last_checked": "2026-01-05", "leechers": 0}} + self.assertFalse( + ft.should_recheck_old_release( + "x", state, date(2026, 1, 8), recheck_interval_days=7 + ) + ) def test_checked_exactly_at_interval_is_due(self): - state = {'x': {'last_checked': '2026-01-01', 'leechers': 0}} - self.assertTrue(ft.should_recheck_old_release('x', state, date(2026, 1, 8), recheck_interval_days=7)) + state = {"x": {"last_checked": "2026-01-01", "leechers": 0}} + self.assertTrue( + ft.should_recheck_old_release( + "x", state, date(2026, 1, 8), recheck_interval_days=7 + ) + ) def test_checked_past_interval_is_due(self): - state = {'x': {'last_checked': '2025-12-01', 'leechers': 0}} - self.assertTrue(ft.should_recheck_old_release('x', state, date(2026, 1, 8), recheck_interval_days=7)) + state = {"x": {"last_checked": "2025-12-01", "leechers": 0}} + self.assertTrue( + ft.should_recheck_old_release( + "x", state, date(2026, 1, 8), recheck_interval_days=7 + ) + ) class OldReleaseCheckStatePersistenceTests(unittest.TestCase): def test_round_trips_through_disk(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_old_release_state_') - path = os.path.join(tmp_dir, 'fetch_torrents_old_release_check_state.json') - state = {'x': {'last_checked': '2026-01-08', 'leechers': 0}} + tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_old_release_state_") + path = os.path.join(tmp_dir, "fetch_torrents_old_release_check_state.json") + state = {"x": {"last_checked": "2026-01-08", "leechers": 0}} ft.save_old_release_check_state(path, state) loaded = ft.load_old_release_check_state(path) @@ -1528,16 +1843,16 @@ def test_round_trips_through_disk(self): self.assertEqual(loaded, state) def test_missing_file_returns_empty_state(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_old_release_state_') - path = os.path.join(tmp_dir, 'does-not-exist.json') + tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_old_release_state_") + path = os.path.join(tmp_dir, "does-not-exist.json") self.assertEqual(ft.load_old_release_check_state(path), {}) def test_corrupt_file_returns_empty_state_instead_of_raising(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_old_release_state_') - path = os.path.join(tmp_dir, 'fetch_torrents_old_release_check_state.json') - with open(path, 'w', encoding='utf-8') as f: - f.write('{not valid json') + tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_old_release_state_") + path = os.path.join(tmp_dir, "fetch_torrents_old_release_check_state.json") + with open(path, "w", encoding="utf-8") as f: + f.write("{not valid json") self.assertEqual(ft.load_old_release_check_state(path), {}) @@ -1546,21 +1861,25 @@ class RecordRemovalTests(unittest.TestCase): def test_adds_entry_without_mutating_input(self): history = {} updated = ft.record_removal( - history, 'kali-linux-2026.1-installer-amd64.iso', date(2026, 2, 1), 'stagnant', 0.42, + history, + "kali-linux-2026.1-installer-amd64.iso", + date(2026, 2, 1), + "stagnant", + 0.42, ) self.assertEqual(history, {}) # input untouched self.assertEqual( - updated['kali-linux-2026.1-installer-amd64.iso'], - {'removed_date': '2026-02-01', 'reason': 'stagnant', 'ratio': 0.42}, + updated["kali-linux-2026.1-installer-amd64.iso"], + {"removed_date": "2026-02-01", "reason": "stagnant", "ratio": 0.42}, ) class RemovedHistoryPersistenceTests(unittest.TestCase): def test_round_trips_through_disk(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_removed_history_') - path = os.path.join(tmp_dir, 'fetch_torrents_removed_history.json') - history = {'x': {'removed_date': '2026-02-01', 'reason': 'stagnant'}} + tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_removed_history_") + path = os.path.join(tmp_dir, "fetch_torrents_removed_history.json") + history = {"x": {"removed_date": "2026-02-01", "reason": "stagnant"}} ft.save_removed_history(path, history) loaded = ft.load_removed_history(path) @@ -1568,16 +1887,16 @@ def test_round_trips_through_disk(self): self.assertEqual(loaded, history) def test_missing_file_returns_empty_history(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_removed_history_') - path = os.path.join(tmp_dir, 'does-not-exist.json') + tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_removed_history_") + path = os.path.join(tmp_dir, "does-not-exist.json") self.assertEqual(ft.load_removed_history(path), {}) def test_corrupt_file_returns_empty_history_instead_of_raising(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_removed_history_') - path = os.path.join(tmp_dir, 'fetch_torrents_removed_history.json') - with open(path, 'w', encoding='utf-8') as f: - f.write('{not valid json') + tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_removed_history_") + path = os.path.join(tmp_dir, "fetch_torrents_removed_history.json") + with open(path, "w", encoding="utf-8") as f: + f.write("{not valid json") self.assertEqual(ft.load_removed_history(path), {}) @@ -1589,21 +1908,34 @@ class CheckOldReleasesForDemandStateTests(unittest.TestCase): network, and must record a fresh check for anything it does evaluate.""" def setUp(self): - self.tmp_watch_dir = tempfile.mkdtemp(prefix='fetch_torrents_test_watch_') - patcher = unittest.mock.patch.object(ft, 'watch_dir', self.tmp_watch_dir) + self.tmp_watch_dir = tempfile.mkdtemp(prefix="fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) def test_recently_checked_candidate_is_skipped_without_network_call(self): - name = 'kali-linux-2026.1-installer-amd64.iso' - candidates = {name: 'https://example.invalid/x.torrent'} - check_state = {name: {'last_checked': '2026-01-05', 'leechers': 0}} - - with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {'kali': lambda: candidates}, clear=True), \ - unittest.mock.patch.object(ft, 'evaluate_old_release_demand') as mock_evaluate: + name = "kali-linux-2026.1-installer-amd64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + check_state = {name: {"last_checked": "2026-01-05", "leechers": 0}} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): added, skipped, new_state = ft.check_old_releases_for_demand( - ['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, - check_state=check_state, removed_history={}, today=date(2026, 1, 8), recheck_interval_days=7, + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state=check_state, + removed_history={}, + today=date(2026, 1, 8), + recheck_interval_days=7, ) mock_evaluate.assert_not_called() @@ -1611,53 +1943,100 @@ def test_recently_checked_candidate_is_skipped_without_network_call(self): self.assertEqual(new_state, check_state) def test_previously_removed_candidate_is_skipped_without_network_call(self): - name = 'kali-linux-2026.1-installer-amd64.iso' - candidates = {name: 'https://example.invalid/x.torrent'} - removed_history = {name: {'removed_date': '2026-01-01', 'reason': 'stagnant'}} - - with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {'kali': lambda: candidates}, clear=True), \ - unittest.mock.patch.object(ft, 'evaluate_old_release_demand') as mock_evaluate: + name = "kali-linux-2026.1-installer-amd64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + removed_history = {name: {"removed_date": "2026-01-01", "reason": "stagnant"}} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): added, skipped, new_state = ft.check_old_releases_for_demand( - ['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, - check_state={}, removed_history=removed_history, today=date(2026, 1, 8), recheck_interval_days=7, + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state={}, + removed_history=removed_history, + today=date(2026, 1, 8), + recheck_interval_days=7, ) mock_evaluate.assert_not_called() self.assertEqual((added, skipped), (0, 1)) def test_due_candidate_is_evaluated_and_recorded(self): - name = 'kali-linux-2026.1-installer-amd64.iso' - candidates = {name: 'https://example.invalid/x.torrent'} - - with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {'kali': lambda: candidates}, clear=True), \ - unittest.mock.patch.object( - ft, 'evaluate_old_release_demand', - return_value=(True, b'bytes', {'tracker': 't', 'seeders': 1, 'leechers': 30, 'completed': 0}), - ): + name = "kali-linux-2026.1-installer-amd64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, + "evaluate_old_release_demand", + return_value=( + True, + b"bytes", + {"tracker": "t", "seeders": 1, "leechers": 30, "completed": 0}, + ), + ), + ): added, skipped, new_state = ft.check_old_releases_for_demand( - ['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, - check_state={}, removed_history={}, today=date(2026, 1, 8), recheck_interval_days=7, + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state={}, + removed_history={}, + today=date(2026, 1, 8), + recheck_interval_days=7, ) self.assertEqual((added, skipped), (1, 0)) - self.assertEqual(new_state[name], {'last_checked': '2026-01-08', 'leechers': 30}) + self.assertEqual( + new_state[name], {"last_checked": "2026-01-08", "leechers": 30} + ) def test_no_demand_result_is_still_recorded_for_future_recheck_gating(self): - name = 'kali-linux-2026.1-installer-arm64.iso' - candidates = {name: 'https://example.invalid/x.torrent'} - - with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {'kali': lambda: candidates}, clear=True), \ - unittest.mock.patch.object( - ft, 'evaluate_old_release_demand', - return_value=(False, b'bytes', {'tracker': 't', 'seeders': 1, 'leechers': 0, 'completed': 0}), - ): + name = "kali-linux-2026.1-installer-arm64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, + "evaluate_old_release_demand", + return_value=( + False, + b"bytes", + {"tracker": "t", "seeders": 1, "leechers": 0, "completed": 0}, + ), + ), + ): added, skipped, new_state = ft.check_old_releases_for_demand( - ['kali'], include_low_demand=False, min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, - check_state={}, removed_history={}, today=date(2026, 1, 8), recheck_interval_days=7, + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state={}, + removed_history={}, + today=date(2026, 1, 8), + recheck_interval_days=7, ) self.assertEqual((added, skipped), (0, 1)) - self.assertEqual(new_state[name], {'last_checked': '2026-01-08', 'leechers': 0}) + self.assertEqual(new_state[name], {"last_checked": "2026-01-08", "leechers": 0}) class CleanupRpcAuthTests(unittest.TestCase): @@ -1670,12 +2049,14 @@ class CleanupRpcAuthTests(unittest.TestCase): failure is only logged, never surfaced.""" def setUp(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_cleanup_auth_') + tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_cleanup_auth_") for attr, filename in ( - ('removed_history_file', 'removed_history.json'), - ('ratio_history_file', 'ratio_history.json'), + ("removed_history_file", "removed_history.json"), + ("ratio_history_file", "ratio_history.json"), ): - patcher = unittest.mock.patch.object(ft, attr, os.path.join(tmp_dir, filename)) + patcher = unittest.mock.patch.object( + ft, attr, os.path.join(tmp_dir, filename) + ) patcher.start() self.addCleanup(patcher.stop) @@ -1685,29 +2066,55 @@ def _mock_client(self): return client def test_cleanup_old_versions_passes_configured_credentials(self): - with unittest.mock.patch.dict(os.environ, { - 'TRANSMISSION_RPC_USERNAME': 'alice', 'TRANSMISSION_RPC_PASSWORD': 'hunter2', - }), unittest.mock.patch.object(ft, 'Client', return_value=self._mock_client()) as mock_client_cls: + with ( + unittest.mock.patch.dict( + os.environ, + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, + ), + unittest.mock.patch.object( + ft, "Client", return_value=self._mock_client() + ) as mock_client_cls, + ): ft.cleanup_old_versions() - mock_client_cls.assert_called_once_with(host='localhost', port=9091, username='alice', password='hunter2') + mock_client_cls.assert_called_once_with( + host="localhost", port=9091, username="alice", password="hunter2" + ) def test_cleanup_old_versions_passes_no_credentials_when_unset(self): with unittest.mock.patch.dict(os.environ, {}, clear=False): - os.environ.pop('TRANSMISSION_RPC_USERNAME', None) - os.environ.pop('TRANSMISSION_RPC_PASSWORD', None) - with unittest.mock.patch.object(ft, 'Client', return_value=self._mock_client()) as mock_client_cls: + os.environ.pop("TRANSMISSION_RPC_USERNAME", None) + os.environ.pop("TRANSMISSION_RPC_PASSWORD", None) + with unittest.mock.patch.object( + ft, "Client", return_value=self._mock_client() + ) as mock_client_cls: ft.cleanup_old_versions() - mock_client_cls.assert_called_once_with(host='localhost', port=9091, username=None, password=None) + mock_client_cls.assert_called_once_with( + host="localhost", port=9091, username=None, password=None + ) def test_cleanup_stagnant_torrents_passes_configured_credentials(self): - with unittest.mock.patch.dict(os.environ, { - 'TRANSMISSION_RPC_USERNAME': 'alice', 'TRANSMISSION_RPC_PASSWORD': 'hunter2', - }), unittest.mock.patch.object(ft, 'Client', return_value=self._mock_client()) as mock_client_cls: + with ( + unittest.mock.patch.dict( + os.environ, + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, + ), + unittest.mock.patch.object( + ft, "Client", return_value=self._mock_client() + ) as mock_client_cls, + ): ft.cleanup_stagnant_torrents() - mock_client_cls.assert_called_once_with(host='localhost', port=9091, username='alice', password='hunter2') + mock_client_cls.assert_called_once_with( + host="localhost", port=9091, username="alice", password="hunter2" + ) class LogSeedRatiosViaHttpTests(unittest.TestCase): @@ -1728,50 +2135,77 @@ def json(self): def test_session_id_request_has_a_timeout(self): with unittest.mock.patch.object( - ft.requests, 'post', + ft.requests, + "post", side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], ) as mock_post: ft.log_seed_ratios_via_http() first_call_kwargs = mock_post.call_args_list[0].kwargs - self.assertIn('timeout', first_call_kwargs) + self.assertIn("timeout", first_call_kwargs) def test_uses_configured_credentials_when_auth_not_explicitly_passed(self): - with unittest.mock.patch.dict(os.environ, { - 'TRANSMISSION_RPC_USERNAME': 'alice', 'TRANSMISSION_RPC_PASSWORD': 'hunter2', - }), unittest.mock.patch.object( - ft.requests, 'post', - side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], - ) as mock_post: + with ( + unittest.mock.patch.dict( + os.environ, + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, + ), + unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[ + self.FakeSessionIdResponse(), + self.FakeTorrentGetResponse(), + ], + ) as mock_post, + ): ft.log_seed_ratios_via_http() second_call_kwargs = mock_post.call_args_list[1].kwargs - self.assertEqual(second_call_kwargs.get('auth'), ('alice', 'hunter2')) + self.assertEqual(second_call_kwargs.get("auth"), ("alice", "hunter2")) def test_no_auth_when_credentials_unset(self): with unittest.mock.patch.dict(os.environ, {}, clear=False): - os.environ.pop('TRANSMISSION_RPC_USERNAME', None) - os.environ.pop('TRANSMISSION_RPC_PASSWORD', None) + os.environ.pop("TRANSMISSION_RPC_USERNAME", None) + os.environ.pop("TRANSMISSION_RPC_PASSWORD", None) with unittest.mock.patch.object( - ft.requests, 'post', - side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], + ft.requests, + "post", + side_effect=[ + self.FakeSessionIdResponse(), + self.FakeTorrentGetResponse(), + ], ) as mock_post: ft.log_seed_ratios_via_http() second_call_kwargs = mock_post.call_args_list[1].kwargs - self.assertIsNone(second_call_kwargs.get('auth')) + self.assertIsNone(second_call_kwargs.get("auth")) def test_explicit_auth_argument_still_wins(self): - with unittest.mock.patch.dict(os.environ, { - 'TRANSMISSION_RPC_USERNAME': 'alice', 'TRANSMISSION_RPC_PASSWORD': 'hunter2', - }), unittest.mock.patch.object( - ft.requests, 'post', - side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], - ) as mock_post: - ft.log_seed_ratios_via_http(auth=('bob', 'other')) + with ( + unittest.mock.patch.dict( + os.environ, + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, + ), + unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[ + self.FakeSessionIdResponse(), + self.FakeTorrentGetResponse(), + ], + ) as mock_post, + ): + ft.log_seed_ratios_via_http(auth=("bob", "other")) second_call_kwargs = mock_post.call_args_list[1].kwargs - self.assertEqual(second_call_kwargs.get('auth'), ('bob', 'other')) + self.assertEqual(second_call_kwargs.get("auth"), ("bob", "other")) class CleanupRemovesWatchFileTests(unittest.TestCase): @@ -1781,67 +2215,73 @@ class CleanupRemovesWatchFileTests(unittest.TestCase): by default) ever cleans them up.""" def setUp(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_cleanup_watchfile_') + tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_cleanup_watchfile_") for attr, filename in ( - ('removed_history_file', 'removed_history.json'), - ('ratio_history_file', 'ratio_history.json'), + ("removed_history_file", "removed_history.json"), + ("ratio_history_file", "ratio_history.json"), ): - patcher = unittest.mock.patch.object(ft, attr, os.path.join(tmp_dir, filename)) + patcher = unittest.mock.patch.object( + ft, attr, os.path.join(tmp_dir, filename) + ) patcher.start() self.addCleanup(patcher.stop) - self.tmp_watch_dir = tempfile.mkdtemp(prefix='fetch_torrents_test_watch_') - patcher = unittest.mock.patch.object(ft, 'watch_dir', self.tmp_watch_dir) + self.tmp_watch_dir = tempfile.mkdtemp(prefix="fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) def _touch_watch_file(self, name): path = os.path.join(self.tmp_watch_dir, f"{name}.torrent") - with open(path, 'w', encoding='utf-8') as f: - f.write('fake torrent bytes') + with open(path, "w", encoding="utf-8") as f: + f.write("fake torrent bytes") return path def test_cleanup_old_versions_removes_the_watch_file(self): - old_path = self._touch_watch_file('ubuntu-23.10-desktop-amd64.iso') - self._touch_watch_file('ubuntu-24.04-desktop-amd64.iso') + old_path = self._touch_watch_file("ubuntu-23.10-desktop-amd64.iso") + self._touch_watch_file("ubuntu-24.04-desktop-amd64.iso") torrents = [ - FakeTorrent('ubuntu-24.04-desktop-amd64.iso', 1, ratio=2.0), - FakeTorrent('ubuntu-23.10-desktop-amd64.iso', 2, ratio=2.0), + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), ] client = MagicMock() client.get_torrents.return_value = torrents - with unittest.mock.patch.dict(os.environ, {'CLEANUP_SKIP_RATIO_CHECK': 'true'}), \ - unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.dict(os.environ, {"CLEANUP_SKIP_RATIO_CHECK": "true"}), + unittest.mock.patch.object(ft, "Client", return_value=client), + ): ft.cleanup_old_versions() client.remove_torrent.assert_called_once_with(2, delete_data=True) self.assertFalse(os.path.exists(old_path)) history = ft.load_removed_history(ft.removed_history_file) - self.assertEqual(history['ubuntu-23.10-desktop-amd64.iso']['ratio'], 2.0) + self.assertEqual(history["ubuntu-23.10-desktop-amd64.iso"]["ratio"], 2.0) def test_missing_watch_file_does_not_raise(self): # No .torrent file created for the superseded version - cleanup must # still succeed and remove the torrent from Transmission. torrents = [ - FakeTorrent('ubuntu-24.04-desktop-amd64.iso', 1, ratio=2.0), - FakeTorrent('ubuntu-23.10-desktop-amd64.iso', 2, ratio=2.0), + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), ] client = MagicMock() client.get_torrents.return_value = torrents - with unittest.mock.patch.dict(os.environ, {'CLEANUP_SKIP_RATIO_CHECK': 'true'}), \ - unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.dict(os.environ, {"CLEANUP_SKIP_RATIO_CHECK": "true"}), + unittest.mock.patch.object(ft, "Client", return_value=client), + ): ft.cleanup_old_versions() # must not raise client.remove_torrent.assert_called_once_with(2, delete_data=True) def test_cleanup_stagnant_torrents_removes_the_watch_file(self): - old_path = self._touch_watch_file('ubuntu-23.10-desktop-amd64.iso') - self._touch_watch_file('ubuntu-24.04-desktop-amd64.iso') + old_path = self._touch_watch_file("ubuntu-23.10-desktop-amd64.iso") + self._touch_watch_file("ubuntu-24.04-desktop-amd64.iso") torrents = [ - FakeTorrent('ubuntu-24.04-desktop-amd64.iso', 1, ratio=2.0), - FakeTorrent('ubuntu-23.10-desktop-amd64.iso', 2, ratio=2.0), + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), ] client = MagicMock() client.get_torrents.return_value = torrents @@ -1853,17 +2293,19 @@ def test_cleanup_stagnant_torrents_removes_the_watch_file(self): # unchanged since then, so plan_stagnation_cleanup() must select this # torrent for removal. anchor_date = (date.today() - timedelta(days=32)).isoformat() - history = {'ubuntu-23.10-desktop-amd64.iso': [{'date': anchor_date, 'ratio': 2.0}]} - with open(ft.ratio_history_file, 'w', encoding='utf-8') as f: + history = { + "ubuntu-23.10-desktop-amd64.iso": [{"date": anchor_date, "ratio": 2.0}] + } + with open(ft.ratio_history_file, "w", encoding="utf-8") as f: json.dump(history, f) - with unittest.mock.patch.object(ft, 'Client', return_value=client): + with unittest.mock.patch.object(ft, "Client", return_value=client): ft.cleanup_stagnant_torrents() client.remove_torrent.assert_called_once_with(2, delete_data=True) self.assertFalse(os.path.exists(old_path)) history = ft.load_removed_history(ft.removed_history_file) - self.assertEqual(history['ubuntu-23.10-desktop-amd64.iso']['ratio'], 2.0) + self.assertEqual(history["ubuntu-23.10-desktop-amd64.iso"]["ratio"], 2.0) class PlanDiskPressureCleanupTests(unittest.TestCase): @@ -1877,24 +2319,45 @@ def test_below_threshold_returns_nothing(self): FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), ] - self.assertEqual(ft.plan_disk_pressure_cleanup(torrents, used_percent=80.0, threshold_percent=95.0), []) + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=80.0, threshold_percent=95.0 + ), + [], + ) def test_disabled_when_threshold_is_zero_or_negative(self): torrents = [ FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), ] - self.assertEqual(ft.plan_disk_pressure_cleanup(torrents, used_percent=99.0, threshold_percent=0), []) - self.assertEqual(ft.plan_disk_pressure_cleanup(torrents, used_percent=99.0, threshold_percent=-1), []) + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=99.0, threshold_percent=0 + ), + [], + ) + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=99.0, threshold_percent=-1 + ), + [], + ) def test_at_or_above_threshold_returns_candidates_lowest_ratio_first(self): torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0), # latest - never a candidate + FakeTorrent( + "ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0 + ), # latest - never a candidate FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), FakeTorrent("ubuntu-22.04.1-live-server-amd64.iso", 3, ratio=0.2), - FakeTorrent("ubuntu-24.04.1-live-server-amd64.iso", 4, ratio=5.0), # latest of this group + FakeTorrent( + "ubuntu-24.04.1-live-server-amd64.iso", 4, ratio=5.0 + ), # latest of this group ] - result = ft.plan_disk_pressure_cleanup(torrents, used_percent=95.0, threshold_percent=95.0) + result = ft.plan_disk_pressure_cleanup( + torrents, used_percent=95.0, threshold_percent=95.0 + ) self.assertEqual( names_of(result), ["ubuntu-22.04.1-live-server-amd64.iso", "ubuntu-23.10-desktop-amd64.iso"], @@ -1904,7 +2367,12 @@ def test_at_or_above_threshold_returns_candidates_lowest_ratio_first(self): def test_no_superseded_candidates_returns_empty(self): torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=0.0)] - self.assertEqual(ft.plan_disk_pressure_cleanup(torrents, used_percent=99.0, threshold_percent=95.0), []) + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=99.0, threshold_percent=95.0 + ), + [], + ) class EnforceDiskUsageLimitTests(unittest.TestCase): @@ -1914,38 +2382,48 @@ class EnforceDiskUsageLimitTests(unittest.TestCase): torrent holds.""" def setUp(self): - tmp_dir = tempfile.mkdtemp(prefix='fetch_torrents_disk_pressure_') + tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_disk_pressure_") for attr, filename in ( - ('removed_history_file', 'removed_history.json'), - ('ratio_history_file', 'ratio_history.json'), + ("removed_history_file", "removed_history.json"), + ("ratio_history_file", "ratio_history.json"), ): - patcher = unittest.mock.patch.object(ft, attr, os.path.join(tmp_dir, filename)) + patcher = unittest.mock.patch.object( + ft, attr, os.path.join(tmp_dir, filename) + ) patcher.start() self.addCleanup(patcher.stop) - self.tmp_watch_dir = tempfile.mkdtemp(prefix='fetch_torrents_test_watch_') - patcher = unittest.mock.patch.object(ft, 'watch_dir', self.tmp_watch_dir) + self.tmp_watch_dir = tempfile.mkdtemp(prefix="fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) def test_disabled_threshold_never_touches_disk_or_rpc(self): - with unittest.mock.patch.object(ft.shutil, 'disk_usage') as mock_disk_usage, \ - unittest.mock.patch.object(ft, 'Client') as mock_client_cls: + with ( + unittest.mock.patch.object(ft.shutil, "disk_usage") as mock_disk_usage, + unittest.mock.patch.object(ft, "Client") as mock_client_cls, + ): ft.enforce_disk_usage_limit(0) mock_disk_usage.assert_not_called() mock_client_cls.assert_not_called() def test_below_threshold_does_not_connect_to_transmission(self): - with unittest.mock.patch.object(ft.shutil, 'disk_usage', return_value=(100, 50, 50)), \ - unittest.mock.patch.object(ft, 'Client') as mock_client_cls: + with ( + unittest.mock.patch.object( + ft.shutil, "disk_usage", return_value=(100, 50, 50) + ), + unittest.mock.patch.object(ft, "Client") as mock_client_cls, + ): ft.enforce_disk_usage_limit(95) mock_client_cls.assert_not_called() def test_removes_lowest_ratio_candidates_until_under_threshold(self): torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0), # latest, never removed + FakeTorrent( + "ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0 + ), # latest, never removed FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), FakeTorrent("ubuntu-22.04-desktop-amd64.iso", 3, ratio=0.2), ] @@ -1955,17 +2433,24 @@ def test_removes_lowest_ratio_candidates_until_under_threshold(self): # 96% initially (over threshold); drops to 90% (under threshold) # after the first removal - the second, higher-ratio candidate must # be left alone. - with unittest.mock.patch.object( - ft.shutil, 'disk_usage', side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], - ), unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.object( + ft.shutil, + "disk_usage", + side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], + ), + unittest.mock.patch.object(ft, "Client", return_value=client), + ): ft.enforce_disk_usage_limit(95) client.remove_torrent.assert_called_once_with(3, delete_data=True) def test_removes_the_watch_file_for_each_removal(self): - watch_path = os.path.join(self.tmp_watch_dir, "ubuntu-22.04-desktop-amd64.iso.torrent") - with open(watch_path, 'w', encoding='utf-8') as f: - f.write('fake torrent bytes') + watch_path = os.path.join( + self.tmp_watch_dir, "ubuntu-22.04-desktop-amd64.iso.torrent" + ) + with open(watch_path, "w", encoding="utf-8") as f: + f.write("fake torrent bytes") torrents = [ FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0), @@ -1975,9 +2460,14 @@ def test_removes_the_watch_file_for_each_removal(self): client = MagicMock() client.get_torrents.return_value = torrents - with unittest.mock.patch.object( - ft.shutil, 'disk_usage', side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], - ), unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.object( + ft.shutil, + "disk_usage", + side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], + ), + unittest.mock.patch.object(ft, "Client", return_value=client), + ): ft.enforce_disk_usage_limit(95) self.assertFalse(os.path.exists(watch_path)) @@ -1987,8 +2477,12 @@ def test_no_candidates_logs_warning_and_does_not_raise(self): client = MagicMock() client.get_torrents.return_value = torrents - with unittest.mock.patch.object(ft.shutil, 'disk_usage', return_value=(100, 96, 4)), \ - unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.object( + ft.shutil, "disk_usage", return_value=(100, 96, 4) + ), + unittest.mock.patch.object(ft, "Client", return_value=client), + ): ft.enforce_disk_usage_limit(95) # must not raise client.remove_torrent.assert_not_called() @@ -2002,14 +2496,21 @@ def test_records_removal_reason_as_disk_pressure(self): client = MagicMock() client.get_torrents.return_value = torrents - with unittest.mock.patch.object( - ft.shutil, 'disk_usage', side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], - ), unittest.mock.patch.object(ft, 'Client', return_value=client): + with ( + unittest.mock.patch.object( + ft.shutil, + "disk_usage", + side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], + ), + unittest.mock.patch.object(ft, "Client", return_value=client), + ): ft.enforce_disk_usage_limit(95) history = ft.load_removed_history(ft.removed_history_file) - self.assertEqual(history["ubuntu-22.04-desktop-amd64.iso"]['reason'], 'disk_pressure') - self.assertEqual(history["ubuntu-22.04-desktop-amd64.iso"]['ratio'], 0.2) + self.assertEqual( + history["ubuntu-22.04-desktop-amd64.iso"]["reason"], "disk_pressure" + ) + self.assertEqual(history["ubuntu-22.04-desktop-amd64.iso"]["ratio"], 0.2) if __name__ == "__main__": diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index fc8b9f2..7ac7e66 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -6,6 +6,7 @@ Run with: python -m unittest discover -s tests """ + import hashlib import os import socket @@ -26,53 +27,53 @@ def bstr(s): """Hand-rolled bencode byte-string, independent of ts.bencode(), so it serves as an oracle rather than testing the implementation against itself.""" - b = s if isinstance(s, bytes) else s.encode('utf-8') - return str(len(b)).encode() + b':' + b + b = s if isinstance(s, bytes) else s.encode("utf-8") + return str(len(b)).encode() + b":" + b def bint(n): - return b'i' + str(n).encode() + b'e' + return b"i" + str(n).encode() + b"e" class BdecodeBencodeTests(unittest.TestCase): def test_bdecode_integer(self): - value, index = ts.bdecode(b'i12345e') + value, index = ts.bdecode(b"i12345e") self.assertEqual(value, 12345) self.assertEqual(index, 7) def test_bdecode_negative_integer(self): - value, index = ts.bdecode(b'i-42e') + value, index = ts.bdecode(b"i-42e") self.assertEqual(value, -42) def test_bdecode_bytestring(self): - value, index = ts.bdecode(b'4:spam trailing') - self.assertEqual(value, b'spam') + value, index = ts.bdecode(b"4:spam trailing") + self.assertEqual(value, b"spam") self.assertEqual(index, 6) def test_bdecode_empty_bytestring(self): - value, index = ts.bdecode(b'0:') - self.assertEqual(value, b'') + value, index = ts.bdecode(b"0:") + self.assertEqual(value, b"") self.assertEqual(index, 2) def test_bdecode_list(self): - value, index = ts.bdecode(b'l4:spam4:eggse') - self.assertEqual(value, [b'spam', b'eggs']) + value, index = ts.bdecode(b"l4:spam4:eggse") + self.assertEqual(value, [b"spam", b"eggs"]) def test_bdecode_dict(self): - value, index = ts.bdecode(b'd3:cow3:moo4:spam4:eggse') - self.assertEqual(value, {b'cow': b'moo', b'spam': b'eggs'}) + value, index = ts.bdecode(b"d3:cow3:moo4:spam4:eggse") + self.assertEqual(value, {b"cow": b"moo", b"spam": b"eggs"}) def test_bdecode_nested(self): - data = b'd4:infod4:name4:test6:lengthi10eee' + data = b"d4:infod4:name4:test6:lengthi10eee" value, _ = ts.bdecode(data) - self.assertEqual(value, {b'info': {b'name': b'test', b'length': 10}}) + self.assertEqual(value, {b"info": {b"name": b"test", b"length": 10}}) def test_bdecode_invalid_prefix_raises(self): with self.assertRaises(ts.BencodeError): - ts.bdecode(b'x123') + ts.bdecode(b"x123") def test_bencode_roundtrip(self): - for value in (0, -5, 12345, b'', b'hello', [b'a', b'b'], {b'a': 1, b'b': b'x'}): + for value in (0, -5, 12345, b"", b"hello", [b"a", b"b"], {b"a": 1, b"b": b"x"}): encoded = ts.bencode(value) decoded, index = ts.bdecode(encoded) self.assertEqual(index, len(encoded)) @@ -80,24 +81,32 @@ def test_bencode_roundtrip(self): def test_bencode_dict_sorts_keys(self): # Insertion order deliberately reversed vs. bencode's required sort order. - encoded = ts.bencode({b'zebra': 1, b'apple': 2}) - self.assertEqual(encoded, b'd5:applei2e5:zebrai1ee') + encoded = ts.bencode({b"zebra": 1, b"apple": 2}) + self.assertEqual(encoded, b"d5:applei2e5:zebrai1ee") class ComputeInfoHashTests(unittest.TestCase): def test_matches_sha1_of_raw_info_bytes(self): info_bytes = ( - b'd6:lengthi12345e4:name8:test.iso12:piece lengthi16384e' - b'6:pieces20:' + b'A' * 20 + b'e' + b"d6:lengthi12345e4:name8:test.iso12:piece lengthi16384e" + b"6:pieces20:" + b"A" * 20 + b"e" + ) + torrent_bytes = ( + b"d" + + bstr("announce") + + bstr("http://tracker.example.com:6969/announce") + + bstr("info") + + info_bytes + + b"e" ) - torrent_bytes = b'd' + bstr('announce') + bstr('http://tracker.example.com:6969/announce') \ - + bstr('info') + info_bytes + b'e' expected = hashlib.sha1(info_bytes).digest() self.assertEqual(ts.compute_info_hash(torrent_bytes), expected) def test_raises_when_info_missing(self): - torrent_bytes = b'd' + bstr('announce') + bstr('http://tracker.example.com/announce') + b'e' + torrent_bytes = ( + b"d" + bstr("announce") + bstr("http://tracker.example.com/announce") + b"e" + ) with self.assertRaises(ts.BencodeError): ts.compute_info_hash(torrent_bytes) @@ -105,26 +114,43 @@ def test_raises_when_info_missing(self): class GetAnnounceUrlsTests(unittest.TestCase): def test_combines_announce_list_and_primary_announce(self): torrent_bytes = ( - b'd' - + bstr('announce') + bstr('http://a.example.com/announce') - + bstr('announce-list') - + b'l' - + b'l' + bstr('http://a.example.com/announce') + b'e' - + b'l' + bstr('http://b.example.com/announce') + b'e' - + b'e' - + bstr('info') + b'de' - + b'e' + b"d" + + bstr("announce") + + bstr("http://a.example.com/announce") + + bstr("announce-list") + + b"l" + + b"l" + + bstr("http://a.example.com/announce") + + b"e" + + b"l" + + bstr("http://b.example.com/announce") + + b"e" + + b"e" + + bstr("info") + + b"de" + + b"e" ) urls = ts.get_announce_urls(torrent_bytes) - self.assertEqual(urls, [ - 'http://a.example.com/announce', - 'http://b.example.com/announce', - ]) + self.assertEqual( + urls, + [ + "http://a.example.com/announce", + "http://b.example.com/announce", + ], + ) def test_falls_back_to_bare_announce(self): - torrent_bytes = b'd' + bstr('announce') + bstr('http://only.example.com/announce') \ - + bstr('info') + b'de' + b'e' - self.assertEqual(ts.get_announce_urls(torrent_bytes), ['http://only.example.com/announce']) + torrent_bytes = ( + b"d" + + bstr("announce") + + bstr("http://only.example.com/announce") + + bstr("info") + + b"de" + + b"e" + ) + self.assertEqual( + ts.get_announce_urls(torrent_bytes), ["http://only.example.com/announce"] + ) class DeriveScrapeUrlTests(unittest.TestCase): @@ -133,7 +159,10 @@ def test_examples_from_the_scrape_convention(self): ("http://example.com/announce", "http://example.com/scrape"), ("http://example.com/x/announce", "http://example.com/x/scrape"), ("http://example.com/announce.php", "http://example.com/scrape.php"), - ("http://example.com/a/announce?x2%0644", "http://example.com/a/scrape?x2%0644"), + ( + "http://example.com/a/announce?x2%0644", + "http://example.com/a/scrape?x2%0644", + ), ("http://example.com/a", None), ] for announce_url, expected in cases: @@ -157,6 +186,7 @@ def test_appends_to_existing_query(self): def test_percent_encoding_roundtrips(self): from urllib.parse import urlsplit, parse_qsl, unquote_to_bytes + info_hash = bytes(range(20)) url = ts.build_scrape_request_url("http://example.com/scrape", info_hash) query = urlsplit(url).query @@ -166,19 +196,25 @@ def test_percent_encoding_roundtrips(self): class ParseScrapeResponseTests(unittest.TestCase): def test_extracts_seeders_leechers_completed(self): - info_hash = b'A' * 20 - response = b'd5:filesd' + bstr(info_hash) + b'd8:completei33e10:incompletei73e10:downloadedi9e' + b'ee' + b'e' + info_hash = b"A" * 20 + response = ( + b"d5:filesd" + + bstr(info_hash) + + b"d8:completei33e10:incompletei73e10:downloadedi9e" + + b"ee" + + b"e" + ) stats = ts.parse_scrape_response(response, info_hash) - self.assertEqual(stats, {'seeders': 33, 'leechers': 73, 'completed': 9}) + self.assertEqual(stats, {"seeders": 33, "leechers": 73, "completed": 9}) def test_returns_none_when_hash_absent(self): - response = b'd5:filesdee' + b'e' - self.assertIsNone(ts.parse_scrape_response(response, b'A' * 20)) + response = b"d5:filesdee" + b"e" + self.assertIsNone(ts.parse_scrape_response(response, b"A" * 20)) def test_raises_on_failure_reason(self): - response = b'd14:failure reason7:blockede' + response = b"d14:failure reason7:blockede" with self.assertRaises(ts.BencodeError): - ts.parse_scrape_response(response, b'A' * 20) + ts.parse_scrape_response(response, b"A" * 20) class ParseUdpTrackerUrlTests(unittest.TestCase): @@ -195,10 +231,14 @@ def test_udp_url_without_path_still_parses(self): ) def test_missing_port_is_none(self): - self.assertIsNone(ts.parse_udp_tracker_url("udp://tracker.example.com/announce")) + self.assertIsNone( + ts.parse_udp_tracker_url("udp://tracker.example.com/announce") + ) def test_non_udp_scheme_is_none(self): - self.assertIsNone(ts.parse_udp_tracker_url("http://tracker.example.com:1337/announce")) + self.assertIsNone( + ts.parse_udp_tracker_url("http://tracker.example.com:1337/announce") + ) def _connect_response(transaction_id, connection_id, action=0): @@ -225,7 +265,9 @@ def test_request_matches_bep15_layout(self): self.assertEqual(transaction_id, 0xAABBCCDD) def test_parses_valid_response(self): - response = _connect_response(transaction_id=42, connection_id=0x1122334455667788) + response = _connect_response( + transaction_id=42, connection_id=0x1122334455667788 + ) connection_id = ts.parse_udp_connect_response(response, transaction_id=42) self.assertEqual(connection_id, 0x1122334455667788) @@ -241,7 +283,7 @@ def test_wrong_action_raises(self): def test_short_response_raises(self): with self.assertRaises(ts.UdpTrackerError): - ts.parse_udp_connect_response(b'\x00' * 8, transaction_id=42) + ts.parse_udp_connect_response(b"\x00" * 8, transaction_id=42) def test_tracker_error_action_raises_with_message(self): response = _error_response(transaction_id=42, message="bad request") @@ -254,7 +296,9 @@ class BuildParseUdpScrapeRequestTests(unittest.TestCase): def test_request_matches_bep15_layout(self): info_hash = bytes(range(20)) request = ts.build_udp_scrape_request( - connection_id=0x1122334455667788, transaction_id=7, info_hash=info_hash, + connection_id=0x1122334455667788, + transaction_id=7, + info_hash=info_hash, ) connection_id, action, transaction_id = struct.unpack(">QII", request[:16]) self.assertEqual(connection_id, 0x1122334455667788) @@ -264,26 +308,34 @@ def test_request_matches_bep15_layout(self): def test_wrong_length_info_hash_raises(self): with self.assertRaises(ts.UdpTrackerError): - ts.build_udp_scrape_request(connection_id=1, transaction_id=1, info_hash=b'short') + ts.build_udp_scrape_request( + connection_id=1, transaction_id=1, info_hash=b"short" + ) def test_parses_valid_response(self): - response = _scrape_response(transaction_id=7, seeders=33, completed=9, leechers=73) + response = _scrape_response( + transaction_id=7, seeders=33, completed=9, leechers=73 + ) stats = ts.parse_udp_scrape_response(response, transaction_id=7) - self.assertEqual(stats, {'seeders': 33, 'leechers': 73, 'completed': 9}) + self.assertEqual(stats, {"seeders": 33, "leechers": 73, "completed": 9}) def test_transaction_id_mismatch_raises(self): - response = _scrape_response(transaction_id=7, seeders=1, completed=1, leechers=1) + response = _scrape_response( + transaction_id=7, seeders=1, completed=1, leechers=1 + ) with self.assertRaises(ts.UdpTrackerError): ts.parse_udp_scrape_response(response, transaction_id=8) def test_wrong_action_raises(self): - response = _scrape_response(transaction_id=7, seeders=1, completed=1, leechers=1, action=0) + response = _scrape_response( + transaction_id=7, seeders=1, completed=1, leechers=1, action=0 + ) with self.assertRaises(ts.UdpTrackerError): ts.parse_udp_scrape_response(response, transaction_id=7) def test_short_response_raises(self): with self.assertRaises(ts.UdpTrackerError): - ts.parse_udp_scrape_response(b'\x00' * 4, transaction_id=7) + ts.parse_udp_scrape_response(b"\x00" * 4, transaction_id=7) def test_tracker_error_action_raises_with_message(self): response = _error_response(transaction_id=7, message="scrape not supported") @@ -303,93 +355,119 @@ def _make_fake_socket(self, recvfrom_side_effect): return fake_sock def test_full_round_trip_returns_stats(self): - info_hash = b'A' * 20 - connect_resp = _connect_response(transaction_id=0x00000001, connection_id=0xCAFEBABE) - scrape_resp = _scrape_response(transaction_id=0x00000001, seeders=5, completed=2, leechers=3) - fake_sock = self._make_fake_socket([ - (connect_resp, ('1.2.3.4', 1337)), - (scrape_resp, ('1.2.3.4', 1337)), - ]) - - with unittest.mock.patch.object(ts.socket, 'socket', return_value=fake_sock), \ - unittest.mock.patch.object(ts.os, 'urandom', return_value=b'\x00\x00\x00\x01'): - stats = ts.scrape_udp_tracker('tracker.example.com', 1337, info_hash, timeout=5) - - self.assertEqual(stats, {'seeders': 5, 'leechers': 3, 'completed': 2}) + info_hash = b"A" * 20 + connect_resp = _connect_response( + transaction_id=0x00000001, connection_id=0xCAFEBABE + ) + scrape_resp = _scrape_response( + transaction_id=0x00000001, seeders=5, completed=2, leechers=3 + ) + fake_sock = self._make_fake_socket( + [ + (connect_resp, ("1.2.3.4", 1337)), + (scrape_resp, ("1.2.3.4", 1337)), + ] + ) + + with ( + unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), + unittest.mock.patch.object( + ts.os, "urandom", return_value=b"\x00\x00\x00\x01" + ), + ): + stats = ts.scrape_udp_tracker( + "tracker.example.com", 1337, info_hash, timeout=5 + ) + + self.assertEqual(stats, {"seeders": 5, "leechers": 3, "completed": 2}) fake_sock.settimeout.assert_called_once_with(5) self.assertEqual(fake_sock.sendto.call_count, 2) # Second (scrape) request must carry the connection_id from the connect response. scrape_request_bytes = fake_sock.sendto.call_args_list[1][0][0] - connection_id_sent, action_sent = struct.unpack(">QI", scrape_request_bytes[:12]) + connection_id_sent, action_sent = struct.unpack( + ">QI", scrape_request_bytes[:12] + ) self.assertEqual(connection_id_sent, 0xCAFEBABE) self.assertEqual(action_sent, 2) fake_sock.close.assert_called_once() def test_connect_transaction_mismatch_raises_and_closes_socket(self): - info_hash = b'A' * 20 + info_hash = b"A" * 20 bad_connect_resp = _connect_response(transaction_id=0x00000002, connection_id=1) - fake_sock = self._make_fake_socket([(bad_connect_resp, ('1.2.3.4', 1337))]) - - with unittest.mock.patch.object(ts.socket, 'socket', return_value=fake_sock), \ - unittest.mock.patch.object(ts.os, 'urandom', return_value=b'\x00\x00\x00\x01'): + fake_sock = self._make_fake_socket([(bad_connect_resp, ("1.2.3.4", 1337))]) + + with ( + unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), + unittest.mock.patch.object( + ts.os, "urandom", return_value=b"\x00\x00\x00\x01" + ), + ): with self.assertRaises(ts.UdpTrackerError): - ts.scrape_udp_tracker('tracker.example.com', 1337, info_hash, timeout=5) + ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) fake_sock.close.assert_called_once() def test_socket_timeout_propagates(self): - info_hash = b'A' * 20 + info_hash = b"A" * 20 fake_sock = unittest.mock.Mock() fake_sock.recvfrom.side_effect = socket.timeout("timed out") - with unittest.mock.patch.object(ts.socket, 'socket', return_value=fake_sock): + with unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock): with self.assertRaises(socket.timeout): - ts.scrape_udp_tracker('tracker.example.com', 1337, info_hash, timeout=5) + ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) fake_sock.close.assert_called_once() class IsDisallowedIpTests(unittest.TestCase): def test_public_ipv4_is_allowed(self): - self.assertFalse(ts._is_disallowed_ip('93.184.216.34')) + self.assertFalse(ts._is_disallowed_ip("93.184.216.34")) def test_private_ipv4_ranges_are_disallowed(self): - for ip in ('10.0.0.1', '172.16.0.1', '192.168.1.1'): + for ip in ("10.0.0.1", "172.16.0.1", "192.168.1.1"): with self.subTest(ip=ip): self.assertTrue(ts._is_disallowed_ip(ip)) def test_loopback_is_disallowed(self): - self.assertTrue(ts._is_disallowed_ip('127.0.0.1')) - self.assertTrue(ts._is_disallowed_ip('::1')) + self.assertTrue(ts._is_disallowed_ip("127.0.0.1")) + self.assertTrue(ts._is_disallowed_ip("::1")) def test_link_local_is_disallowed(self): # Cloud metadata endpoint (AWS/GCP/Azure) lives here. - self.assertTrue(ts._is_disallowed_ip('169.254.169.254')) + self.assertTrue(ts._is_disallowed_ip("169.254.169.254")) def test_public_ipv6_is_allowed(self): - self.assertFalse(ts._is_disallowed_ip('2001:4860:4860::8888')) + self.assertFalse(ts._is_disallowed_ip("2001:4860:4860::8888")) class IsSafeScrapeHostTests(unittest.TestCase): def test_public_address_is_safe(self): - with unittest.mock.patch.object(ts, '_resolve_addresses', return_value=['93.184.216.34']): - self.assertTrue(ts._is_safe_scrape_host('tracker.example.com')) + with unittest.mock.patch.object( + ts, "_resolve_addresses", return_value=["93.184.216.34"] + ): + self.assertTrue(ts._is_safe_scrape_host("tracker.example.com")) def test_private_address_is_unsafe(self): - with unittest.mock.patch.object(ts, '_resolve_addresses', return_value=['10.1.2.3']): - self.assertFalse(ts._is_safe_scrape_host('tracker.internal')) + with unittest.mock.patch.object( + ts, "_resolve_addresses", return_value=["10.1.2.3"] + ): + self.assertFalse(ts._is_safe_scrape_host("tracker.internal")) def test_any_disallowed_address_among_several_makes_host_unsafe(self): - with unittest.mock.patch.object(ts, '_resolve_addresses', return_value=['93.184.216.34', '127.0.0.1']): - self.assertFalse(ts._is_safe_scrape_host('multi-homed.example.com')) + with unittest.mock.patch.object( + ts, "_resolve_addresses", return_value=["93.184.216.34", "127.0.0.1"] + ): + self.assertFalse(ts._is_safe_scrape_host("multi-homed.example.com")) def test_resolution_failure_is_unsafe(self): - with unittest.mock.patch.object(ts, '_resolve_addresses', side_effect=OSError('not known')): - self.assertFalse(ts._is_safe_scrape_host('nonexistent.invalid')) + with unittest.mock.patch.object( + ts, "_resolve_addresses", side_effect=OSError("not known") + ): + self.assertFalse(ts._is_safe_scrape_host("nonexistent.invalid")) def test_empty_hostname_is_unsafe(self): self.assertFalse(ts._is_safe_scrape_host(None)) - self.assertFalse(ts._is_safe_scrape_host('')) + self.assertFalse(ts._is_safe_scrape_host("")) class ScrapeTorrentTests(unittest.TestCase): @@ -399,178 +477,239 @@ def setUp(self): # hosts to a public address so existing tracker-selection behavior # doesn't depend on real DNS. Tests of the SSRF guard itself override # this per-case. - patcher = unittest.mock.patch.object(ts, '_resolve_addresses', return_value=['93.184.216.34']) + patcher = unittest.mock.patch.object( + ts, "_resolve_addresses", return_value=["93.184.216.34"] + ) patcher.start() self.addCleanup(patcher.stop) def _write_torrent(self, announce_urls): - tiers = b''.join(b'l' + bstr(u) + b'e' for u in announce_urls) - info_bytes = b'd6:lengthi1e4:name4:teste' + tiers = b"".join(b"l" + bstr(u) + b"e" for u in announce_urls) + info_bytes = b"d6:lengthi1e4:name4:teste" torrent_bytes = ( - b'd' - + bstr('announce') + bstr(announce_urls[0]) - + bstr('announce-list') + b'l' + tiers + b'e' - + bstr('info') + info_bytes - + b'e' + b"d" + + bstr("announce") + + bstr(announce_urls[0]) + + bstr("announce-list") + + b"l" + + tiers + + b"e" + + bstr("info") + + info_bytes + + b"e" ) - fd, path = tempfile.mkstemp(suffix='.torrent') - with os.fdopen(fd, 'wb') as f: + fd, path = tempfile.mkstemp(suffix=".torrent") + with os.fdopen(fd, "wb") as f: f.write(torrent_bytes) return path def _scrape_response_bytes(self, info_hash, seeders, leechers): - return b'd5:filesd' + bstr(info_hash) + \ - f'd8:completei{seeders}e10:incompletei{leechers}e10:downloadedi0ee'.encode() + b'ee' + b'e' + return ( + b"d5:filesd" + + bstr(info_hash) + + f"d8:completei{seeders}e10:incompletei{leechers}e10:downloadedi0ee".encode() + + b"ee" + + b"e" + ) def test_uses_udp_tracker_when_it_succeeds(self): - path = self._write_torrent([ - 'udp://tracker.example.com:80/announce', - 'http://tracker.example.com:6969/announce', - ]) - with unittest.mock.patch.object(ts, 'scrape_udp_tracker', return_value={ - 'seeders': 5, 'leechers': 7, 'completed': 1, - }) as mock_udp_scrape, unittest.mock.patch.object(ts.requests, 'get') as mock_get: + path = self._write_torrent( + [ + "udp://tracker.example.com:80/announce", + "http://tracker.example.com:6969/announce", + ] + ) + with ( + unittest.mock.patch.object( + ts, + "scrape_udp_tracker", + return_value={ + "seeders": 5, + "leechers": 7, + "completed": 1, + }, + ) as mock_udp_scrape, + unittest.mock.patch.object(ts.requests, "get") as mock_get, + ): result = ts.scrape_torrent(path) mock_udp_scrape.assert_called_once() mock_get.assert_not_called() - self.assertEqual(result['seeders'], 5) - self.assertEqual(result['leechers'], 7) - self.assertEqual(result['tracker'], 'udp://tracker.example.com:80') + self.assertEqual(result["seeders"], 5) + self.assertEqual(result["leechers"], 7) + self.assertEqual(result["tracker"], "udp://tracker.example.com:80") def test_falls_through_from_failing_udp_tracker_to_http_tracker(self): - path = self._write_torrent([ - 'udp://tracker.example.com:80/announce', - 'http://tracker.example.com:6969/announce', - ]) - with open(path, 'rb') as f: + path = self._write_torrent( + [ + "udp://tracker.example.com:80/announce", + "http://tracker.example.com:6969/announce", + ] + ) + with open(path, "rb") as f: info_hash = ts.compute_info_hash(f.read()) fake_response = unittest.mock.Mock() fake_response.content = self._scrape_response_bytes(info_hash, 5, 7) fake_response.raise_for_status = lambda: None - with unittest.mock.patch.object(ts, 'scrape_udp_tracker', side_effect=OSError("unreachable")), \ - unittest.mock.patch.object(ts.requests, 'get', return_value=fake_response) as mock_get: + with ( + unittest.mock.patch.object( + ts, "scrape_udp_tracker", side_effect=OSError("unreachable") + ), + unittest.mock.patch.object( + ts.requests, "get", return_value=fake_response + ) as mock_get, + ): result = ts.scrape_torrent(path) mock_get.assert_called_once() - self.assertIn('tracker.example.com:6969/scrape', mock_get.call_args[0][0]) - self.assertEqual(result['seeders'], 5) - self.assertEqual(result['leechers'], 7) - self.assertEqual(result['tracker'], 'http://tracker.example.com:6969/scrape') + self.assertIn("tracker.example.com:6969/scrape", mock_get.call_args[0][0]) + self.assertEqual(result["seeders"], 5) + self.assertEqual(result["leechers"], 7) + self.assertEqual(result["tracker"], "http://tracker.example.com:6969/scrape") def test_skips_udp_tracker_with_no_port(self): """BEP 15 requires an explicit port; an announce URL without one can't be dialed, so it should be skipped like any other unusable tracker rather than raising.""" - path = self._write_torrent([ - 'udp://tracker.example.com/announce', - 'http://tracker.example.com:6969/announce', - ]) - with open(path, 'rb') as f: + path = self._write_torrent( + [ + "udp://tracker.example.com/announce", + "http://tracker.example.com:6969/announce", + ] + ) + with open(path, "rb") as f: info_hash = ts.compute_info_hash(f.read()) fake_response = unittest.mock.Mock() fake_response.content = self._scrape_response_bytes(info_hash, 5, 7) fake_response.raise_for_status = lambda: None - with unittest.mock.patch.object(ts, 'scrape_udp_tracker') as mock_udp_scrape, \ - unittest.mock.patch.object(ts.requests, 'get', return_value=fake_response): + with ( + unittest.mock.patch.object(ts, "scrape_udp_tracker") as mock_udp_scrape, + unittest.mock.patch.object(ts.requests, "get", return_value=fake_response), + ): result = ts.scrape_torrent(path) mock_udp_scrape.assert_not_called() - self.assertEqual(result['tracker'], 'http://tracker.example.com:6969/scrape') + self.assertEqual(result["tracker"], "http://tracker.example.com:6969/scrape") def test_falls_through_to_next_tracker_on_failure(self): - path = self._write_torrent([ - 'http://dead.example.com/announce', - 'http://alive.example.com/announce', - ]) - with open(path, 'rb') as f: + path = self._write_torrent( + [ + "http://dead.example.com/announce", + "http://alive.example.com/announce", + ] + ) + with open(path, "rb") as f: info_hash = ts.compute_info_hash(f.read()) fake_response = unittest.mock.Mock() fake_response.content = self._scrape_response_bytes(info_hash, 1, 2) fake_response.raise_for_status = lambda: None def fake_get(url, timeout=None): - if 'dead' in url: + if "dead" in url: raise ConnectionError("nope") return fake_response - with unittest.mock.patch.object(ts.requests, 'get', side_effect=fake_get): + with unittest.mock.patch.object(ts.requests, "get", side_effect=fake_get): result = ts.scrape_torrent(path) - self.assertEqual(result['seeders'], 1) - self.assertEqual(result['tracker'], 'http://alive.example.com/scrape') + self.assertEqual(result["seeders"], 1) + self.assertEqual(result["tracker"], "http://alive.example.com/scrape") def test_returns_none_when_no_tracker_usable(self): - path = self._write_torrent(['udp://only.example.com:1337/announce']) - with unittest.mock.patch.object(ts, 'scrape_udp_tracker', side_effect=OSError("unreachable")), \ - unittest.mock.patch.object(ts.requests, 'get') as mock_get: + path = self._write_torrent(["udp://only.example.com:1337/announce"]) + with ( + unittest.mock.patch.object( + ts, "scrape_udp_tracker", side_effect=OSError("unreachable") + ), + unittest.mock.patch.object(ts.requests, "get") as mock_get, + ): result = ts.scrape_torrent(path) mock_get.assert_not_called() self.assertIsNone(result) def test_skips_udp_tracker_resolving_to_a_private_address(self): - path = self._write_torrent([ - 'udp://internal.example.com:1337/announce', - 'http://real-tracker.example.com/announce', - ]) - with open(path, 'rb') as f: + path = self._write_torrent( + [ + "udp://internal.example.com:1337/announce", + "http://real-tracker.example.com/announce", + ] + ) + with open(path, "rb") as f: info_hash = ts.compute_info_hash(f.read()) fake_response = unittest.mock.Mock() fake_response.content = self._scrape_response_bytes(info_hash, 3, 4) fake_response.raise_for_status = lambda: None def fake_resolve(hostname): - if hostname == 'internal.example.com': - return ['127.0.0.1'] - return ['93.184.216.34'] - - with unittest.mock.patch.object(ts, '_resolve_addresses', side_effect=fake_resolve), \ - unittest.mock.patch.object(ts, 'scrape_udp_tracker') as mock_udp_scrape, \ - unittest.mock.patch.object(ts.requests, 'get', return_value=fake_response) as mock_get: + if hostname == "internal.example.com": + return ["127.0.0.1"] + return ["93.184.216.34"] + + with ( + unittest.mock.patch.object( + ts, "_resolve_addresses", side_effect=fake_resolve + ), + unittest.mock.patch.object(ts, "scrape_udp_tracker") as mock_udp_scrape, + unittest.mock.patch.object( + ts.requests, "get", return_value=fake_response + ) as mock_get, + ): result = ts.scrape_torrent(path) mock_udp_scrape.assert_not_called() mock_get.assert_called_once() - self.assertEqual(result['tracker'], 'http://real-tracker.example.com/scrape') + self.assertEqual(result["tracker"], "http://real-tracker.example.com/scrape") def test_skips_tracker_resolving_to_a_private_address(self): """A .torrent whose announce URL resolves to an internal address (e.g. a compromised upstream mirror pointing at cloud metadata or this container's own services) must never be scraped - no request should go out to it at all.""" - path = self._write_torrent([ - 'http://internal.example.com/announce', - 'http://real-tracker.example.com/announce', - ]) - with open(path, 'rb') as f: + path = self._write_torrent( + [ + "http://internal.example.com/announce", + "http://real-tracker.example.com/announce", + ] + ) + with open(path, "rb") as f: info_hash = ts.compute_info_hash(f.read()) fake_response = unittest.mock.Mock() fake_response.content = self._scrape_response_bytes(info_hash, 3, 4) fake_response.raise_for_status = lambda: None def fake_resolve(hostname): - if hostname == 'internal.example.com': - return ['127.0.0.1'] - return ['93.184.216.34'] - - with unittest.mock.patch.object(ts, '_resolve_addresses', side_effect=fake_resolve), \ - unittest.mock.patch.object(ts.requests, 'get', return_value=fake_response) as mock_get: + if hostname == "internal.example.com": + return ["127.0.0.1"] + return ["93.184.216.34"] + + with ( + unittest.mock.patch.object( + ts, "_resolve_addresses", side_effect=fake_resolve + ), + unittest.mock.patch.object( + ts.requests, "get", return_value=fake_response + ) as mock_get, + ): result = ts.scrape_torrent(path) mock_get.assert_called_once() - self.assertIn('real-tracker.example.com', mock_get.call_args[0][0]) - self.assertEqual(result['tracker'], 'http://real-tracker.example.com/scrape') + self.assertIn("real-tracker.example.com", mock_get.call_args[0][0]) + self.assertEqual(result["tracker"], "http://real-tracker.example.com/scrape") def test_returns_none_when_every_tracker_resolves_privately(self): - path = self._write_torrent(['http://internal.example.com/announce']) - with unittest.mock.patch.object(ts, '_resolve_addresses', return_value=['169.254.169.254']), \ - unittest.mock.patch.object(ts.requests, 'get') as mock_get: + path = self._write_torrent(["http://internal.example.com/announce"]) + with ( + unittest.mock.patch.object( + ts, "_resolve_addresses", return_value=["169.254.169.254"] + ), + unittest.mock.patch.object(ts.requests, "get") as mock_get, + ): result = ts.scrape_torrent(path) mock_get.assert_not_called() self.assertIsNone(result) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/torrent_scrape.py b/torrent_scrape.py index f1e460d..92ebb29 100644 --- a/torrent_scrape.py +++ b/torrent_scrape.py @@ -11,6 +11,7 @@ (BEP 15) are supported. Any other scheme is skipped; callers should just move on to the torrent's next tracker. """ + import hashlib import ipaddress import os @@ -30,6 +31,7 @@ class UdpTrackerError(ValueError): response, transaction ID mismatch, or an explicit tracker error action) - callers should catch it the same as any other scrape failure and move on to the next tracker.""" + pass @@ -40,31 +42,31 @@ def bdecode(data, index=0): if index >= len(data): raise BencodeError("Unexpected end of data") - prefix = data[index:index + 1] + prefix = data[index : index + 1] - if prefix == b'i': - end = data.index(b'e', index) - return int(data[index + 1:end]), end + 1 + if prefix == b"i": + end = data.index(b"e", index) + return int(data[index + 1 : end]), end + 1 if prefix.isdigit(): - colon = data.index(b':', index) + colon = data.index(b":", index) length = int(data[index:colon]) start = colon + 1 end = start + length return data[start:end], end - if prefix == b'l': + if prefix == b"l": index += 1 items = [] - while data[index:index + 1] != b'e': + while data[index : index + 1] != b"e": item, index = bdecode(data, index) items.append(item) return items, index + 1 - if prefix == b'd': + if prefix == b"d": index += 1 result = {} - while data[index:index + 1] != b'e': + while data[index : index + 1] != b"e": key, index = bdecode(data, index) if not isinstance(key, bytes): raise BencodeError("Dict keys must be byte strings") @@ -82,24 +84,26 @@ def bencode(value): if isinstance(value, bool): raise BencodeError("bool is not a bencode type") if isinstance(value, int): - return b'i' + str(value).encode() + b'e' + return b"i" + str(value).encode() + b"e" if isinstance(value, (bytes, bytearray)): - return str(len(value)).encode() + b':' + bytes(value) + return str(len(value)).encode() + b":" + bytes(value) if isinstance(value, str): - encoded = value.encode('utf-8') - return str(len(encoded)).encode() + b':' + encoded + encoded = value.encode("utf-8") + return str(len(encoded)).encode() + b":" + encoded if isinstance(value, list): - return b'l' + b''.join(bencode(v) for v in value) + b'e' + return b"l" + b"".join(bencode(v) for v in value) + b"e" if isinstance(value, dict): + def key_bytes(k): - return k if isinstance(k, bytes) else k.encode('utf-8') + return k if isinstance(k, bytes) else k.encode("utf-8") + items = sorted(value.items(), key=lambda kv: key_bytes(kv[0])) - parts = [b'd'] + parts = [b"d"] for k, v in items: parts.append(bencode(key_bytes(k))) parts.append(bencode(v)) - parts.append(b'e') - return b''.join(parts) + parts.append(b"e") + return b"".join(parts) raise BencodeError(f"Cannot bencode type {type(value)}") @@ -107,9 +111,9 @@ def compute_info_hash(torrent_bytes): """The 20-byte SHA-1 hash that identifies a torrent, computed from the raw bytes of its 'info' dict.""" metainfo, _ = bdecode(torrent_bytes) - if not isinstance(metainfo, dict) or b'info' not in metainfo: + if not isinstance(metainfo, dict) or b"info" not in metainfo: raise BencodeError("Not a valid .torrent file: missing 'info' dictionary") - return hashlib.sha1(bencode(metainfo[b'info'])).digest() + return hashlib.sha1(bencode(metainfo[b"info"])).digest() def get_announce_urls(torrent_bytes): @@ -117,15 +121,15 @@ def get_announce_urls(torrent_bytes): order) then the primary 'announce' if not already present, de-duped.""" metainfo, _ = bdecode(torrent_bytes) urls = [] - for tier in metainfo.get(b'announce-list', []): + for tier in metainfo.get(b"announce-list", []): for url in tier: - decoded = url.decode('utf-8') + decoded = url.decode("utf-8") if decoded not in urls: urls.append(decoded) - primary = metainfo.get(b'announce') + primary = metainfo.get(b"announce") if primary is not None: - decoded = primary.decode('utf-8') + decoded = primary.decode("utf-8") if decoded not in urls: urls.insert(0, decoded) @@ -138,20 +142,24 @@ def derive_scrape_url(announce_url): None if the tracker doesn't support scrape (no 'announce' in its last path segment) or isn't HTTP(S).""" parts = urlsplit(announce_url) - if parts.scheme not in ('http', 'https'): + if parts.scheme not in ("http", "https"): return None - last_slash = parts.path.rfind('/') - segment = parts.path[last_slash + 1:] - if 'announce' not in segment: + last_slash = parts.path.rfind("/") + segment = parts.path[last_slash + 1 :] + if "announce" not in segment: return None - new_segment = segment.replace('announce', 'scrape', 1) - new_path = parts.path[:last_slash + 1] + new_segment - return urlunsplit((parts.scheme, parts.netloc, new_path, parts.query, parts.fragment)) + new_segment = segment.replace("announce", "scrape", 1) + new_path = parts.path[: last_slash + 1] + new_segment + return urlunsplit( + (parts.scheme, parts.netloc, new_path, parts.query, parts.fragment) + ) -UDP_TRACKER_PROTOCOL_ID = 0x41727101980 # BEP 15 magic connection_id for the initial connect request +UDP_TRACKER_PROTOCOL_ID = ( + 0x41727101980 # BEP 15 magic connection_id for the initial connect request +) UDP_ACTION_CONNECT = 0 UDP_ACTION_SCRAPE = 2 UDP_ACTION_ERROR = 3 @@ -162,7 +170,7 @@ def parse_udp_tracker_url(announce_url): isn't udp or no port is present - BEP 15 has no default port, so a tracker URL without one can't be dialed.""" parts = urlsplit(announce_url) - if parts.scheme != 'udp': + if parts.scheme != "udp": return None if not parts.hostname or not parts.port: return None @@ -170,7 +178,9 @@ def parse_udp_tracker_url(announce_url): def build_udp_connect_request(transaction_id): - return struct.pack(">QII", UDP_TRACKER_PROTOCOL_ID, UDP_ACTION_CONNECT, transaction_id) + return struct.pack( + ">QII", UDP_TRACKER_PROTOCOL_ID, UDP_ACTION_CONNECT, transaction_id + ) def _check_for_udp_error_action(data, transaction_id): @@ -180,7 +190,9 @@ def _check_for_udp_error_action(data, transaction_id): if len(data) >= 8: action, resp_transaction_id = struct.unpack(">II", data[:8]) if action == UDP_ACTION_ERROR and resp_transaction_id == transaction_id: - raise UdpTrackerError(f"Tracker error: {data[8:].decode('utf-8', 'replace')}") + raise UdpTrackerError( + f"Tracker error: {data[8:].decode('utf-8', 'replace')}" + ) def parse_udp_connect_response(data, transaction_id): @@ -198,7 +210,10 @@ def parse_udp_connect_response(data, transaction_id): def build_udp_scrape_request(connection_id, transaction_id, info_hash): if len(info_hash) != 20: raise UdpTrackerError(f"info_hash must be 20 bytes, got {len(info_hash)}") - return struct.pack(">QII", connection_id, UDP_ACTION_SCRAPE, transaction_id) + info_hash + return ( + struct.pack(">QII", connection_id, UDP_ACTION_SCRAPE, transaction_id) + + info_hash + ) def parse_udp_scrape_response(data, transaction_id): @@ -213,11 +228,11 @@ def parse_udp_scrape_response(data, transaction_id): if resp_transaction_id != transaction_id: raise UdpTrackerError("Transaction ID mismatch in scrape response") seeders, completed, leechers = struct.unpack(">III", data[8:20]) - return {'seeders': seeders, 'leechers': leechers, 'completed': completed} + return {"seeders": seeders, "leechers": leechers, "completed": completed} def _random_transaction_id(): - return int.from_bytes(os.urandom(4), 'big') + return int.from_bytes(os.urandom(4), "big") def scrape_udp_tracker(host, port, info_hash, timeout=15): @@ -237,7 +252,10 @@ def scrape_udp_tracker(host, port, info_hash, timeout=15): connection_id = parse_udp_connect_response(data, connect_transaction_id) scrape_transaction_id = _random_transaction_id() - sock.sendto(build_udp_scrape_request(connection_id, scrape_transaction_id, info_hash), addr) + sock.sendto( + build_udp_scrape_request(connection_id, scrape_transaction_id, info_hash), + addr, + ) data, _ = sock.recvfrom(2048) return parse_udp_scrape_response(data, scrape_transaction_id) finally: @@ -251,8 +269,12 @@ def _is_disallowed_ip(ip_str): container's own services (127.0.0.1) alongside RFC1918 space.""" ip = ipaddress.ip_address(ip_str) return ( - ip.is_private or ip.is_loopback or ip.is_link_local - or ip.is_multicast or ip.is_reserved or ip.is_unspecified + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + or ip.is_unspecified ) @@ -280,8 +302,8 @@ def _is_safe_scrape_host(hostname): def build_scrape_request_url(scrape_url, info_hash): - encoded_hash = quote_from_bytes(info_hash, safe='') - separator = '&' if '?' in scrape_url else '?' + encoded_hash = quote_from_bytes(info_hash, safe="") + separator = "&" if "?" in scrape_url else "?" return f"{scrape_url}{separator}info_hash={encoded_hash}" @@ -290,17 +312,17 @@ def parse_scrape_response(data, info_hash): 'completed'} for info_hash, or None if the tracker's response doesn't include stats for it.""" parsed, _ = bdecode(data) - if b'failure reason' in parsed: - raise BencodeError(parsed[b'failure reason'].decode('utf-8', 'replace')) + if b"failure reason" in parsed: + raise BencodeError(parsed[b"failure reason"].decode("utf-8", "replace")) - stats = parsed.get(b'files', {}).get(info_hash) + stats = parsed.get(b"files", {}).get(info_hash) if stats is None: return None return { - 'seeders': stats.get(b'complete', 0), - 'leechers': stats.get(b'incomplete', 0), - 'completed': stats.get(b'downloaded', 0), + "seeders": stats.get(b"complete", 0), + "leechers": stats.get(b"incomplete", 0), + "completed": stats.get(b"downloaded", 0), } @@ -310,7 +332,7 @@ def scrape_torrent(torrent_path, timeout=15): until one returns usable stats. Returns {'tracker', 'seeders', 'leechers', 'completed'}, or None if no tracker could be scraped (all UDP-only, all unreachable, or none had stats for this hash).""" - with open(torrent_path, 'rb') as f: + with open(torrent_path, "rb") as f: torrent_bytes = f.read() info_hash = compute_info_hash(torrent_bytes) @@ -325,7 +347,7 @@ def scrape_torrent(torrent_path, timeout=15): stats = scrape_udp_tracker(host, port, info_hash, timeout=timeout) except Exception: continue - return {'tracker': f'udp://{host}:{port}', **stats} + return {"tracker": f"udp://{host}:{port}", **stats} scrape_url = derive_scrape_url(announce_url) if not scrape_url: @@ -342,6 +364,6 @@ def scrape_torrent(torrent_path, timeout=15): continue if stats is not None: - return {'tracker': scrape_url, **stats} + return {"tracker": scrape_url, **stats} return None From b3882fef6e295a73c283ea20b418ac0de004a50b Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 07:14:32 +0200 Subject: [PATCH 02/46] Apply ruff's safe autofixes (import sorting, unused imports) Co-Authored-By: Claude Opus 5.5 --- fetch_torrents.py | 11 ++++++----- tests/test_torrent_scrape.py | 9 ++++----- torrent_scrape.py | 3 +-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/fetch_torrents.py b/fetch_torrents.py index 78fb3df..8f1366f 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -1,15 +1,16 @@ #!/usr/bin/env python3 -import os -import re import json -import requests import logging import logging.handlers -import tempfile -import time +import os +import re import shutil import socket +import tempfile +import time from datetime import date + +import requests from bs4 import BeautifulSoup from transmission_rpc import Client diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index 7ac7e66..9d3bc54 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -185,7 +185,7 @@ def test_appends_to_existing_query(self): self.assertIn("?x=1&info_hash=", url) def test_percent_encoding_roundtrips(self): - from urllib.parse import urlsplit, parse_qsl, unquote_to_bytes + from urllib.parse import unquote_to_bytes, urlsplit info_hash = bytes(range(20)) url = ts.build_scrape_request_url("http://example.com/scrape", info_hash) @@ -400,17 +400,16 @@ def test_connect_transaction_mismatch_raises_and_closes_socket(self): unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), unittest.mock.patch.object( ts.os, "urandom", return_value=b"\x00\x00\x00\x01" - ), + ),self.assertRaises(ts.UdpTrackerError) ): - with self.assertRaises(ts.UdpTrackerError): - ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) + ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) fake_sock.close.assert_called_once() def test_socket_timeout_propagates(self): info_hash = b"A" * 20 fake_sock = unittest.mock.Mock() - fake_sock.recvfrom.side_effect = socket.timeout("timed out") + fake_sock.recvfrom.side_effect = TimeoutError("timed out") with unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock): with self.assertRaises(socket.timeout): diff --git a/torrent_scrape.py b/torrent_scrape.py index 92ebb29..46ea5c5 100644 --- a/torrent_scrape.py +++ b/torrent_scrape.py @@ -17,7 +17,7 @@ import os import socket import struct -from urllib.parse import urlsplit, urlunsplit, quote_from_bytes +from urllib.parse import quote_from_bytes, urlsplit, urlunsplit import requests @@ -32,7 +32,6 @@ class UdpTrackerError(ValueError): action) - callers should catch it the same as any other scrape failure and move on to the next tracker.""" - pass def bdecode(data, index=0): From 5800ad23d42923584bb0dd8e698904a83cbb3f3d Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 07:17:33 +0200 Subject: [PATCH 03/46] Isolate tests from env vars, the clock and temp-dir leaks (task 12) - tests/helpers.py: isolated_env()/scrubbed_environ() strip SKIP_RATIO_CHECK, CLEANUP_*, TRANSMISSION_* and FETCH_TORRENTS_* (except the log dir), and make_temp_dir() removes its dir on cleanup. - test_fetch_torrents applies isolated_env() module-wide, so an exported SKIP_RATIO_CHECK=true no longer fails 9 ratio-gate tests. - The stagnation watch-file test pins ft.date instead of using date.today(). - The unsafe-name watch-file test asserts on disk instead of patching the process-wide os.remove (which log rotation also calls). - Every mkdtemp/mkstemp in tests is now removed after the test. - CI runs the suite a second time with a hostile environment. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/docker-image.yml | 14 +++ tests/helpers.py | 61 +++++++++++++ tests/test_configure_transmission.py | 8 +- tests/test_fetch_torrents.py | 128 ++++++++++++++++++--------- tests/test_torrent_scrape.py | 4 +- 5 files changed, 169 insertions(+), 46 deletions(-) create mode 100644 tests/helpers.py diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 3b74153..3edf80f 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -45,6 +45,20 @@ jobs: - name: Run tests run: python -m unittest discover -s tests -v + # Same suite again with env vars exported that would flip outcomes if any + # test leaked the developer's shell environment into production code. + - name: Run tests with a hostile environment + run: python -m unittest discover -s tests + env: + SKIP_RATIO_CHECK: "true" + CLEANUP_MIN_RATIO: "5" + CLEANUP_SKIP_RATIO_CHECK: "true" + TRANSMISSION_RPC_USERNAME: x + TRANSMISSION_RPC_PASSWORD: y + FETCH_TORRENTS_DISTROS: arch + FETCH_TORRENTS_LOG_MAX_BYTES: "7" + FETCH_TORRENTS_ALWAYS_LOG: "false" + build: needs: test runs-on: ubuntu-latest diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 0000000..9991311 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,61 @@ +"""Shared test helpers. + +Not a test module itself (doesn't match the test*.py discovery pattern); +`unittest discover -s tests` puts tests/ on sys.path, so test modules import +this as `import helpers`. +""" + +import os +import shutil +import tempfile +import unittest.mock + +# Env vars production code reads at call time. A developer who happens to +# have any of these exported (e.g. SKIP_RATIO_CHECK=true) must not be able +# to flip test outcomes, so isolated_env() removes them unless a test sets +# them explicitly. +_ISOLATED_PREFIXES = ( + "SKIP_RATIO_CHECK", + "CLEANUP_", + "TRANSMISSION_", + "FETCH_TORRENTS_", +) +# Read at import time to choose where fetch_torrents writes its logs; the +# test modules set it to a temp dir before importing, so it must survive. +_PRESERVED = frozenset({"FETCH_TORRENTS_LOG_DIR"}) + + +def scrubbed_environ(**overrides): + """A copy of os.environ without the isolated keys, plus overrides.""" + env = { + key: value + for key, value in os.environ.items() + if key in _PRESERVED or not key.startswith(_ISOLATED_PREFIXES) + } + env.update(overrides) + return env + + +def isolated_env(**overrides): + """patch.dict replacing os.environ with scrubbed_environ(**overrides). + + Usable as a context manager, or started/stopped manually (see + use_isolated_env()). The scrubbed copy is taken when this is called, not + when it is entered, so call it at the point of use.""" + return unittest.mock.patch.dict( + os.environ, scrubbed_environ(**overrides), clear=True + ) + + +def use_isolated_env(testcase, **overrides): + """Apply isolated_env() for the rest of a test (call from setUp).""" + patcher = isolated_env(**overrides) + patcher.start() + testcase.addCleanup(patcher.stop) + + +def make_temp_dir(testcase, prefix): + """mkdtemp() that is removed again when the test finishes.""" + path = tempfile.mkdtemp(prefix=prefix) + testcase.addCleanup(shutil.rmtree, path, ignore_errors=True) + return path diff --git a/tests/test_configure_transmission.py b/tests/test_configure_transmission.py index f01cd8a..75a0e6f 100644 --- a/tests/test_configure_transmission.py +++ b/tests/test_configure_transmission.py @@ -5,7 +5,6 @@ import os import sys -import tempfile import unittest REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -13,6 +12,7 @@ sys.path.insert(0, REPO_ROOT) import configure_transmission as ct +import helpers class BuildRpcOverridesTests(unittest.TestCase): @@ -228,7 +228,7 @@ def test_overrides_take_precedence_but_other_keys_survive(self): class SettingsFilePersistenceTests(unittest.TestCase): def test_round_trips_through_disk(self): - tmp_dir = tempfile.mkdtemp(prefix="configure_transmission_") + tmp_dir = helpers.make_temp_dir(self, "configure_transmission_") path = os.path.join(tmp_dir, "settings.json") settings = {"rpc-whitelist": "10.0.0.*"} @@ -238,13 +238,13 @@ def test_round_trips_through_disk(self): self.assertEqual(loaded, settings) def test_missing_file_returns_empty_settings(self): - tmp_dir = tempfile.mkdtemp(prefix="configure_transmission_") + tmp_dir = helpers.make_temp_dir(self, "configure_transmission_") path = os.path.join(tmp_dir, "does-not-exist.json") self.assertEqual(ct.load_settings(path), {}) def test_corrupt_file_returns_empty_settings_instead_of_raising(self): - tmp_dir = tempfile.mkdtemp(prefix="configure_transmission_") + tmp_dir = helpers.make_temp_dir(self, "configure_transmission_") path = os.path.join(tmp_dir, "settings.json") with open(path, "w", encoding="utf-8") as f: f.write("{not valid json") diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index 22848ef..8ad3b38 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -11,6 +11,7 @@ import json import os +import shutil import subprocess import sys import tempfile @@ -25,17 +26,37 @@ stub.Client = MagicMock() sys.modules["transmission_rpc"] = stub -os.environ.setdefault( - "FETCH_TORRENTS_LOG_DIR", tempfile.mkdtemp(prefix="fetch_torrents_test_logs_") -) +_TEST_LOG_DIR = tempfile.mkdtemp(prefix="fetch_torrents_test_logs_") +os.environ.setdefault("FETCH_TORRENTS_LOG_DIR", _TEST_LOG_DIR) REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if REPO_ROOT not in sys.path: sys.path.insert(0, REPO_ROOT) +import helpers + import fetch_torrents as ft +def setUpModule(): + # Baseline for every test here: the developer's exported + # SKIP_RATIO_CHECK/CLEANUP_*/TRANSMISSION_*/FETCH_TORRENTS_* can't leak in + # (including into the subprocess tests, which copy os.environ). + patcher = helpers.isolated_env() + patcher.start() + unittest.addModuleCleanup(patcher.stop) + + +def tearDownModule(): + # The import-time log dir can only be removed once ft's handlers have + # released their files (Windows refuses to delete open files). + for handler in (ft.file_handler, ft.ratio_handler): + ft.logger.removeHandler(handler) + handler.close() + if os.environ.get("FETCH_TORRENTS_LOG_DIR") == _TEST_LOG_DIR: + shutil.rmtree(_TEST_LOG_DIR, ignore_errors=True) + + class FakeTorrent: def __init__(self, name, id_, ratio=0.0, added_date=None): self.name = name @@ -47,6 +68,17 @@ def __repr__(self): return f"FakeTorrent({self.name!r}, ratio={self.ratio})" +def fixed_date(today): + """A drop-in for fetch_torrents' `date` whose today() is pinned.""" + + class FixedDate(date): + @classmethod + def today(cls): + return today + + return FixedDate + + def names_of(torrents): return sorted(t.name for t in torrents) @@ -462,7 +494,7 @@ class RatioLogSurvivesRestartTests(unittest.TestCase): about what happens between import and the first get_previous_ratios() call.""" def test_previous_ratios_are_readable_after_a_fresh_start(self): - log_dir = tempfile.mkdtemp(prefix="fetch_torrents_ratio_restart_") + log_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_restart_") ratio_log_path = os.path.join(log_dir, "fetch_torrents_ratios.log") with open(ratio_log_path, "w", encoding="utf-8") as f: f.write( @@ -510,7 +542,7 @@ class LogFileRotationTests(unittest.TestCase): was in effect during test collection).""" def _run(self, extra_env=None): - log_dir = tempfile.mkdtemp(prefix="fetch_torrents_log_rotation_") + log_dir = helpers.make_temp_dir(self, "fetch_torrents_log_rotation_") script = ( "import os, sys, types\n" "from unittest.mock import MagicMock\n" @@ -766,8 +798,7 @@ class ParseSupportedDistrosTests(unittest.TestCase): fetched just because FETCH_TORRENTS_DISTROS is unset.""" def test_default_excludes_opt_in_distros(self): - with unittest.mock.patch.dict(os.environ, {}, clear=False): - os.environ.pop("FETCH_TORRENTS_DISTROS", None) + with helpers.isolated_env(): result = ft.parse_supported_distros() self.assertNotIn("arch", result) self.assertNotIn("devuan", result) @@ -892,7 +923,7 @@ class DownloadTorrent404Tests(unittest.TestCase): same way as a genuine download failure.""" def setUp(self): - self.tmp_watch_dir = tempfile.mkdtemp(prefix="fetch_torrents_test_watch_") + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) @@ -986,7 +1017,7 @@ def test_rejects_empty_or_dot_names(self): class DownloadTorrentUnsafeNameTests(unittest.TestCase): def setUp(self): - self.tmp_watch_dir = tempfile.mkdtemp(prefix="fetch_torrents_test_watch_") + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) @@ -1000,10 +1031,20 @@ def test_download_torrent_refuses_unsafe_name_without_any_request(self): self.assertEqual(os.listdir(self.tmp_watch_dir), []) def test_remove_watch_file_refuses_unsafe_name_without_touching_disk(self): - with unittest.mock.patch.object(ft.os, "remove") as mock_remove: + # Asserted on disk rather than by patching os.remove: that patch is + # process-wide, so a log rotation triggered by the warning would + # trip it too. + parent = helpers.make_temp_dir(self, "fetch_torrents_test_escape_") + watch_dir = os.path.join(parent, "watch") + os.mkdir(watch_dir) + outside = os.path.join(parent, "evil.torrent") + with open(outside, "w", encoding="utf-8") as f: + f.write("must survive") + + with unittest.mock.patch.object(ft, "watch_dir", watch_dir): ft._remove_watch_file("../evil") - mock_remove.assert_not_called() + self.assertTrue(os.path.exists(outside)) class UpdateRatioHistoryTests(unittest.TestCase): @@ -1201,7 +1242,7 @@ class RatioHistoryPersistenceTests(unittest.TestCase): tolerate a missing or corrupt file rather than crashing a run.""" def test_round_trips_through_disk(self): - tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_ratio_history_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") path = os.path.join(tmp_dir, "fetch_torrents_ratio_history.json") history = { "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-29", "ratio": 1.2}] @@ -1213,13 +1254,13 @@ def test_round_trips_through_disk(self): self.assertEqual(loaded, history) def test_missing_file_returns_empty_history(self): - tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_ratio_history_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") path = os.path.join(tmp_dir, "does-not-exist.json") self.assertEqual(ft.load_ratio_history(path), {}) def test_corrupt_file_returns_empty_history_instead_of_raising(self): - tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_ratio_history_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") path = os.path.join(tmp_dir, "fetch_torrents_ratio_history.json") with open(path, "w", encoding="utf-8") as f: f.write("{not valid json") @@ -1602,7 +1643,7 @@ def raise_for_status(self): class CheckOldReleasesForDemandTests(unittest.TestCase): def setUp(self): - self.tmp_watch_dir = tempfile.mkdtemp(prefix="fetch_torrents_test_watch_") + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) @@ -1833,7 +1874,7 @@ def test_checked_past_interval_is_due(self): class OldReleaseCheckStatePersistenceTests(unittest.TestCase): def test_round_trips_through_disk(self): - tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_old_release_state_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") path = os.path.join(tmp_dir, "fetch_torrents_old_release_check_state.json") state = {"x": {"last_checked": "2026-01-08", "leechers": 0}} @@ -1843,13 +1884,13 @@ def test_round_trips_through_disk(self): self.assertEqual(loaded, state) def test_missing_file_returns_empty_state(self): - tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_old_release_state_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") path = os.path.join(tmp_dir, "does-not-exist.json") self.assertEqual(ft.load_old_release_check_state(path), {}) def test_corrupt_file_returns_empty_state_instead_of_raising(self): - tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_old_release_state_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") path = os.path.join(tmp_dir, "fetch_torrents_old_release_check_state.json") with open(path, "w", encoding="utf-8") as f: f.write("{not valid json") @@ -1877,7 +1918,7 @@ def test_adds_entry_without_mutating_input(self): class RemovedHistoryPersistenceTests(unittest.TestCase): def test_round_trips_through_disk(self): - tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_removed_history_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") path = os.path.join(tmp_dir, "fetch_torrents_removed_history.json") history = {"x": {"removed_date": "2026-02-01", "reason": "stagnant"}} @@ -1887,13 +1928,13 @@ def test_round_trips_through_disk(self): self.assertEqual(loaded, history) def test_missing_file_returns_empty_history(self): - tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_removed_history_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") path = os.path.join(tmp_dir, "does-not-exist.json") self.assertEqual(ft.load_removed_history(path), {}) def test_corrupt_file_returns_empty_history_instead_of_raising(self): - tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_removed_history_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") path = os.path.join(tmp_dir, "fetch_torrents_removed_history.json") with open(path, "w", encoding="utf-8") as f: f.write("{not valid json") @@ -1908,7 +1949,7 @@ class CheckOldReleasesForDemandStateTests(unittest.TestCase): network, and must record a fresh check for anything it does evaluate.""" def setUp(self): - self.tmp_watch_dir = tempfile.mkdtemp(prefix="fetch_torrents_test_watch_") + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) @@ -2049,7 +2090,7 @@ class CleanupRpcAuthTests(unittest.TestCase): failure is only logged, never surfaced.""" def setUp(self): - tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_cleanup_auth_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_auth_") for attr, filename in ( ("removed_history_file", "removed_history.json"), ("ratio_history_file", "ratio_history.json"), @@ -2085,13 +2126,13 @@ def test_cleanup_old_versions_passes_configured_credentials(self): ) def test_cleanup_old_versions_passes_no_credentials_when_unset(self): - with unittest.mock.patch.dict(os.environ, {}, clear=False): - os.environ.pop("TRANSMISSION_RPC_USERNAME", None) - os.environ.pop("TRANSMISSION_RPC_PASSWORD", None) - with unittest.mock.patch.object( + with ( + helpers.isolated_env(), + unittest.mock.patch.object( ft, "Client", return_value=self._mock_client() - ) as mock_client_cls: - ft.cleanup_old_versions() + ) as mock_client_cls, + ): + ft.cleanup_old_versions() mock_client_cls.assert_called_once_with( host="localhost", port=9091, username=None, password=None @@ -2168,18 +2209,18 @@ def test_uses_configured_credentials_when_auth_not_explicitly_passed(self): self.assertEqual(second_call_kwargs.get("auth"), ("alice", "hunter2")) def test_no_auth_when_credentials_unset(self): - with unittest.mock.patch.dict(os.environ, {}, clear=False): - os.environ.pop("TRANSMISSION_RPC_USERNAME", None) - os.environ.pop("TRANSMISSION_RPC_PASSWORD", None) - with unittest.mock.patch.object( + with ( + helpers.isolated_env(), + unittest.mock.patch.object( ft.requests, "post", side_effect=[ self.FakeSessionIdResponse(), self.FakeTorrentGetResponse(), ], - ) as mock_post: - ft.log_seed_ratios_via_http() + ) as mock_post, + ): + ft.log_seed_ratios_via_http() second_call_kwargs = mock_post.call_args_list[1].kwargs self.assertIsNone(second_call_kwargs.get("auth")) @@ -2215,7 +2256,7 @@ class CleanupRemovesWatchFileTests(unittest.TestCase): by default) ever cleans them up.""" def setUp(self): - tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_cleanup_watchfile_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_watchfile_") for attr, filename in ( ("removed_history_file", "removed_history.json"), ("ratio_history_file", "ratio_history.json"), @@ -2226,7 +2267,7 @@ def setUp(self): patcher.start() self.addCleanup(patcher.stop) - self.tmp_watch_dir = tempfile.mkdtemp(prefix="fetch_torrents_test_watch_") + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) @@ -2292,14 +2333,19 @@ def test_cleanup_stagnant_torrents_removes_the_watch_file(self): # history - before plan_stagnation_cleanup() ever sees it. Ratio # unchanged since then, so plan_stagnation_cleanup() must select this # torrent for removal. - anchor_date = (date.today() - timedelta(days=32)).isoformat() + # The clock is pinned so a run crossing midnight can't shift the offset. + today = date(2026, 8, 1) + anchor_date = (today - timedelta(days=32)).isoformat() history = { "ubuntu-23.10-desktop-amd64.iso": [{"date": anchor_date, "ratio": 2.0}] } with open(ft.ratio_history_file, "w", encoding="utf-8") as f: json.dump(history, f) - with unittest.mock.patch.object(ft, "Client", return_value=client): + with ( + unittest.mock.patch.object(ft, "Client", return_value=client), + unittest.mock.patch.object(ft, "date", fixed_date(today)), + ): ft.cleanup_stagnant_torrents() client.remove_torrent.assert_called_once_with(2, delete_data=True) @@ -2382,7 +2428,7 @@ class EnforceDiskUsageLimitTests(unittest.TestCase): torrent holds.""" def setUp(self): - tmp_dir = tempfile.mkdtemp(prefix="fetch_torrents_disk_pressure_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_disk_pressure_") for attr, filename in ( ("removed_history_file", "removed_history.json"), ("ratio_history_file", "ratio_history.json"), @@ -2393,7 +2439,7 @@ def setUp(self): patcher.start() self.addCleanup(patcher.stop) - self.tmp_watch_dir = tempfile.mkdtemp(prefix="fetch_torrents_test_watch_") + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) patcher.start() self.addCleanup(patcher.stop) diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index 9d3bc54..b15f178 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -400,7 +400,8 @@ def test_connect_transaction_mismatch_raises_and_closes_socket(self): unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), unittest.mock.patch.object( ts.os, "urandom", return_value=b"\x00\x00\x00\x01" - ),self.assertRaises(ts.UdpTrackerError) + ), + self.assertRaises(ts.UdpTrackerError), ): ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) @@ -498,6 +499,7 @@ def _write_torrent(self, announce_urls): + b"e" ) fd, path = tempfile.mkstemp(suffix=".torrent") + self.addCleanup(os.remove, path) with os.fdopen(fd, "wb") as f: f.write(torrent_bytes) return path From de5802f20a971d3739db6d47b64346316eac7583 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 07:18:54 +0200 Subject: [PATCH 04/46] Pin ratio-gate boundaries and bypass branches (task 10) Adds ShouldFetchTorrentBoundaryTests: ratio exactly 1.0 vs 0.999 (standard and Devuan), only-newer / same-version-only records, highest older version deciding, other-arch isolation, Devuan ignoring non-Devuan and same-release entries, live-over-removed priority both ways, SKIP_RATIO_CHECK, unknown distro and unparseable names. Every ShouldFetchTorrent* class now pins SKIP_RATIO_CHECK=false in setUp. Hand-applied mutants listed in the task are all killed; the one survivor (dropping 'n in ratios or' in the removed-history loop) is equivalent, since setdefault() already keeps the live ratio. Co-Authored-By: Claude Opus 5.5 --- tests/test_fetch_torrents.py | 154 +++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index 8ad3b38..5886013 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -278,6 +278,10 @@ class ShouldFetchTorrentRatioKeyTests(unittest.TestCase): actually reports (which include the arch suffix and .iso extension), for all supported distros.""" + def setUp(self): + # Pinned explicitly: SKIP_RATIO_CHECK=true would make every case True. + helpers.use_isolated_env(self, SKIP_RATIO_CHECK="false") + def test_finds_previous_version_and_honors_low_ratio_ubuntu(self): ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.5} self.assertFalse( @@ -347,6 +351,10 @@ class ShouldFetchTorrentRemovedHistoryFallbackTests(unittest.TestCase): the ratio gate for exactly the versions cleanup already gave up on. removed_history (record_removal()'s output) is the fallback source.""" + def setUp(self): + # Pinned explicitly: SKIP_RATIO_CHECK=true would make every case True. + helpers.use_isolated_env(self, SKIP_RATIO_CHECK="false") + def test_falls_back_to_removed_history_ratio_when_not_live(self): removed_history = { "ubuntu-23.10-desktop-amd64.iso": { @@ -399,6 +407,23 @@ def test_live_ratio_takes_priority_over_removed_history(self): ) ) + def test_low_live_ratio_is_not_overridden_by_high_removed_history(self): + removed_history = { + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 1.5, + } + } + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.4} + self.assertFalse( + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios, + removed_history=removed_history, + ) + ) + def test_removed_history_entry_without_ratio_is_ignored(self): # Legacy entries recorded before this field existed have no 'ratio' # key - must not crash, and must not count as a usable prior version. @@ -439,6 +464,10 @@ class ShouldFetchTorrentDevuanRatioGateTests(unittest.TestCase): fetching a new Devuan release on the ratio achieved by whichever Devuan release we currently hold, using the same 1.0 floor as other distros.""" + def setUp(self): + # Pinned explicitly: SKIP_RATIO_CHECK=true would make every case True. + helpers.use_isolated_env(self, SKIP_RATIO_CHECK="false") + def test_no_previous_devuan_release_fetches(self): self.assertTrue(ft.should_fetch_torrent("devuan_excalibur", ratios={})) @@ -485,6 +514,131 @@ def test_other_distros_removed_history_not_consulted_for_devuan(self): ) +class ShouldFetchTorrentBoundaryTests(unittest.TestCase): + """The 1.0 floor is inclusive, only strictly older versions count as the + "previous" one, and the branches that bypass the gate entirely.""" + + def setUp(self): + helpers.use_isolated_env(self, SKIP_RATIO_CHECK="false") + + def test_previous_ratio_exactly_at_floor_allows_fetch(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.0} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_previous_ratio_just_below_floor_blocks_fetch(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.999} + self.assertFalse( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_devuan_previous_ratio_exactly_at_floor_allows_fetch(self): + self.assertTrue( + ft.should_fetch_torrent("devuan_excalibur", {"devuan_daedalus": 1.0}) + ) + + def test_devuan_previous_ratio_just_below_floor_blocks_fetch(self): + self.assertFalse( + ft.should_fetch_torrent("devuan_excalibur", {"devuan_daedalus": 0.999}) + ) + + def test_only_newer_versions_on_record_fetches(self): + # A newer release with a low ratio isn't a "previous" one, so there + # is nothing to gate on. + ratios = {"ubuntu-24.10-desktop-amd64.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_same_version_on_record_is_not_its_own_predecessor(self): + ratios = {"ubuntu-24.04-desktop-amd64.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_highest_older_version_decides_not_any_older_version(self): + ratios = { + "ubuntu-22.04-desktop-amd64.iso": 5.0, + "ubuntu-23.10-desktop-amd64.iso": 0.4, + } + self.assertFalse( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_other_architecture_ratio_is_not_consulted(self): + ratios = {"ubuntu-23.10-desktop-arm64.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_devuan_ignores_non_devuan_ratios(self): + ratios = {"debian-12.4.0-amd64-DVD-1.iso": 5.0, "devuan_daedalus": 0.4} + self.assertFalse(ft.should_fetch_torrent("devuan_excalibur", ratios)) + + def test_devuan_removed_history_for_the_same_release_is_not_a_predecessor(self): + removed_history = { + "devuan_excalibur": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + } + } + self.assertTrue( + ft.should_fetch_torrent( + "devuan_excalibur", ratios={}, removed_history=removed_history + ) + ) + + def test_devuan_live_ratio_beats_removed_history(self): + removed_history = { + "devuan_chimaera": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + } + } + self.assertTrue( + ft.should_fetch_torrent( + "devuan_excalibur", + {"devuan_daedalus": 1.5}, + removed_history=removed_history, + ) + ) + + def test_skip_ratio_check_fetches_despite_low_previous_ratio(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.1} + with helpers.isolated_env(SKIP_RATIO_CHECK="TRUE"): + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_unknown_distro_fetches(self): + ratios = {"slackware-15.0-install-dvd.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent("slackware-15.1-install-dvd.iso", ratios) + ) + + def test_unparseable_name_of_known_distro_fetches_and_logs_error(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.1} + with self.assertLogs(ft.logger, level="ERROR") as logs: + result = ft.should_fetch_torrent("ubuntu-noble-desktop-amd64.iso", ratios) + + self.assertTrue(result) + self.assertEqual(len(logs.records), 1) + self.assertIn("ubuntu-noble-desktop-amd64.iso", logs.output[0]) + + def test_unparseable_stored_ratio_entry_is_skipped(self): + ratios = { + "ubuntu-mantic-desktop-amd64.iso": 0.1, + "ubuntu-23.10-desktop-amd64.iso": 1.5, + } + with self.assertLogs(ft.logger, level="WARNING"): + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + class RatioLogSurvivesRestartTests(unittest.TestCase): """Each container run is a fresh `python fetch_torrents.py` process: the ratio log from the *previous* run is what should_fetch_torrent() needs in From d86f7639a5ce834373ed9461bc2546187d2ae766 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 07:19:49 +0200 Subject: [PATCH 05/46] Test get_previous_ratios in-process (task 11) Covers missing file, no markers, last-segment-wins, START without END, END before the last START, name stripping, integer ratios not matching, and duplicate names. get_previous_ratios is now fully covered by the normal unittest run; the subprocess restart test stays as-is. Co-Authored-By: Claude Opus 5.5 --- tests/test_fetch_torrents.py | 99 +++++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index 5886013..265782f 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -33,10 +33,10 @@ if REPO_ROOT not in sys.path: sys.path.insert(0, REPO_ROOT) -import helpers - import fetch_torrents as ft +import helpers + def setUpModule(): # Baseline for every test here: the developer's exported @@ -687,6 +687,101 @@ def test_previous_ratios_are_readable_after_a_fresh_start(self): ) +class GetPreviousRatiosTests(unittest.TestCase): + """get_previous_ratios() is what feeds should_fetch_torrent()'s ratios: + only the last START/END segment of the previous run's ratio log counts, + so a ratio from an older run can't satisfy (or block) the gate.""" + + def _write_log(self, *messages): + path = os.path.join( + helpers.make_temp_dir(self, "fetch_torrents_prev_ratios_"), + "fetch_torrents_ratios.log", + ) + with open(path, "w", encoding="utf-8") as f: + f.writelines(f"2026-08-01 00:00:00,000 INFO: {message}\n" for message in messages) + return path + + def test_missing_file_returns_empty(self): + missing = os.path.join( + helpers.make_temp_dir(self, "fetch_torrents_prev_ratios_"), "nope.log" + ) + self.assertEqual(ft.get_previous_ratios(missing), {}) + + def test_without_markers_every_ratio_line_is_parsed(self): + path = self._write_log( + "[ratio] ubuntu-24.04-desktop-amd64.iso → 1.250", + "unrelated line", + "[ratio] debian-12.5.0-amd64-DVD-1.iso → 0.664", + ) + self.assertEqual( + ft.get_previous_ratios(path), + { + "ubuntu-24.04-desktop-amd64.iso": 1.25, + "debian-12.5.0-amd64-DVD-1.iso": 0.664, + }, + ) + + def test_only_the_last_complete_segment_is_returned(self): + path = self._write_log( + "[ratio] RATIOS START", + "[ratio] old-only.iso → 0.100", + "[ratio] shared.iso → 0.200", + "[ratio] RATIOS END", + "[ratio] RATIOS START", + "[ratio] shared.iso → 1.500", + "[ratio] RATIOS END", + "[ratio] after-end.iso → 0.300", + ) + self.assertEqual(ft.get_previous_ratios(path), {"shared.iso": 1.5}) + + def test_start_without_end_parses_to_end_of_file(self): + # A run that crashed mid-way never wrote its END marker. + path = self._write_log( + "[ratio] RATIOS START", + "[ratio] old.iso → 0.100", + "[ratio] RATIOS END", + "[ratio] RATIOS START", + "[ratio] crashed-run.iso → 0.700", + ) + self.assertEqual(ft.get_previous_ratios(path), {"crashed-run.iso": 0.7}) + + def test_end_before_the_last_start_is_ignored(self): + path = self._write_log( + "[ratio] RATIOS END", + "[ratio] before-start.iso → 0.100", + "[ratio] RATIOS START", + "[ratio] current.iso → 0.900", + ) + self.assertEqual(ft.get_previous_ratios(path), {"current.iso": 0.9}) + + def test_padded_names_are_stripped(self): + path = self._write_log( + "[ratio] archlinux-2026.04.01-x86_64.iso → 0.664", + ) + self.assertEqual( + ft.get_previous_ratios(path), {"archlinux-2026.04.01-x86_64.iso": 0.664} + ) + + def test_names_with_internal_spaces_keep_them(self): + path = self._write_log("[ratio] Some Torrent Name → 2.000") + self.assertEqual(ft.get_previous_ratios(path), {"Some Torrent Name": 2.0}) + + def test_ratio_without_decimal_point_is_not_matched(self): + # Pins current behavior: log_seed_ratios_via_http() always writes + # three decimals, so an integer ratio means a line it didn't write. + path = self._write_log("[ratio] whole.iso → 1") + self.assertEqual(ft.get_previous_ratios(path), {}) + + def test_duplicate_name_in_a_segment_last_one_wins(self): + path = self._write_log( + "[ratio] RATIOS START", + "[ratio] dup.iso → 0.100", + "[ratio] dup.iso → 1.100", + "[ratio] RATIOS END", + ) + self.assertEqual(ft.get_previous_ratios(path), {"dup.iso": 1.1}) + + class LogFileRotationTests(unittest.TestCase): """fetch_torrents.log must be bounded (RotatingFileHandler), not a plain FileHandler appending forever for the life of a 'deploy and forget' From 36c8ca7fc4c472eed1a56e0af709269c26522446 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 07:21:02 +0200 Subject: [PATCH 06/46] Make bdecode reject truncated and malformed input (task 13) bdecode silently accepted a byte string whose declared length ran past the end of the data (b'4:ab' -> b'ab'), so a truncated .torrent decoded into a wrong info dict and hash. Missing 'e'/':' terminators and non-numeric integers/lengths leaked bare ValueErrors from bytes.index() and int(); int() also accepted '1_0' and whitespace. All of these now raise BencodeError naming the problem and its index. BencodeError still subclasses ValueError, so existing callers are unaffected. Co-Authored-By: Claude Opus 5.5 --- tests/test_torrent_scrape.py | 27 +++++++++++++++++++++++++++ torrent_scrape.py | 32 ++++++++++++++++++++++++++------ 2 files changed, 53 insertions(+), 6 deletions(-) diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index b15f178..c0e8d21 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -72,6 +72,33 @@ def test_bdecode_invalid_prefix_raises(self): with self.assertRaises(ts.BencodeError): ts.bdecode(b"x123") + def test_bdecode_malformed_input_raises_bencode_error(self): + # A truncated .torrent must fail loudly: decoding it silently gives + # a wrong info dict, hence a wrong info-hash and a bogus scrape. + cases = { + b"4:ab": "declares 4 bytes, has 2", + b"i12": "integer without terminating e", + b"iabce": "non-numeric integer", + b"i1_0e": "int() accepts underscores, bencode doesn't", + b"i e": "blank integer", + b"2x:ab": "non-numeric length", + b"12": "length without colon", + b"l4:spam": "unterminated list", + b"d3:cow3:moo": "unterminated dict", + } + for data, problem in cases.items(): + with self.subTest(problem, data=data): + with self.assertRaises(ts.BencodeError) as ctx: + ts.bdecode(data) + self.assertRegex(str(ctx.exception), r"index \d+") + + def test_bdecode_truncated_string_inside_container_raises(self): + with self.assertRaises(ts.BencodeError): + ts.bdecode(b"d4:info10:shorte") + + def test_bdecode_string_ending_exactly_at_end_of_data_is_accepted(self): + self.assertEqual(ts.bdecode(b"4:spam"), (b"spam", 6)) + def test_bencode_roundtrip(self): for value in (0, -5, 12345, b"", b"hello", [b"a", b"b"], {b"a": 1, b"b": b"x"}): encoded = ts.bencode(value) diff --git a/torrent_scrape.py b/torrent_scrape.py index 46ea5c5..b955579 100644 --- a/torrent_scrape.py +++ b/torrent_scrape.py @@ -15,6 +15,7 @@ import hashlib import ipaddress import os +import re import socket import struct from urllib.parse import quote_from_bytes, urlsplit, urlunsplit @@ -33,25 +34,44 @@ class UdpTrackerError(ValueError): failure and move on to the next tracker.""" +# Stricter than int(), which also accepts whitespace, '+' and underscores. +_BENCODE_INT = re.compile(rb"-?\d+") + def bdecode(data, index=0): """Decode a single bencoded value starting at index. Returns (value, next_index). Byte strings decode to bytes, dict keys must be byte strings (per spec) and dicts decode with keys as bytes.""" if index >= len(data): - raise BencodeError("Unexpected end of data") + raise BencodeError(f"Unexpected end of data at index {index}") prefix = data[index : index + 1] if prefix == b"i": - end = data.index(b"e", index) - return int(data[index + 1 : end]), end + 1 + end = data.find(b"e", index) + if end == -1: + raise BencodeError(f"Unterminated integer at index {index}") + digits = data[index + 1 : end] + if not _BENCODE_INT.fullmatch(digits): + raise BencodeError(f"Invalid integer {digits!r} at index {index}") + return int(digits), end + 1 if prefix.isdigit(): - colon = data.index(b":", index) - length = int(data[index:colon]) + colon = data.find(b":", index) + if colon == -1: + raise BencodeError(f"Byte string length without ':' at index {index}") + digits = data[index:colon] + if not digits.isdigit(): + raise BencodeError( + f"Invalid byte string length {digits!r} at index {index}" + ) start = colon + 1 - end = start + length + end = start + int(digits) + if end > len(data): + raise BencodeError( + f"Byte string at index {index} declares {int(digits)} bytes, " + f"only {len(data) - start} remain" + ) return data[start:end], end if prefix == b"l": From 336201e861301d4962426ec3b84995e26a237395 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 08:04:43 +0200 Subject: [PATCH 07/46] Kill remaining ratio-gate and bdecode survivors from mutmut re-run A full mutmut 2.5.1 re-run after tasks 10-13 left four killable survivors in the touched functions: - should_fetch_torrent: 'continue' -> 'break' and 'or' -> 'and' in the removed-history skip (a skipped legacy entry first, and no parse-error warning for entries without a ratio). - bdecode: '>=' -> '>' on the end-of-data guard and the remaining-bytes arithmetic in the truncation message. Co-Authored-By: Claude Opus 5.5 --- tests/test_fetch_torrents.py | 32 ++++++++++++++++++++++++++++---- tests/test_torrent_scrape.py | 12 ++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index 265782f..0fba246 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -33,10 +33,10 @@ if REPO_ROOT not in sys.path: sys.path.insert(0, REPO_ROOT) -import fetch_torrents as ft - import helpers +import fetch_torrents as ft + def setUpModule(): # Baseline for every test here: the developer's exported @@ -433,7 +433,29 @@ def test_removed_history_entry_without_ratio_is_ignored(self): "reason": "stagnant", }, } - self.assertTrue( + # Skipped up front, not by tripping over the missing key and logging + # a parse-error warning. + with self.assertNoLogs(ft.logger, level="WARNING"): + result = ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) + self.assertTrue(result) + + def test_skipped_removed_history_entry_does_not_hide_later_ones(self): + removed_history = { + "ubuntu-23.04-desktop-amd64.iso": { + "removed_date": "2025-06-01", + "reason": "stagnant", + }, + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.4, + }, + } + self.assertFalse( ft.should_fetch_torrent( "ubuntu-24.04-desktop-amd64.iso", ratios={}, @@ -698,7 +720,9 @@ def _write_log(self, *messages): "fetch_torrents_ratios.log", ) with open(path, "w", encoding="utf-8") as f: - f.writelines(f"2026-08-01 00:00:00,000 INFO: {message}\n" for message in messages) + f.writelines( + f"2026-08-01 00:00:00,000 INFO: {message}\n" for message in messages + ) return path def test_missing_file_returns_empty(self): diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index c0e8d21..5813f14 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -92,6 +92,18 @@ def test_bdecode_malformed_input_raises_bencode_error(self): ts.bdecode(data) self.assertRegex(str(ctx.exception), r"index \d+") + def test_bdecode_truncated_string_reports_declared_and_remaining_bytes(self): + with self.assertRaisesRegex( + ts.BencodeError, r"index 0 declares 4 bytes, only 2 remain" + ): + ts.bdecode(b"4:ab") + + def test_bdecode_unterminated_list_reports_end_of_data(self): + with self.assertRaisesRegex( + ts.BencodeError, r"Unexpected end of data at index 7" + ): + ts.bdecode(b"l4:spam") + def test_bdecode_truncated_string_inside_container_raises(self): with self.assertRaises(ts.BencodeError): ts.bdecode(b"d4:info10:shorte") From 10f9ce6db1040410ea6de0369b28b89761062722 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 08:18:46 +0200 Subject: [PATCH 08/46] Simplify get_previous_ratios segment selection The old loop kept an end_index that was reset on every START, which made the 'end_index > start_index' check and the initial 'end_index = None' redundant - mutmut reported both as surviving (equivalent) mutants. Selecting 'after the last START, up to the last END after it' directly removes them. A new test pins the existing 'last END wins' behavior before the rewrite; mutmut now kills 28/28 mutants in the function. Co-Authored-By: Claude Opus 5.5 --- fetch_torrents.py | 24 +++++++----------------- tests/test_fetch_torrents.py | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/fetch_torrents.py b/fetch_torrents.py index 8f1366f..6493bd0 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -11,11 +11,10 @@ from datetime import date import requests +import torrent_scrape from bs4 import BeautifulSoup from transmission_rpc import Client -import torrent_scrape - # Configure logging log_dir = os.getenv("FETCH_TORRENTS_LOG_DIR", "/logs") log_file = os.path.join(log_dir, "fetch_torrents.log") @@ -185,21 +184,12 @@ def get_previous_ratios(log_file): # Only parse the latest ratio segment between the last start and end markers. start_marker = "[ratio] RATIOS START" end_marker = "[ratio] RATIOS END" - start_index = None - end_index = None - - for idx, line in enumerate(lines): - if start_marker in line: - start_index = idx - end_index = None - elif start_index is not None and end_marker in line: - end_index = idx - - if start_index is not None: - if end_index is not None and end_index > start_index: - lines = lines[start_index + 1 : end_index] - else: - lines = lines[start_index + 1 :] + starts = [idx for idx, line in enumerate(lines) if start_marker in line] + if starts: + lines = lines[starts[-1] + 1 :] + ends = [idx for idx, line in enumerate(lines) if end_marker in line] + if ends: + lines = lines[: ends[-1]] for line in lines: match = re.search(r"\[ratio\]\s+(.+?)\s+→\s+(\d+\.\d+)", line) diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index 0fba246..41db1d7 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -778,6 +778,20 @@ def test_end_before_the_last_start_is_ignored(self): ) self.assertEqual(ft.get_previous_ratios(path), {"current.iso": 0.9}) + def test_last_end_after_the_last_start_closes_the_segment(self): + # Not written by a normal run; pins current behavior. + path = self._write_log( + "[ratio] RATIOS START", + "[ratio] first.iso → 0.100", + "[ratio] RATIOS END", + "[ratio] second.iso → 0.200", + "[ratio] RATIOS END", + "[ratio] after.iso → 0.300", + ) + self.assertEqual( + ft.get_previous_ratios(path), {"first.iso": 0.1, "second.iso": 0.2} + ) + def test_padded_names_are_stripped(self): path = self._write_log( "[ratio] archlinux-2026.04.01-x86_64.iso → 0.664", From 61622a002cd00140494b8565d11a638cc167c926 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 08:27:07 +0200 Subject: [PATCH 09/46] Document mutation testing and how to handle surviving mutants Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2023dc1..dbd1c0c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,57 @@ When making changes to the fetch logic or features: - Verify regex patterns match all intended torrent names (e.g., Ubuntu variants) - Ratio checking is per ISO type (e.g., `installer-amd64`) rather than per distro +## Mutation testing + +Mutation testing checks that the tests actually pin behavior, not just +execute lines. It uses mutmut 2.x (`mutmut<3`): + +``` +mutmut run --paths-to-mutate fetch_torrents.py,torrent_scrape.py,configure_transmission.py \ + --tests-dir tests/ --runner "python -m unittest discover -s tests -f -q" +mutmut results # list surviving mutants +mutmut show # show one mutant's diff +``` + +mutmut edits the source files in place while it runs, so run it on a clean +working tree (or a copy of the repo). A full run takes a while (~40 minutes on +Windows). + +### Handling surviving mutants + +A survivor is either a missing test or a mutant that cannot change behavior +(an *equivalent* mutant). Deal with them in this order: + +1. **Write the missing test.** Most survivors are real gaps: an untested + boundary (`>=` vs `>`), a `continue` that could be a `break`, a branch whose + result is never asserted. Confirm the new test fails with the mutant applied + (`mutmut apply `, run the test, then `git checkout` the file). +2. **If no test can kill it, remove the cause.** An equivalent mutant usually + points at redundant code: a variable initialised but never read, or a + condition that an earlier step already guarantees. Simplify the code so the + mutant can no longer be generated. Pin the current behavior with a test + before refactoring. +3. **If the code is already as simple as it should be, exclude the line** with + `# pragma: no mutate` and say why on the same line, so the exclusion can be + reviewed: + + ```python + if os.getenv("SKIP_RATIO_CHECK", "false").lower() == "true": # pragma: no mutate (any default but "true" is equivalent) + ``` + + Don't use the pragma to hide a gap you could test. + +**Log and error-message strings.** mutmut changes string literals +(`"Could not…"` → `"XXCould not…XX"`), and most of these survive because tests +don't assert exact log text. They're expected and are not worth a pragma on +every log line. Assert the message only where its content matters, for example +an error that names the bad input. + +**In CI**, don't gate on zero survivors. Use a score threshold for the critical +modules, a committed baseline of known survivors (fail only on new ones), or +mutate only the lines changed in the PR. Run it on a schedule rather than on +every push, given how long it takes. + ## Cutting a release Releases and container publishing are automated from Git tags: From 1e2550b0029b203e439428dffc2bd2b0a27e2d3a Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 12:22:31 +0200 Subject: [PATCH 10/46] Pin cleanup-planner boundaries and skip ordering (task 14) Edge-value tests for plan_cleanup (ratio == min_ratio, default floor), plan_stagnation_cleanup / _stagnation_anchor (anchor exactly window_days old, delta == min_ratio_delta, oldest qualifying sample, defaults), update_ratio_history (sample interval and prune horizon edges) and plan_disk_pressure_cleanup (usage == threshold, small positive threshold). Skipped torrents (Devuan, unknown distro, unparseable, no anchor) are put before valid candidates so continue->break mutants fail, and missing/None ratios and added_dates are asserted to count as zero/oldest. A focused mutmut run kills every mutant in these seven functions. Co-Authored-By: Claude Opus 5.5 --- tests/test_fetch_torrents.py | 342 +++++++++++++++++++++++++++++++++++ 1 file changed, 342 insertions(+) diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index 41db1d7..400a47b 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -205,6 +205,87 @@ def test_unparseable_name_for_matched_distro_is_skipped_not_raised(self): self.assertEqual(to_remove, []) self.assertEqual(to_keep_low_ratio, []) + def test_ratio_exactly_at_min_ratio_is_removed(self): + torrents = [ + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.0), + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) + + self.assertEqual(names_of(to_remove), ["ubuntu-23.10-desktop-amd64.iso"]) + self.assertEqual(to_keep_low_ratio, []) + + def test_ratio_just_below_min_ratio_is_kept(self): + older = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.999) + torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), older] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) + + self.assertEqual(to_remove, []) + self.assertEqual(to_keep_low_ratio, [(older, 0.999)]) + + def test_default_min_ratio_is_one(self): + older = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5) + torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), older] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) + + self.assertEqual(to_remove, [older]) + self.assertEqual(to_keep_low_ratio, []) + + def test_missing_or_none_ratio_counts_as_zero(self): + no_attr = types.SimpleNamespace(name="ubuntu-23.04-desktop-amd64.iso", id=2) + none_ratio = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 3, ratio=None) + torrents = [ + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + no_attr, + none_ratio, + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) + + self.assertEqual(to_remove, []) + self.assertEqual( + sorted(to_keep_low_ratio, key=lambda entry: entry[0].name), + [(no_attr, 0.0), (none_ratio, 0.0)], + ) + + def test_skipped_torrents_listed_first_do_not_stop_planning(self): + # Each kind of torrent _old_version_candidates() skips (or routes + # elsewhere) is placed *before* a valid pair, so a 'continue' that + # became 'break' would drop the pair. + leading = { + "devuan": FakeTorrent("devuan_excalibur", 10, ratio=2.0, added_date=1), + "unknown distro": FakeTorrent("slackware-15.0-install-dvd.iso", 11), + "unparseable": FakeTorrent("ubuntu-noble-desktop-amd64.iso", 12), + } + for kind, first in leading.items(): + with self.subTest(kind): + older = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0) + torrents = [ + first, + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + older, + ] + + with self.assertNoLogs(ft.logger, level="ERROR"): + to_remove, _ = ft.plan_cleanup(torrents) + + self.assertEqual(to_remove, [older]) + + def test_unparseable_name_is_skipped_with_a_warning(self): + torrents = [ + FakeTorrent("ubuntu-noble-desktop-amd64.iso", 1, ratio=2.0), + FakeTorrent("ubuntu-mantic-desktop-amd64.iso", 2, ratio=2.0), + ] + + with self.assertLogs(ft.logger, level="WARNING") as logs: + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) + + self.assertEqual((to_remove, to_keep_low_ratio), ([], [])) + self.assertEqual(len(logs.records), 2) + class PlanCleanupDevuanGroupingTests(unittest.TestCase): """Devuan releases (e.g. "devuan_excalibur") carry no dotted version to @@ -272,6 +353,40 @@ def test_missing_added_date_does_not_crash(self): self.assertEqual(len(to_remove), 1) + def test_release_without_added_date_sorts_as_oldest(self): + undated = FakeTorrent("devuan_daedalus", 1, ratio=2.0, added_date=None) + torrents = [ + undated, + FakeTorrent("devuan_excalibur", 2, ratio=2.0, added_date=0.5), + ] + + to_remove, _ = ft.plan_cleanup(torrents) + + self.assertEqual(to_remove, [undated]) + + def test_datetime_added_dates_are_ordered_by_timestamp(self): + from datetime import datetime, timezone + + older = FakeTorrent( + "devuan_daedalus", + 1, + ratio=2.0, + added_date=datetime(2026, 1, 1, tzinfo=timezone.utc), + ) + torrents = [ + FakeTorrent( + "devuan_excalibur", + 2, + ratio=2.0, + added_date=datetime(2026, 6, 1, tzinfo=timezone.utc), + ), + older, + ] + + to_remove, _ = ft.plan_cleanup(torrents) + + self.assertEqual(to_remove, [older]) + class ShouldFetchTorrentRatioKeyTests(unittest.TestCase): """Candidate names must key into the same ratio-lookup bucket as the names Transmission @@ -1416,6 +1531,59 @@ def test_torrent_no_longer_seeding_is_dropped(self): self.assertEqual(updated, {}) + def _days_ago(self, today, days): + return (today - timedelta(days=days)).isoformat() + + def test_sample_exactly_one_interval_old_triggers_a_new_sample(self): + today = date(2026, 8, 29) + history = {"a.iso": [{"date": self._days_ago(today, 7), "ratio": 1.0}]} + + updated = ft.update_ratio_history( + history, {"a.iso": 1.2}, today, sample_interval_days=7, window_days=30 + ) + + self.assertEqual( + updated["a.iso"], + [ + {"date": self._days_ago(today, 7), "ratio": 1.0}, + {"date": today.isoformat(), "ratio": 1.2}, + ], + ) + + def test_sample_just_under_one_interval_old_adds_nothing(self): + today = date(2026, 8, 29) + history = {"a.iso": [{"date": self._days_ago(today, 6), "ratio": 1.0}]} + + updated = ft.update_ratio_history( + history, {"a.iso": 1.2}, today, sample_interval_days=7, window_days=30 + ) + + self.assertEqual( + updated["a.iso"], [{"date": self._days_ago(today, 6), "ratio": 1.0}] + ) + + def test_prune_horizon_defaults_to_window_plus_interval(self): + # Defaults: window 30 + interval 7 = 37 days. A sample exactly 37 + # days old is kept; 38 days old is pruned. + today = date(2026, 8, 29) + history = { + "a.iso": [ + {"date": self._days_ago(today, 38), "ratio": 0.5}, + {"date": self._days_ago(today, 37), "ratio": 0.6}, + {"date": self._days_ago(today, 1), "ratio": 0.9}, + ] + } + + updated = ft.update_ratio_history(history, {"a.iso": 1.0}, today) + + self.assertEqual( + updated["a.iso"], + [ + {"date": self._days_ago(today, 37), "ratio": 0.6}, + {"date": self._days_ago(today, 1), "ratio": 0.9}, + ], + ) + class PlanStagnationCleanupTests(unittest.TestCase): """plan_stagnation_cleanup() removes only superseded (non-latest) @@ -1523,6 +1691,137 @@ def test_unrelated_single_torrent_groups_are_ignored(self): self.assertEqual(to_remove, []) self.assertEqual(to_keep, []) + def _pair(self, older_ratio): + older = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=older_ratio) + return older, [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), older] + + def _history(self, today, *samples): + return { + "ubuntu-23.10-desktop-amd64.iso": [ + {"date": (today - timedelta(days=days)).isoformat(), "ratio": ratio} + for days, ratio in samples + ] + } + + def test_anchor_exactly_window_days_old_is_used(self): + today = date(2026, 8, 29) + older, torrents = self._pair(1.0) + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, self._history(today, (30, 1.0)), today, 30, 0.02 + ) + + self.assertEqual((to_remove, to_keep), ([older], [])) + + def test_anchor_one_day_short_of_window_is_too_new_to_judge(self): + today = date(2026, 8, 29) + older, torrents = self._pair(1.0) + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, self._history(today, (29, 1.0)), today, 30, 0.02 + ) + + self.assertEqual((to_remove, to_keep), ([], [(older, None)])) + + def test_oldest_qualifying_sample_is_the_anchor(self): + # Listed newest-first, so picking the first old-enough sample + # instead of the oldest would give a different delta. + today = date(2026, 8, 29) + older, torrents = self._pair(1.0) + history = self._history(today, (35, 0.99), (45, 0.5)) + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, 30, 0.02 + ) + + self.assertEqual((to_remove, to_keep), ([], [(older, 0.5)])) + + def test_delta_exactly_at_min_ratio_delta_is_kept(self): + today = date(2026, 8, 29) + older, torrents = self._pair(1.5) + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, self._history(today, (40, 1.0)), today, 30, 0.5 + ) + + self.assertEqual((to_remove, to_keep), ([], [(older, 0.5)])) + + def test_delta_just_below_min_ratio_delta_is_removed(self): + today = date(2026, 8, 29) + older, torrents = self._pair(1.25) + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, self._history(today, (40, 1.0)), today, 30, 0.5 + ) + + self.assertEqual((to_remove, to_keep), ([older], [])) + + def test_defaults_are_a_30_day_window_and_0_02_delta(self): + today = date(2026, 8, 29) + growing, torrents = self._pair(1.5) + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, self._history(today, (30, 1.0)), today + ) + + self.assertEqual((to_remove, to_keep), ([], [(growing, 0.5)])) + + stagnant, torrents = self._pair(1.01) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, self._history(today, (30, 1.0)), today + ) + + self.assertEqual((to_remove, to_keep), ([stagnant], [])) + + def test_missing_or_none_ratio_counts_as_zero(self): + today = date(2026, 8, 29) + torrents = [ + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), + types.SimpleNamespace(name="ubuntu-23.04-desktop-amd64.iso", id=2), + FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 3, ratio=None), + ] + history = { + name: [{"date": (today - timedelta(days=40)).isoformat(), "ratio": -0.5}] + for name in ( + "ubuntu-23.04-desktop-amd64.iso", + "ubuntu-23.10-desktop-amd64.iso", + ) + } + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, 30, 0.02 + ) + + self.assertEqual(to_remove, []) + self.assertEqual( + sorted((t.name, delta) for t, delta in to_keep), + [ + ("ubuntu-23.04-desktop-amd64.iso", 0.5), + ("ubuntu-23.10-desktop-amd64.iso", 0.5), + ], + ) + + def test_candidate_without_anchor_does_not_stop_later_ones(self): + today = date(2026, 8, 29) + no_history = FakeTorrent("ubuntu-23.04-desktop-amd64.iso", 3, ratio=1.0) + stagnant = FakeTorrent("ubuntu-22.10-desktop-amd64.iso", 4, ratio=1.0) + torrents = [ + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), + no_history, + stagnant, + ] + history = { + "ubuntu-22.10-desktop-amd64.iso": [ + {"date": (today - timedelta(days=40)).isoformat(), "ratio": 1.0} + ] + } + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, 30, 0.02 + ) + + self.assertEqual((to_remove, to_keep), ([stagnant], [(no_history, None)])) + class RatioHistoryPersistenceTests(unittest.TestCase): """load_ratio_history()/save_ratio_history() must round-trip cleanly and @@ -2707,6 +3006,49 @@ def test_no_superseded_candidates_returns_empty(self): [], ) + def test_any_positive_threshold_is_enabled(self): + older = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1) + torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), older] + + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=50.0, threshold_percent=1 + ), + [older], + ) + + def test_usage_exactly_at_threshold_triggers_cleanup(self): + older = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1) + torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), older] + + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=95.0, threshold_percent=95.0 + ), + [older], + ) + + def test_missing_or_none_ratio_sorts_as_zero(self): + half = FakeTorrent("ubuntu-22.10-desktop-amd64.iso", 2, ratio=0.5) + no_attr = types.SimpleNamespace(name="ubuntu-23.04-desktop-amd64.iso", id=3) + none_ratio = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 4, ratio=None) + torrents = [ + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), + half, + no_attr, + none_ratio, + ] + + planned = ft.plan_disk_pressure_cleanup( + torrents, used_percent=99.0, threshold_percent=95.0 + ) + + self.assertEqual(planned[-1], half) + self.assertEqual( + sorted(t.name for t in planned[:2]), + ["ubuntu-23.04-desktop-amd64.iso", "ubuntu-23.10-desktop-amd64.iso"], + ) + class EnforceDiskUsageLimitTests(unittest.TestCase): """enforce_disk_usage_limit() wires plan_disk_pressure_cleanup() to a From 9e6d1cd2e75274eab562b70fc8ab8e81889043e1 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 12:24:46 +0200 Subject: [PATCH 11/46] Gate CI on ruff, ruff format and mypy (task 15) - pyproject.toml holds ruff and mypy settings so local runs and CI agree. BLE001 is ignored project-wide (broad except-and-log is the run's error-handling design); transmission_rpc gets a scoped ignore_missing_imports. - Real findings fixed: an unused variable in fetch_ubuntu_lts, the NamedTemporaryFile in evaluate_old_release_demand now uses a context manager (and also cleans up if the write fails), and the RPC payload is typed so requests' json= accepts it. - Test-side findings fixed (explicit check=False, HTTPError instead of a bare Exception, instance attribute instead of a mutable class default, unused unpacked values, combined with-statements). - ruff and mypy are pinned in requirements-dev.txt; the CI test job installs it and runs the three checks before the tests. Co-Authored-By: Claude Opus 5.5 --- .claude/CLAUDE.md | 3 ++- .claude/skills/pr/SKILL.md | 2 +- .github/workflows/docker-image.yml | 13 +++++++++++-- fetch_torrents.py | 21 +++++++++++++-------- pyproject.toml | 25 +++++++++++++++++++++++++ requirements-dev.txt | 5 +++-- tests/test_configure_transmission.py | 3 ++- tests/test_fetch_torrents.py | 19 +++++++++++-------- tests/test_torrent_scrape.py | 14 ++++++++------ torrent_scrape.py | 4 ++-- 10 files changed, 78 insertions(+), 31 deletions(-) create mode 100644 pyproject.toml diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index bb10fe7..09f4146 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -10,7 +10,8 @@ Testing rules and the import-stubbing pattern live in the root [CLAUDE.md](../CL ## Commands - Test: `python -m unittest discover -s tests -v` (Windows: `py -m unittest discover -s tests -v`) - Lint/format: `ruff check .` / `ruff format .` -- Typecheck: `mypy fetch_torrents.py torrent_scrape.py configure_transmission.py` +- Typecheck: `mypy` (files and settings in `pyproject.toml`) +- CI's `test` job runs ruff check, ruff format --check and mypy before the tests; versions are pinned in `requirements-dev.txt` - Mutation: `mutmut run` (mutmut<3) - Dev deps: `pip install -r requirements-dev.txt` (includes `requirements-test.txt`) diff --git a/.claude/skills/pr/SKILL.md b/.claude/skills/pr/SKILL.md index 852d97f..5a3a618 100644 --- a/.claude/skills/pr/SKILL.md +++ b/.claude/skills/pr/SKILL.md @@ -9,6 +9,6 @@ description: Verify and open a pull request for linux-iso-seeder — runs lint, 2. Run, in order, and stop on the first failure (report its output; do not open the PR): - `ruff check .` - `ruff format --check .` - - `mypy fetch_torrents.py torrent_scrape.py configure_transmission.py` — advisory; report findings but do not block. + - `mypy` (files and settings come from `pyproject.toml`) - `python -m unittest discover -s tests -v` (Windows: `py -m unittest discover -s tests -v`) — must pass; CI gates the Docker build on this. 3. Push the branch and run `gh pr create --base main` with a summary of the change and the test result. diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 3edf80f..fc79ae3 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -39,8 +39,17 @@ jobs: with: python-version: "3.x" - - name: Install test dependencies - run: pip install -r requirements-test.txt + - name: Install dev dependencies + run: pip install -r requirements-dev.txt + + - name: Lint + run: ruff check . + + - name: Check formatting + run: ruff format --check . + + - name: Type-check + run: mypy - name: Run tests run: python -m unittest discover -s tests -v diff --git a/fetch_torrents.py b/fetch_torrents.py index 6493bd0..73aba18 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -9,12 +9,14 @@ import tempfile import time from datetime import date +from typing import Any import requests -import torrent_scrape from bs4 import BeautifulSoup from transmission_rpc import Client +import torrent_scrape + # Configure logging log_dir = os.getenv("FETCH_TORRENTS_LOG_DIR", "/logs") log_file = os.path.join(log_dir, "fetch_torrents.log") @@ -522,7 +524,6 @@ def download_torrent(name, url): def fetch_ubuntu_lts(): - url = "https://releases.ubuntu.com/" try: text = requests.get( "https://changelogs.ubuntu.com/meta-release-lts", timeout=30 @@ -889,16 +890,20 @@ def evaluate_old_release_demand( return False, None, None torrent_bytes = r.content - tmp = tempfile.NamedTemporaryFile(suffix=".torrent", delete=False) + tmp_path = None try: - tmp.write(torrent_bytes) - tmp.close() - scrape_result = torrent_scrape.scrape_torrent(tmp.name, timeout=timeout) + # Closed before scraping: scrape_torrent() reopens it by name, which + # Windows refuses while the file is still open. + with tempfile.NamedTemporaryFile(suffix=".torrent", delete=False) as tmp: + tmp_path = tmp.name + tmp.write(torrent_bytes) + scrape_result = torrent_scrape.scrape_torrent(tmp_path, timeout=timeout) except Exception as exc: logger.warning("Could not scrape %s: %s", name, exc) scrape_result = None finally: - os.remove(tmp.name) + if tmp_path is not None: + os.remove(tmp_path) keep = has_unmet_demand( scrape_result, min_leechers, min_leecher_ratio, allow_zero_seeders @@ -1135,7 +1140,7 @@ def log_seed_ratios_via_http( logger.info("Querying Transmission RPC for seed ratios...") r = requests.post(rpc_url, timeout=15) headers = {"X-Transmission-Session-Id": r.headers["X-Transmission-Session-Id"]} - payload = { + payload: dict[str, Any] = { "method": "torrent-get", "arguments": {"fields": ["name", "uploadRatio"]}, } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..016276a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +# Tool configuration only - this project isn't packaged. The container runs +# the scripts directly with Alpine's python3 (see Dockerfile). + +[tool.ruff] +# Alpine's python3 at the time of writing; also what CI's "3.x" must stay +# compatible with. +target-version = "py312" + +[tool.ruff.lint] +ignore = [ + # Broad `except Exception` is the error-handling design here: each + # distro fetcher, cleanup step and tracker scrape must log and carry on + # rather than abort the whole daily run (see the __main__ block). + "BLE001", +] + +[tool.mypy] +python_version = "3.12" +files = ["fetch_torrents.py", "torrent_scrape.py", "configure_transmission.py"] + +[[tool.mypy.overrides]] +# Installed from Alpine's py3-transmission-rpc in the container; not a test +# dependency, so there's nothing for mypy to read locally or in CI. +module = "transmission_rpc" +ignore_missing_imports = true diff --git a/requirements-dev.txt b/requirements-dev.txt index e2c0841..59905e2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ -r requirements-test.txt -ruff -mypy +# Pinned so new upstream lint/type rules can't break CI unannounced. +ruff==0.17.1 +mypy==2.4.0 diff --git a/tests/test_configure_transmission.py b/tests/test_configure_transmission.py index 75a0e6f..913a4a9 100644 --- a/tests/test_configure_transmission.py +++ b/tests/test_configure_transmission.py @@ -11,9 +11,10 @@ if REPO_ROOT not in sys.path: sys.path.insert(0, REPO_ROOT) -import configure_transmission as ct import helpers +import configure_transmission as ct + class BuildRpcOverridesTests(unittest.TestCase): def test_no_relevant_env_vars_yields_no_overrides(self): diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index 400a47b..54c9d00 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -18,7 +18,7 @@ import types import unittest import unittest.mock -from datetime import date, timedelta +from datetime import UTC, date, timedelta from unittest.mock import MagicMock if "transmission_rpc" not in sys.modules: @@ -365,20 +365,20 @@ def test_release_without_added_date_sorts_as_oldest(self): self.assertEqual(to_remove, [undated]) def test_datetime_added_dates_are_ordered_by_timestamp(self): - from datetime import datetime, timezone + from datetime import datetime older = FakeTorrent( "devuan_daedalus", 1, ratio=2.0, - added_date=datetime(2026, 1, 1, tzinfo=timezone.utc), + added_date=datetime(2026, 1, 1, tzinfo=UTC), ) torrents = [ FakeTorrent( "devuan_excalibur", 2, ratio=2.0, - added_date=datetime(2026, 6, 1, tzinfo=timezone.utc), + added_date=datetime(2026, 6, 1, tzinfo=UTC), ), older, ] @@ -807,6 +807,7 @@ def test_previous_ratios_are_readable_after_a_fresh_start(self): env = dict(os.environ, FETCH_TORRENTS_LOG_DIR=log_dir) result = subprocess.run( [sys.executable, "-c", script], + check=False, env=env, capture_output=True, text=True, @@ -960,6 +961,7 @@ def _run(self, extra_env=None): env = dict(os.environ, FETCH_TORRENTS_LOG_DIR=log_dir, **(extra_env or {})) result = subprocess.run( [sys.executable, "-c", script], + check=False, env=env, capture_output=True, text=True, @@ -1353,7 +1355,7 @@ class FakeResponse: status_code = 500 def raise_for_status(self): - raise Exception("500 Server Error") + raise ft.requests.HTTPError("500 Server Error") with unittest.mock.patch.object( ft.requests, "get", return_value=FakeResponse() @@ -2215,7 +2217,7 @@ def raise_for_status(self): }, ), ): - keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( + keep, _torrent_bytes, scrape_result = ft.evaluate_old_release_demand( "ubuntu-14.04.6-desktop-amd64.iso", "https://example.invalid/x.torrent", min_leechers=1, @@ -2582,7 +2584,7 @@ def test_previously_removed_candidate_is_skipped_without_network_call(self): ft, "evaluate_old_release_demand" ) as mock_evaluate, ): - added, skipped, new_state = ft.check_old_releases_for_demand( + added, skipped, _new_state = ft.check_old_releases_for_demand( ["kali"], include_low_demand=False, min_leechers=1, @@ -2751,7 +2753,8 @@ class LogSeedRatiosViaHttpTests(unittest.TestCase): the caller doesn't explicitly override auth.""" class FakeSessionIdResponse: - headers = {"X-Transmission-Session-Id": "abc"} + def __init__(self): + self.headers = {"X-Transmission-Session-Id": "abc"} class FakeTorrentGetResponse: def raise_for_status(self): diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index 5813f14..1f3a3c6 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -42,7 +42,7 @@ def test_bdecode_integer(self): self.assertEqual(index, 7) def test_bdecode_negative_integer(self): - value, index = ts.bdecode(b"i-42e") + value, _index = ts.bdecode(b"i-42e") self.assertEqual(value, -42) def test_bdecode_bytestring(self): @@ -56,11 +56,11 @@ def test_bdecode_empty_bytestring(self): self.assertEqual(index, 2) def test_bdecode_list(self): - value, index = ts.bdecode(b"l4:spam4:eggse") + value, _index = ts.bdecode(b"l4:spam4:eggse") self.assertEqual(value, [b"spam", b"eggs"]) def test_bdecode_dict(self): - value, index = ts.bdecode(b"d3:cow3:moo4:spam4:eggse") + value, _index = ts.bdecode(b"d3:cow3:moo4:spam4:eggse") self.assertEqual(value, {b"cow": b"moo", b"spam": b"eggs"}) def test_bdecode_nested(self): @@ -451,9 +451,11 @@ def test_socket_timeout_propagates(self): fake_sock = unittest.mock.Mock() fake_sock.recvfrom.side_effect = TimeoutError("timed out") - with unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock): - with self.assertRaises(socket.timeout): - ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) + with ( + unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), + self.assertRaises(socket.timeout), + ): + ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) fake_sock.close.assert_called_once() diff --git a/torrent_scrape.py b/torrent_scrape.py index b955579..226d98a 100644 --- a/torrent_scrape.py +++ b/torrent_scrape.py @@ -364,7 +364,7 @@ def scrape_torrent(torrent_path, timeout=15): continue try: stats = scrape_udp_tracker(host, port, info_hash, timeout=timeout) - except Exception: + except Exception: # noqa: S112 - any failing tracker: try the next one continue return {"tracker": f"udp://{host}:{port}", **stats} @@ -379,7 +379,7 @@ def scrape_torrent(torrent_path, timeout=15): response = requests.get(request_url, timeout=timeout) response.raise_for_status() stats = parse_scrape_response(response.content, info_hash) - except Exception: + except Exception: # noqa: S112 - any failing tracker: try the next one continue if stats is not None: From ce7787a38045dbe550818363c4cc5fca7f353243 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 12:26:35 +0200 Subject: [PATCH 12/46] Add coverage and mutmut tooling (task 16) - requirements-dev.txt pins coverage and mutmut 2.5.1. - pyproject.toml configures branch coverage over the three modules and mutmut's paths/runner, so 'coverage run' and 'mutmut run' need no flags. - .gitignore covers mutmut's cache and output. - CI runs the suite under coverage and prints the report (no threshold yet; task 27 sets one). Co-Authored-By: Claude Opus 5.5 --- .claude/CLAUDE.md | 3 ++- .github/workflows/docker-image.yml | 7 ++++++- .gitignore | 4 ++++ CONTRIBUTING.md | 7 ++++--- pyproject.toml | 15 +++++++++++++++ requirements-dev.txt | 2 ++ 6 files changed, 33 insertions(+), 5 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 09f4146..987d9ff 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -12,7 +12,8 @@ Testing rules and the import-stubbing pattern live in the root [CLAUDE.md](../CL - Lint/format: `ruff check .` / `ruff format .` - Typecheck: `mypy` (files and settings in `pyproject.toml`) - CI's `test` job runs ruff check, ruff format --check and mypy before the tests; versions are pinned in `requirements-dev.txt` -- Mutation: `mutmut run` (mutmut<3) +- Coverage: `coverage run -m unittest discover -s tests` then `coverage report` (branch coverage; config in `pyproject.toml`) +- Mutation: `mutmut run` (mutmut<3; config in `pyproject.toml`, see CONTRIBUTING.md) - Dev deps: `pip install -r requirements-dev.txt` (includes `requirements-test.txt`) ## Active agent templates diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index fc79ae3..a2f7ca8 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -52,7 +52,12 @@ jobs: run: mypy - name: Run tests - run: python -m unittest discover -s tests -v + run: coverage run -m unittest discover -s tests -v + + # Report only for now; settings (branch coverage, which modules) are in + # pyproject.toml. + - name: Coverage report + run: coverage report # Same suite again with env vars exported that would flip outcomes if any # test leaked the developer's shell environment into production code. diff --git a/.gitignore b/.gitignore index b55e166..e6e4ae8 100644 --- a/.gitignore +++ b/.gitignore @@ -219,3 +219,7 @@ plans/ # CodeGraph local index (machine-local) .codegraph/ + +# Mutation testing (mutmut 2.x) +.mutmut-cache +mutants/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbd1c0c..6dba329 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,11 +25,12 @@ When making changes to the fetch logic or features: ## Mutation testing Mutation testing checks that the tests actually pin behavior, not just -execute lines. It uses mutmut 2.x (`mutmut<3`): +execute lines. It uses mutmut 2.x (pinned in `requirements-dev.txt`), configured +under `[tool.mutmut]` in `pyproject.toml`: ``` -mutmut run --paths-to-mutate fetch_torrents.py,torrent_scrape.py,configure_transmission.py \ - --tests-dir tests/ --runner "python -m unittest discover -s tests -f -q" +pip install -r requirements-dev.txt +mutmut run # mutate all three modules, run the suite per mutant mutmut results # list surviving mutants mutmut show # show one mutant's diff ``` diff --git a/pyproject.toml b/pyproject.toml index 016276a..12d9b9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,3 +23,18 @@ files = ["fetch_torrents.py", "torrent_scrape.py", "configure_transmission.py"] # dependency, so there's nothing for mypy to read locally or in CI. module = "transmission_rpc" ignore_missing_imports = true + +[tool.coverage.run] +branch = true +source_pkgs = ["fetch_torrents", "torrent_scrape", "configure_transmission"] +omit = ["tests/*"] + +[tool.coverage.report] +show_missing = true + +[tool.mutmut] +# mutmut<3 (see requirements-dev.txt). It edits these files in place while +# it runs, so run it on a clean working tree. See CONTRIBUTING.md. +paths_to_mutate = "fetch_torrents.py,torrent_scrape.py,configure_transmission.py" +tests_dir = "tests/" +runner = "python -m unittest discover -s tests -f -q" diff --git a/requirements-dev.txt b/requirements-dev.txt index 59905e2..9b55b0e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,5 @@ # Pinned so new upstream lint/type rules can't break CI unannounced. ruff==0.17.1 mypy==2.4.0 +coverage==7.16.2 +mutmut==2.5.1 From 07cbe1652cfea043e2a7842eeb9800beb83bf525 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 12:30:16 +0200 Subject: [PATCH 13/46] Test configure_transmission.main end to end (task 25) Covers first run with credentials (exact file contents, port forwarding defaulted off), existing files keeping unrelated keys and their port forwarding setting, no-override runs leaving the file untouched, the open-whitelist warning on stderr, invalid peer limits being dropped silently (current behavior), corrupt files starting fresh, the default /config path, and running the module as a script. configure_transmission.py is now at 100% line and branch coverage. Co-Authored-By: Claude Opus 5.5 --- tests/test_configure_transmission.py | 132 +++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/tests/test_configure_transmission.py b/tests/test_configure_transmission.py index 913a4a9..9207246 100644 --- a/tests/test_configure_transmission.py +++ b/tests/test_configure_transmission.py @@ -3,9 +3,14 @@ Run with: python -m unittest discover -s tests """ +import contextlib +import io +import json import os +import runpy import sys import unittest +import unittest.mock REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if REPO_ROOT not in sys.path: @@ -253,5 +258,132 @@ def test_corrupt_file_returns_empty_settings_instead_of_raising(self): self.assertEqual(ct.load_settings(path), {}) +class MainTests(unittest.TestCase): + """main() runs at every container start, before the daemon: env vars in, + settings.json out.""" + + def setUp(self): + self.path = os.path.join( + helpers.make_temp_dir(self, "configure_transmission_main_"), + "settings.json", + ) + + def _run(self, **env): + stderr = io.StringIO() + with ( + unittest.mock.patch.dict( + os.environ, {"TRANSMISSION_SETTINGS_FILE": self.path, **env}, clear=True + ), + contextlib.redirect_stderr(stderr), + ): + ct.main() + return stderr.getvalue() + + def _write_settings(self, settings): + with open(self.path, "w", encoding="utf-8") as f: + json.dump(settings, f) + + def _read_settings(self): + with open(self.path, encoding="utf-8") as f: + return json.load(f) + + def test_first_run_with_credentials_writes_exactly_the_overrides(self): + stderr = self._run( + TRANSMISSION_RPC_USERNAME="alice", TRANSMISSION_RPC_PASSWORD="hunter2" + ) + + self.assertEqual( + self._read_settings(), + { + "rpc-authentication-required": True, + "rpc-username": "alice", + "rpc-password": "hunter2", + "port-forwarding-enabled": False, + }, + ) + self.assertEqual(stderr, "") + + def test_existing_file_keeps_unrelated_keys_and_port_forwarding(self): + self._write_settings( + {"download-dir": "/downloads", "port-forwarding-enabled": True} + ) + + self._run(TRANSMISSION_PEER_LIMIT_GLOBAL="500") + + self.assertEqual( + self._read_settings(), + { + "download-dir": "/downloads", + "port-forwarding-enabled": True, + "peer-limit-global": 500, + }, + ) + + def test_existing_file_and_no_overrides_is_left_untouched(self): + self._write_settings({"download-dir": "/downloads"}) + with open(self.path, encoding="utf-8") as f: + before = f.read() + + self._run() + + with open(self.path, encoding="utf-8") as f: + self.assertEqual(f.read(), before) + + def test_whitelist_without_credentials_warns_on_stderr(self): + stderr = self._run(TRANSMISSION_RPC_WHITELIST="10.0.0.*") + + self.assertTrue(stderr.startswith("WARNING: TRANSMISSION_RPC_WHITELIST is set")) + self.assertEqual(self._read_settings()["rpc-whitelist"], "10.0.0.*") + + def test_invalid_peer_limits_are_dropped_silently(self): + # Pins current behavior; whether these should warn is a product call. + for value in ("abc", "0"): + with self.subTest(value=value): + if os.path.exists(self.path): + os.remove(self.path) + + stderr = self._run(TRANSMISSION_PEER_LIMIT_GLOBAL=value) + + self.assertEqual( + self._read_settings(), {"port-forwarding-enabled": False} + ) + self.assertEqual(stderr, "") + + def test_corrupt_settings_file_starts_fresh(self): + with open(self.path, "w", encoding="utf-8") as f: + f.write("{not valid json") + + stderr = self._run(TRANSMISSION_RPC_WHITELIST="10.0.0.*") + + self.assertIn("starting fresh", stderr) + self.assertEqual( + self._read_settings(), + {"rpc-whitelist-enabled": True, "rpc-whitelist": "10.0.0.*"}, + ) + + def test_running_the_script_calls_main(self): + # entrypoint.sh runs it as a script, not as an import. + with ( + unittest.mock.patch.dict( + os.environ, {"TRANSMISSION_SETTINGS_FILE": self.path}, clear=True + ), + ): + runpy.run_path(ct.__file__, run_name="__main__") + + self.assertEqual(self._read_settings(), {"port-forwarding-enabled": False}) + + def test_defaults_to_config_settings_path(self): + with ( + unittest.mock.patch.dict(os.environ, {}, clear=True), + unittest.mock.patch.object(ct, "save_settings") as save, + unittest.mock.patch.object(ct, "load_settings", return_value={}), + ): + ct.main() + + save.assert_called_once_with( + "/config/settings.json", {"port-forwarding-enabled": False} + ) + + if __name__ == "__main__": unittest.main() From ce7a90b8d37c8cd519c2ccdeb8fef4fdf581b774 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 12:33:43 +0200 Subject: [PATCH 14/46] Characterize the daily run via runpy (task 17) tests/test_daily_run.py executes fetch_torrents.py as __main__ with HTTP, the Transmission client, the RPC port check, disk usage and the clock faked at their boundaries, and state/log files in a temp dir. It pins: per-outcome download counting and the summary line, distro selection, the ratio gate fed from the previous run's ratio log (blocked below 1.0, allowed at 1.0), one failing distro not stopping the others, which cleanup runs for CLEANUP_KEEP_ONLY_LATEST_VERSION, the disk-usage limit running after cleanup and its errors being logged, and seed ratios being written for the next run. No production code changed; /watch is redirected in the test because watch_dir is fixed at import time. Co-Authored-By: Claude Opus 5.5 --- tests/test_daily_run.py | 387 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 387 insertions(+) create mode 100644 tests/test_daily_run.py diff --git a/tests/test_daily_run.py b/tests/test_daily_run.py new file mode 100644 index 0000000..d63a763 --- /dev/null +++ b/tests/test_daily_run.py @@ -0,0 +1,387 @@ +"""Characterization tests for fetch_torrents.py's `__main__` block - the +whole daily run - executed via runpy exactly as the container runs it. + +Everything external is faked at its boundary: HTTP (requests.get/post), +the Transmission client (the transmission_rpc stub), the RPC port check +(socket.create_connection), disk usage and the clock. Log/state files live +in a per-test temp dir via FETCH_TORRENTS_LOG_DIR. + +`watch_dir = "/watch"` is assigned at import time, and runpy executes the +module in a fresh namespace, so it can't be patched like the other module +globals. _redirect_watch_dir() reroutes file access under /watch to a temp +dir instead. (That friction is what tasks/18 removes by extracting run().) + +Run with: python -m unittest discover -s tests +""" + +import builtins +import contextlib +import io +import logging +import os +import runpy +import shutil +import socket +import sys +import time +import types +import unittest +import unittest.mock +from unittest.mock import MagicMock + +if "transmission_rpc" not in sys.modules: + stub = types.ModuleType("transmission_rpc") + stub.Client = MagicMock() + sys.modules["transmission_rpc"] = stub + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +SCRIPT = os.path.join(REPO_ROOT, "fetch_torrents.py") +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + +import helpers + +UBUNTU_META = ( + "Dist: jammy\nName: Jammy Jellyfish\nVersion: 22.04 LTS\nSupported: 0\n\n" + "Dist: noble\nName: Noble Numbat\nVersion: 24.04 LTS\nSupported: 1\n" +) +UBUNTU = "https://releases.ubuntu.com/noble/" +LUBUNTU = "https://cdimage.ubuntu.com/lubuntu/releases/noble/release/" +XUBUNTU = "https://torrent.ubuntu.com/xubuntu/releases/noble/release/" +ARCH_HTML = ( + "" + "" + "" + "
t
" +) +ARCH_TORRENT = "https://archlinux.org/releng/releases/2026.09.01/torrent/" + + +class FakeResponse: + def __init__(self, status_code=200, text="", content=b"", headers=None, body=None): + self.status_code = status_code + self.text = text + self.content = content + self.headers = headers or {} + self._body = body + + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError(f"HTTP {self.status_code}") + + def json(self): + return self._body + + +class FakeTransmission: + """In-memory Transmission: get_torrents() returns what's held, + remove_torrent() drops it and records the call in `events`.""" + + def __init__(self, torrents, events): + self.torrents = list(torrents) + self.events = events + + def get_torrents(self): + self.events.append("get_torrents") + return list(self.torrents) + + def remove_torrent(self, torrent_id, delete_data=False): + self.events.append(("remove", torrent_id)) + self.torrents = [t for t in self.torrents if t.id != torrent_id] + + +def torrent(name, id_, ratio): + return types.SimpleNamespace(name=name, id=id_, ratio=ratio, added_date=None) + + +class DailyRunCharacterizationTests(unittest.TestCase): + def setUp(self): + self.log_dir = helpers.make_temp_dir(self, "daily_run_logs_") + self.watch_dir = helpers.make_temp_dir(self, "daily_run_watch_") + self.requested = [] + self.events = [] + self.routes = {} + self.disk_used_percent = 10 + self.transmission = FakeTransmission([], self.events) + + # -- fakes -------------------------------------------------------------- + + def _get(self, url, timeout=None): + self.requested.append(url) + route = self.routes.get(url) + if route is None: + return FakeResponse(404) + if isinstance(route, Exception): + raise route + return route + + def _post(self, url, json=None, **kwargs): + if json is None: + return FakeResponse(headers={"X-Transmission-Session-Id": "sid"}) + torrents = [ + {"name": t.name, "uploadRatio": t.ratio} for t in self.transmission.torrents + ] + return FakeResponse(body={"arguments": {"torrents": torrents}}) + + def _disk_usage(self, path): + self.events.append("disk_usage") + total = 100 * 2**30 + used = total * self.disk_used_percent // 100 + return (total, used, total - used) + + def _redirect_watch_dir(self): + real_open, real_exists, real_remove = builtins.open, os.path.exists, os.remove + + def redirect(path): + if isinstance(path, str) and path.replace("\\", "/").startswith("/watch/"): + return os.path.join( + self.watch_dir, path.replace("\\", "/")[len("/watch/") :] + ) + return path + + return [ + unittest.mock.patch.object( + builtins, "open", lambda p, *a, **k: real_open(redirect(p), *a, **k) + ), + unittest.mock.patch.object( + os.path, "exists", lambda p: real_exists(redirect(p)) + ), + unittest.mock.patch.object( + os, "remove", lambda p: real_remove(redirect(p)) + ), + ] + + def _write_ratio_log(self, ratios): + with open( + os.path.join(self.log_dir, "fetch_torrents_ratios.log"), + "w", + encoding="utf-8", + ) as f: + f.write("2026-08-01 00:00:00,000 INFO: [ratio] RATIOS START\n") + f.writelines( + f"2026-08-01 00:00:00,000 INFO: [ratio] {name} → {ratio:.3f}\n" + for name, ratio in ratios.items() + ) + f.write("2026-08-01 00:00:00,000 INFO: [ratio] RATIOS END\n") + + def _run(self, **env): + """Runs the script as __main__; returns its log messages.""" + logger = logging.getLogger("fetch_torrents") + handlers_before = list(logger.handlers) + + @contextlib.contextmanager + def connect(*args, **kwargs): + yield MagicMock() + + with contextlib.ExitStack() as stack: + stack.enter_context( + helpers.isolated_env(FETCH_TORRENTS_LOG_DIR=self.log_dir, **env) + ) + stack.enter_context(unittest.mock.patch("requests.get", self._get)) + stack.enter_context(unittest.mock.patch("requests.post", self._post)) + stack.enter_context( + unittest.mock.patch.object( + sys.modules["transmission_rpc"], + "Client", + side_effect=lambda **kwargs: self.transmission, + ) + ) + stack.enter_context( + unittest.mock.patch.object(socket, "create_connection", connect) + ) + stack.enter_context( + unittest.mock.patch.object(shutil, "disk_usage", self._disk_usage) + ) + stack.enter_context( + unittest.mock.patch.object(time, "time", return_value=1000.0) + ) + for patcher in self._redirect_watch_dir(): + stack.enter_context(patcher) + stack.enter_context(contextlib.redirect_stderr(io.StringIO())) + logs = stack.enter_context(self.assertLogs(logger, level="DEBUG")) + try: + runpy.run_path(SCRIPT, run_name="__main__") + finally: + # assertLogs puts the logger's original handlers back on exit, + # dropping the ones this run added - close their files first. + for handler in logger.handlers: + if isinstance(handler, logging.FileHandler): + handler.close() + self.assertEqual(logger.handlers, handlers_before) + return [record.getMessage() for record in logs.records] + + def _route_ubuntu(self): + self.routes.update( + { + "https://changelogs.ubuntu.com/meta-release-lts": FakeResponse( + text=UBUNTU_META + ), + UBUNTU + "ubuntu-24.04-desktop-amd64.iso.torrent": FakeResponse( + content=b"d1" + ), + # live-server: not routed -> 404 + LUBUNTU + "lubuntu-24.04-desktop-amd64.iso.torrent": FakeResponse(500), + XUBUNTU + + "desktop/xubuntu-24.04-desktop-amd64.iso.torrent": FakeResponse( + content=b"x1" + ), + XUBUNTU + + "minimal/xubuntu-24.04-minimal-amd64.iso.torrent": FakeResponse( + content=b"x2" + ), + } + ) + with open( + os.path.join(self.watch_dir, "xubuntu-24.04-minimal-amd64.iso.torrent"), + "wb", + ) as f: + f.write(b"already here") + + def _route_arch(self): + self.routes.update( + { + "https://archlinux.org/releng/releases/": FakeResponse(text=ARCH_HTML), + ARCH_TORRENT: FakeResponse(content=b"a1"), + } + ) + + # -- tests -------------------------------------------------------------- + + def test_counts_each_download_outcome_and_reports_them_in_the_summary(self): + self._route_ubuntu() + + messages = self._run(FETCH_TORRENTS_DISTROS="ubuntu") + + self.assertEqual( + sorted(os.listdir(self.watch_dir)), + [ + "ubuntu-24.04-desktop-amd64.iso.torrent", + "xubuntu-24.04-desktop-amd64.iso.torrent", + "xubuntu-24.04-minimal-amd64.iso.torrent", + ], + ) + self.assertEqual( + messages[-1], + "Run complete in 0.00 seconds. 2 added, 1 existing, " + "1 not found upstream, 1 failed.", + ) + + def test_distros_not_selected_are_never_fetched(self): + self._route_ubuntu() + self._route_arch() + + self._run(FETCH_TORRENTS_DISTROS="arch") + + self.assertEqual( + self.requested, ["https://archlinux.org/releng/releases/", ARCH_TORRENT] + ) + + def test_low_previous_ratio_blocks_the_new_release(self): + self._route_arch() + self._write_ratio_log({"archlinux-2026.08.01": 0.4}) + + messages = self._run(FETCH_TORRENTS_DISTROS="arch") + + self.assertNotIn(ARCH_TORRENT, self.requested) + self.assertIn( + "Skipping archlinux-2026.09.01 due to low ratio on previous version.", + messages, + ) + self.assertEqual(os.listdir(self.watch_dir), []) + + def test_previous_ratio_at_the_floor_allows_the_new_release(self): + self._route_arch() + self._write_ratio_log({"archlinux-2026.08.01": 1.0}) + + self._run(FETCH_TORRENTS_DISTROS="arch") + + self.assertEqual(os.listdir(self.watch_dir), ["archlinux-2026.09.01.torrent"]) + + def test_a_failing_distro_does_not_stop_the_others(self): + self._route_arch() + self.routes["https://changelogs.ubuntu.com/meta-release-lts"] = ConnectionError( + "down" + ) + + messages = self._run(FETCH_TORRENTS_DISTROS="ubuntu,arch") + + self.assertEqual(os.listdir(self.watch_dir), ["archlinux-2026.09.01.torrent"]) + self.assertEqual( + messages[-1], + "Run complete in 0.00 seconds. 1 added, 0 existing, " + "0 not found upstream, 1 failed.", + ) + + def test_keep_only_latest_selects_the_old_version_cleanup(self): + self.transmission.torrents = [ + torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), + torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), + ] + + self._run( + FETCH_TORRENTS_DISTROS="arch", CLEANUP_KEEP_ONLY_LATEST_VERSION="true" + ) + + self.assertEqual( + [t.name for t in self.transmission.torrents], + ["ubuntu-24.04-desktop-amd64.iso"], + ) + + def test_default_cleanup_is_stagnation_based(self): + # No ratio history yet, so the stagnation cleanup can't judge the old + # version and keeps it - unlike keep-only-latest, which removes it. + self.transmission.torrents = [ + torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), + torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), + ] + + self._run(FETCH_TORRENTS_DISTROS="arch") + + self.assertEqual(len(self.transmission.torrents), 2) + self.assertTrue( + os.path.exists( + os.path.join(self.log_dir, "fetch_torrents_ratio_history.json") + ) + ) + + def test_disk_usage_limit_runs_after_cleanup(self): + self.transmission.torrents = [ + torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), + torrent("ubuntu-23.10-desktop-amd64.iso", 2, 0.1), + ] + self.disk_used_percent = 99 + + self._run( + FETCH_TORRENTS_DISTROS="arch", CLEANUP_DISK_USAGE_THRESHOLD_PERCENT="95" + ) + + cleanup_listing = self.events.index("get_torrents") + first_disk_check = self.events.index("disk_usage") + self.assertLess(cleanup_listing, first_disk_check) + self.assertIn(("remove", 2), self.events[first_disk_check:]) + + def test_disk_usage_limit_error_is_logged_not_raised(self): + messages = self._run( + FETCH_TORRENTS_DISTROS="arch", CLEANUP_DISK_USAGE_THRESHOLD_PERCENT="lots" + ) + + self.assertTrue( + any(m.startswith("Could not enforce disk usage limit:") for m in messages) + ) + self.assertTrue(messages[-1].startswith("Run complete in")) + + def test_seed_ratios_are_logged_for_the_next_run(self): + self.transmission.torrents = [torrent("archlinux-2026.08.01", 1, 0.664)] + + self._run(FETCH_TORRENTS_DISTROS="arch") + + with open( + os.path.join(self.log_dir, "fetch_torrents_ratios.log"), encoding="utf-8" + ) as f: + text = f.read() + self.assertIn("[ratio] archlinux-2026.08.01", text) + self.assertIn("→ 0.664", text) + + +if __name__ == "__main__": + unittest.main() From d2dff9836f2d4f1fd06d440a83839bc94a3294ae Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 12:34:30 +0200 Subject: [PATCH 15/46] Recognise lubuntu and xubuntu torrents as Ubuntu The ubuntu distro pattern was '^ubuntu-|^lbuntu-|^xbuntu-', but fetch_ubuntu_lts() names them lubuntu-*/xubuntu-*. get_distro() returned None for both flavours, so they bypassed the previous-version ratio gate and were never grouped for old-version/stagnation/disk-pressure cleanup. Behavior change: after this, a new Lubuntu/Xubuntu release is only fetched once the previous one reached ratio 1.0, and superseded Lubuntu/Xubuntu releases become cleanup candidates like Ubuntu's. Co-Authored-By: Claude Opus 5.5 --- fetch_torrents.py | 2 +- tests/test_fetch_torrents.py | 97 ++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/fetch_torrents.py b/fetch_torrents.py index 73aba18..f04c73c 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -203,7 +203,7 @@ def get_previous_ratios(log_file): distro_patterns = { - "ubuntu": re.compile(r"^ubuntu-|^lbuntu-|^xbuntu-"), + "ubuntu": re.compile(r"^(ubuntu|lubuntu|xubuntu)-"), "debian": re.compile(r"^debian-"), "kali": re.compile(r"^kali-linux-"), "arch": re.compile(r"^archlinux-"), diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index 54c9d00..9f9f4e8 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -388,6 +388,103 @@ def test_datetime_added_dates_are_ordered_by_timestamp(self): self.assertEqual(to_remove, [older]) +class ParseVersionTypeTests(unittest.TestCase): + """(version, type_) is the grouping key for the ratio gate and cleanup: + two names share a ratio history only when their type_ is equal.""" + + CASES = ( + ( + "ubuntu-24.04-desktop-amd64.iso", + "ubuntu", + ("24.04", "ubuntu-desktop-amd64.iso"), + ), + ( + "ubuntu-24.04-desktop-arm64.iso", + "ubuntu", + ("24.04", "ubuntu-desktop-arm64.iso"), + ), + ( + "xubuntu-24.04-desktop-amd64.iso", + "ubuntu", + ("24.04", "xubuntu-desktop-amd64.iso"), + ), + ("debian-12.5.0-amd64-DVD-1.iso", "debian", ("12.5.0", "amd64-DVD-1.iso")), + ("debian-12.5.0-arm64-DVD-1.iso", "debian", ("12.5.0", "arm64-DVD-1.iso")), + ( + "kali-linux-2024.1-installer-amd64.iso", + "kali", + ("2024.1", "installer-amd64.iso"), + ), + ( + "kali-linux-2024.1-installer-arm64.iso", + "kali", + ("2024.1", "installer-arm64.iso"), + ), + ("archlinux-2024.05.01-x86_64.iso", "arch", ("2024.05.01", "")), + ("linuxmint-22.3-cinnamon-64bit.iso", "mint", ("22.3", "cinnamon-64bit.iso")), + ("Fedora-Workstation-Live-x86_64-44", "fedora", ("44", "x86_64")), + ("Fedora-Workstation-Live-aarch64-44", "fedora", ("44", "aarch64")), + ("anything-1.0", "not-a-distro", ("", "")), + ) + + def test_every_ubuntu_flavour_fetched_is_recognised_as_ubuntu(self): + # fetch_ubuntu_lts() fetches ubuntu, lubuntu and xubuntu images; a + # flavour get_distro() doesn't recognise skips the ratio gate and + # cleanup grouping entirely. + for name in ( + "ubuntu-24.04-desktop-amd64.iso", + "lubuntu-24.04-desktop-amd64.iso", + "xubuntu-24.04-desktop-amd64.iso", + ): + with self.subTest(name): + self.assertEqual(ft.get_distro(name), "ubuntu") + + def test_lubuntu_is_ratio_gated_on_its_own_previous_release(self): + ratios = { + "lubuntu-23.10-desktop-amd64.iso": 0.2, + "ubuntu-23.10-desktop-amd64.iso": 5.0, + } + with helpers.isolated_env(): + self.assertFalse( + ft.should_fetch_torrent("lubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_version_and_type_per_distro(self): + for name, distro, expected in self.CASES: + with self.subTest(name): + self.assertEqual(ft.parse_version_type(name, distro), expected) + + def test_architectures_of_the_same_release_get_different_types(self): + pairs = [ + ( + "ubuntu-24.04-desktop-amd64.iso", + "ubuntu-24.04-desktop-arm64.iso", + "ubuntu", + ), + ( + "debian-12.5.0-amd64-DVD-1.iso", + "debian-12.5.0-arm64-DVD-1.iso", + "debian", + ), + ( + "kali-linux-2024.1-installer-amd64.iso", + "kali-linux-2024.1-installer-arm64.iso", + "kali", + ), + ( + "Fedora-Workstation-Live-x86_64-44", + "Fedora-Workstation-Live-aarch64-44", + "fedora", + ), + ] + for a, b, distro in pairs: + with self.subTest(distro): + self.assertNotEqual( + ft.parse_version_type(a, distro)[1], + ft.parse_version_type(b, distro)[1], + ) + + class ShouldFetchTorrentRatioKeyTests(unittest.TestCase): """Candidate names must key into the same ratio-lookup bucket as the names Transmission actually reports (which include the arch suffix and .iso extension), From a15ffdb59e48eb63fb89dd557306574dcbfa586c Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 12:46:41 +0200 Subject: [PATCH 16/46] Extract run(), configure_logging() and load_config() (tasks 18, 19) The daily run was ~150 lines of __main__ reading os.environ, date.today() and a live Transmission client inline, and importing the module opened (and truncated) log files under /logs. Now: - configure_logging(log_dir, env) attaches the stream, rotating file and ratio handlers; import opens nothing. It replaces only its own handlers. - load_config(env) parses every env var once into a frozen Config. A bad number is recorded and raised by Config.require() where it is used, so cleanup and the disk limit still just log and skip, as before. - run(config, client_factory=, today=, now=, fetchers=, disk_usage=, download_path=) is the daily run, returning a RunSummary. - cleanup_old_versions / cleanup_stagnant_torrents take the client and today; enforce_disk_usage_limit takes a client factory, disk_usage and the download path. make_client_factory() carries the RPC credentials. The runpy characterization tests from task 17 passed unchanged against the refactor before being rewritten to call run() directly with FakeTransmission/FakeDisk; one runpy test keeps the __main__ wiring. Also adds LoadConfigTests, parse_version_type arch-variant cases (task 24) and updates CLAUDE.md's test-setup notes. fetch_torrents.py branch coverage: 91%. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 22 +- fetch_torrents.py | 576 +++++++++++++++++---------- tests/helpers.py | 46 ++- tests/test_daily_run.py | 520 +++++++++++++++++-------- tests/test_fetch_torrents.py | 731 ++++++++++++++++++++--------------- 5 files changed, 1220 insertions(+), 675 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 942530b..0c39171 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,12 +34,22 @@ CI runs this same command on every push and PR (`.github/workflows/docker-image. ## Why the test setup looks the way it does `fetch_torrents.py` is written to run inside the container: it imports -`transmission_rpc` (an apk-only package, not pip-installable) and opens log -files under `/logs` at import time. `tests/test_fetch_torrents.py` stubs -`transmission_rpc` via `sys.modules` and points `FETCH_TORRENTS_LOG_DIR` at a -temp directory before importing the module, so tests run without a container -or a live Transmission daemon. Follow that pattern for any new test file that -needs to import `fetch_torrents`. +`transmission_rpc` (an apk-only package, not pip-installable). +`tests/test_fetch_torrents.py` stubs `transmission_rpc` via `sys.modules` +before importing the module, so tests run without a container or a live +Transmission daemon. Follow that pattern for any new test file that needs to +import `fetch_torrents`. + +Importing the module opens no files: log handlers are only attached by +`configure_logging()`, which the `__main__` block calls. Tests that need it +should restore the logger afterwards (see `configure_logging_for_test()`). +The daily run itself is `run(config, client_factory=..., today=...)` with +its I/O (Transmission client, clock, disk usage) injected, so it is tested +directly with fakes from `tests/helpers.py` (`FakeTransmission`, `FakeDisk`); +`tests/test_daily_run.py` keeps one runpy test for the `__main__` wiring. +Env vars are read once by `load_config(env)`, and `tests/helpers.py` +`isolated_env()` scrubs the developer's own `CLEANUP_*`, `FETCH_TORRENTS_*` +etc. so they can't change test outcomes. Business logic that needs to be unit-testable should be kept as pure functions (taking plain data in, returning plain data out) separate from diff --git a/fetch_torrents.py b/fetch_torrents.py index f04c73c..ae4c85f 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -8,6 +8,7 @@ import socket import tempfile import time +from dataclasses import dataclass, field from datetime import date from typing import Any @@ -17,10 +18,12 @@ import torrent_scrape -# Configure logging +# State and log file locations. Plain paths only - nothing is opened at +# import time; configure_logging() creates the log handlers. log_dir = os.getenv("FETCH_TORRENTS_LOG_DIR", "/logs") -log_file = os.path.join(log_dir, "fetch_torrents.log") -ratio_log_file = os.path.join(log_dir, "fetch_torrents_ratios.log") +LOG_FILE_NAME = "fetch_torrents.log" +RATIO_LOG_FILE_NAME = "fetch_torrents_ratios.log" +ratio_log_file = os.path.join(log_dir, RATIO_LOG_FILE_NAME) ratio_history_file = os.path.join(log_dir, "fetch_torrents_ratio_history.json") old_release_check_state_file = os.path.join( log_dir, "fetch_torrents_old_release_check_state.json" @@ -28,8 +31,8 @@ removed_history_file = os.path.join(log_dir, "fetch_torrents_removed_history.json") -def parse_log_level(env_var: str, default: int = logging.INFO) -> int: - value = os.getenv(env_var, "").strip() +def parse_log_level(env_var: str, default: int = logging.INFO, env=None) -> int: + value = (os.environ if env is None else env).get(env_var, "").strip() if not value: return default if value.isdigit(): @@ -41,15 +44,15 @@ def parse_log_level(env_var: str, default: int = logging.INFO) -> int: return logging._nameToLevel.get(level, default) -def parse_bool(env_var: str, default: bool = False) -> bool: - value = os.getenv(env_var, "").strip().lower() +def parse_bool(env_var: str, default: bool = False, env=None) -> bool: + value = (os.environ if env is None else env).get(env_var, "").strip().lower() if not value: return default return value in ("1", "true", "yes", "on") -def parse_int(env_var: str, default: int) -> int: - value = os.getenv(env_var, "").strip() +def parse_int(env_var: str, default: int, env=None) -> int: + value = (os.environ if env is None else env).get(env_var, "").strip() if not value: return default try: @@ -58,28 +61,6 @@ def parse_int(env_var: str, default: int) -> int: return default -log_level = parse_log_level( - "FETCH_TORRENTS_LOG_LEVEL", parse_log_level("LOG_LEVEL", logging.INFO) -) -formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s") - -stream_handler = logging.StreamHandler() -stream_handler.setFormatter(formatter) - -# Rotating rather than a plain FileHandler: fetch_torrents.log otherwise -# grows without bound for the lifetime of a "deploy and forget" container - -# see FETCH_TORRENTS_LOG_MAX_BYTES/FETCH_TORRENTS_LOG_BACKUP_COUNT in the -# README. 5MB x 3 backups keeps a generous amount of history bounded to ~20MB. -log_max_bytes = parse_int("FETCH_TORRENTS_LOG_MAX_BYTES", 5 * 1024 * 1024) -log_backup_count = parse_int("FETCH_TORRENTS_LOG_BACKUP_COUNT", 3) -file_handler = logging.handlers.RotatingFileHandler( - log_file, - maxBytes=log_max_bytes, - backupCount=log_backup_count, -) -file_handler.setFormatter(formatter) - - class RatioOnlyFilter(logging.Filter): def filter(self, record: logging.LogRecord) -> bool: return record.getMessage().startswith("[ratio]") @@ -118,36 +99,72 @@ def get_always_log_prefixes(): ] -always_log_enabled = parse_bool("FETCH_TORRENTS_ALWAYS_LOG", True) -important_prefixes = get_always_log_prefixes() if always_log_enabled else [] - -stream_handler.setLevel(logging.DEBUG) -stream_handler.addFilter(NonRatioFilter()) -if always_log_enabled: - stream_handler.addFilter(ImportantMessageFilter(log_level, important_prefixes)) -else: - stream_handler.setLevel(log_level) - -file_handler.setLevel(logging.DEBUG) -file_handler.addFilter(NonRatioFilter()) -if always_log_enabled: - file_handler.addFilter(ImportantMessageFilter(log_level, important_prefixes)) -else: - file_handler.setLevel(log_level) +logger = logging.getLogger("fetch_torrents") +# Silent until configure_logging() runs, so importing this module (tests, +# other scripts) neither writes log files nor falls back to stderr. +_configured_handlers: list[logging.Handler] = [logging.NullHandler()] +logger.addHandler(_configured_handlers[0]) + + +def configure_logging(log_dir, env=None): + """Attach the daily run's handlers to `logger`, replacing the ones a + previous call (or the import-time NullHandler) attached: + stderr and a rotating fetch_torrents.log (both filtered by log level and + FETCH_TORRENTS_ALWAYS_LOG), plus fetch_torrents_ratios.log holding only + the "[ratio]" lines. Returns (stream_handler, file_handler, ratio_handler). + + The ratio log is opened with mode="w" but delay=True, so it's only + truncated on the first "[ratio]" line of this run - after + get_previous_ratios() has already read the previous run's ratios.""" + log_level = parse_log_level( + "FETCH_TORRENTS_LOG_LEVEL", + parse_log_level("LOG_LEVEL", logging.INFO, env=env), + env=env, + ) + always_log_enabled = parse_bool("FETCH_TORRENTS_ALWAYS_LOG", True, env=env) + important_prefixes = get_always_log_prefixes() if always_log_enabled else [] + formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s") + + stream_handler = logging.StreamHandler() + # Rotating rather than a plain FileHandler: fetch_torrents.log otherwise + # grows without bound for the lifetime of a "deploy and forget" container - + # see FETCH_TORRENTS_LOG_MAX_BYTES/FETCH_TORRENTS_LOG_BACKUP_COUNT in the + # README. 5MB x 3 backups keeps a generous amount of history bounded to ~20MB. + file_handler = logging.handlers.RotatingFileHandler( + os.path.join(log_dir, LOG_FILE_NAME), + maxBytes=parse_int("FETCH_TORRENTS_LOG_MAX_BYTES", 5 * 1024 * 1024, env=env), + backupCount=parse_int("FETCH_TORRENTS_LOG_BACKUP_COUNT", 3, env=env), + ) + for handler in (stream_handler, file_handler): + handler.setFormatter(formatter) + handler.setLevel(logging.DEBUG) + handler.addFilter(NonRatioFilter()) + if always_log_enabled: + handler.addFilter(ImportantMessageFilter(log_level, important_prefixes)) + else: + handler.setLevel(log_level) -ratio_handler = logging.FileHandler( - ratio_log_file, mode="w", delay=True, encoding="utf-8" -) -ratio_handler.setLevel(logging.INFO) -ratio_handler.addFilter(RatioOnlyFilter()) -ratio_handler.setFormatter(formatter) + ratio_handler = logging.FileHandler( + os.path.join(log_dir, RATIO_LOG_FILE_NAME), + mode="w", + delay=True, + encoding="utf-8", + ) + ratio_handler.setLevel(logging.INFO) + ratio_handler.addFilter(RatioOnlyFilter()) + ratio_handler.setFormatter(formatter) + + while _configured_handlers: + handler = _configured_handlers.pop() + logger.removeHandler(handler) + handler.close() + logger.setLevel(logging.DEBUG) + for handler in (stream_handler, file_handler, ratio_handler): + logger.addHandler(handler) + _configured_handlers.append(handler) + logger.propagate = False + return stream_handler, file_handler, ratio_handler -logger = logging.getLogger("fetch_torrents") -logger.setLevel(logging.DEBUG) -logger.addHandler(stream_handler) -logger.addHandler(file_handler) -logger.addHandler(ratio_handler) -logger.propagate = False watch_dir = "/watch" @@ -242,8 +259,8 @@ def get_previous_ratios(log_file): DEFAULT_DISTROS = tuple(d for d in ALL_DISTROS if d not in OPT_IN_DISTROS) -def parse_supported_distros(env_var="FETCH_TORRENTS_DISTROS"): - value = os.getenv(env_var, "").strip() +def parse_supported_distros(env_var="FETCH_TORRENTS_DISTROS", env=None): + value = (os.environ if env is None else env).get(env_var, "").strip() if not value: return list(DEFAULT_DISTROS) @@ -393,7 +410,7 @@ def _should_fetch_devuan(name, ratios, removed_history): return max(prior_ratios) >= 1.0 -def should_fetch_torrent(name, ratios, removed_history=None): +def should_fetch_torrent(name, ratios, removed_history=None, *, skip_ratio_check=False): """removed_history (record_removal()'s output) is consulted as a fallback for versions ratios doesn't cover: ratios only reflects torrents Transmission currently holds (see get_previous_ratios()), so a @@ -401,8 +418,8 @@ def should_fetch_torrent(name, ratios, removed_history=None): out of this check and fall into the "no previous version, fetch" branch below - bypassing the ratio gate for exactly the versions cleanup already gave up on. Live ratios still take priority when a name somehow - appears in both.""" - if os.getenv("SKIP_RATIO_CHECK", "false").lower() == "true": + appears in both. skip_ratio_check (SKIP_RATIO_CHECK) bypasses the gate.""" + if skip_ratio_check: return True distro = get_distro(name) if not distro: @@ -1250,17 +1267,13 @@ def plan_cleanup(torrents, skip_ratio_check=False, min_ratio=1.0): return to_remove, to_keep_low_ratio -def cleanup_old_versions(): +def cleanup_old_versions(tc, *, today, skip_ratio_check=False, min_ratio=1.0): """Space-constrained option (CLEANUP_KEEP_ONLY_LATEST_VERSION=true): - remove superseded versions as soon as they clear the ratio floor, - regardless of whether anyone's still downloading them.""" - username, password = get_rpc_credentials() - tc = Client(host="localhost", port=9091, username=username, password=password) + remove superseded versions as soon as they clear the ratio floor + (CLEANUP_MIN_RATIO, unless CLEANUP_SKIP_RATIO_CHECK), regardless of + whether anyone's still downloading them. tc is a Transmission client.""" torrents = tc.get_torrents() - skip_ratio_check = parse_bool("CLEANUP_SKIP_RATIO_CHECK", False) - min_ratio = float(os.getenv("CLEANUP_MIN_RATIO", "1.0")) - to_remove, to_keep_low_ratio = plan_cleanup(torrents, skip_ratio_check, min_ratio) for torrent, ratio in to_keep_low_ratio: @@ -1272,7 +1285,6 @@ def cleanup_old_versions(): ) removed_history = load_removed_history(removed_history_file) - today = date.today() for torrent in to_remove: logger.info(f"Removing old version: {torrent.name}") tc.remove_torrent(torrent.id, delete_data=True) @@ -1304,20 +1316,27 @@ def plan_disk_pressure_cleanup(torrents, used_percent, threshold_percent): return candidates -def enforce_disk_usage_limit(threshold_percent, download_path="/downloads"): - """Runs after the normal cleanup pass. Only connects to Transmission and - touches disk usage at all once threshold_percent is actually exceeded, +def enforce_disk_usage_limit( + client_factory, + threshold_percent, + *, + today, + disk_usage=shutil.disk_usage, + download_path="/downloads", +): + """Runs after the normal cleanup pass. Only checks disk usage at all + when enabled (threshold_percent > 0), and only calls client_factory() + to connect to Transmission once threshold_percent is actually exceeded, so a deployment that never gets close to full pays no extra cost.""" if threshold_percent <= 0: return - total, used, _free = shutil.disk_usage(download_path) + total, used, _free = disk_usage(download_path) used_percent = (used / total * 100) if total else 0.0 if used_percent < threshold_percent: return - username, password = get_rpc_credentials() - tc = Client(host="localhost", port=9091, username=username, password=password) + tc = client_factory() torrents = tc.get_torrents() candidates = plan_disk_pressure_cleanup(torrents, used_percent, threshold_percent) @@ -1331,9 +1350,8 @@ def enforce_disk_usage_limit(threshold_percent, download_path="/downloads"): return removed_history = load_removed_history(removed_history_file) - today = date.today() for torrent in candidates: - total, used, _free = shutil.disk_usage(download_path) + total, used, _free = disk_usage(download_path) used_percent = (used / total * 100) if total else 0.0 if used_percent < threshold_percent: break @@ -1446,15 +1464,13 @@ def plan_stagnation_cleanup( return to_remove, to_keep -def cleanup_stagnant_torrents(): - username, password = get_rpc_credentials() - tc = Client(host="localhost", port=9091, username=username, password=password) +def cleanup_stagnant_torrents(tc, *, today, window_days=30, min_ratio_delta=0.02): + """Default cleanup path: records today's ratios in the ratio history, + then removes superseded versions whose ratio grew by less than + min_ratio_delta over window_days (see plan_stagnation_cleanup()). tc is + a Transmission client.""" torrents = tc.get_torrents() - window_days = int(os.getenv("CLEANUP_STAGNATION_WINDOW_DAYS", "30")) - min_ratio_delta = float(os.getenv("CLEANUP_STAGNATION_MIN_RATIO_DELTA", "0.02")) - today = date.today() - history = load_ratio_history(ratio_history_file) current_ratios = {t.name: float(getattr(t, "ratio", 0.0) or 0.0) for t in torrents} history = update_ratio_history( @@ -1500,38 +1516,156 @@ def cleanup_stagnant_torrents(): save_ratio_history(ratio_history_file, history) -if __name__ == "__main__": - start_time = time.time() - logger.info("Starting torrent fetch run.") - - ratios = get_previous_ratios(ratio_log_file) - removed_history = load_removed_history(removed_history_file) +@dataclass(frozen=True) +class Config: + """Every env var the daily run reads, parsed once by load_config(). + + A numeric setting that doesn't parse is None here, with the problem + recorded in `invalid`; require() raises it at the point the setting is + actually used. So a bad CLEANUP_* or CLEANUP_DISK_USAGE_THRESHOLD_PERCENT + value still only skips that step (logged), and a bad + FETCH_TORRENTS_OLD_RELEASE_* value still only fails once the old-release + check runs - the same as when each was parsed in place.""" + + distros: tuple[str, ...] = DEFAULT_DISTROS + include_low_demand: bool = False + skip_ratio_check: bool = False + check_old_releases: bool = False + old_release_min_leechers: int | None = 10 + old_release_min_leecher_ratio: float | None = 0.1 + old_release_allow_zero_seeders: bool = False + old_release_recheck_days: int | None = 7 + old_release_skip_removed: bool = True + cleanup_keep_only_latest: bool = False + cleanup_skip_ratio_check: bool = False + cleanup_min_ratio: float | None = 1.0 + stagnation_window_days: int | None = 30 + stagnation_min_ratio_delta: float | None = 0.02 + disk_usage_threshold_percent: float | None = 95.0 + rpc_username: str | None = None + rpc_password: str | None = None + invalid: dict[str, str] = field(default_factory=dict) + + def require(self, *fields): + for name in fields: + if name in self.invalid: + raise ValueError(self.invalid[name]) + + @property + def rpc_auth(self): + if self.rpc_username and self.rpc_password: + return (self.rpc_username, self.rpc_password) + return None - success_count = 0 - existing_count = 0 - failure_count = 0 - not_found_count = 0 - - distro_funcs = [ - ("ubuntu", fetch_ubuntu_lts), - ("debian", fetch_debian_stable), - ("kali", fetch_kali_latest), - ("arch", fetch_arch_latest), - ("mint", fetch_linuxmint_cinnamon), - ("fedora", fetch_fedora_workstation), - ("devuan", fetch_devuan_latest), - ] - selected_distros = parse_supported_distros() - logger.info("Selected distros for this run: %s", ", ".join(selected_distros)) +def load_config(env): + invalid = {} - include_low_demand = parse_bool("FETCH_TORRENTS_INCLUDE_LOW_DEMAND", False) - logger.info( - "Including low-demand image families (arm64/cloud/etc.): %s", include_low_demand + def number(field_name, key, default, convert): + raw = env.get(key) + if raw is None: + return default + try: + return convert(raw) + except ValueError as exc: + invalid[field_name] = f"{key}: {exc}" + return None + + return Config( + distros=tuple(parse_supported_distros(env=env)), + include_low_demand=parse_bool("FETCH_TORRENTS_INCLUDE_LOW_DEMAND", False, env), + skip_ratio_check=parse_bool("SKIP_RATIO_CHECK", False, env), + check_old_releases=parse_bool("FETCH_TORRENTS_CHECK_OLD_RELEASES", False, env), + old_release_min_leechers=number( + "old_release_min_leechers", + "FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS", + 10, + int, + ), + old_release_min_leecher_ratio=number( + "old_release_min_leecher_ratio", + "FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO", + 0.1, + float, + ), + old_release_allow_zero_seeders=parse_bool( + "FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS", False, env + ), + old_release_recheck_days=number( + "old_release_recheck_days", + "FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS", + 7, + int, + ), + old_release_skip_removed=parse_bool( + "FETCH_TORRENTS_OLD_RELEASE_SKIP_REMOVED", True, env + ), + cleanup_keep_only_latest=parse_bool( + "CLEANUP_KEEP_ONLY_LATEST_VERSION", False, env + ), + cleanup_skip_ratio_check=parse_bool("CLEANUP_SKIP_RATIO_CHECK", False, env), + cleanup_min_ratio=number("cleanup_min_ratio", "CLEANUP_MIN_RATIO", 1.0, float), + stagnation_window_days=number( + "stagnation_window_days", "CLEANUP_STAGNATION_WINDOW_DAYS", 30, int + ), + stagnation_min_ratio_delta=number( + "stagnation_min_ratio_delta", + "CLEANUP_STAGNATION_MIN_RATIO_DELTA", + 0.02, + float, + ), + disk_usage_threshold_percent=number( + "disk_usage_threshold_percent", + "CLEANUP_DISK_USAGE_THRESHOLD_PERCENT", + 95.0, + float, + ), + rpc_username=env.get("TRANSMISSION_RPC_USERNAME", "").strip() or None, + rpc_password=env.get("TRANSMISSION_RPC_PASSWORD", "").strip() or None, + invalid=invalid, ) - for distro, func in distro_funcs: - if distro not in selected_distros: + +def make_client_factory(config): + """Connects to the local daemon with the credentials configure_transmission.py + configured it with (see get_rpc_credentials() for why that matters).""" + + def connect(): + return Client( + host="localhost", + port=9091, + username=config.rpc_username, + password=config.rpc_password, + ) + + return connect + + +DEFAULT_FETCHERS = ( + ("ubuntu", fetch_ubuntu_lts), + ("debian", fetch_debian_stable), + ("kali", fetch_kali_latest), + ("arch", fetch_arch_latest), + ("mint", fetch_linuxmint_cinnamon), + ("fedora", fetch_fedora_workstation), + ("devuan", fetch_devuan_latest), +) + + +@dataclass +class RunSummary: + elapsed: float = 0.0 + added: int = 0 + existing: int = 0 + not_found: int = 0 + failed: int = 0 + old_added: int | None = None + old_skipped: int | None = None + + +def _fetch_latest(config, fetchers, ratios, removed_history, summary): + for distro, func in fetchers: + if distro not in config.distros: logger.info( "Skipping distro %s because it is not enabled by FETCH_TORRENTS_DISTROS.", distro, @@ -1540,79 +1674,121 @@ def cleanup_stagnant_torrents(): logger.info(f"Fetching latest {distro} torrents...") torrents = func() - if torrents: - torrents = filter_low_demand(torrents, include_low_demand) - for name, url in torrents.items(): - if should_fetch_torrent(name, ratios, removed_history): - status = download_torrent(name, url) - if status == "added": - success_count += 1 - elif status == "existing": - existing_count += 1 - elif status == "not_found": - not_found_count += 1 - else: - failure_count += 1 - else: - logger.info( - f"Skipping {name} due to low ratio on previous version." - ) - else: - failure_count += 1 + if not torrents: + summary.failed += 1 + continue + torrents = filter_low_demand(torrents, config.include_low_demand) + for name, url in torrents.items(): + if not should_fetch_torrent( + name, + ratios, + removed_history, + skip_ratio_check=config.skip_ratio_check, + ): + logger.info(f"Skipping {name} due to low ratio on previous version.") + continue + status = download_torrent(name, url) + if status == "added": + summary.added += 1 + elif status == "existing": + summary.existing += 1 + elif status == "not_found": + summary.not_found += 1 + else: + summary.failed += 1 - old_added = old_skipped = None - check_old_releases = parse_bool("FETCH_TORRENTS_CHECK_OLD_RELEASES", False) - if check_old_releases: - old_release_min_leechers = int( - os.getenv("FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS", "10") - ) - old_release_min_leecher_ratio = float( - os.getenv("FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO", "0.1") - ) - old_release_allow_zero_seeders = parse_bool( - "FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS", False - ) - old_release_recheck_days = int( - os.getenv("FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS", "7") - ) - skip_previously_removed = parse_bool( - "FETCH_TORRENTS_OLD_RELEASE_SKIP_REMOVED", True - ) - check_state = load_old_release_check_state(old_release_check_state_file) - removed_history = ( - load_removed_history(removed_history_file) - if skip_previously_removed - else {} - ) +def _check_old_releases(config, today, summary): + config.require( + "old_release_min_leechers", + "old_release_min_leecher_ratio", + "old_release_recheck_days", + ) + check_state = load_old_release_check_state(old_release_check_state_file) + removed_history = ( + load_removed_history(removed_history_file) + if config.old_release_skip_removed + else {} + ) + logger.info( + "Checking old releases for unmet demand (min leechers: %d, min leecher/seeder ratio: %.1fx, " + "allow zero-seeder swarms: %s, recheck interval: %d days)...", + config.old_release_min_leechers, + config.old_release_min_leecher_ratio, + config.old_release_allow_zero_seeders, + config.old_release_recheck_days, + ) + summary.old_added, summary.old_skipped, check_state = check_old_releases_for_demand( + list(config.distros), + config.include_low_demand, + config.old_release_min_leechers, + config.old_release_min_leecher_ratio, + config.old_release_allow_zero_seeders, + check_state, + removed_history, + today, + config.old_release_recheck_days, + ) + save_old_release_check_state(old_release_check_state_file, check_state) + logger.info( + "Old-release check complete: %d added, %d skipped.", + summary.old_added, + summary.old_skipped, + ) - logger.info( - "Checking old releases for unmet demand (min leechers: %d, min leecher/seeder ratio: %.1fx, " - "allow zero-seeder swarms: %s, recheck interval: %d days)...", - old_release_min_leechers, - old_release_min_leecher_ratio, - old_release_allow_zero_seeders, - old_release_recheck_days, - ) - old_added, old_skipped, check_state = check_old_releases_for_demand( - selected_distros, - include_low_demand, - old_release_min_leechers, - old_release_min_leecher_ratio, - old_release_allow_zero_seeders, - check_state, - removed_history, - date.today(), - old_release_recheck_days, + +def _cleanup(config, client_factory, today): + if config.cleanup_keep_only_latest: + config.require("cleanup_min_ratio") + cleanup_old_versions( + client_factory(), + today=today, + skip_ratio_check=config.cleanup_skip_ratio_check, + min_ratio=config.cleanup_min_ratio, ) - save_old_release_check_state(old_release_check_state_file, check_state) - logger.info( - "Old-release check complete: %d added, %d skipped.", old_added, old_skipped + else: + config.require("stagnation_window_days", "stagnation_min_ratio_delta") + cleanup_stagnant_torrents( + client_factory(), + today=today, + window_days=config.stagnation_window_days, + min_ratio_delta=config.stagnation_min_ratio_delta, ) + +def run( + config, + *, + client_factory, + today, + now=time.time, + fetchers=DEFAULT_FETCHERS, + disk_usage=shutil.disk_usage, + download_path="/downloads", +): + """The daily run: fetch new releases (ratio-gated on the previous run's + ratio log), optionally check old releases for demand, log seed ratios + for the next run, clean up, then enforce the disk-usage limit. Each + step after fetching logs its own failure and lets the rest carry on.""" + start_time = now() + logger.info("Starting torrent fetch run.") + logger.info("Selected distros for this run: %s", ", ".join(config.distros)) + logger.info( + "Including low-demand image families (arm64/cloud/etc.): %s", + config.include_low_demand, + ) + + summary = RunSummary() + ratios = get_previous_ratios(ratio_log_file) + removed_history = load_removed_history(removed_history_file) + _fetch_latest(config, fetchers, ratios, removed_history, summary) + + if config.check_old_releases: + _check_old_releases(config, today, summary) + if wait_for_transmission_rpc(): try: - log_seed_ratios_via_http() + log_seed_ratios_via_http(auth=config.rpc_auth) except Exception as exc: logger.error("Could not query Transmission: %s", exc) else: @@ -1621,35 +1797,43 @@ def cleanup_stagnant_torrents(): ) try: - if parse_bool("CLEANUP_KEEP_ONLY_LATEST_VERSION", False): - cleanup_old_versions() - else: - cleanup_stagnant_torrents() + _cleanup(config, client_factory, today) except Exception as exc: logger.error("Could not clean up old versions: %s", exc) try: - disk_usage_threshold = float( - os.getenv("CLEANUP_DISK_USAGE_THRESHOLD_PERCENT", "95") + config.require("disk_usage_threshold_percent") + enforce_disk_usage_limit( + client_factory, + config.disk_usage_threshold_percent, + today=today, + disk_usage=disk_usage, + download_path=download_path, ) - enforce_disk_usage_limit(disk_usage_threshold) except Exception as exc: logger.error("Could not enforce disk usage limit: %s", exc) - total, used, free = shutil.disk_usage("/downloads") + total, used, _free = disk_usage(download_path) logger.info( f"Downloads folder usage: {used // (2**30)} GB used / {total // (2**30)} GB total" ) - elapsed = time.time() - start_time + summary.elapsed = now() - start_time logger.info( format_run_summary( - elapsed, - success_count, - existing_count, - not_found_count, - failure_count, - old_added, - old_skipped, + summary.elapsed, + summary.added, + summary.existing, + summary.not_found, + summary.failed, + summary.old_added, + summary.old_skipped, ) ) + return summary + + +if __name__ == "__main__": + configure_logging(log_dir) + config = load_config(os.environ) + run(config, client_factory=make_client_factory(config), today=date.today()) diff --git a/tests/helpers.py b/tests/helpers.py index 9991311..55de79b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -20,8 +20,8 @@ "TRANSMISSION_", "FETCH_TORRENTS_", ) -# Read at import time to choose where fetch_torrents writes its logs; the -# test modules set it to a temp dir before importing, so it must survive. +# Where fetch_torrents' state and log files go; a test that sets it (e.g. +# for a subprocess or runpy run) must not have it scrubbed. _PRESERVED = frozenset({"FETCH_TORRENTS_LOG_DIR"}) @@ -59,3 +59,45 @@ def make_temp_dir(testcase, prefix): path = tempfile.mkdtemp(prefix=prefix) testcase.addCleanup(shutil.rmtree, path, ignore_errors=True) return path + + +class FakeTransmission: + """In-memory stand-in for transmission_rpc.Client: get_torrents() + returns what it currently holds, remove_torrent() drops that torrent. + `calls` records both, in order, for tests that care about sequencing.""" + + def __init__(self, torrents=()): + self.torrents = list(torrents) + self.calls = [] + + def get_torrents(self): + self.calls.append("get_torrents") + return list(self.torrents) + + def remove_torrent(self, ids, delete_data=False): + self.calls.append(("remove_torrent", ids, delete_data)) + self.torrents = [t for t in self.torrents if t.id != ids] + + @property + def names(self): + return sorted(t.name for t in self.torrents) + + +class FakeDisk: + """shutil.disk_usage() stand-in whose used space is `base_used` plus the + size of every torrent `transmission` still holds - so removals free + space the way they would for real.""" + + def __init__(self, transmission, total, sizes, base_used=0): + self.transmission = transmission + self.total = total + self.sizes = sizes + self.base_used = base_used + self.paths = [] + + def __call__(self, path): + self.paths.append(path) + used = self.base_used + sum( + self.sizes.get(t.id, 0) for t in self.transmission.torrents + ) + return (self.total, used, self.total - used) diff --git a/tests/test_daily_run.py b/tests/test_daily_run.py index d63a763..a5fbe00 100644 --- a/tests/test_daily_run.py +++ b/tests/test_daily_run.py @@ -1,20 +1,18 @@ -"""Characterization tests for fetch_torrents.py's `__main__` block - the -whole daily run - executed via runpy exactly as the container runs it. +"""Tests for the daily run: fetch_torrents.run() and the `__main__` wiring +around it. -Everything external is faked at its boundary: HTTP (requests.get/post), -the Transmission client (the transmission_rpc stub), the RPC port check -(socket.create_connection), disk usage and the clock. Log/state files live -in a per-test temp dir via FETCH_TORRENTS_LOG_DIR. +RunTests call run() directly with everything external faked at its +boundary: HTTP (ft.requests.get/post), the Transmission client (a +client_factory returning helpers.FakeTransmission), the RPC port check +(socket.create_connection), disk usage and the clock. State files live in a +per-test temp dir. -`watch_dir = "/watch"` is assigned at import time, and runpy executes the -module in a fresh namespace, so it can't be patched like the other module -globals. _redirect_watch_dir() reroutes file access under /watch to a temp -dir instead. (That friction is what tasks/18 removes by extracting run().) +MainWiringTests runs the script via runpy exactly as the container does, to +check that `__main__` sets up logging, reads the config and calls run(). Run with: python -m unittest discover -s tests """ -import builtins import contextlib import io import logging @@ -23,10 +21,10 @@ import shutil import socket import sys -import time import types import unittest import unittest.mock +from datetime import date from unittest.mock import MagicMock if "transmission_rpc" not in sys.modules: @@ -41,6 +39,8 @@ import helpers +import fetch_torrents as ft + UBUNTU_META = ( "Dist: jammy\nName: Jammy Jellyfish\nVersion: 22.04 LTS\nSupported: 0\n\n" "Dist: noble\nName: Noble Numbat\nVersion: 24.04 LTS\nSupported: 1\n" @@ -48,6 +48,7 @@ UBUNTU = "https://releases.ubuntu.com/noble/" LUBUNTU = "https://cdimage.ubuntu.com/lubuntu/releases/noble/release/" XUBUNTU = "https://torrent.ubuntu.com/xubuntu/releases/noble/release/" +ARCH_RELEASES = "https://archlinux.org/releng/releases/" ARCH_HTML = ( "" "" @@ -67,46 +68,29 @@ def __init__(self, status_code=200, text="", content=b"", headers=None, body=Non def raise_for_status(self): if self.status_code >= 400: - raise RuntimeError(f"HTTP {self.status_code}") + raise ft.requests.HTTPError(f"HTTP {self.status_code}") def json(self): return self._body -class FakeTransmission: - """In-memory Transmission: get_torrents() returns what's held, - remove_torrent() drops it and records the call in `events`.""" - - def __init__(self, torrents, events): - self.torrents = list(torrents) - self.events = events - - def get_torrents(self): - self.events.append("get_torrents") - return list(self.torrents) - - def remove_torrent(self, torrent_id, delete_data=False): - self.events.append(("remove", torrent_id)) - self.torrents = [t for t in self.torrents if t.id != torrent_id] - - def torrent(name, id_, ratio): return types.SimpleNamespace(name=name, id=id_, ratio=ratio, added_date=None) -class DailyRunCharacterizationTests(unittest.TestCase): - def setUp(self): - self.log_dir = helpers.make_temp_dir(self, "daily_run_logs_") - self.watch_dir = helpers.make_temp_dir(self, "daily_run_watch_") +class FakeWorld: + """HTTP routes, a Transmission daemon and a disk, all in memory.""" + + def __init__(self, testcase): + self.state_dir = helpers.make_temp_dir(testcase, "daily_run_state_") + self.watch_dir = helpers.make_temp_dir(testcase, "daily_run_watch_") + self.routes = {} self.requested = [] self.events = [] - self.routes = {} + self.transmission = helpers.FakeTransmission() self.disk_used_percent = 10 - self.transmission = FakeTransmission([], self.events) - - # -- fakes -------------------------------------------------------------- - def _get(self, url, timeout=None): + def get(self, url, timeout=None): self.requested.append(url) route = self.routes.get(url) if route is None: @@ -115,7 +99,7 @@ def _get(self, url, timeout=None): raise route return route - def _post(self, url, json=None, **kwargs): + def post(self, url, json=None, **kwargs): if json is None: return FakeResponse(headers={"X-Transmission-Session-Id": "sid"}) torrents = [ @@ -123,39 +107,22 @@ def _post(self, url, json=None, **kwargs): ] return FakeResponse(body={"arguments": {"torrents": torrents}}) - def _disk_usage(self, path): + def connect(self): + self.events.append("connect") + return self.transmission + + def disk_usage(self, path): self.events.append("disk_usage") total = 100 * 2**30 used = total * self.disk_used_percent // 100 return (total, used, total - used) - def _redirect_watch_dir(self): - real_open, real_exists, real_remove = builtins.open, os.path.exists, os.remove + def state_path(self, name): + return os.path.join(self.state_dir, name) - def redirect(path): - if isinstance(path, str) and path.replace("\\", "/").startswith("/watch/"): - return os.path.join( - self.watch_dir, path.replace("\\", "/")[len("/watch/") :] - ) - return path - - return [ - unittest.mock.patch.object( - builtins, "open", lambda p, *a, **k: real_open(redirect(p), *a, **k) - ), - unittest.mock.patch.object( - os.path, "exists", lambda p: real_exists(redirect(p)) - ), - unittest.mock.patch.object( - os, "remove", lambda p: real_remove(redirect(p)) - ), - ] - - def _write_ratio_log(self, ratios): + def write_ratio_log(self, ratios): with open( - os.path.join(self.log_dir, "fetch_torrents_ratios.log"), - "w", - encoding="utf-8", + self.state_path("fetch_torrents_ratios.log"), "w", encoding="utf-8" ) as f: f.write("2026-08-01 00:00:00,000 INFO: [ratio] RATIOS START\n") f.writelines( @@ -164,53 +131,7 @@ def _write_ratio_log(self, ratios): ) f.write("2026-08-01 00:00:00,000 INFO: [ratio] RATIOS END\n") - def _run(self, **env): - """Runs the script as __main__; returns its log messages.""" - logger = logging.getLogger("fetch_torrents") - handlers_before = list(logger.handlers) - - @contextlib.contextmanager - def connect(*args, **kwargs): - yield MagicMock() - - with contextlib.ExitStack() as stack: - stack.enter_context( - helpers.isolated_env(FETCH_TORRENTS_LOG_DIR=self.log_dir, **env) - ) - stack.enter_context(unittest.mock.patch("requests.get", self._get)) - stack.enter_context(unittest.mock.patch("requests.post", self._post)) - stack.enter_context( - unittest.mock.patch.object( - sys.modules["transmission_rpc"], - "Client", - side_effect=lambda **kwargs: self.transmission, - ) - ) - stack.enter_context( - unittest.mock.patch.object(socket, "create_connection", connect) - ) - stack.enter_context( - unittest.mock.patch.object(shutil, "disk_usage", self._disk_usage) - ) - stack.enter_context( - unittest.mock.patch.object(time, "time", return_value=1000.0) - ) - for patcher in self._redirect_watch_dir(): - stack.enter_context(patcher) - stack.enter_context(contextlib.redirect_stderr(io.StringIO())) - logs = stack.enter_context(self.assertLogs(logger, level="DEBUG")) - try: - runpy.run_path(SCRIPT, run_name="__main__") - finally: - # assertLogs puts the logger's original handlers back on exit, - # dropping the ones this run added - close their files first. - for handler in logger.handlers: - if isinstance(handler, logging.FileHandler): - handler.close() - self.assertEqual(logger.handlers, handlers_before) - return [record.getMessage() for record in logs.records] - - def _route_ubuntu(self): + def route_ubuntu(self): self.routes.update( { "https://changelogs.ubuntu.com/meta-release-lts": FakeResponse( @@ -237,29 +158,76 @@ def _route_ubuntu(self): ) as f: f.write(b"already here") - def _route_arch(self): + def route_arch(self): self.routes.update( { - "https://archlinux.org/releng/releases/": FakeResponse(text=ARCH_HTML), + ARCH_RELEASES: FakeResponse(text=ARCH_HTML), ARCH_TORRENT: FakeResponse(content=b"a1"), } ) - # -- tests -------------------------------------------------------------- - def test_counts_each_download_outcome_and_reports_them_in_the_summary(self): - self._route_ubuntu() +class RunTests(unittest.TestCase): + def setUp(self): + self.world = FakeWorld(self) + patches = [ + unittest.mock.patch.object(ft.requests, "get", self.world.get), + unittest.mock.patch.object(ft.requests, "post", self.world.post), + unittest.mock.patch.object( + ft.socket, "create_connection", self._rpc_port_open + ), + unittest.mock.patch.object(ft, "watch_dir", self.world.watch_dir), + ] + for attr, filename in ( + ("ratio_log_file", "fetch_torrents_ratios.log"), + ("ratio_history_file", "fetch_torrents_ratio_history.json"), + ("removed_history_file", "fetch_torrents_removed_history.json"), + ( + "old_release_check_state_file", + "fetch_torrents_old_release_check_state.json", + ), + ): + patches.append( + unittest.mock.patch.object(ft, attr, self.world.state_path(filename)) + ) + for patcher in patches: + patcher.start() + self.addCleanup(patcher.stop) - messages = self._run(FETCH_TORRENTS_DISTROS="ubuntu") + @staticmethod + @contextlib.contextmanager + def _rpc_port_open(*args, **kwargs): + yield MagicMock() + + def _run(self, **env): + """Runs ft.run() with load_config(env); returns (summary, messages).""" + with self.assertLogs(ft.logger, level="DEBUG") as logs: + summary = ft.run( + ft.load_config(env), + client_factory=self.world.connect, + today=date(2026, 9, 2), + now=lambda: 1000.0, + disk_usage=self.world.disk_usage, + ) + return summary, [record.getMessage() for record in logs.records] + + def test_counts_each_download_outcome(self): + self.world.route_ubuntu() + + summary, messages = self._run(FETCH_TORRENTS_DISTROS="ubuntu") self.assertEqual( - sorted(os.listdir(self.watch_dir)), + sorted(os.listdir(self.world.watch_dir)), [ "ubuntu-24.04-desktop-amd64.iso.torrent", "xubuntu-24.04-desktop-amd64.iso.torrent", "xubuntu-24.04-minimal-amd64.iso.torrent", ], ) + self.assertEqual( + summary, + ft.RunSummary(elapsed=0.0, added=2, existing=1, not_found=1, failed=1), + ) self.assertEqual( messages[-1], "Run complete in 0.00 seconds. 2 added, 1 existing, " @@ -267,53 +235,74 @@ def test_counts_each_download_outcome_and_reports_them_in_the_summary(self): ) def test_distros_not_selected_are_never_fetched(self): - self._route_ubuntu() - self._route_arch() + self.world.route_ubuntu() + self.world.route_arch() self._run(FETCH_TORRENTS_DISTROS="arch") - self.assertEqual( - self.requested, ["https://archlinux.org/releng/releases/", ARCH_TORRENT] - ) + self.assertEqual(self.world.requested, [ARCH_RELEASES, ARCH_TORRENT]) def test_low_previous_ratio_blocks_the_new_release(self): - self._route_arch() - self._write_ratio_log({"archlinux-2026.08.01": 0.4}) + self.world.route_arch() + self.world.write_ratio_log({"archlinux-2026.08.01": 0.4}) - messages = self._run(FETCH_TORRENTS_DISTROS="arch") + _summary, messages = self._run(FETCH_TORRENTS_DISTROS="arch") - self.assertNotIn(ARCH_TORRENT, self.requested) + self.assertNotIn(ARCH_TORRENT, self.world.requested) self.assertIn( "Skipping archlinux-2026.09.01 due to low ratio on previous version.", messages, ) - self.assertEqual(os.listdir(self.watch_dir), []) + self.assertEqual(os.listdir(self.world.watch_dir), []) def test_previous_ratio_at_the_floor_allows_the_new_release(self): - self._route_arch() - self._write_ratio_log({"archlinux-2026.08.01": 1.0}) + self.world.route_arch() + self.world.write_ratio_log({"archlinux-2026.08.01": 1.0}) self._run(FETCH_TORRENTS_DISTROS="arch") - self.assertEqual(os.listdir(self.watch_dir), ["archlinux-2026.09.01.torrent"]) + self.assertEqual( + os.listdir(self.world.watch_dir), ["archlinux-2026.09.01.torrent"] + ) + + def test_skip_ratio_check_fetches_despite_a_low_previous_ratio(self): + self.world.route_arch() + self.world.write_ratio_log({"archlinux-2026.08.01": 0.4}) + + self._run(FETCH_TORRENTS_DISTROS="arch", SKIP_RATIO_CHECK="true") + + self.assertEqual( + os.listdir(self.world.watch_dir), ["archlinux-2026.09.01.torrent"] + ) def test_a_failing_distro_does_not_stop_the_others(self): - self._route_arch() - self.routes["https://changelogs.ubuntu.com/meta-release-lts"] = ConnectionError( - "down" + self.world.route_arch() + self.world.routes["https://changelogs.ubuntu.com/meta-release-lts"] = ( + ConnectionError("down") ) - messages = self._run(FETCH_TORRENTS_DISTROS="ubuntu,arch") + summary, _messages = self._run(FETCH_TORRENTS_DISTROS="ubuntu,arch") - self.assertEqual(os.listdir(self.watch_dir), ["archlinux-2026.09.01.torrent"]) self.assertEqual( - messages[-1], - "Run complete in 0.00 seconds. 1 added, 0 existing, " - "0 not found upstream, 1 failed.", + os.listdir(self.world.watch_dir), ["archlinux-2026.09.01.torrent"] ) + self.assertEqual((summary.added, summary.failed), (1, 1)) + + def test_custom_fetchers_replace_the_default_set(self): + summary = ft.run( + ft.load_config({"FETCH_TORRENTS_DISTROS": "arch"}), + client_factory=self.world.connect, + today=date(2026, 9, 2), + now=lambda: 1000.0, + disk_usage=self.world.disk_usage, + fetchers=(("arch", lambda: {"archlinux-2026.10.01": ARCH_TORRENT}),), + ) + + self.assertEqual(self.world.requested, [ARCH_TORRENT]) + self.assertEqual(summary.not_found, 1) def test_keep_only_latest_selects_the_old_version_cleanup(self): - self.transmission.torrents = [ + self.world.transmission.torrents = [ torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), ] @@ -323,64 +312,261 @@ def test_keep_only_latest_selects_the_old_version_cleanup(self): ) self.assertEqual( - [t.name for t in self.transmission.torrents], - ["ubuntu-24.04-desktop-amd64.iso"], + self.world.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"] ) + def test_keep_only_latest_honors_min_ratio(self): + self.world.transmission.torrents = [ + torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), + torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), + ] + + self._run( + FETCH_TORRENTS_DISTROS="arch", + CLEANUP_KEEP_ONLY_LATEST_VERSION="true", + CLEANUP_MIN_RATIO="3", + ) + + self.assertEqual(len(self.world.transmission.torrents), 2) + def test_default_cleanup_is_stagnation_based(self): # No ratio history yet, so the stagnation cleanup can't judge the old # version and keeps it - unlike keep-only-latest, which removes it. - self.transmission.torrents = [ + self.world.transmission.torrents = [ torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), ] self._run(FETCH_TORRENTS_DISTROS="arch") - self.assertEqual(len(self.transmission.torrents), 2) - self.assertTrue( - os.path.exists( - os.path.join(self.log_dir, "fetch_torrents_ratio_history.json") - ) + self.assertEqual(len(self.world.transmission.torrents), 2) + self.assertEqual( + ft.load_ratio_history(ft.ratio_history_file), + { + "ubuntu-24.04-desktop-amd64.iso": [ + {"date": "2026-09-02", "ratio": 2.0} + ], + "ubuntu-23.10-desktop-amd64.iso": [ + {"date": "2026-09-02", "ratio": 2.0} + ], + }, ) + def test_invalid_cleanup_setting_skips_cleanup_and_continues(self): + self.world.transmission.torrents = [ + torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), + torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), + ] + + _summary, messages = self._run( + FETCH_TORRENTS_DISTROS="arch", + CLEANUP_KEEP_ONLY_LATEST_VERSION="true", + CLEANUP_MIN_RATIO="high", + ) + + self.assertEqual(len(self.world.transmission.torrents), 2) + self.assertIn( + "Could not clean up old versions: CLEANUP_MIN_RATIO: could not " + "convert string to float: 'high'", + messages, + ) + self.assertTrue(messages[-1].startswith("Run complete in")) + def test_disk_usage_limit_runs_after_cleanup(self): - self.transmission.torrents = [ + self.world.transmission.torrents = [ torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), torrent("ubuntu-23.10-desktop-amd64.iso", 2, 0.1), ] - self.disk_used_percent = 99 + self.world.disk_used_percent = 99 self._run( FETCH_TORRENTS_DISTROS="arch", CLEANUP_DISK_USAGE_THRESHOLD_PERCENT="95" ) - cleanup_listing = self.events.index("get_torrents") - first_disk_check = self.events.index("disk_usage") - self.assertLess(cleanup_listing, first_disk_check) - self.assertIn(("remove", 2), self.events[first_disk_check:]) + events = self.world.events + cleanup_connect = events.index("connect") + first_disk_check = events.index("disk_usage") + self.assertLess(cleanup_connect, first_disk_check) + self.assertEqual( + self.world.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"] + ) def test_disk_usage_limit_error_is_logged_not_raised(self): - messages = self._run( + _summary, messages = self._run( FETCH_TORRENTS_DISTROS="arch", CLEANUP_DISK_USAGE_THRESHOLD_PERCENT="lots" ) - self.assertTrue( - any(m.startswith("Could not enforce disk usage limit:") for m in messages) + self.assertIn( + "Could not enforce disk usage limit: " + "CLEANUP_DISK_USAGE_THRESHOLD_PERCENT: could not convert string to " + "float: 'lots'", + messages, ) self.assertTrue(messages[-1].startswith("Run complete in")) + def test_rpc_unavailable_skips_ratio_logging_but_still_cleans_up(self): + def refuse(*args, **kwargs): + raise ConnectionRefusedError + + self.world.transmission.torrents = [ + torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), + torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), + ] + with ( + unittest.mock.patch.object( + ft, "wait_for_transmission_rpc", return_value=False + ), + unittest.mock.patch.object(ft.socket, "create_connection", refuse), + ): + _summary, messages = self._run( + FETCH_TORRENTS_DISTROS="arch", CLEANUP_KEEP_ONLY_LATEST_VERSION="true" + ) + + self.assertIn( + "Transmission RPC not available on localhost:9091; skipping ratio logs.", + messages, + ) + self.assertFalse(any(m.startswith("[ratio]") for m in messages)) + self.assertEqual( + self.world.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"] + ) + def test_seed_ratios_are_logged_for_the_next_run(self): - self.transmission.torrents = [torrent("archlinux-2026.08.01", 1, 0.664)] + self.world.transmission.torrents = [torrent("archlinux-2026.08.01", 1, 0.664)] - self._run(FETCH_TORRENTS_DISTROS="arch") + _summary, messages = self._run(FETCH_TORRENTS_DISTROS="arch") - with open( - os.path.join(self.log_dir, "fetch_torrents_ratios.log"), encoding="utf-8" - ) as f: - text = f.read() - self.assertIn("[ratio] archlinux-2026.08.01", text) - self.assertIn("→ 0.664", text) + start = messages.index("[ratio] RATIOS START") + self.assertEqual( + messages[start : start + 3], + [ + "[ratio] RATIOS START", + f"[ratio] {'archlinux-2026.08.01':<50} → 0.664", + "[ratio] RATIOS END", + ], + ) + + def test_rpc_credentials_from_config_are_used_for_ratio_logging(self): + with unittest.mock.patch.object(ft, "log_seed_ratios_via_http") as log_ratios: + self._run( + FETCH_TORRENTS_DISTROS="arch", + TRANSMISSION_RPC_USERNAME="alice", + TRANSMISSION_RPC_PASSWORD="hunter2", + ) + + log_ratios.assert_called_once_with(auth=("alice", "hunter2")) + + def test_old_release_check_runs_only_when_enabled(self): + with unittest.mock.patch.object( + ft, "check_old_releases_for_demand", return_value=(2, 3, {"x": "y"}) + ) as check: + disabled, _ = self._run(FETCH_TORRENTS_DISTROS="arch") + check.assert_not_called() + + enabled, messages = self._run( + FETCH_TORRENTS_DISTROS="arch", + FETCH_TORRENTS_CHECK_OLD_RELEASES="true", + FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS="4", + ) + + self.assertEqual((disabled.old_added, disabled.old_skipped), (None, None)) + self.assertEqual((enabled.old_added, enabled.old_skipped), (2, 3)) + self.assertEqual(check.call_args.args[2], 4) + self.assertEqual(check.call_args.args[7], date(2026, 9, 2)) + self.assertEqual( + ft.load_old_release_check_state(ft.old_release_check_state_file), + {"x": "y"}, + ) + self.assertTrue( + messages[-1].endswith( + "Old releases (FETCH_TORRENTS_CHECK_OLD_RELEASES): 2 added, 3 skipped." + ) + ) + + def test_invalid_old_release_setting_aborts_the_run(self): + # Unlike cleanup and the disk limit, the old-release check has never + # been guarded - a bad value fails the run at that point. + with self.assertRaisesRegex( + ValueError, "FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS" + ): + self._run( + FETCH_TORRENTS_DISTROS="arch", + FETCH_TORRENTS_CHECK_OLD_RELEASES="true", + FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS="weekly", + ) + + +class MainWiringTests(unittest.TestCase): + """`python fetch_torrents.py` must configure logging (so this run's ratio + log is written for the next run to gate on), load the config from the + environment and call run() with a real Transmission client factory.""" + + def test_running_the_script_performs_a_daily_run(self): + world = FakeWorld(self) + world.transmission.torrents = [ + torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), + torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), + ] + log_dir = helpers.make_temp_dir(self, "daily_run_main_logs_") + logger = logging.getLogger("fetch_torrents") + handlers_before = list(logger.handlers) + + @contextlib.contextmanager + def rpc_port_open(*args, **kwargs): + yield MagicMock() + + with contextlib.ExitStack() as stack: + stack.enter_context( + helpers.isolated_env( + FETCH_TORRENTS_LOG_DIR=log_dir, + FETCH_TORRENTS_DISTROS="arch", + CLEANUP_KEEP_ONLY_LATEST_VERSION="true", + TRANSMISSION_RPC_USERNAME="alice", + TRANSMISSION_RPC_PASSWORD="hunter2", + ) + ) + stack.enter_context(unittest.mock.patch("requests.get", world.get)) + stack.enter_context(unittest.mock.patch("requests.post", world.post)) + client_cls = stack.enter_context( + unittest.mock.patch.object( + sys.modules["transmission_rpc"], + "Client", + side_effect=lambda **kwargs: world.transmission, + ) + ) + stack.enter_context( + unittest.mock.patch.object(socket, "create_connection", rpc_port_open) + ) + stack.enter_context( + unittest.mock.patch.object(shutil, "disk_usage", world.disk_usage) + ) + stack.enter_context(contextlib.redirect_stderr(io.StringIO())) + logs = stack.enter_context(self.assertLogs(logger, level="DEBUG")) + try: + runpy.run_path(SCRIPT, run_name="__main__") + finally: + # assertLogs puts the logger's original handlers back on exit, + # dropping the ones configure_logging() added - close them. + for handler in logger.handlers: + if isinstance(handler, logging.FileHandler): + handler.close() + self.assertEqual(logger.handlers, handlers_before) + + messages = [record.getMessage() for record in logs.records] + self.assertIn("Selected distros for this run: arch", messages) + self.assertTrue(messages[-1].startswith("Run complete in")) + client_cls.assert_called_with( + host="localhost", port=9091, username="alice", password="hunter2" + ) + self.assertEqual(world.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + self.assertEqual( + ft.get_previous_ratios(os.path.join(log_dir, "fetch_torrents_ratios.log")), + { + "ubuntu-24.04-desktop-amd64.iso": 2.0, + "ubuntu-23.10-desktop-amd64.iso": 2.0, + }, + ) + self.assertTrue(os.path.exists(os.path.join(log_dir, "fetch_torrents.log"))) if __name__ == "__main__": diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index 9f9f4e8..6dbeffd 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -1,20 +1,19 @@ -"""Tests for fetch_torrents.py, focused on the cleanup_old_versions() logic. +"""Tests for fetch_torrents.py. -fetch_torrents.py is written to run inside the container: it imports -transmission_rpc (an apk-only package not available via pip) and opens log -files under /logs at import time. To make the module importable in a plain -test environment, this file stubs transmission_rpc and points -FETCH_TORRENTS_LOG_DIR at a temp directory before importing it. +fetch_torrents.py imports transmission_rpc, which the container installs +from Alpine's py3-transmission-rpc and isn't a test dependency, so this file +stubs it in sys.modules before importing the module. Importing it touches no +files (logging is only set up by configure_logging()). Run with: python -m unittest discover -s tests """ import json +import logging +import logging.handlers import os -import shutil import subprocess import sys -import tempfile import types import unittest import unittest.mock @@ -26,9 +25,6 @@ stub.Client = MagicMock() sys.modules["transmission_rpc"] = stub -_TEST_LOG_DIR = tempfile.mkdtemp(prefix="fetch_torrents_test_logs_") -os.environ.setdefault("FETCH_TORRENTS_LOG_DIR", _TEST_LOG_DIR) - REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if REPO_ROOT not in sys.path: sys.path.insert(0, REPO_ROOT) @@ -47,16 +43,6 @@ def setUpModule(): unittest.addModuleCleanup(patcher.stop) -def tearDownModule(): - # The import-time log dir can only be removed once ft's handlers have - # released their files (Windows refuses to delete open files). - for handler in (ft.file_handler, ft.ratio_handler): - ft.logger.removeHandler(handler) - handler.close() - if os.environ.get("FETCH_TORRENTS_LOG_DIR") == _TEST_LOG_DIR: - shutil.rmtree(_TEST_LOG_DIR, ignore_errors=True) - - class FakeTorrent: def __init__(self, name, id_, ratio=0.0, added_date=None): self.name = name @@ -444,10 +430,9 @@ def test_lubuntu_is_ratio_gated_on_its_own_previous_release(self): "lubuntu-23.10-desktop-amd64.iso": 0.2, "ubuntu-23.10-desktop-amd64.iso": 5.0, } - with helpers.isolated_env(): - self.assertFalse( - ft.should_fetch_torrent("lubuntu-24.04-desktop-amd64.iso", ratios) - ) + self.assertFalse( + ft.should_fetch_torrent("lubuntu-24.04-desktop-amd64.iso", ratios) + ) def test_version_and_type_per_distro(self): for name, distro, expected in self.CASES: @@ -490,10 +475,6 @@ class ShouldFetchTorrentRatioKeyTests(unittest.TestCase): actually reports (which include the arch suffix and .iso extension), for all supported distros.""" - def setUp(self): - # Pinned explicitly: SKIP_RATIO_CHECK=true would make every case True. - helpers.use_isolated_env(self, SKIP_RATIO_CHECK="false") - def test_finds_previous_version_and_honors_low_ratio_ubuntu(self): ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.5} self.assertFalse( @@ -563,10 +544,6 @@ class ShouldFetchTorrentRemovedHistoryFallbackTests(unittest.TestCase): the ratio gate for exactly the versions cleanup already gave up on. removed_history (record_removal()'s output) is the fallback source.""" - def setUp(self): - # Pinned explicitly: SKIP_RATIO_CHECK=true would make every case True. - helpers.use_isolated_env(self, SKIP_RATIO_CHECK="false") - def test_falls_back_to_removed_history_ratio_when_not_live(self): removed_history = { "ubuntu-23.10-desktop-amd64.iso": { @@ -698,10 +675,6 @@ class ShouldFetchTorrentDevuanRatioGateTests(unittest.TestCase): fetching a new Devuan release on the ratio achieved by whichever Devuan release we currently hold, using the same 1.0 floor as other distros.""" - def setUp(self): - # Pinned explicitly: SKIP_RATIO_CHECK=true would make every case True. - helpers.use_isolated_env(self, SKIP_RATIO_CHECK="false") - def test_no_previous_devuan_release_fetches(self): self.assertTrue(ft.should_fetch_torrent("devuan_excalibur", ratios={})) @@ -752,9 +725,6 @@ class ShouldFetchTorrentBoundaryTests(unittest.TestCase): """The 1.0 floor is inclusive, only strictly older versions count as the "previous" one, and the branches that bypass the gate entirely.""" - def setUp(self): - helpers.use_isolated_env(self, SKIP_RATIO_CHECK="false") - def test_previous_ratio_exactly_at_floor_allows_fetch(self): ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.0} self.assertTrue( @@ -842,10 +812,11 @@ def test_devuan_live_ratio_beats_removed_history(self): def test_skip_ratio_check_fetches_despite_low_previous_ratio(self): ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.1} - with helpers.isolated_env(SKIP_RATIO_CHECK="TRUE"): - self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + self.assertTrue( + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", ratios, skip_ratio_check=True ) + ) def test_unknown_distro_fetches(self): ratios = {"slackware-15.0-install-dvd.iso": 0.1} @@ -873,53 +844,92 @@ def test_unparseable_stored_ratio_entry_is_skipped(self): ) -class RatioLogSurvivesRestartTests(unittest.TestCase): - """Each container run is a fresh `python fetch_torrents.py` process: the - ratio log from the *previous* run is what should_fetch_torrent() needs in - order to gate a new version's download. Simulate a real restart (fresh - interpreter, pre-existing ratio log on disk) rather than reusing the - already-imported module, since the bug this guards against is specifically - about what happens between import and the first get_previous_ratios() call.""" +def configure_logging_for_test(testcase, log_dir, **env): + """ft.configure_logging() with only `env`, undone when the test ends.""" + saved = ( + list(ft.logger.handlers), + list(ft._configured_handlers), + ft.logger.level, + ft.logger.propagate, + ) + + def restore(): + for handler in ft._configured_handlers: + handler.close() + ft.logger.handlers[:] = saved[0] + ft._configured_handlers[:] = saved[1] + ft.logger.setLevel(saved[2]) + ft.logger.propagate = saved[3] + + testcase.addCleanup(restore) + return ft.configure_logging(log_dir, env=env) + - def test_previous_ratios_are_readable_after_a_fresh_start(self): - log_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_restart_") - ratio_log_path = os.path.join(log_dir, "fetch_torrents_ratios.log") - with open(ratio_log_path, "w", encoding="utf-8") as f: +class RatioLogSurvivesConfigureLoggingTests(unittest.TestCase): + """Each container run is a fresh process: the ratio log from the + *previous* run is what should_fetch_torrent() needs in order to gate a + new version's download. configure_logging() opens that same file with + mode="w" for this run's ratios, so it must not truncate it before + get_previous_ratios() has read it - otherwise the low-ratio fetch gate + is silently defeated for every distro.""" + + def setUp(self): + self.log_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_restart_") + self.path = os.path.join(self.log_dir, "fetch_torrents_ratios.log") + with open(self.path, "w", encoding="utf-8") as f: f.write( "2026-08-01 00:00:00,000 INFO: [ratio] RATIOS START\n" "2026-08-01 00:00:00,000 INFO: [ratio] archlinux-2026.04.01-x86_64.iso → 0.664\n" "2026-08-01 00:00:00,000 INFO: [ratio] RATIOS END\n" ) + def test_previous_ratios_are_readable_after_configure_logging(self): + configure_logging_for_test(self, self.log_dir) + + self.assertEqual( + ft.get_previous_ratios(self.path), + {"archlinux-2026.04.01-x86_64.iso": 0.664}, + ) + + def test_first_ratio_line_of_the_run_replaces_the_previous_log(self): + _stream, _file, ratio_handler = configure_logging_for_test(self, self.log_dir) + + ft.logger.info("[ratio] %-50s → %.3f", "new.iso", 1.0) + ratio_handler.flush() + + self.assertEqual(ft.get_previous_ratios(self.path), {"new.iso": 1.0}) + + +class ImportOpensNoFilesTests(unittest.TestCase): + """Importing fetch_torrents (tests, other scripts) must not create or + truncate anything under FETCH_TORRENTS_LOG_DIR - only + configure_logging() does. Needs a fresh interpreter, since this process + imported the module long ago.""" + + def test_import_leaves_the_log_dir_untouched(self): + log_dir = helpers.make_temp_dir(self, "fetch_torrents_import_") script = ( - "import os, sys, types\n" + "import sys, types\n" "from unittest.mock import MagicMock\n" "stub = types.ModuleType('transmission_rpc')\n" "stub.Client = MagicMock()\n" "sys.modules['transmission_rpc'] = stub\n" f"sys.path.insert(0, {REPO_ROOT!r})\n" "import fetch_torrents as ft\n" - "print(ft.get_previous_ratios(ft.ratio_log_file))\n" + "ft.logger.warning('logged before configure_logging()')\n" ) - env = dict(os.environ, FETCH_TORRENTS_LOG_DIR=log_dir) result = subprocess.run( [sys.executable, "-c", script], check=False, - env=env, + env=dict(os.environ, FETCH_TORRENTS_LOG_DIR=log_dir), capture_output=True, text=True, timeout=30, ) self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn( - "archlinux-2026.04.01-x86_64.iso", - result.stdout, - "get_previous_ratios() found nothing from the prior run's ratio " - "log - it was likely wiped by the ratio FileHandler's mode='w' " - "open at import time, before ever being read. That silently " - "defeats the low-ratio fetch gate for every distro.", - ) + self.assertEqual(os.listdir(log_dir), []) + self.assertEqual(result.stderr, "") class GetPreviousRatiosTests(unittest.TestCase): @@ -1033,66 +1043,49 @@ def test_duplicate_name_in_a_segment_last_one_wins(self): self.assertEqual(ft.get_previous_ratios(path), {"dup.iso": 1.1}) -class LogFileRotationTests(unittest.TestCase): +class ConfigureLoggingTests(unittest.TestCase): """fetch_torrents.log must be bounded (RotatingFileHandler), not a plain FileHandler appending forever for the life of a 'deploy and forget' - container. Import-time behavior driven by env vars, so this uses a fresh - subprocess like RatioLogSurvivesRestartTests, rather than the - already-imported module (whose handler was built once, at whatever env - was in effect during test collection).""" + container.""" - def _run(self, extra_env=None): + def _configure(self, **env): log_dir = helpers.make_temp_dir(self, "fetch_torrents_log_rotation_") - script = ( - "import os, sys, types\n" - "from unittest.mock import MagicMock\n" - "stub = types.ModuleType('transmission_rpc')\n" - "stub.Client = MagicMock()\n" - "sys.modules['transmission_rpc'] = stub\n" - f"sys.path.insert(0, {REPO_ROOT!r})\n" - "import fetch_torrents as ft\n" - "print(type(ft.file_handler).__name__)\n" - "print(ft.file_handler.maxBytes)\n" - "print(ft.file_handler.backupCount)\n" - ) - env = dict(os.environ, FETCH_TORRENTS_LOG_DIR=log_dir, **(extra_env or {})) - result = subprocess.run( - [sys.executable, "-c", script], - check=False, - env=env, - capture_output=True, - text=True, - timeout=30, - ) - self.assertEqual(result.returncode, 0, result.stderr) - lines = result.stdout.splitlines() - return lines[0], int(lines[1]), int(lines[2]) + return configure_logging_for_test(self, log_dir, **env) def test_defaults_to_a_bounded_rotating_handler(self): - handler_type, max_bytes, backup_count = self._run() - self.assertEqual(handler_type, "RotatingFileHandler") - self.assertEqual(max_bytes, 5 * 1024 * 1024) - self.assertEqual(backup_count, 3) + _stream, file_handler, _ratio = self._configure() + self.assertIsInstance(file_handler, logging.handlers.RotatingFileHandler) + self.assertEqual(file_handler.maxBytes, 5 * 1024 * 1024) + self.assertEqual(file_handler.backupCount, 3) def test_env_vars_override_rotation_settings(self): - _handler_type, max_bytes, backup_count = self._run( - { - "FETCH_TORRENTS_LOG_MAX_BYTES": "1000", - "FETCH_TORRENTS_LOG_BACKUP_COUNT": "1", - } + _stream, file_handler, _ratio = self._configure( + FETCH_TORRENTS_LOG_MAX_BYTES="1000", FETCH_TORRENTS_LOG_BACKUP_COUNT="1" ) - self.assertEqual(max_bytes, 1000) - self.assertEqual(backup_count, 1) + self.assertEqual(file_handler.maxBytes, 1000) + self.assertEqual(file_handler.backupCount, 1) def test_invalid_env_vars_fall_back_to_defaults(self): - _handler_type, max_bytes, backup_count = self._run( - { - "FETCH_TORRENTS_LOG_MAX_BYTES": "not-a-number", - "FETCH_TORRENTS_LOG_BACKUP_COUNT": "also-bad", - } + _stream, file_handler, _ratio = self._configure( + FETCH_TORRENTS_LOG_MAX_BYTES="not-a-number", + FETCH_TORRENTS_LOG_BACKUP_COUNT="also-bad", + ) + self.assertEqual(file_handler.maxBytes, 5 * 1024 * 1024) + self.assertEqual(file_handler.backupCount, 3) + + def test_calling_again_replaces_its_own_handlers_only(self): + other = logging.NullHandler() + ft.logger.addHandler(other) + self.addCleanup(ft.logger.removeHandler, other) + + first = self._configure() + second = self._configure() + + self.assertEqual( + [h for h in ft.logger.handlers if h is not other], list(second) ) - self.assertEqual(max_bytes, 5 * 1024 * 1024) - self.assertEqual(backup_count, 3) + self.assertIn(other, ft.logger.handlers) + self.assertTrue(all(h not in ft.logger.handlers for h in first)) class FetchUbuntuLtsNamingTests(unittest.TestCase): @@ -2765,83 +2758,194 @@ def test_no_demand_result_is_still_recorded_for_future_recheck_gating(self): self.assertEqual(new_state[name], {"last_checked": "2026-01-08", "leechers": 0}) -class CleanupRpcAuthTests(unittest.TestCase): - """cleanup_old_versions()/cleanup_stagnant_torrents() must authenticate - their own Transmission RPC connection the same way - configure_transmission.py configured the daemon. Without this, a user - who sets TRANSMISSION_RPC_USERNAME/PASSWORD (as the README requires once - TRANSMISSION_RPC_WHITELIST is widened beyond localhost) finds cleanup - silently stops working forever: every RPC call fails with 401 and the - failure is only logged, never surfaced.""" +class LoadConfigTests(unittest.TestCase): + """load_config() is the one place the daily run reads its environment: + each env var must land on the right field, and a bad number must be + deferred (recorded, then raised by require()) rather than crash the + whole run up front.""" - def setUp(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_auth_") - for attr, filename in ( - ("removed_history_file", "removed_history.json"), - ("ratio_history_file", "ratio_history.json"), - ): - patcher = unittest.mock.patch.object( - ft, attr, os.path.join(tmp_dir, filename) - ) - patcher.start() - self.addCleanup(patcher.stop) + def test_empty_env_gives_the_documented_defaults(self): + self.assertEqual(ft.load_config({}), ft.Config()) + self.assertEqual( + ft.Config(), + ft.Config( + distros=ft.DEFAULT_DISTROS, + include_low_demand=False, + skip_ratio_check=False, + check_old_releases=False, + old_release_min_leechers=10, + old_release_min_leecher_ratio=0.1, + old_release_allow_zero_seeders=False, + old_release_recheck_days=7, + old_release_skip_removed=True, + cleanup_keep_only_latest=False, + cleanup_skip_ratio_check=False, + cleanup_min_ratio=1.0, + stagnation_window_days=30, + stagnation_min_ratio_delta=0.02, + disk_usage_threshold_percent=95.0, + rpc_username=None, + rpc_password=None, + invalid={}, + ), + ) - def _mock_client(self): - client = MagicMock() - client.get_torrents.return_value = [] - return client + def test_each_env_var_sets_its_field(self): + config = ft.load_config( + { + "FETCH_TORRENTS_DISTROS": "devuan, arch", + "FETCH_TORRENTS_INCLUDE_LOW_DEMAND": "true", + "SKIP_RATIO_CHECK": "yes", + "FETCH_TORRENTS_CHECK_OLD_RELEASES": "1", + "FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS": "3", + "FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO": "0.5", + "FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS": "true", + "FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS": "2", + "FETCH_TORRENTS_OLD_RELEASE_SKIP_REMOVED": "false", + "CLEANUP_KEEP_ONLY_LATEST_VERSION": "true", + "CLEANUP_SKIP_RATIO_CHECK": "true", + "CLEANUP_MIN_RATIO": "1.5", + "CLEANUP_STAGNATION_WINDOW_DAYS": "14", + "CLEANUP_STAGNATION_MIN_RATIO_DELTA": "0.1", + "CLEANUP_DISK_USAGE_THRESHOLD_PERCENT": "80", + "TRANSMISSION_RPC_USERNAME": " alice ", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + } + ) - def test_cleanup_old_versions_passes_configured_credentials(self): - with ( - unittest.mock.patch.dict( - os.environ, + self.assertEqual( + config, + ft.Config( + distros=tuple(d for d in ft.ALL_DISTROS if d in {"arch", "devuan"}), + include_low_demand=True, + skip_ratio_check=True, + check_old_releases=True, + old_release_min_leechers=3, + old_release_min_leecher_ratio=0.5, + old_release_allow_zero_seeders=True, + old_release_recheck_days=2, + old_release_skip_removed=False, + cleanup_keep_only_latest=True, + cleanup_skip_ratio_check=True, + cleanup_min_ratio=1.5, + stagnation_window_days=14, + stagnation_min_ratio_delta=0.1, + disk_usage_threshold_percent=80.0, + rpc_username="alice", + rpc_password="hunter2", + ), + ) + + def test_reads_only_the_env_it_is_given(self): + with helpers.isolated_env(SKIP_RATIO_CHECK="true", CLEANUP_MIN_RATIO="9"): + config = ft.load_config({}) + + self.assertFalse(config.skip_ratio_check) + self.assertEqual(config.cleanup_min_ratio, 1.0) + + def test_invalid_number_is_recorded_not_raised(self): + config = ft.load_config( + {"CLEANUP_MIN_RATIO": "high", "CLEANUP_STAGNATION_WINDOW_DAYS": "1.5"} + ) + + self.assertIsNone(config.cleanup_min_ratio) + self.assertIsNone(config.stagnation_window_days) + self.assertEqual( + config.invalid, + { + "cleanup_min_ratio": ( + "CLEANUP_MIN_RATIO: could not convert string to float: 'high'" + ), + "stagnation_window_days": ( + "CLEANUP_STAGNATION_WINDOW_DAYS: invalid literal for int() " + "with base 10: '1.5'" + ), + }, + ) + + def test_require_raises_the_first_invalid_field_named(self): + config = ft.load_config({"CLEANUP_MIN_RATIO": "high"}) + + config.require("stagnation_window_days") # valid: no error + with self.assertRaisesRegex(ValueError, "^CLEANUP_MIN_RATIO: "): + config.require("stagnation_window_days", "cleanup_min_ratio") + + def test_rpc_auth_needs_both_username_and_password(self): + cases = ( + ({}, None), + ({"TRANSMISSION_RPC_USERNAME": "alice"}, None), + ({"TRANSMISSION_RPC_PASSWORD": "hunter2"}, None), + ( + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": " ", + }, + None, + ), + ( { "TRANSMISSION_RPC_USERNAME": "alice", "TRANSMISSION_RPC_PASSWORD": "hunter2", }, + ("alice", "hunter2"), ), - unittest.mock.patch.object( - ft, "Client", return_value=self._mock_client() - ) as mock_client_cls, - ): - ft.cleanup_old_versions() + ) + for env, expected in cases: + with self.subTest(env=env): + self.assertEqual(ft.load_config(env).rpc_auth, expected) - mock_client_cls.assert_called_once_with( + +class MakeClientFactoryTests(unittest.TestCase): + """The daily run's own Transmission connections (cleanup, disk-usage + limit) must authenticate the same way configure_transmission.py + configured the daemon. Without this, a user who sets + TRANSMISSION_RPC_USERNAME/PASSWORD (as the README requires once + TRANSMISSION_RPC_WHITELIST is widened beyond localhost) finds cleanup + silently stops working forever: every RPC call fails with 401 and the + failure is only logged, never surfaced.""" + + def _connect(self, env): + config = ft.load_config(env) + with unittest.mock.patch.object(ft, "Client") as client_cls: + client = ft.make_client_factory(config)() + return client_cls, client + + def test_passes_configured_credentials(self): + client_cls, client = self._connect( + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + } + ) + + client_cls.assert_called_once_with( host="localhost", port=9091, username="alice", password="hunter2" ) + self.assertIs(client, client_cls.return_value) - def test_cleanup_old_versions_passes_no_credentials_when_unset(self): - with ( - helpers.isolated_env(), - unittest.mock.patch.object( - ft, "Client", return_value=self._mock_client() - ) as mock_client_cls, - ): - ft.cleanup_old_versions() + def test_passes_no_credentials_when_unset(self): + client_cls, _client = self._connect({}) - mock_client_cls.assert_called_once_with( + client_cls.assert_called_once_with( host="localhost", port=9091, username=None, password=None ) - def test_cleanup_stagnant_torrents_passes_configured_credentials(self): - with ( - unittest.mock.patch.dict( - os.environ, - { - "TRANSMISSION_RPC_USERNAME": "alice", - "TRANSMISSION_RPC_PASSWORD": "hunter2", - }, - ), - unittest.mock.patch.object( - ft, "Client", return_value=self._mock_client() - ) as mock_client_cls, - ): - ft.cleanup_stagnant_torrents() + def test_blank_credentials_count_as_unset(self): + client_cls, _client = self._connect( + {"TRANSMISSION_RPC_USERNAME": " ", "TRANSMISSION_RPC_PASSWORD": ""} + ) - mock_client_cls.assert_called_once_with( - host="localhost", port=9091, username="alice", password="hunter2" + client_cls.assert_called_once_with( + host="localhost", port=9091, username=None, password=None ) + def test_factory_connects_only_when_called(self): + with unittest.mock.patch.object(ft, "Client") as client_cls: + factory = ft.make_client_factory(ft.load_config({})) + + client_cls.assert_not_called() + self.assertTrue(callable(factory)) + class LogSeedRatiosViaHttpTests(unittest.TestCase): """log_seed_ratios_via_http() must not hang the daily run forever on a @@ -2942,7 +3046,7 @@ class CleanupRemovesWatchFileTests(unittest.TestCase): by default) ever cleans them up.""" def setUp(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_watchfile_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_") for attr, filename in ( ("removed_history_file", "removed_history.json"), ("ratio_history_file", "ratio_history.json"), @@ -2964,54 +3068,65 @@ def _touch_watch_file(self, name): f.write("fake torrent bytes") return path + def _transmission(self): + return helpers.FakeTransmission( + [ + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), + ] + ) + def test_cleanup_old_versions_removes_the_watch_file(self): old_path = self._touch_watch_file("ubuntu-23.10-desktop-amd64.iso") - self._touch_watch_file("ubuntu-24.04-desktop-amd64.iso") - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), - ] - client = MagicMock() - client.get_torrents.return_value = torrents + new_path = self._touch_watch_file("ubuntu-24.04-desktop-amd64.iso") + transmission = self._transmission() - with ( - unittest.mock.patch.dict(os.environ, {"CLEANUP_SKIP_RATIO_CHECK": "true"}), - unittest.mock.patch.object(ft, "Client", return_value=client), - ): - ft.cleanup_old_versions() + ft.cleanup_old_versions( + transmission, today=date(2026, 8, 1), skip_ratio_check=True + ) - client.remove_torrent.assert_called_once_with(2, delete_data=True) + self.assertEqual(transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + self.assertIn(("remove_torrent", 2, True), transmission.calls) self.assertFalse(os.path.exists(old_path)) - history = ft.load_removed_history(ft.removed_history_file) - self.assertEqual(history["ubuntu-23.10-desktop-amd64.iso"]["ratio"], 2.0) + self.assertTrue(os.path.exists(new_path)) + self.assertEqual( + ft.load_removed_history(ft.removed_history_file), + { + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-08-01", + "reason": "keep_only_latest", + "ratio": 2.0, + } + }, + ) def test_missing_watch_file_does_not_raise(self): # No .torrent file created for the superseded version - cleanup must # still succeed and remove the torrent from Transmission. - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), - ] - client = MagicMock() - client.get_torrents.return_value = torrents + transmission = self._transmission() - with ( - unittest.mock.patch.dict(os.environ, {"CLEANUP_SKIP_RATIO_CHECK": "true"}), - unittest.mock.patch.object(ft, "Client", return_value=client), - ): - ft.cleanup_old_versions() # must not raise + ft.cleanup_old_versions( + transmission, today=date(2026, 8, 1), skip_ratio_check=True + ) + + self.assertEqual(transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + + def test_cleanup_old_versions_keeps_versions_below_min_ratio(self): + transmission = helpers.FakeTransmission( + [ + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), + ] + ) - client.remove_torrent.assert_called_once_with(2, delete_data=True) + ft.cleanup_old_versions(transmission, today=date(2026, 8, 1), min_ratio=2.0) + + self.assertEqual(len(transmission.torrents), 2) def test_cleanup_stagnant_torrents_removes_the_watch_file(self): old_path = self._touch_watch_file("ubuntu-23.10-desktop-amd64.iso") self._touch_watch_file("ubuntu-24.04-desktop-amd64.iso") - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), - ] - client = MagicMock() - client.get_torrents.return_value = torrents + transmission = self._transmission() # 32 days old: past the default 30-day stagnation window, but still # within update_ratio_history()'s own 37-day (window + sample @@ -3019,7 +3134,6 @@ def test_cleanup_stagnant_torrents_removes_the_watch_file(self): # history - before plan_stagnation_cleanup() ever sees it. Ratio # unchanged since then, so plan_stagnation_cleanup() must select this # torrent for removal. - # The clock is pinned so a run crossing midnight can't shift the offset. today = date(2026, 8, 1) anchor_date = (today - timedelta(days=32)).isoformat() history = { @@ -3028,16 +3142,34 @@ def test_cleanup_stagnant_torrents_removes_the_watch_file(self): with open(ft.ratio_history_file, "w", encoding="utf-8") as f: json.dump(history, f) - with ( - unittest.mock.patch.object(ft, "Client", return_value=client), - unittest.mock.patch.object(ft, "date", fixed_date(today)), - ): - ft.cleanup_stagnant_torrents() + ft.cleanup_stagnant_torrents(transmission, today=today) - client.remove_torrent.assert_called_once_with(2, delete_data=True) + self.assertEqual(transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) self.assertFalse(os.path.exists(old_path)) - history = ft.load_removed_history(ft.removed_history_file) - self.assertEqual(history["ubuntu-23.10-desktop-amd64.iso"]["ratio"], 2.0) + removed = ft.load_removed_history(ft.removed_history_file) + self.assertEqual( + removed["ubuntu-23.10-desktop-amd64.iso"], + {"removed_date": "2026-08-01", "reason": "stagnant", "ratio": 2.0}, + ) + + def test_cleanup_stagnant_torrents_records_todays_sample(self): + transmission = self._transmission() + + ft.cleanup_stagnant_torrents( + transmission, today=date(2026, 8, 1), window_days=10 + ) + + self.assertEqual( + ft.load_ratio_history(ft.ratio_history_file), + { + "ubuntu-24.04-desktop-amd64.iso": [ + {"date": "2026-08-01", "ratio": 2.0} + ], + "ubuntu-23.10-desktop-amd64.iso": [ + {"date": "2026-08-01", "ratio": 2.0} + ], + }, + ) class PlanDiskPressureCleanupTests(unittest.TestCase): @@ -3152,12 +3284,12 @@ def test_missing_or_none_ratio_sorts_as_zero(self): class EnforceDiskUsageLimitTests(unittest.TestCase): """enforce_disk_usage_limit() wires plan_disk_pressure_cleanup() to a - live Transmission connection and real disk usage, re-checking usage - between removals since ratio data alone can't say how many bytes each - torrent holds.""" + Transmission connection and real disk usage, re-checking usage between + removals since ratio data alone can't say how many bytes each torrent + holds.""" def setUp(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_disk_pressure_") + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_") for attr, filename in ( ("removed_history_file", "removed_history.json"), ("ratio_history_file", "ratio_history.json"), @@ -3173,52 +3305,76 @@ def setUp(self): patcher.start() self.addCleanup(patcher.stop) + self.transmission = helpers.FakeTransmission( + [ + FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0), + FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), + FakeTorrent("ubuntu-22.04-desktop-amd64.iso", 3, ratio=0.2), + ] + ) + self.connections = 0 + + def _connect(self): + self.connections += 1 + return self.transmission + + def _disk(self, base_used, sizes=None): + # 100 GB disk; each torrent's size in GB. + return helpers.FakeDisk( + self.transmission, 100, sizes or {1: 2, 2: 2, 3: 6}, base_used + ) + + def _enforce(self, threshold, disk): + ft.enforce_disk_usage_limit( + self._connect, + threshold, + today=date(2026, 8, 1), + disk_usage=disk, + download_path="/downloads", + ) + def test_disabled_threshold_never_touches_disk_or_rpc(self): - with ( - unittest.mock.patch.object(ft.shutil, "disk_usage") as mock_disk_usage, - unittest.mock.patch.object(ft, "Client") as mock_client_cls, - ): - ft.enforce_disk_usage_limit(0) + disk = self._disk(base_used=99) + + self._enforce(0, disk) - mock_disk_usage.assert_not_called() - mock_client_cls.assert_not_called() + self.assertEqual((disk.paths, self.connections), ([], 0)) def test_below_threshold_does_not_connect_to_transmission(self): - with ( - unittest.mock.patch.object( - ft.shutil, "disk_usage", return_value=(100, 50, 50) - ), - unittest.mock.patch.object(ft, "Client") as mock_client_cls, - ): - ft.enforce_disk_usage_limit(95) + disk = self._disk(base_used=40) # 50% - mock_client_cls.assert_not_called() + self._enforce(95, disk) + + self.assertEqual(disk.paths, ["/downloads"]) + self.assertEqual(self.connections, 0) def test_removes_lowest_ratio_candidates_until_under_threshold(self): - torrents = [ - FakeTorrent( - "ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0 - ), # latest, never removed - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), - FakeTorrent("ubuntu-22.04-desktop-amd64.iso", 3, ratio=0.2), - ] - client = MagicMock() - client.get_torrents.return_value = torrents + # 86 + 10 = 96% used. Removing 22.04 (lowest ratio, 6 GB) gets to + # 90%, under the threshold, so 23.10 must be left alone. + disk = self._disk(base_used=86) - # 96% initially (over threshold); drops to 90% (under threshold) - # after the first removal - the second, higher-ratio candidate must - # be left alone. - with ( - unittest.mock.patch.object( - ft.shutil, - "disk_usage", - side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], - ), - unittest.mock.patch.object(ft, "Client", return_value=client), - ): - ft.enforce_disk_usage_limit(95) + self._enforce(95, disk) - client.remove_torrent.assert_called_once_with(3, delete_data=True) + self.assertEqual( + self.transmission.names, + ["ubuntu-23.10-desktop-amd64.iso", "ubuntu-24.04-desktop-amd64.iso"], + ) + + def test_keeps_removing_while_still_over_threshold(self): + # 96%; removing 22.04 (1 GB) leaves 95% - still at the threshold - + # so 23.10 (4 GB) goes too. + disk = self._disk(base_used=90, sizes={1: 1, 2: 4, 3: 1}) + + self._enforce(95, disk) + + self.assertEqual(self.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + + def test_never_removes_the_latest_version(self): + disk = self._disk(base_used=99) + + self._enforce(95, disk) + + self.assertEqual(self.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) def test_removes_the_watch_file_for_each_removal(self): watch_path = os.path.join( @@ -3227,65 +3383,32 @@ def test_removes_the_watch_file_for_each_removal(self): with open(watch_path, "w", encoding="utf-8") as f: f.write("fake torrent bytes") - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), - FakeTorrent("ubuntu-22.04-desktop-amd64.iso", 3, ratio=0.2), - ] - client = MagicMock() - client.get_torrents.return_value = torrents - - with ( - unittest.mock.patch.object( - ft.shutil, - "disk_usage", - side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], - ), - unittest.mock.patch.object(ft, "Client", return_value=client), - ): - ft.enforce_disk_usage_limit(95) + self._enforce(95, self._disk(base_used=86)) self.assertFalse(os.path.exists(watch_path)) def test_no_candidates_logs_warning_and_does_not_raise(self): - torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0)] - client = MagicMock() - client.get_torrents.return_value = torrents + self.transmission.torrents = self.transmission.torrents[:1] - with ( - unittest.mock.patch.object( - ft.shutil, "disk_usage", return_value=(100, 96, 4) - ), - unittest.mock.patch.object(ft, "Client", return_value=client), - ): - ft.enforce_disk_usage_limit(95) # must not raise + with self.assertLogs(ft.logger, level="WARNING") as logs: + self._enforce(95, self._disk(base_used=96)) - client.remove_torrent.assert_not_called() + self.assertEqual(self.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + self.assertIn("no superseded versions are available", logs.output[0]) def test_records_removal_reason_as_disk_pressure(self): - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), - FakeTorrent("ubuntu-22.04-desktop-amd64.iso", 3, ratio=0.2), - ] - client = MagicMock() - client.get_torrents.return_value = torrents + self._enforce(95, self._disk(base_used=86)) - with ( - unittest.mock.patch.object( - ft.shutil, - "disk_usage", - side_effect=[(100, 96, 4), (100, 96, 4), (100, 90, 10)], - ), - unittest.mock.patch.object(ft, "Client", return_value=client), - ): - ft.enforce_disk_usage_limit(95) - - history = ft.load_removed_history(ft.removed_history_file) self.assertEqual( - history["ubuntu-22.04-desktop-amd64.iso"]["reason"], "disk_pressure" + ft.load_removed_history(ft.removed_history_file), + { + "ubuntu-22.04-desktop-amd64.iso": { + "removed_date": "2026-08-01", + "reason": "disk_pressure", + "ratio": 0.2, + } + }, ) - self.assertEqual(history["ubuntu-22.04-desktop-amd64.iso"]["ratio"], 0.2) if __name__ == "__main__": From f02e787c4f075438f57d4237c422727d173af62b Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 12:48:15 +0200 Subject: [PATCH 17/46] Close torrent_scrape parser gaps found by mutmut (task 24) Adds tests for bencode str encoding, missing scrape counts, UDP connect and scrape responses that are short, carry trailing bytes, an error action or another transaction's ID, distinct connect/scrape transaction IDs, the default timeout, and address resolution. Also pins the primary announce going first, the single 'announce' substitution in derive_scrape_url and scrape_torrent's default timeout. The UDP receive buffer size becomes UDP_RECV_BUFFER_BYTES with `# pragma: no mutate`: any size above the largest response read works, so no test can tell 2048 from 2049. Focused mutmut run on torrent_scrape.py: every function at 100% (string-literal-only survivors excluded). Co-Authored-By: Claude Opus 5.5 --- tests/test_torrent_scrape.py | 150 ++++++++++++++++++++++++++++++++++- torrent_scrape.py | 7 +- 2 files changed, 152 insertions(+), 5 deletions(-) diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index 1f3a3c6..c6d9ba6 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -124,6 +124,28 @@ def test_bencode_dict_sorts_keys(self): self.assertEqual(encoded, b"d5:applei2e5:zebrai1ee") +class BencodeStrTests(unittest.TestCase): + def test_str_encodes_as_utf8_byte_string(self): + self.assertEqual(ts.bencode("abc"), b"3:abc") + # Length is in encoded bytes, not characters. + self.assertEqual(ts.bencode("æ"), b"2:" + "æ".encode()) + + def test_str_keys_encode_like_bytes_keys(self): + self.assertEqual( + ts.bencode({"cow": b"moo", "spam": 1}), + ts.bencode({b"cow": b"moo", b"spam": 1}), + ) + + def test_mixed_keys_are_sorted_by_byte_value(self): + # "B" (0x42) sorts before "a" (0x61) by byte value. + self.assertEqual(ts.bencode({"a": 1, b"B": 2}), b"d1:Bi2e1:ai1ee") + + def test_unsupported_types_raise(self): + for value in (True, 1.5, None, ("tuple",)): + with self.subTest(value=value), self.assertRaises(ts.BencodeError): + ts.bencode(value) + + class ComputeInfoHashTests(unittest.TestCase): def test_matches_sha1_of_raw_info_bytes(self): info_bytes = ( @@ -191,6 +213,30 @@ def test_falls_back_to_bare_announce(self): ts.get_announce_urls(torrent_bytes), ["http://only.example.com/announce"] ) + def test_primary_announce_missing_from_the_list_goes_first(self): + torrent_bytes = ( + b"d" + + bstr("announce") + + bstr("http://primary.example.com/announce") + + bstr("announce-list") + + b"ll" + + bstr("http://a.example.com/announce") + + b"el" + + bstr("http://b.example.com/announce") + + b"ee" + + bstr("info") + + b"de" + + b"e" + ) + self.assertEqual( + ts.get_announce_urls(torrent_bytes), + [ + "http://primary.example.com/announce", + "http://a.example.com/announce", + "http://b.example.com/announce", + ], + ) + class DeriveScrapeUrlTests(unittest.TestCase): def test_examples_from_the_scrape_convention(self): @@ -203,6 +249,11 @@ def test_examples_from_the_scrape_convention(self): "http://example.com/a/scrape?x2%0644", ), ("http://example.com/a", None), + # Only the first 'announce' in the segment is substituted. + ( + "http://example.com/announce_announce", + "http://example.com/scrape_announce", + ), ] for announce_url, expected in cases: with self.subTest(announce_url=announce_url): @@ -246,6 +297,14 @@ def test_extracts_seeders_leechers_completed(self): stats = ts.parse_scrape_response(response, info_hash) self.assertEqual(stats, {"seeders": 33, "leechers": 73, "completed": 9}) + def test_missing_counts_default_to_zero(self): + info_hash = b"A" * 20 + response = b"d5:filesd" + bstr(info_hash) + b"dee" + b"e" + self.assertEqual( + ts.parse_scrape_response(response, info_hash), + {"seeders": 0, "leechers": 0, "completed": 0}, + ) + def test_returns_none_when_hash_absent(self): response = b"d5:filesdee" + b"e" self.assertIsNone(ts.parse_scrape_response(response, b"A" * 20)) @@ -324,6 +383,27 @@ def test_short_response_raises(self): with self.assertRaises(ts.UdpTrackerError): ts.parse_udp_connect_response(b"\x00" * 8, transaction_id=42) + def test_short_response_reports_its_length(self): + with self.assertRaisesRegex(ts.UdpTrackerError, r"too short \(15 bytes\)"): + ts.parse_udp_connect_response(b"\x00" * 15, transaction_id=42) + + def test_trailing_bytes_after_connect_response_are_ignored(self): + response = _connect_response(transaction_id=42, connection_id=7) + b"xx" + self.assertEqual(ts.parse_udp_connect_response(response, transaction_id=42), 7) + + def test_error_action_with_empty_message_is_still_a_tracker_error(self): + # Exactly 8 bytes: action + transaction id, no message text. + with self.assertRaisesRegex(ts.UdpTrackerError, r"^Tracker error: $"): + ts.parse_udp_connect_response( + _error_response(transaction_id=42, message=""), transaction_id=42 + ) + + def test_error_action_for_another_transaction_is_not_reported_as_ours(self): + response = _error_response(transaction_id=99, message="not for you") + with self.assertRaises(ts.UdpTrackerError) as ctx: + ts.parse_udp_connect_response(response, transaction_id=42) + self.assertNotIn("not for you", str(ctx.exception)) + def test_tracker_error_action_raises_with_message(self): response = _error_response(transaction_id=42, message="bad request") with self.assertRaises(ts.UdpTrackerError) as ctx: @@ -376,6 +456,20 @@ def test_short_response_raises(self): with self.assertRaises(ts.UdpTrackerError): ts.parse_udp_scrape_response(b"\x00" * 4, transaction_id=7) + def test_response_one_byte_short_of_one_hash_raises(self): + response = _scrape_response(7, seeders=1, completed=1, leechers=1)[:19] + with self.assertRaisesRegex(ts.UdpTrackerError, r"too short \(19 bytes\)"): + ts.parse_udp_scrape_response(response, transaction_id=7) + + def test_trailing_bytes_after_first_hash_are_ignored(self): + response = ( + _scrape_response(7, seeders=1, completed=2, leechers=3) + b"\x00" * 12 + ) + self.assertEqual( + ts.parse_udp_scrape_response(response, transaction_id=7), + {"seeders": 1, "leechers": 3, "completed": 2}, + ) + def test_tracker_error_action_raises_with_message(self): response = _error_response(transaction_id=7, message="scrape not supported") with self.assertRaises(ts.UdpTrackerError) as ctx: @@ -399,7 +493,7 @@ def test_full_round_trip_returns_stats(self): transaction_id=0x00000001, connection_id=0xCAFEBABE ) scrape_resp = _scrape_response( - transaction_id=0x00000001, seeders=5, completed=2, leechers=3 + transaction_id=0x00000002, seeders=5, completed=2, leechers=3 ) fake_sock = self._make_fake_socket( [ @@ -411,7 +505,7 @@ def test_full_round_trip_returns_stats(self): with ( unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), unittest.mock.patch.object( - ts.os, "urandom", return_value=b"\x00\x00\x00\x01" + ts.os, "urandom", side_effect=[b"\x00\x00\x00\x01", b"\x00\x00\x00\x02"] ), ): stats = ts.scrape_udp_tracker( @@ -420,7 +514,10 @@ def test_full_round_trip_returns_stats(self): self.assertEqual(stats, {"seeders": 5, "leechers": 3, "completed": 2}) fake_sock.settimeout.assert_called_once_with(5) - self.assertEqual(fake_sock.sendto.call_count, 2) + self.assertEqual( + [call.args[1] for call in fake_sock.sendto.call_args_list], + [("tracker.example.com", 1337)] * 2, + ) # Second (scrape) request must carry the connection_id from the connect response. scrape_request_bytes = fake_sock.sendto.call_args_list[1][0][0] connection_id_sent, action_sent = struct.unpack( @@ -430,6 +527,37 @@ def test_full_round_trip_returns_stats(self): self.assertEqual(action_sent, 2) fake_sock.close.assert_called_once() + def test_scrape_response_echoing_the_connect_id_raises(self): + info_hash = b"A" * 20 + fake_sock = self._make_fake_socket( + [ + (_connect_response(transaction_id=1, connection_id=9), None), + (_scrape_response(1, seeders=5, completed=2, leechers=3), None), + ] + ) + + with ( + unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), + unittest.mock.patch.object( + ts.os, "urandom", side_effect=[b"\x00\x00\x00\x01", b"\x00\x00\x00\x02"] + ), + self.assertRaisesRegex(ts.UdpTrackerError, "mismatch in scrape response"), + ): + ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) + + fake_sock.close.assert_called_once() + + def test_default_timeout_is_15_seconds(self): + fake_sock = self._make_fake_socket(TimeoutError("timed out")) + + with ( + unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), + self.assertRaises(socket.timeout), + ): + ts.scrape_udp_tracker("tracker.example.com", 1337, b"A" * 20) + + fake_sock.settimeout.assert_called_once_with(15) + def test_connect_transaction_mismatch_raises_and_closes_socket(self): info_hash = b"A" * 20 bad_connect_resp = _connect_response(transaction_id=0x00000002, connection_id=1) @@ -481,6 +609,21 @@ def test_public_ipv6_is_allowed(self): self.assertFalse(ts._is_disallowed_ip("2001:4860:4860::8888")) +class ResolveAddressesTests(unittest.TestCase): + def test_returns_the_ip_of_every_getaddrinfo_entry(self): + infos = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", 0)), + (socket.AF_INET6, socket.SOCK_STREAM, 6, "", ("2606:2800::1", 0, 0, 0)), + ] + with unittest.mock.patch.object( + ts.socket, "getaddrinfo", return_value=infos + ) as getaddrinfo: + addresses = ts._resolve_addresses("tracker.example.com") + + self.assertEqual(addresses, ["93.184.216.34", "2606:2800::1"]) + getaddrinfo.assert_called_once_with("tracker.example.com", None) + + class IsSafeScrapeHostTests(unittest.TestCase): def test_public_address_is_safe(self): with unittest.mock.patch.object( @@ -576,6 +719,7 @@ def test_uses_udp_tracker_when_it_succeeds(self): result = ts.scrape_torrent(path) mock_udp_scrape.assert_called_once() + self.assertEqual(mock_udp_scrape.call_args.kwargs["timeout"], 15) mock_get.assert_not_called() self.assertEqual(result["seeders"], 5) self.assertEqual(result["leechers"], 7) diff --git a/torrent_scrape.py b/torrent_scrape.py index 226d98a..c5b4eb5 100644 --- a/torrent_scrape.py +++ b/torrent_scrape.py @@ -182,6 +182,9 @@ def derive_scrape_url(announce_url): UDP_ACTION_CONNECT = 0 UDP_ACTION_SCRAPE = 2 UDP_ACTION_ERROR = 3 +# Comfortably larger than any single-hash response (20 bytes) or error +# message, so any value >= that size behaves identically. +UDP_RECV_BUFFER_BYTES = 2048 # pragma: no mutate def parse_udp_tracker_url(announce_url): @@ -267,7 +270,7 @@ def scrape_udp_tracker(host, port, info_hash, timeout=15): connect_transaction_id = _random_transaction_id() sock.sendto(build_udp_connect_request(connect_transaction_id), addr) - data, _ = sock.recvfrom(2048) + data, _ = sock.recvfrom(UDP_RECV_BUFFER_BYTES) connection_id = parse_udp_connect_response(data, connect_transaction_id) scrape_transaction_id = _random_transaction_id() @@ -275,7 +278,7 @@ def scrape_udp_tracker(host, port, info_hash, timeout=15): build_udp_scrape_request(connection_id, scrape_transaction_id, info_hash), addr, ) - data, _ = sock.recvfrom(2048) + data, _ = sock.recvfrom(UDP_RECV_BUFFER_BYTES) return parse_udp_scrape_response(data, scrape_transaction_id) finally: sock.close() From c92584fb406af7503d7e8b5ed452c227c30a10aa Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 12:51:57 +0200 Subject: [PATCH 18/46] Check Transmission doubles against the real library (task 20) transmission_rpc isn't apk-only: alpine:latest (3.24) ships py3-transmission-rpc 7.0.11, which is also on PyPI. Pin that version in requirements-test.txt and drop the sys.modules MagicMock stubs: - helpers.FakeTorrent builds a real transmission_rpc.Torrent from RPC fields, so tests only read attributes the library has. DuckTorrent stays for the few tests of production's defensive None/float branches. - helpers.FakeTransmission binds each call against the real Client signature, so an argument the library wouldn't accept raises TypeError (tests/test_transmission_double.py pins this). - make_client_factory tests patch Client with autospec=True. - log_seed_ratios_via_http's request is checked against the library's RpcMethod and TORRENT_GET_ARGS tables, and its ratio ordering/None handling is pinned. - mypy now type-checks against the real library (override removed). Renaming Client's username= or remove_torrent's delete_data= in production now fails the suite. Co-Authored-By: Claude Opus 5.5 --- .claude/CLAUDE.md | 2 +- CLAUDE.md | 18 +++-- pyproject.toml | 6 -- requirements-test.txt | 4 + tests/helpers.py | 48 ++++++++++-- tests/test_daily_run.py | 8 +- tests/test_fetch_torrents.py | 125 +++++++++++++++++++++++------- tests/test_transmission_double.py | 58 ++++++++++++++ 8 files changed, 214 insertions(+), 55 deletions(-) create mode 100644 tests/test_transmission_double.py diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 987d9ff..47710a2 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -3,7 +3,7 @@ Testing rules and the import-stubbing pattern live in the root [CLAUDE.md](../CLAUDE.md); this file only records dev-team toolchain conventions. ## Stack -- Python 3 (runs in an Alpine container; `transmission_rpc` is apk-only) +- Python 3 (runs in an Alpine container; `transmission_rpc` comes from apk there, pinned from PyPI for tests) - Docker (`Dockerfile`, `entrypoint.sh`) - Tests: stdlib `unittest` (`tests/`) diff --git a/CLAUDE.md b/CLAUDE.md index 0c39171..028b9b6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,12 +33,14 @@ CI runs this same command on every push and PR (`.github/workflows/docker-image. ## Why the test setup looks the way it does -`fetch_torrents.py` is written to run inside the container: it imports -`transmission_rpc` (an apk-only package, not pip-installable). -`tests/test_fetch_torrents.py` stubs `transmission_rpc` via `sys.modules` -before importing the module, so tests run without a container or a live -Transmission daemon. Follow that pattern for any new test file that needs to -import `fetch_torrents`. +`fetch_torrents.py` is written to run inside the container, which installs +`transmission_rpc` from Alpine's `py3-transmission-rpc`. Tests use the same +version from PyPI (`transmission-rpc`, pinned in `requirements-test.txt`; +bump the pin when `alpine:latest` moves), so they run without a container +or a live Transmission daemon. Never talk to a real `Client` in a test: use +`tests/helpers.py` `FakeTransmission` and `FakeTorrent`, which are checked +against the pinned library's `Client` signatures and `Torrent` fields (see +`tests/test_transmission_double.py`), so they can't drift from it. Importing the module opens no files: log handlers are only attached by `configure_logging()`, which the `__main__` block calls. Tests that need it @@ -56,7 +58,7 @@ functions (taking plain data in, returning plain data out) separate from functions that perform I/O (RPC calls, network requests, file writes) — see `plan_cleanup()` vs. `cleanup_old_versions()` for the pattern. -Test-only dependencies (currently just `requests` and `beautifulsoup4`, -needed to import the module — `unittest` itself is stdlib) live in +Test-only dependencies (`requests`, `beautifulsoup4` and `transmission-rpc`, +needed to import the modules — `unittest` itself is stdlib) live in `requirements-test.txt`, separate from the apk packages the container actually runs with. diff --git a/pyproject.toml b/pyproject.toml index 12d9b9c..e8f3ee0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,12 +18,6 @@ ignore = [ python_version = "3.12" files = ["fetch_torrents.py", "torrent_scrape.py", "configure_transmission.py"] -[[tool.mypy.overrides]] -# Installed from Alpine's py3-transmission-rpc in the container; not a test -# dependency, so there's nothing for mypy to read locally or in CI. -module = "transmission_rpc" -ignore_missing_imports = true - [tool.coverage.run] branch = true source_pkgs = ["fetch_torrents", "torrent_scrape", "configure_transmission"] diff --git a/requirements-test.txt b/requirements-test.txt index 1190bd8..21f42cb 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,6 @@ requests beautifulsoup4 +# The container gets this from Alpine's py3-transmission-rpc; keep the pin at +# the version alpine:latest ships (`apk info py3-transmission-rpc`) so the +# test doubles in tests/helpers.py are checked against the same API. +transmission-rpc==7.0.11 diff --git a/tests/helpers.py b/tests/helpers.py index 55de79b..a7bb565 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -5,11 +5,14 @@ this as `import helpers`. """ +import inspect import os import shutil import tempfile import unittest.mock +import transmission_rpc + # Env vars production code reads at call time. A developer who happens to # have any of these exported (e.g. SKIP_RATIO_CHECK=true) must not be able # to flip test outcomes, so isolated_env() removes them unless a test sets @@ -61,22 +64,57 @@ def make_temp_dir(testcase, prefix): return path +def FakeTorrent(name, id_, ratio=0.0, added_date=0): + """A real transmission_rpc.Torrent holding the RPC fields production + code reads (via .name, .id, .ratio, .added_date), so a test can't rely + on an attribute the library doesn't have. `added_date` is a Unix time.""" + return transmission_rpc.Torrent( + fields={ + "id": id_, + "name": name, + "uploadRatio": ratio, + "addedDate": added_date, + } + ) + + +# Captured at import: a test may patch transmission_rpc.Client itself (e.g. +# to hand a FakeTransmission to a runpy'd script). +_REAL_CLIENT = transmission_rpc.Client + + +def _bind_to_client(method_name, args, kwargs): + """The arguments of a call to transmission_rpc.Client., by + parameter name with defaults filled in. Raises TypeError for a call the + real (pinned) Client wouldn't accept.""" + signature = inspect.signature(getattr(_REAL_CLIENT, method_name)) + bound = signature.bind(None, *args, **kwargs) + bound.apply_defaults() + return bound.arguments + + class FakeTransmission: """In-memory stand-in for transmission_rpc.Client: get_torrents() returns what it currently holds, remove_torrent() drops that torrent. - `calls` records both, in order, for tests that care about sequencing.""" + `calls` records both, in order, for tests that care about sequencing. + + Every call is bound against the real Client's signature first, so an + argument the pinned library wouldn't accept raises TypeError here too + (see tests/test_transmission_double.py).""" def __init__(self, torrents=()): self.torrents = list(torrents) self.calls = [] - def get_torrents(self): + def get_torrents(self, *args, **kwargs): + _bind_to_client("get_torrents", args, kwargs) self.calls.append("get_torrents") return list(self.torrents) - def remove_torrent(self, ids, delete_data=False): - self.calls.append(("remove_torrent", ids, delete_data)) - self.torrents = [t for t in self.torrents if t.id != ids] + def remove_torrent(self, *args, **kwargs): + call = _bind_to_client("remove_torrent", args, kwargs) + self.calls.append(("remove_torrent", call["ids"], call["delete_data"])) + self.torrents = [t for t in self.torrents if t.id != call["ids"]] @property def names(self): diff --git a/tests/test_daily_run.py b/tests/test_daily_run.py index a5fbe00..6be9dd6 100644 --- a/tests/test_daily_run.py +++ b/tests/test_daily_run.py @@ -21,17 +21,11 @@ import shutil import socket import sys -import types import unittest import unittest.mock from datetime import date from unittest.mock import MagicMock -if "transmission_rpc" not in sys.modules: - stub = types.ModuleType("transmission_rpc") - stub.Client = MagicMock() - sys.modules["transmission_rpc"] = stub - REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SCRIPT = os.path.join(REPO_ROOT, "fetch_torrents.py") if REPO_ROOT not in sys.path: @@ -75,7 +69,7 @@ def json(self): def torrent(name, id_, ratio): - return types.SimpleNamespace(name=name, id=id_, ratio=ratio, added_date=None) + return helpers.FakeTorrent(name, id_, ratio=ratio) class FakeWorld: diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index 6dbeffd..cb50c0c 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -1,9 +1,8 @@ """Tests for fetch_torrents.py. -fetch_torrents.py imports transmission_rpc, which the container installs -from Alpine's py3-transmission-rpc and isn't a test dependency, so this file -stubs it in sys.modules before importing the module. Importing it touches no -files (logging is only set up by configure_logging()). +Importing fetch_torrents touches no files (logging is only set up by +configure_logging()). transmission_rpc is the real library, pinned in +requirements-test.txt to the version Alpine's py3-transmission-rpc ships. Run with: python -m unittest discover -s tests """ @@ -18,18 +17,13 @@ import unittest import unittest.mock from datetime import UTC, date, timedelta -from unittest.mock import MagicMock - -if "transmission_rpc" not in sys.modules: - stub = types.ModuleType("transmission_rpc") - stub.Client = MagicMock() - sys.modules["transmission_rpc"] = stub REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if REPO_ROOT not in sys.path: sys.path.insert(0, REPO_ROOT) import helpers +import transmission_rpc.constants import fetch_torrents as ft @@ -43,7 +37,14 @@ def setUpModule(): unittest.addModuleCleanup(patcher.stop) -class FakeTorrent: +FakeTorrent = helpers.FakeTorrent + + +class DuckTorrent: + """A torrent-shaped object a real transmission_rpc.Torrent can't be - + ratio None, or added_date None/a float/a datetime - only for exercising + the defensive getattr()/`or 0.0` branches in production code.""" + def __init__(self, name, id_, ratio=0.0, added_date=None): self.name = name self.id = id_ @@ -51,7 +52,7 @@ def __init__(self, name, id_, ratio=0.0, added_date=None): self.added_date = added_date def __repr__(self): - return f"FakeTorrent({self.name!r}, ratio={self.ratio})" + return f"DuckTorrent({self.name!r}, ratio={self.ratio})" def fixed_date(today): @@ -222,7 +223,7 @@ def test_default_min_ratio_is_one(self): def test_missing_or_none_ratio_counts_as_zero(self): no_attr = types.SimpleNamespace(name="ubuntu-23.04-desktop-amd64.iso", id=2) - none_ratio = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 3, ratio=None) + none_ratio = DuckTorrent("ubuntu-23.10-desktop-amd64.iso", 3, ratio=None) torrents = [ FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), no_attr, @@ -331,8 +332,8 @@ def test_missing_added_date_does_not_crash(self): # Real Transmission torrents always carry added_date; this just # guards _devuan_sort_key() against a hypothetical missing attribute. torrents = [ - FakeTorrent("devuan_daedalus", 1, ratio=2.0, added_date=None), - FakeTorrent("devuan_excalibur", 2, ratio=2.0, added_date=None), + DuckTorrent("devuan_daedalus", 1, ratio=2.0, added_date=None), + DuckTorrent("devuan_excalibur", 2, ratio=2.0, added_date=None), ] to_remove, _ = ft.plan_cleanup(torrents) @@ -340,10 +341,10 @@ def test_missing_added_date_does_not_crash(self): self.assertEqual(len(to_remove), 1) def test_release_without_added_date_sorts_as_oldest(self): - undated = FakeTorrent("devuan_daedalus", 1, ratio=2.0, added_date=None) + undated = DuckTorrent("devuan_daedalus", 1, ratio=2.0, added_date=None) torrents = [ undated, - FakeTorrent("devuan_excalibur", 2, ratio=2.0, added_date=0.5), + DuckTorrent("devuan_excalibur", 2, ratio=2.0, added_date=0.5), ] to_remove, _ = ft.plan_cleanup(torrents) @@ -353,14 +354,14 @@ def test_release_without_added_date_sorts_as_oldest(self): def test_datetime_added_dates_are_ordered_by_timestamp(self): from datetime import datetime - older = FakeTorrent( + older = DuckTorrent( "devuan_daedalus", 1, ratio=2.0, added_date=datetime(2026, 1, 1, tzinfo=UTC), ) torrents = [ - FakeTorrent( + DuckTorrent( "devuan_excalibur", 2, ratio=2.0, @@ -909,11 +910,7 @@ class ImportOpensNoFilesTests(unittest.TestCase): def test_import_leaves_the_log_dir_untouched(self): log_dir = helpers.make_temp_dir(self, "fetch_torrents_import_") script = ( - "import sys, types\n" - "from unittest.mock import MagicMock\n" - "stub = types.ModuleType('transmission_rpc')\n" - "stub.Client = MagicMock()\n" - "sys.modules['transmission_rpc'] = stub\n" + "import sys\n" f"sys.path.insert(0, {REPO_ROOT!r})\n" "import fetch_torrents as ft\n" "ft.logger.warning('logged before configure_logging()')\n" @@ -1870,7 +1867,7 @@ def test_missing_or_none_ratio_counts_as_zero(self): torrents = [ FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), types.SimpleNamespace(name="ubuntu-23.04-desktop-amd64.iso", id=2), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 3, ratio=None), + DuckTorrent("ubuntu-23.10-desktop-amd64.iso", 3, ratio=None), ] history = { name: [{"date": (today - timedelta(days=40)).isoformat(), "ratio": -0.5}] @@ -2906,7 +2903,7 @@ class MakeClientFactoryTests(unittest.TestCase): def _connect(self, env): config = ft.load_config(env) - with unittest.mock.patch.object(ft, "Client") as client_cls: + with unittest.mock.patch.object(ft, "Client", autospec=True) as client_cls: client = ft.make_client_factory(config)() return client_cls, client @@ -2940,7 +2937,7 @@ def test_blank_credentials_count_as_unset(self): ) def test_factory_connects_only_when_called(self): - with unittest.mock.patch.object(ft, "Client") as client_cls: + with unittest.mock.patch.object(ft, "Client", autospec=True) as client_cls: factory = ft.make_client_factory(ft.load_config({})) client_cls.assert_not_called() @@ -2964,6 +2961,78 @@ def raise_for_status(self): def json(self): return {"arguments": {"torrents": []}} + def test_requests_follow_the_transmission_rpc_protocol(self): + # The pinned transmission_rpc's own tables of methods and torrent-get + # fields stand in for the RPC spec, so a renamed field fails here. + with unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], + ) as mock_post: + ft.log_seed_ratios_via_http(auth=("alice", "hunter2")) + + session_call, get_call = mock_post.call_args_list + self.assertEqual( + session_call, + unittest.mock.call("http://localhost:9091/transmission/rpc", timeout=15), + ) + self.assertEqual( + get_call, + unittest.mock.call( + "http://localhost:9091/transmission/rpc", + json={ + "method": "torrent-get", + "arguments": {"fields": ["name", "uploadRatio"]}, + }, + headers={"X-Transmission-Session-Id": "abc"}, + auth=("alice", "hunter2"), + timeout=15, + ), + ) + payload = get_call.kwargs["json"] + self.assertEqual( + payload["method"], transmission_rpc.constants.RpcMethod.TorrentGet.value + ) + self.assertLessEqual( + set(payload["arguments"]["fields"]), + set(transmission_rpc.constants.TORRENT_GET_ARGS), + ) + + def test_logs_ratios_highest_first_with_none_as_zero(self): + response = self.FakeTorrentGetResponse() + response.json = lambda: { + "arguments": { + "torrents": [ + {"name": "low.iso", "uploadRatio": 0.5}, + {"name": "unset.iso", "uploadRatio": None}, + {"name": "high.iso", "uploadRatio": 2.25}, + ] + } + } + with ( + unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[self.FakeSessionIdResponse(), response], + ), + self.assertLogs(ft.logger, level="INFO") as logs, + ): + ft.log_seed_ratios_via_http(auth=None) + + ratio_lines = [ + r.getMessage() for r in logs.records if r.getMessage().startswith("[ratio]") + ] + self.assertEqual( + ratio_lines, + [ + "[ratio] RATIOS START", + f"[ratio] {'high.iso':<50} → 2.250", + f"[ratio] {'low.iso':<50} → 0.500", + f"[ratio] {'unset.iso':<50} → 0.000", + "[ratio] RATIOS END", + ], + ) + def test_session_id_request_has_a_timeout(self): with unittest.mock.patch.object( ft.requests, @@ -3263,7 +3332,7 @@ def test_usage_exactly_at_threshold_triggers_cleanup(self): def test_missing_or_none_ratio_sorts_as_zero(self): half = FakeTorrent("ubuntu-22.10-desktop-amd64.iso", 2, ratio=0.5) no_attr = types.SimpleNamespace(name="ubuntu-23.04-desktop-amd64.iso", id=3) - none_ratio = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 4, ratio=None) + none_ratio = DuckTorrent("ubuntu-23.10-desktop-amd64.iso", 4, ratio=None) torrents = [ FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), half, diff --git a/tests/test_transmission_double.py b/tests/test_transmission_double.py new file mode 100644 index 0000000..2db5c2e --- /dev/null +++ b/tests/test_transmission_double.py @@ -0,0 +1,58 @@ +"""The Transmission doubles in tests/helpers.py must not drift from the real +transmission_rpc library (pinned in requirements-test.txt to the version +Alpine's py3-transmission-rpc ships): a call production code makes that the +real Client wouldn't accept has to fail here, not in the container. + +Run with: python -m unittest discover -s tests +""" + +import unittest + +import helpers +import transmission_rpc + + +class FakeTransmissionMatchesClientTests(unittest.TestCase): + def test_accepts_every_call_shape_the_real_client_accepts(self): + transmission = helpers.FakeTransmission( + [helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1)] + ) + + transmission.get_torrents(ids=[1], arguments=["name"], timeout=5) + transmission.remove_torrent(ids=1, delete_data=True, timeout=5) + + self.assertEqual(transmission.torrents, []) + + def test_rejects_keywords_the_real_client_lacks(self): + transmission = helpers.FakeTransmission() + + with self.assertRaises(TypeError): + transmission.remove_torrent(1, delete=True) + with self.assertRaises(TypeError): + transmission.get_torrents(fields=["name"]) + + def test_every_public_method_exists_on_the_real_client(self): + own = { + name + for name, value in vars(helpers.FakeTransmission).items() + if callable(value) and not name.startswith("_") + } + + self.assertEqual(own - set(dir(transmission_rpc.Client)), set()) + + +class FakeTorrentTests(unittest.TestCase): + def test_is_a_real_torrent_reading_real_rpc_field_names(self): + torrent = helpers.FakeTorrent( + "devuan_excalibur", 7, ratio=1.25, added_date=2000 + ) + + self.assertIsInstance(torrent, transmission_rpc.Torrent) + self.assertEqual( + (torrent.name, torrent.id, torrent.ratio, torrent.added_date.timestamp()), + ("devuan_excalibur", 7, 1.25, 2000.0), + ) + + +if __name__ == "__main__": + unittest.main() From aa5ef930e3b4ed3031d98c8093e5eb3dbbbe1479 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 12:54:13 +0200 Subject: [PATCH 19/46] Smoke-test the image and run the suite inside it before pushing (task 22) The image was built and pushed without ever being started, and the test job only exercised pip packages on Ubuntu, never the apk ones the container runs with. - scripts/smoke_test.sh starts the image with --network none, as root and with RUN_AS_NON_ROOT=true PUID=PGID=1234. It checks entrypoint.sh got going, configure_transmission.py applied an override to settings.json, transmission-daemon is running (as uid 1234 in the non-root case), /config is owned by 1234:1234, and every module imports. About 5 s. - A Dockerfile `test` stage runs the unit suite on the image's own packages. BuildKit skips it in a normal build, and the final stage is the runtime image unchanged. - The build job runs both before any push step, so PRs get them too. Checked locally against a deliberately broken image (configure step exiting 1): the smoke test fails with the container logs. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/docker-image.yml | 13 +++++ CONTRIBUTING.md | 17 ++++++ Dockerfile | 13 ++++- scripts/smoke_test.sh | 94 ++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 1 deletion(-) create mode 100755 scripts/smoke_test.sh diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index a2f7ca8..89afc40 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -83,6 +83,19 @@ jobs: steps: - uses: actions/checkout@v7 + # The test job uses pip packages on Ubuntu; this reruns the suite on the + # apk versions the container actually ships (Dockerfile `test` stage). + - name: Run unit tests inside the image + run: docker build --target test . + + - name: Build image for smoke test + run: docker build -t linux-iso-seeder:smoke . + + # Starts the image (root and RUN_AS_NON_ROOT) and checks it comes up; + # nothing below pushes unless this passes. + - name: Smoke-test the image + run: sh scripts/smoke_test.sh linux-iso-seeder:smoke + - name: Log in to GitHub Container Registry uses: docker/login-action@v4 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6dba329..bca1a9a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,6 +22,23 @@ When making changes to the fetch logic or features: - Verify regex patterns match all intended torrent names (e.g., Ubuntu variants) - Ratio checking is per ISO type (e.g., `installer-amd64`) rather than per distro +## Testing the image + +CI checks the built image before anything is pushed. To run the same checks +locally (Docker required): + +```sh +docker build --target test . # unit suite on the image's apk packages +docker build -t linux-iso-seeder:smoke . +sh scripts/smoke_test.sh linux-iso-seeder:smoke +``` + +The smoke test starts the image with `--network none`, both as root and +with `RUN_AS_NON_ROOT=true`, and checks that `configure_transmission.py` +applied its overrides, that `transmission-daemon` is running, and that every +module imports. When `alpine:latest` moves to a new `py3-transmission-rpc`, +bump the `transmission-rpc` pin in `requirements-test.txt` to match. + ## Mutation testing Mutation testing checks that the tests actually pin behavior, not just diff --git a/Dockerfile b/Dockerfile index 2cc64e2..7bde249 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:latest +FROM alpine:latest AS runtime ARG VERSION=latest LABEL org.opencontainers.image.source=https://github.com/squazz/linux-iso-seeder LABEL org.opencontainers.image.version=$VERSION @@ -37,3 +37,14 @@ RUN mkdir -p /config /downloads /watch /logs VOLUME ["/config", "/downloads", "/watch", "/logs"] ENTRYPOINT ["/entrypoint.sh"] + +# Unit suite against the image's own apk package versions (CI builds this +# with --target test). BuildKit skips this stage for a normal build, and the +# final stage below is the runtime image unchanged - tests/ never ships. +FROM runtime AS test +WORKDIR /src +COPY fetch_torrents.py torrent_scrape.py configure_transmission.py ./ +COPY tests ./tests +RUN python3 -m unittest discover -s tests + +FROM runtime diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh new file mode 100755 index 0000000..8f6485c --- /dev/null +++ b/scripts/smoke_test.sh @@ -0,0 +1,94 @@ +#!/bin/sh +# Starts a built image the way a deployment would and checks it comes up: +# entrypoint.sh runs, configure_transmission.py applies its overrides, +# transmission-daemon is running, RUN_AS_NON_ROOT hands /config to PUID:PGID, +# and every Python module imports against the image's apk packages. +# +# Usage: scripts/smoke_test.sh +# +# --network none keeps it deterministic: the daily fetch inside the +# container logs network errors, which is expected - only startup is checked. +set -eu + +IMAGE=$1 +TIMEOUT_SECONDS=${SMOKE_TIMEOUT_SECONDS:-30} +# Git Bash on Windows would otherwise rewrite /config etc. into Windows paths. +export MSYS_NO_PATHCONV=1 + +CONTAINERS="" +cleanup() { + for cid in $CONTAINERS; do + docker rm -f -v "$cid" >/dev/null 2>&1 || true + done +} +trap cleanup EXIT + +fail() { + echo "SMOKE TEST FAILED: $1" >&2 + if [ -n "${2:-}" ]; then + echo "--- container logs ---" >&2 + docker logs "$2" >&2 2>&1 || true + fi + exit 1 +} + +# Usage: start_container ; prints the container id. +start_container() { + docker run -d --network none \ + -e SKIP_PACKAGE_UPDATES=true \ + -e TRANSMISSION_PEER_LIMIT_GLOBAL=123 \ + "$@" "$IMAGE" +} + +wait_until_started() { + cid=$1 + elapsed=0 + while [ "$elapsed" -lt "$TIMEOUT_SECONDS" ]; do + if [ "$(docker inspect -f '{{.State.Running}}' "$cid")" != "true" ]; then + fail "container exited during startup" "$cid" + fi + if docker logs "$cid" 2>&1 | grep -q "Starting linux-iso-seeder container" && + docker exec "$cid" pgrep -x transmission-daemon >/dev/null; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + fail "transmission-daemon not running after ${TIMEOUT_SECONDS}s" "$cid" +} + +# configure_transmission.py only writes settings.json once it has parsed the +# env and applied every override, so the override being there means it ran +# to completion. +check_configure_ran() { + cid=$1 + docker exec "$cid" python3 -c " +import json, sys +settings = json.load(open('/config/settings.json')) +sys.exit(settings.get('peer-limit-global') != 123) +" || fail "configure_transmission.py did not apply TRANSMISSION_PEER_LIMIT_GLOBAL" "$cid" +} + +echo "== default (root) startup" +cid=$(start_container) +CONTAINERS="$CONTAINERS $cid" +wait_until_started "$cid" +check_configure_ran "$cid" + +echo "== RUN_AS_NON_ROOT=true PUID=1234 PGID=1234 startup" +cid=$(start_container -e RUN_AS_NON_ROOT=true -e PUID=1234 -e PGID=1234) +CONTAINERS="$CONTAINERS $cid" +wait_until_started "$cid" +check_configure_ran "$cid" +owner=$(docker exec "$cid" stat -c %u:%g /config) +[ "$owner" = "1234:1234" ] || fail "/config owned by $owner, expected 1234:1234" "$cid" +daemon_uid=$(docker exec "$cid" sh -c 'stat -c %u /proc/$(pgrep -x transmission-daemon)') +[ "$daemon_uid" = "1234" ] || fail "transmission-daemon runs as uid $daemon_uid, expected 1234" "$cid" + +echo "== imports against the image's apk packages" +docker run --rm --network none -e FETCH_TORRENTS_LOG_DIR=/tmp \ + -w /usr/local/bin --entrypoint python3 "$IMAGE" \ + -c "import fetch_torrents, torrent_scrape, configure_transmission, transmission_rpc, requests, bs4" || + fail "a module failed to import inside the image" + +echo "Smoke test passed." From f69f52251f6868f4ce9b1a8e7063e97dd3cb8e93 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 12:59:14 +0200 Subject: [PATCH 20/46] Replay recorded upstream pages in scraper contract tests (task 21) Each fetcher was only tested against a hand-written ~5-line snippet of what its page was assumed to look like, and fetch_kali_latest had no test at all. - scripts/record_fixtures.py captures every page the fetch_* scrapers read into tests/fixtures/pages//., with the URL and capture time in a .meta.json. Captured 2026-10-19. - tests/test_scrapers.py replays them. Each fetcher (including both *_old_versions) must produce exactly the candidates read off the snapshot by hand. Also covers connection errors, HTTP 404/500, and pages missing the element a scraper looks for. - helpers.fake_response() returns a real requests.Response and replaces the 19 ad-hoc FakeResponse classes, plus the one in test_daily_run. helpers.RecordedPages serves the fixtures by URL and records any unexpected request. - helpers.block_network() makes every test module refuse real socket connections. Bug fixed: fetch_ubuntu_lts, fetch_kali_latest, fetch_kali_old_versions, fetch_devuan_latest and fetch_linuxmint_cinnamon never checked the HTTP status. An error page became "no releases" ({} with nothing logged, for Ubuntu and Kali old versions) or a misleading "could not detect a version" warning. They now call raise_for_status() and log the HTTP error like the other fetchers. fetch_torrents.py branch coverage: 91% -> 96%. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 2 + fetch_torrents.py | 22 +- scripts/record_fixtures.py | 96 + .../fixtures/pages/arch/releng-releases.html | 2933 +++++++++++++++++ .../pages/arch/releng-releases.meta.json | 5 + tests/fixtures/pages/debian/amd64-bt-cd.html | 159 + .../pages/debian/amd64-bt-cd.meta.json | 5 + tests/fixtures/pages/debian/amd64-bt-dvd.html | 143 + .../pages/debian/amd64-bt-dvd.meta.json | 5 + tests/fixtures/pages/debian/arm64-bt-cd.html | 137 + .../pages/debian/arm64-bt-cd.meta.json | 5 + tests/fixtures/pages/debian/arm64-bt-dvd.html | 143 + .../pages/debian/arm64-bt-dvd.meta.json | 5 + tests/fixtures/pages/devuan/get-devuan.html | 421 +++ .../pages/devuan/get-devuan.meta.json | 5 + tests/fixtures/pages/fedora/torrents.html | 116 + .../fixtures/pages/fedora/torrents.meta.json | 5 + tests/fixtures/pages/kali/cdimage.html | 22 + tests/fixtures/pages/kali/cdimage.meta.json | 5 + tests/fixtures/pages/kali/get-kali.html | 330 ++ tests/fixtures/pages/kali/get-kali.meta.json | 5 + tests/fixtures/pages/mint/download.html | 446 +++ tests/fixtures/pages/mint/download.meta.json | 5 + .../pages/ubuntu/meta-release-lts.meta.json | 5 + .../pages/ubuntu/meta-release-lts.txt | 127 + tests/helpers.py | 85 + tests/test_configure_transmission.py | 4 + tests/test_daily_run.py | 58 +- tests/test_fetch_torrents.py | 163 +- tests/test_scrapers.py | 418 +++ tests/test_torrent_scrape.py | 11 +- 31 files changed, 5725 insertions(+), 166 deletions(-) create mode 100644 scripts/record_fixtures.py create mode 100644 tests/fixtures/pages/arch/releng-releases.html create mode 100644 tests/fixtures/pages/arch/releng-releases.meta.json create mode 100644 tests/fixtures/pages/debian/amd64-bt-cd.html create mode 100644 tests/fixtures/pages/debian/amd64-bt-cd.meta.json create mode 100644 tests/fixtures/pages/debian/amd64-bt-dvd.html create mode 100644 tests/fixtures/pages/debian/amd64-bt-dvd.meta.json create mode 100644 tests/fixtures/pages/debian/arm64-bt-cd.html create mode 100644 tests/fixtures/pages/debian/arm64-bt-cd.meta.json create mode 100644 tests/fixtures/pages/debian/arm64-bt-dvd.html create mode 100644 tests/fixtures/pages/debian/arm64-bt-dvd.meta.json create mode 100644 tests/fixtures/pages/devuan/get-devuan.html create mode 100644 tests/fixtures/pages/devuan/get-devuan.meta.json create mode 100644 tests/fixtures/pages/fedora/torrents.html create mode 100644 tests/fixtures/pages/fedora/torrents.meta.json create mode 100644 tests/fixtures/pages/kali/cdimage.html create mode 100644 tests/fixtures/pages/kali/cdimage.meta.json create mode 100644 tests/fixtures/pages/kali/get-kali.html create mode 100644 tests/fixtures/pages/kali/get-kali.meta.json create mode 100644 tests/fixtures/pages/mint/download.html create mode 100644 tests/fixtures/pages/mint/download.meta.json create mode 100644 tests/fixtures/pages/ubuntu/meta-release-lts.meta.json create mode 100644 tests/fixtures/pages/ubuntu/meta-release-lts.txt create mode 100644 tests/test_scrapers.py diff --git a/.gitattributes b/.gitattributes index dfdb8b7..383b679 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,3 @@ *.sh text eol=lf +# Recorded upstream pages: keep byte-for-byte as captured. +tests/fixtures/pages/** -text diff --git a/fetch_torrents.py b/fetch_torrents.py index ae4c85f..c047263 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -542,9 +542,9 @@ def download_torrent(name, url): def fetch_ubuntu_lts(): try: - text = requests.get( - "https://changelogs.ubuntu.com/meta-release-lts", timeout=30 - ).text + r = requests.get("https://changelogs.ubuntu.com/meta-release-lts", timeout=30) + r.raise_for_status() + text = r.text blocks = [b for b in text.strip().split("\n\n") if "Supported: 1" in b] results = {} @@ -628,7 +628,9 @@ def _kali_torrent_urls(ver): def fetch_kali_latest(): url = "https://www.kali.org/get-kali/#kali-installer-images" try: - html = requests.get(url, timeout=30).text + r = requests.get(url, timeout=30) + r.raise_for_status() + html = r.text matches = re.findall(r"kali-linux-(\d+\.\d+)-installer-", html) if not matches: @@ -662,7 +664,9 @@ def fetch_kali_old_versions(): current release to find.""" url = "https://cdimage.kali.org/" try: - html = requests.get(url, timeout=30).text + r = requests.get(url, timeout=30) + r.raise_for_status() + html = r.text soup = BeautifulSoup(html, "html.parser") versions = [] @@ -744,7 +748,9 @@ def fetch_devuan_latest(): release codename (e.g. devuan_excalibur.torrent).""" url = "https://www.devuan.org/get-devuan" try: - text = requests.get(url, timeout=30).text + r = requests.get(url, timeout=30) + r.raise_for_status() + text = r.text match = re.search(r"https://files\.devuan\.org/devuan_\w+\.torrent", text) if not match: logger.warning("Could not find a Devuan release torrent link on %s", url) @@ -761,7 +767,9 @@ def fetch_devuan_latest(): def fetch_linuxmint_cinnamon(): url = "https://www.linuxmint.com/download.php" try: - text = requests.get(url, timeout=30).text + r = requests.get(url, timeout=30) + r.raise_for_status() + text = r.text match = re.search(r"Download Linux Mint ([\d.]+)", text) if not match: logger.warning("Could not detect a Linux Mint version on %s", url) diff --git a/scripts/record_fixtures.py b/scripts/record_fixtures.py new file mode 100644 index 0000000..e331aaa --- /dev/null +++ b/scripts/record_fixtures.py @@ -0,0 +1,96 @@ +"""Captures the upstream pages fetch_torrents' scrapers read into +tests/fixtures/pages/, for tests/test_scrapers.py to replay. + +Usage: python scripts/record_fixtures.py [distro ...] + +Each capture is written as <distro>/<slug>.<ext> plus <slug>.meta.json +holding the URL it came from and when. Re-recording overwrites both, so +`git diff tests/fixtures/pages` shows exactly what changed upstream; the +expected scraper results in tests/test_scrapers.py then need updating to +match the new snapshot (that's the point - review that they still make +sense before accepting them). +""" + +import json +import os +import sys +from datetime import UTC, datetime + +import requests + +FIXTURES_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "tests", + "fixtures", + "pages", +) + +# (distro, slug, url, extension) - every URL a fetch_* scraper requests. +ENDPOINTS = ( + ( + "ubuntu", + "meta-release-lts", + "https://changelogs.ubuntu.com/meta-release-lts", + "txt", + ), + ( + "debian", + "amd64-bt-dvd", + "https://cdimage.debian.org/debian-cd/current/amd64/bt-dvd/", + "html", + ), + ( + "debian", + "arm64-bt-dvd", + "https://cdimage.debian.org/debian-cd/current/arm64/bt-dvd/", + "html", + ), + ( + "debian", + "amd64-bt-cd", + "https://cdimage.debian.org/debian-cd/current/amd64/bt-cd/", + "html", + ), + ( + "debian", + "arm64-bt-cd", + "https://cdimage.debian.org/debian-cd/current/arm64/bt-cd/", + "html", + ), + ("kali", "get-kali", "https://www.kali.org/get-kali/", "html"), + ("kali", "cdimage", "https://cdimage.kali.org/", "html"), + ("arch", "releng-releases", "https://archlinux.org/releng/releases/", "html"), + ("devuan", "get-devuan", "https://www.devuan.org/get-devuan", "html"), + ("mint", "download", "https://www.linuxmint.com/download.php", "html"), + ("fedora", "torrents", "https://torrent.fedoraproject.org/torrents/", "html"), +) + + +def record(distro, slug, url, ext, now): + response = requests.get(url, timeout=30) + response.raise_for_status() + directory = os.path.join(FIXTURES_DIR, distro) + os.makedirs(directory, exist_ok=True) + with open(os.path.join(directory, f"{slug}.{ext}"), "wb") as f: + f.write(response.content) + meta = { + "url": url, + "captured": now.isoformat(timespec="seconds"), + "content_type": response.headers.get("Content-Type"), + } + with open(os.path.join(directory, f"{slug}.meta.json"), "w", encoding="utf-8") as f: + json.dump(meta, f, indent=2) + f.write("\n") + print(f"{distro}/{slug}.{ext}: {len(response.content)} bytes from {url}") + + +def main(argv): + wanted = set(argv) + now = datetime.now(UTC) + for distro, slug, url, ext in ENDPOINTS: + if not wanted or distro in wanted: + record(distro, slug, url, ext, now) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tests/fixtures/pages/arch/releng-releases.html b/tests/fixtures/pages/arch/releng-releases.html new file mode 100644 index 0000000..69cea54 --- /dev/null +++ b/tests/fixtures/pages/arch/releng-releases.html @@ -0,0 +1,2933 @@ +<!DOCTYPE html> + +<html lang="en"> +<head> + <meta charset="utf-8" /> + <meta name="theme-color" content="#08C" /> + <title>Arch Linux - Releases + + + + + + + + + + + + + + +
+ + +
+
+
+ +
+ + +
+

Releases

+ +

This is a list of ISO releases made by the Arch Linux release + engineering team. These are typically done on a monthly cadence, containing + the latest kernel and base packages from the package repositories. Click + the version of each release to read any additional notes or details about + each release.

+

Torrents and magnet URIs are available to download the releases. + Releases listed as not available may still be seeded by peers, but may no + longer be available from WebSeeds. We always recommend running the latest + available release.

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Release DateVersionKernel VersionAvailable?Download Size
+ Torrent +   + Magnet + 2026-10-012026.10.017.2.7Yes1.5 GB
+ Torrent +   + Magnet + 2026-09-012026.09.017.2.2Yes1.5 GB
+ Torrent +   + Magnet + 2026-08-012026.08.017.1.5Yes1.5 GB
+ Torrent +   + Magnet + 2026-07-012026.07.017.0.14No1.5 GB
+ Torrent +   + Magnet + 2026-06-012026.06.017.0.10No1.5 GB
+ Torrent +   + Magnet + 2026-05-012026.05.017.0.3No1.4 GB
+ Torrent +   + Magnet + 2026-04-012026.04.016.19.10No1.4 GB
+ Torrent +   + Magnet + 2026-03-012026.03.016.18.13No1.4 GB
+ Torrent +   + Magnet + 2026-02-012026.02.016.18.7No1.4 GB
+ Torrent +   + Magnet + 2026-01-012026.01.016.18.2No1.4 GB
+ Torrent +   + Magnet + 2025-12-012025.12.016.17.9No1.4 GB
+ Torrent +   + Magnet + 2025-11-012025.11.016.17.6No1.4 GB
+ Torrent +   + Magnet + 2025-10-012025.10.016.16.8No1.4 GB
+ Torrent +   + Magnet + 2025-09-012025.09.016.16.4No1.4 GB
+ Torrent +   + Magnet + 2025-08-012025.08.016.15.8No1.3 GB
+ Torrent +   + Magnet + 2025-07-012025.07.016.15.4No1.3 GB
+ Torrent +   + Magnet + 2025-06-012025.06.016.14.9No1.2 GB
+ Torrent +   + Magnet + 2025-05-012025.05.016.14.4No1.2 GB
+ Torrent +   + Magnet + 2025-04-012025.04.016.13.8No1.2 GB
+ Torrent +   + Magnet + 2025-03-012025.03.016.13.5No1.2 GB
+ Torrent +   + Magnet + 2025-02-012025.02.016.12.10No1.2 GB
+ Torrent +   + Magnet + 2025-01-012025.01.016.12.7No1.1 GB
+ Torrent +   + Magnet + 2024-12-012024.12.016.12.1No1.1 GB
+ Torrent +   + Magnet + 2024-11-012024.11.016.11.5No1.1 GB
+ Torrent +   + Magnet + 2024-10-012024.10.016.10.10No1.1 GB
+ Torrent +   + Magnet + 2024-09-012024.09.016.10.7No1.1 GB
+ Torrent +   + Magnet + 2024-08-012024.08.016.10.2No1.1 GB
+ Torrent +   + Magnet + 2024-07-012024.07.016.9.7No1.1 GB
+ Torrent +   + Magnet + 2024-06-012024.06.016.9.3No1.1 GB
+ Torrent +   + Magnet + 2024-05-012024.05.016.8.8No1.1 GB
+ Torrent +   + Magnet + 2024-04-012024.04.016.8.2No955.3 MB
+ Torrent +   + Magnet + 2024-03-292024.03.296.8.2No955.3 MB
+ Torrent +   + Magnet + 2024-03-012024.03.016.7.6No942.3 MB
+ Torrent +   + Magnet + 2024-02-012024.02.016.7.2No932.3 MB
+ Torrent +   + Magnet + 2024-01-012024.01.016.6.8No883.3 MB
+ Torrent +   + Magnet + 2023-12-012023.12.016.6.3No872.3 MB
+ Torrent +   + Magnet + 2023-11-012023.11.016.5.9No807.3 MB
+ Torrent +   + Magnet + 2023-10-142023.10.146.5.7No808.3 MB
+ Torrent +   + Magnet + 2023-09-012023.09.016.4.12No804.3 MB
+ Torrent +   + Magnet + 2023-08-012023.08.016.4.7No793.3 MB
+ Torrent +   + Magnet + 2023-07-012023.07.016.3.9No813.3 MB
+ Torrent +   + Magnet + 2023-06-012023.06.016.3.5No790.3 MB
+ Torrent +   + Magnet + 2023-05-032023.05.036.2.13No813.3 MB
+ Torrent +   + Magnet + 2023-04-012023.04.016.2.8No809.3 MB
+ Torrent +   + Magnet + 2023-03-012023.03.016.2.1No810.3 MB
+ Torrent +   + Magnet + 2023-02-012023.02.016.1.8No824.3 MB
+ Torrent +   + Magnet + 2023-01-012023.01.016.1.1No816.3 MB
+ Torrent +   + Magnet + 2022-12-012022.12.016.0.10No818.3 MB
+ Torrent +   + Magnet + 2022-11-012022.11.016.0.6No782.3 MB
+ Torrent +   + Magnet + 2022-10-012022.10.015.19.12No798.3 MB
+ Torrent +   + Magnet + 2022-09-032022.09.035.19.6No794.3 MB
+ Torrent +   + Magnet + 2022-08-052022.08.055.18.16No786.3 MB
+ Torrent +   + Magnet + 2022-07-012022.07.015.18.7No795.3 MB
+ Torrent +   + Magnet + 2022-06-012022.06.015.18.1No861.3 MB
+ Torrent +   + Magnet + 2022-05-012022.05.015.17.5No826.3 MB
+ Torrent +   + Magnet + 2022-04-052022.04.055.17.1No824.3 MB
+ Torrent +   + Magnet + 2022-04-012022.04.015.17.1No824.3 MB
+ Torrent +   + Magnet + 2022-03-012022.03.015.16.11No807.3 MB
+ Torrent +   + Magnet + 2022-02-012022.02.015.16.4No812.3 MB
+ Torrent +   + Magnet + 2022-01-012022.01.015.15.12No864.3 MB
+ Torrent +   + Magnet + 2021-12-012021.12.015.15.5No850.3 MB
+ Torrent +   + Magnet + 2021-11-012021.11.015.14.15No846.3 MB
+ Torrent +   + Magnet + 2021-10-012021.10.015.14.8No846.3 MB
+ Torrent +   + Magnet + 2021-09-012021.09.015.13.13No832.3 MB
+ Torrent +   + Magnet + 2021-08-012021.08.015.13.6No831.3 MB
+ Torrent +   + Magnet + 2021-07-012021.07.015.12.13No779.3 MB
+ Torrent +   + Magnet + 2021-06-012021.06.015.12.8No774.3 MB
+ Torrent +   + Magnet + 2021-05-012021.05.015.11.16No755.3 MB
+ Torrent +   + Magnet + 2021-04-012021.04.015.11.11No750.3 MB
+ Torrent +   + Magnet + 2021-03-012021.03.015.11.2No734.3 MB
+ Torrent +   + Magnet + 2021-02-012021.02.015.10.11No695.3 MB
+ Torrent +   + Magnet + 2021-01-012021.01.015.10.3No690.3 MB
+ Torrent +   + Magnet + 2020-12-012020.12.015.9.11No682.3 MB
+ Torrent +   + Magnet + 2020-11-012020.11.015.9.2No731.3 MB
+ Torrent +   + Magnet + 2020-10-012020.10.015.8.12No681.0 MB
+ Torrent +   + Magnet + 2020-09-012020.09.015.8.5No679.0 MB
+ Torrent +   + Magnet + 2020-08-012020.08.015.7.11No671.0 MB
+ Torrent +   + Magnet + 2020-07-012020.07.015.7.6No647.0 MB
+ Torrent +   + Magnet + 2020-06-012020.06.015.6.15No662.0 MB
+ Torrent +   + Magnet + 2020-05-012020.05.015.6.8No652.0 MB
+ Torrent +   + Magnet + 2020-04-012020.04.015.5.13No649.0 MB
+ Torrent +   + Magnet + 2020-03-012020.03.015.5.6No651.0 MB
+ Torrent +   + Magnet + 2020-02-012020.02.015.4.15No646.0 MB
+ Torrent +   + Magnet + 2020-01-012020.01.015.4.6No656.0 MB
+ Torrent +   + Magnet + 2019-12-012019.12.015.3.13No639.0 MB
+ Torrent +   + Magnet + 2019-11-012019.11.015.3.8No635.0 MB
+ Torrent +   + Magnet + 2019-10-012019.10.015.3.1No627.0 MB
+ Torrent +   + Magnet + 2019-09-012019.09.015.2.11No623.0 MB
+ Torrent +   + Magnet + 2019-08-012019.08.015.2.5No622.0 MB
+ Torrent +   + Magnet + 2019-07-012019.07.015.1.15No615.0 MB
+ Torrent +   + Magnet + 2019-06-012019.06.015.1.5No614.0 MB
+ Torrent +   + Magnet + 2019-05-022019.05.025.0.10No609.0 MB
+ Torrent +   + Magnet + 2019-05-012019.05.015.0.10No608.0 MB
+ Torrent +   + Magnet + 2019-04-012019.04.015.0.5No604.0 MB
+ Torrent +   + Magnet + 2019-03-012019.03.014.20.13No602.0 MB
+ Torrent +   + Magnet + 2019-02-012019.02.014.20.6No600.0 MB
+ Torrent +   + Magnet + 2019-01-012019.01.014.20.0No602.0 MB
+ Torrent +   + Magnet + 2018-12-012018.12.014.19.4No588.0 MB
+ Torrent +   + Magnet + 2018-11-012018.11.014.18.16No586.0 MB
+ Torrent +   + Magnet + 2018-10-012018.10.014.18.9No577.0 MB
+ Torrent +   + Magnet + 2018-09-012018.09.014.18.5No571.0 MB
+ Torrent +   + Magnet + 2018-08-012018.08.014.17.11No573.0 MB
+ Torrent +   + Magnet + 2018-07-012018.07.014.17.3No574.0 MB
+ Torrent +   + Magnet + 2018-06-012018.06.014.16.12No571.0 MB
+ Torrent +   + Magnet + 2018-05-012018.05.014.16.5No560.0 MB
+ Torrent +   + Magnet + 2018-04-012018.04.014.15.14No556.0 MB
+ Torrent +   + Magnet + 2018-03-012018.03.014.15.6No551.0 MB
+ Torrent +   + Magnet + 2018-02-012018.02.014.14.15No539.0 MB
+ Torrent +   + Magnet + 2018-01-012018.01.014.14.9No522.0 MB
+ Torrent +   + Magnet + 2017-12-012017.12.014.13.12No516.0 MB
+ Torrent +   + Magnet + 2017-11-012017.11.014.13.9No523.0 MB
+ Torrent +   + Magnet + 2017-10-012017.10.014.13.3No523.0 MB
+ Torrent +   + Magnet + 2017-09-012017.09.014.12.8No518.0 MB
+ Torrent +   + Magnet + 2017-08-012017.08.014.12.3No516.0 MB
+ Torrent +   + Magnet + 2017-07-012017.07.014.11.7No510.0 MB
+ Torrent +   + Magnet + 2017-06-012017.06.014.11.3No488.0 MB
+ Torrent +   + Magnet + 2017-05-012017.05.014.10.13No481.0 MB
+ Torrent +   + Magnet + 2017-04-012017.04.014.10.6No478.0 MB
+ Torrent +   + Magnet + 2017-03-012017.03.014.9.11No476.0 MB
+ Torrent +   + Magnet + 2017-02-012017.02.014.9.6No870.0 MB
+ Torrent +   + Magnet + 2017-01-012017.01.014.8.13No867.0 MB
+ Torrent +   + Magnet + 2016-12-012016.12.014.8.11No799.0 MB
+ Torrent +   + Magnet + 2016-11-012016.11.014.8.6No797.0 MB
+ Torrent +   + Magnet + 2016-10-012016.10.014.7.5No792.0 MB
+ Torrent +   + Magnet + 2016-09-032016.09.034.7.2No756.0 MB
+ Torrent +   + Magnet + 2016-08-012016.08.014.6.4No742.0 MB
+ Torrent +   + Magnet + 2016-07-012016.07.014.6.3No749.0 MB
+ Torrent +   + Magnet + 2016-06-012016.06.014.5.4No744.0 MB
+ Torrent +   + Magnet + 2016-05-012016.05.014.5.1No734.0 MB
+ Torrent +   + Magnet + 2016-04-012016.04.014.4.5No724.0 MB
+ Torrent +   + Magnet + 2016-03-012016.03.014.4.1No709.0 MB
+ Torrent +   + Magnet + 2016-02-012016.02.014.3.3No701.0 MB
+ Torrent +   + Magnet + 2016-01-012016.01.014.3.3No686.0 MB
+ Torrent +   + Magnet + 2015-12-012015.12.014.2.5No663.0 MB
+ Torrent +   + Magnet + 2015-11-012015.11.014.2.5No659.0 MB
+ Torrent +   + Magnet + 2015-10-022015.10.014.2.2No658.0 MB
+ Torrent +   + Magnet + 2015-09-012015.09.014.1.6No658.0 MB
+ Torrent +   + Magnet + 2015-08-012015.08.014.1.3No657.0 MB
+ Torrent +   + Magnet + 2015-07-012015.07.014.0.7No646.0 MB
+ Torrent +   + Magnet + 2015-06-012015.06.014.0.4No637.0 MB
+ Torrent +   + Magnet + 2015-05-012015.05.014.0.1No626.0 MB
+ Torrent +   + Magnet + 2015-04-012015.04.013.19.2No599.0 MB
+ Torrent +   + Magnet + 2015-03-012015.03.013.18.6No595.0 MB
+ Torrent +   + Magnet + 2015-02-012015.02.013.18.4No588.0 MB
+ Torrent +   + Magnet + 2015-01-012015.01.013.17.6No587.0 MB
+ Torrent +   + Magnet + 2014-12-012014.12.013.17.4No578.0 MB
+ Torrent +   + Magnet + 2014-11-012014.11.013.17.1No577.0 MB
+ Torrent +   + Magnet + 2014-10-012014.10.013.16.3No572.0 MB
+ Torrent +   + Magnet + 2014-09-032014.09.033.16.1No574.0 MB
+ Torrent +   + Magnet + 2014-08-012014.08.013.15.7No559.0 MB
+ Torrent +   + Magnet + 2014-07-032014.07.033.15.3No558.0 MB
+ Torrent +   + Magnet + 2014-06-012014.06.013.14.4No547.0 MB
+ Torrent +   + Magnet + 2014-05-012014.05.013.14.1No539.0 MB
+ Torrent +   + Magnet + 2014-04-012014.04.013.13.7No534.0 MB
+ Torrent +   + Magnet + 2014-03-012014.03.013.13.5No527.0 MB
+ Torrent +   + Magnet + 2014-02-012014.02.013.12.9No539.0 MB
+ Torrent +   + Magnet + 2014-01-052014.01.053.12.6No528.0 MB
+ Torrent +   + Magnet + 2013-12-012013.12.013.12.1No522.0 MB
+ Torrent +   + Magnet + 2013-11-012013.11.013.11.6No516.0 MB
+ Torrent +   + Magnet + 2013-10-012013.10.013.11.2No529.0 MB
+ Torrent +   + Magnet + 2013-09-012013.09.013.10.10No525.0 MB
+ Torrent +   + Magnet + 2013-08-012013.08.013.10.3No526.0 MB
+ Torrent +   + Magnet + 2013-07-012013.07.013.9.8No522.0 MB
+ Torrent +   + Magnet + 2013-06-012013.06.013.9.4No523.0 MB
+ Torrent +   + Magnet + 2013-05-012013.05.013.8.10No509.0 MB
+ Torrent +   + Magnet + 2013-04-022013.04.013.8.4No506.0 MB
+ Torrent +   + Magnet + 2013-03-012013.03.013.7.9No501.0 MB
+ Torrent +   + Magnet + 2013-02-012013.02.013.7.5No498.0 MB
+ Torrent +   + Magnet + 2013-01-042013.01.043.6.11No492.0 MB
+ Torrent +   + Magnet + 2012-12-012012.12.013.6.8No429.0 MB
+ Torrent +   + Magnet + 2012-11-012012.11.013.6.4No417.0 MB
+ Torrent +   + Magnet + 2012-10-062012.10.063.5.5No415.0 MB
+ Torrent +   + Magnet + 2012-09-072012.09.073.5.3No392.0 MB
+ Torrent +   + Magnet + 2012-08-042012.08.043.4.7No384.0 MB
+ Torrent +   + Magnet + 2012-07-152012.07.153.4.4No370.0 MB
+ Torrent +   + Magnet + 2011-08-192011.08.193.0.3No673.0 MB
+ Torrent +   + Magnet + 2010-05-162010.052.6.33.4No659.0 MB
2009-08-172009.082.6.30.4No
2009-02-162009.022.6.28No
2008-06-242008.062.6.25.6No
2007-10-072007.08-22.6.22.9No
2007-09-102007.08.12.6.22.6No
2007-08-052007.082.6.22.1No
2007-05-282007.05-Linuxtag20072.6.21.3No
2007-05-172007.052.6.21.1No
2007-03-280.82.6.20.4No
2006-05-180.7.22.6.16.16No
2006-01-050.7.12.6.15No
2005-01-240.72.6.10No
2004-03-010.62.6.3No
2003-07-210.52.4.21No
2002-12-180.42.4.20No
2002-08-070.32.4.19No
2002-04-170.22.4.18No
2002-03-120.12.4.18No
+ + + + + + + + + + + + diff --git a/tests/fixtures/pages/arch/releng-releases.meta.json b/tests/fixtures/pages/arch/releng-releases.meta.json new file mode 100644 index 0000000..105d1b5 --- /dev/null +++ b/tests/fixtures/pages/arch/releng-releases.meta.json @@ -0,0 +1,5 @@ +{ + "url": "https://archlinux.org/releng/releases/", + "captured": "2026-10-19T10:54:57+00:00", + "content_type": "text/html; charset=utf-8" +} diff --git a/tests/fixtures/pages/debian/amd64-bt-cd.html b/tests/fixtures/pages/debian/amd64-bt-cd.html new file mode 100644 index 0000000..a10c187 --- /dev/null +++ b/tests/fixtures/pages/debian/amd64-bt-cd.html @@ -0,0 +1,159 @@ + + + + Index of /debian-cd/current/amd64/bt-cd + + + + +

What's in this directory?

+ + +

These are files containing the installer and other software + for the Debian GNU/Linux operating system. + + + The files in this directory are specifically for + the amd64 architecture. + + +

+ + +

How do I use these files?

+ + +

The files here are bittorrent files. Use + a bittorrent + client program to download the contents of the complete ISO image + files in a peer-to-peer manner. You should end up with an exact + copy of each ISO image as though you'd downloaded it directly via + HTTP or FTP.

+ + +

Once you have downloaded all the ISO images you want, you will + typically need to write them to installation media.

+ +

What size and type of media will I need?

+ +

The images described here are sized to be written to + + writeable CD media + + at a minimum, but may be written to larger media if needed.

+ + +

For extra convenience, these images may also be written + directly to a USB stick. So long as your computer will + boot directly from that USB stick, it should start the Debian + installer that way. + +

+ + +

There are lots of files here! Do I need all of them?

+ +

In most cases it is not necessary to download and + use all of these images to be able to install + Debian on your computer. Debian comes with a massive set of + software packages, hence why it takes so many disks for a + complete set. Most typical users only need a small subset of + those software packages.

+ + + + +

What is a netinst image?

+ +

The netinst CD here is a small CD image that + contains just the core Debian installer code and a small core set + of text-mode programs (known as "standard" in Debian). To install + a desktop or other common software, you'll also need either an + Internet connection or some other Debian CD/DVD images.

+ + + +

What is an edu netinst image?

+ +

The edu netinst CD here is a special version of + the netinst CD image that is targeted specifically at 64-bit + Intel machines. It provides a menu to install the Debian Edu Pure + Blend. See the + Debian Wiki for more information.

+ + + + +

What is a mac netinst image?

+ +

The mac netinst CD here is a special version of + the netinst CD image that is targeted specifically at older + 64-bit Intel Macintosh machines. It will likely work on most + other amd64 machines too, but it does not contain UEFI boot + files that some people + need. See + the Debian Wiki for more information.

+ + + + + +

How can I verify my download is correct and exactly what has + been created by Debian?

+ +

There are files here (SHA512SUMS, etc.) which contain checksums + of the images. These checksum files are also signed - see the + matching .sign files. Once you've downloaded an image, you can + check:

+ +
    +
  • that its checksum matches that expected from the checksum file; and +
  • that the checksum file has not been tampered with. +
+ +

For more information about how to do these steps, read + the verification guide. + + + +

Only the first few images are available! Where are the rest?

+ +

We don't store/serve the full set of ISO images for all + architectures, to reduce the amount of space taken up on the + mirrors. You can + use the jigdo + tool to recreate the missing ISO images instead.

+ + +

Non-free Firmware

+ +

This Debian image build only includes Free Software where + possible. However, many systems include hardware which depends on + non-free firmware to function properly so this build also includes + those firmware files for those cases. See the Debian + Wiki non-free + firmware page for more information. + +

Other questions?

+ +

See the Debian CD FAQ +for lots more information about Debian CDs and installation.

+ +

The images here were put together by +the Debian CD team +, using debian-cd and other software.

+ + + + + + + + + + + + +
[ICO]NameLast modifiedSize

[PARENTDIR]Parent Directory  -
[SUM]SHA256SUMS2026-09-12 18:16 628
[CRT]SHA256SUMS.sign2026-09-12 18:20 833
[SUM]SHA512SUMS2026-09-12 18:16 1.0K
[CRT]SHA512SUMS.sign2026-09-12 18:20 833
[P2P]debian-13.7.0-amd64-netinst.iso.torrent2026-09-12 13:32 60K
[P2P]debian-edu-13.7.0-amd64-netinst.iso.torrent2026-09-12 13:32 60K
[P2P]debian-mac-13.7.0-amd64-netinst.iso.torrent2026-09-12 13:32 60K

+
Apache/2.4.68 (Unix) Server at cdimage.debian.org Port 443
+ diff --git a/tests/fixtures/pages/debian/amd64-bt-cd.meta.json b/tests/fixtures/pages/debian/amd64-bt-cd.meta.json new file mode 100644 index 0000000..47bcd4c --- /dev/null +++ b/tests/fixtures/pages/debian/amd64-bt-cd.meta.json @@ -0,0 +1,5 @@ +{ + "url": "https://cdimage.debian.org/debian-cd/current/amd64/bt-cd/", + "captured": "2026-10-19T10:54:57+00:00", + "content_type": "text/html;charset=UTF-8" +} diff --git a/tests/fixtures/pages/debian/amd64-bt-dvd.html b/tests/fixtures/pages/debian/amd64-bt-dvd.html new file mode 100644 index 0000000..7979077 --- /dev/null +++ b/tests/fixtures/pages/debian/amd64-bt-dvd.html @@ -0,0 +1,143 @@ + + + + Index of /debian-cd/current/amd64/bt-dvd + + + + +

What's in this directory?

+ + +

These are files containing the installer and other software + for the Debian GNU/Linux operating system. + + + The files in this directory are specifically for + the amd64 architecture. + + +

+ + +

How do I use these files?

+ + +

The files here are bittorrent files. Use + a bittorrent + client program to download the contents of the complete ISO image + files in a peer-to-peer manner. You should end up with an exact + copy of each ISO image as though you'd downloaded it directly via + HTTP or FTP.

+ + +

Once you have downloaded all the ISO images you want, you will + typically need to write them to installation media.

+ +

What size and type of media will I need?

+ +

The images described here are sized to be written to + + writeable DVD media + + at a minimum, but may be written to larger media if needed.

+ + +

For extra convenience, these images may also be written + directly to a USB stick. So long as your computer will + boot directly from that USB stick, it should start the Debian + installer that way. + + + The first DVD in this set is also deliberately limited in size + so it should fit on a standard-sized 4GB USB stick. + +

+ + +

There are lots of files here! Do I need all of them?

+ +

In most cases it is not necessary to download and + use all of these images to be able to install + Debian on your computer. Debian comes with a massive set of + software packages, hence why it takes so many disks for a + complete set. Most typical users only need a small subset of + those software packages.

+ + +

Initially, you will only need to download and use + the first image of a set (labelled as + debian-something-1 to be able to start the + Debian installer and set up Debian on your computer. If there are + more images available here + (labelled debian-something-2, + debian-something-3, etc.), they contain the + extra packages that can be installed on a Debian system (as + mentioned previously). They will not be + bootable and are entirely optional. If you have a fast Internet + connection, you're most likely better off installing any + desired extra packages directly from the Debian mirrors on the + Internet instead of by using these extra images.

+ + + + + + +

How can I verify my download is correct and exactly what has + been created by Debian?

+ +

There are files here (SHA512SUMS, etc.) which contain checksums + of the images. These checksum files are also signed - see the + matching .sign files. Once you've downloaded an image, you can + check:

+ +
    +
  • that its checksum matches that expected from the checksum file; and +
  • that the checksum file has not been tampered with. +
+ +

For more information about how to do these steps, read + the verification guide. + + + +

Only the first few images are available! Where are the rest?

+ +

We don't store/serve the full set of ISO images for all + architectures, to reduce the amount of space taken up on the + mirrors. You can + use the jigdo + tool to recreate the missing ISO images instead.

+ + +

Non-free Firmware

+ +

This Debian image build only includes Free Software where + possible. However, many systems include hardware which depends on + non-free firmware to function properly so this build also includes + those firmware files for those cases. See the Debian + Wiki non-free + firmware page for more information. + +

Other questions?

+ +

See the Debian CD FAQ +for lots more information about Debian CDs and installation.

+ +

The images here were put together by +the Debian CD team +, using debian-cd and other software.

+ + + + + + + + + + +
[ICO]NameLast modifiedSize

[PARENTDIR]Parent Directory  -
[SUM]SHA256SUMS2026-09-12 18:16 2.7K
[CRT]SHA256SUMS.sign2026-09-12 18:20 833
[SUM]SHA512SUMS2026-09-12 18:16 4.4K
[CRT]SHA512SUMS.sign2026-09-12 18:20 833
[P2P]debian-13.7.0-amd64-DVD-1.iso.torrent2026-09-12 13:32 298K

+
Apache/2.4.68 (Unix) Server at cdimage.debian.org Port 443
+ diff --git a/tests/fixtures/pages/debian/amd64-bt-dvd.meta.json b/tests/fixtures/pages/debian/amd64-bt-dvd.meta.json new file mode 100644 index 0000000..0d6c786 --- /dev/null +++ b/tests/fixtures/pages/debian/amd64-bt-dvd.meta.json @@ -0,0 +1,5 @@ +{ + "url": "https://cdimage.debian.org/debian-cd/current/amd64/bt-dvd/", + "captured": "2026-10-19T10:54:57+00:00", + "content_type": "text/html;charset=UTF-8" +} diff --git a/tests/fixtures/pages/debian/arm64-bt-cd.html b/tests/fixtures/pages/debian/arm64-bt-cd.html new file mode 100644 index 0000000..3d252aa --- /dev/null +++ b/tests/fixtures/pages/debian/arm64-bt-cd.html @@ -0,0 +1,137 @@ + + + + Index of /debian-cd/current/arm64/bt-cd + + + + +

What's in this directory?

+ + +

These are files containing the installer and other software + for the Debian GNU/Linux operating system. + + + The files in this directory are specifically for + the arm64 architecture. + + +

+ + +

How do I use these files?

+ + +

The files here are bittorrent files. Use + a bittorrent + client program to download the contents of the complete ISO image + files in a peer-to-peer manner. You should end up with an exact + copy of each ISO image as though you'd downloaded it directly via + HTTP or FTP.

+ + +

Once you have downloaded all the ISO images you want, you will + typically need to write them to installation media.

+ +

What size and type of media will I need?

+ +

The images described here are sized to be written to + + writeable CD media + + at a minimum, but may be written to larger media if needed.

+ + +

For extra convenience, these images may also be written + directly to a USB stick. So long as your computer will + boot directly from that USB stick, it should start the Debian + installer that way. + +

+ + +

There are lots of files here! Do I need all of them?

+ +

In most cases it is not necessary to download and + use all of these images to be able to install + Debian on your computer. Debian comes with a massive set of + software packages, hence why it takes so many disks for a + complete set. Most typical users only need a small subset of + those software packages.

+ + + + +

What is a netinst image?

+ +

The netinst CD here is a small CD image that + contains just the core Debian installer code and a small core set + of text-mode programs (known as "standard" in Debian). To install + a desktop or other common software, you'll also need either an + Internet connection or some other Debian CD/DVD images.

+ + + + + + + + +

How can I verify my download is correct and exactly what has + been created by Debian?

+ +

There are files here (SHA512SUMS, etc.) which contain checksums + of the images. These checksum files are also signed - see the + matching .sign files. Once you've downloaded an image, you can + check:

+ +
    +
  • that its checksum matches that expected from the checksum file; and +
  • that the checksum file has not been tampered with. +
+ +

For more information about how to do these steps, read + the verification guide. + + + +

Only the first few images are available! Where are the rest?

+ +

We don't store/serve the full set of ISO images for all + architectures, to reduce the amount of space taken up on the + mirrors. You can + use the jigdo + tool to recreate the missing ISO images instead.

+ + +

Non-free Firmware

+ +

This Debian image build only includes Free Software where + possible. However, many systems include hardware which depends on + non-free firmware to function properly so this build also includes + those firmware files for those cases. See the Debian + Wiki non-free + firmware page for more information. + +

Other questions?

+ +

See the Debian CD FAQ +for lots more information about Debian CDs and installation.

+ +

The images here were put together by +the Debian CD team +, using debian-cd and other software.

+ + + + + + + + + + +
[ICO]NameLast modifiedSize

[PARENTDIR]Parent Directory  -
[SUM]SHA256SUMS2026-09-12 18:16 204
[CRT]SHA256SUMS.sign2026-09-12 18:20 833
[SUM]SHA512SUMS2026-09-12 18:16 332
[CRT]SHA512SUMS.sign2026-09-12 18:20 833
[P2P]debian-13.7.0-arm64-netinst.iso.torrent2026-09-12 13:26 55K

+
Apache/2.4.68 (Unix) Server at cdimage.debian.org Port 443
+ diff --git a/tests/fixtures/pages/debian/arm64-bt-cd.meta.json b/tests/fixtures/pages/debian/arm64-bt-cd.meta.json new file mode 100644 index 0000000..76972f4 --- /dev/null +++ b/tests/fixtures/pages/debian/arm64-bt-cd.meta.json @@ -0,0 +1,5 @@ +{ + "url": "https://cdimage.debian.org/debian-cd/current/arm64/bt-cd/", + "captured": "2026-10-19T10:54:57+00:00", + "content_type": "text/html;charset=UTF-8" +} diff --git a/tests/fixtures/pages/debian/arm64-bt-dvd.html b/tests/fixtures/pages/debian/arm64-bt-dvd.html new file mode 100644 index 0000000..20497c8 --- /dev/null +++ b/tests/fixtures/pages/debian/arm64-bt-dvd.html @@ -0,0 +1,143 @@ + + + + Index of /debian-cd/current/arm64/bt-dvd + + + + +

What's in this directory?

+ + +

These are files containing the installer and other software + for the Debian GNU/Linux operating system. + + + The files in this directory are specifically for + the arm64 architecture. + + +

+ + +

How do I use these files?

+ + +

The files here are bittorrent files. Use + a bittorrent + client program to download the contents of the complete ISO image + files in a peer-to-peer manner. You should end up with an exact + copy of each ISO image as though you'd downloaded it directly via + HTTP or FTP.

+ + +

Once you have downloaded all the ISO images you want, you will + typically need to write them to installation media.

+ +

What size and type of media will I need?

+ +

The images described here are sized to be written to + + writeable DVD media + + at a minimum, but may be written to larger media if needed.

+ + +

For extra convenience, these images may also be written + directly to a USB stick. So long as your computer will + boot directly from that USB stick, it should start the Debian + installer that way. + + + The first DVD in this set is also deliberately limited in size + so it should fit on a standard-sized 4GB USB stick. + +

+ + +

There are lots of files here! Do I need all of them?

+ +

In most cases it is not necessary to download and + use all of these images to be able to install + Debian on your computer. Debian comes with a massive set of + software packages, hence why it takes so many disks for a + complete set. Most typical users only need a small subset of + those software packages.

+ + +

Initially, you will only need to download and use + the first image of a set (labelled as + debian-something-1 to be able to start the + Debian installer and set up Debian on your computer. If there are + more images available here + (labelled debian-something-2, + debian-something-3, etc.), they contain the + extra packages that can be installed on a Debian system (as + mentioned previously). They will not be + bootable and are entirely optional. If you have a fast Internet + connection, you're most likely better off installing any + desired extra packages directly from the Debian mirrors on the + Internet instead of by using these extra images.

+ + + + + + +

How can I verify my download is correct and exactly what has + been created by Debian?

+ +

There are files here (SHA512SUMS, etc.) which contain checksums + of the images. These checksum files are also signed - see the + matching .sign files. Once you've downloaded an image, you can + check:

+ +
    +
  • that its checksum matches that expected from the checksum file; and +
  • that the checksum file has not been tampered with. +
+ +

For more information about how to do these steps, read + the verification guide. + + + +

Only the first few images are available! Where are the rest?

+ +

We don't store/serve the full set of ISO images for all + architectures, to reduce the amount of space taken up on the + mirrors. You can + use the jigdo + tool to recreate the missing ISO images instead.

+ + +

Non-free Firmware

+ +

This Debian image build only includes Free Software where + possible. However, many systems include hardware which depends on + non-free firmware to function properly so this build also includes + those firmware files for those cases. See the Debian + Wiki non-free + firmware page for more information. + +

Other questions?

+ +

See the Debian CD FAQ +for lots more information about Debian CDs and installation.

+ +

The images here were put together by +the Debian CD team +, using debian-cd and other software.

+ + + + + + + + + + +
[ICO]NameLast modifiedSize

[PARENTDIR]Parent Directory  -
[SUM]SHA256SUMS2026-09-12 18:16 2.7K
[CRT]SHA256SUMS.sign2026-09-12 18:20 833
[SUM]SHA512SUMS2026-09-12 18:16 4.4K
[CRT]SHA512SUMS.sign2026-09-12 18:20 833
[P2P]debian-13.7.0-arm64-DVD-1.iso.torrent2026-09-12 13:26 298K

+
Apache/2.4.68 (Unix) Server at cdimage.debian.org Port 443
+ diff --git a/tests/fixtures/pages/debian/arm64-bt-dvd.meta.json b/tests/fixtures/pages/debian/arm64-bt-dvd.meta.json new file mode 100644 index 0000000..85e013a --- /dev/null +++ b/tests/fixtures/pages/debian/arm64-bt-dvd.meta.json @@ -0,0 +1,5 @@ +{ + "url": "https://cdimage.debian.org/debian-cd/current/arm64/bt-dvd/", + "captured": "2026-10-19T10:54:57+00:00", + "content_type": "text/html;charset=UTF-8" +} diff --git a/tests/fixtures/pages/devuan/get-devuan.html b/tests/fixtures/pages/devuan/get-devuan.html new file mode 100644 index 0000000..f49aeef --- /dev/null +++ b/tests/fixtures/pages/devuan/get-devuan.html @@ -0,0 +1,421 @@ + + + + + Get Devuan | Devuan GNU+Linux Free Operating System + + + + + + + + + + + + + + + + + + + + +
+

Devuan GNU+Linux is a free software operating system for your computer. Free software means you are free to use, copy and distribute, study, change the software, and share your modifications with the community.

+
+ + +
+
+
+

 Download Devuan From Mirrors

+

ISO Guide for amd64

+

desktop-live: Explore the default Xfce desktop before installing. Then install easily and quickly from the live session using the refractainstaller. Firmware is installed but can easily be removed.

+

netinstall: Preferred by more experienced users to create servers, minimal systems but also includes choice of several Desktop Environments (DEs). Not to be used for offline install. Internet connection and available bandwidth needed because all but the core components are downloaded during the installation process. The “expert install” allows user to opt out of firmware installation and a choice of alternate bootloader. A default install will provide firmware.

+ +

server: CD1 of a 5 CD set that allows for a complete off-line server/minimal installation. The remaining CDs offer several desktop choices and a limited selection of additional software.

+
    +
  • CD2: Additional server packages.
  • +
  • CD3: Basic desktop packages and Openbox window manager.
  • +
  • CD4: MATE and XFCE desktop.
  • +
  • CD5: LXDE and LXQt desktops.
  • +
+

desktop: Use this DVD if there is no network available and/or a need for multiple offline installations. The image contains several desktop choices and additional software options. LXQt and KDE are only available on DVD.

+

minimal-live: A full-featured, console-based recovery tool with a focus on accessibility for visually-impaired and blind users. Uses the refractainstaller.

+

Note that these Devuan installation ISOs are signed with the key of the developer in charge of them. All developers keys are listed on the Team page.

+

Installation Media for amd64, arm64, armel, armhf, and ppc64el

+ +

netboot: Files for PXE install, mini.isos and installer disk images for some specific boards can be found in the appropriate directory for your $ARCH at https://pkgmaster.devuan.org/devuan/dists/excalibur/main/installer-$ARCH/current/images/

+

embedded: The Devuan community provides discussion and information about available ARM images and download locations.

+
+ + + + +
+ + + + + +

The torrent contains all files for a given release. You can select which files to download if you have limited bandwidth or storage. Please seed if you can (~16.21GB)!

+

Distributions Based on Devuan

+

The official Devuan installation options shouldn’t be considered the only way to use Devuan. A number of derivatives have already adopted Devuan as a base. When reviewing Devuan we do recommend taking them into consideration. They harness the power of our base distribution by targeting specific usage which is exactly what we hope to achieve with Devuan. Please go to the dev1galaxy.org forum for Devuan derivatives discussion and support.

+
+
+

This site is free of cookies and javascript

+ + + diff --git a/tests/fixtures/pages/devuan/get-devuan.meta.json b/tests/fixtures/pages/devuan/get-devuan.meta.json new file mode 100644 index 0000000..bb2454a --- /dev/null +++ b/tests/fixtures/pages/devuan/get-devuan.meta.json @@ -0,0 +1,5 @@ +{ + "url": "https://www.devuan.org/get-devuan", + "captured": "2026-10-19T10:54:57+00:00", + "content_type": "text/html" +} diff --git a/tests/fixtures/pages/fedora/torrents.html b/tests/fixtures/pages/fedora/torrents.html new file mode 100644 index 0000000..1b5fb7f --- /dev/null +++ b/tests/fixtures/pages/fedora/torrents.html @@ -0,0 +1,116 @@ + + + + Index of /torrents + + +

Index of /torrents

+
Icon  Name                                                  Last modified      Size  Description
[PARENTDIR] Parent Directory - +[   ] Fedora-Budgie-Live-x86_64-43.torrent 2025-10-29 17:20 219K +[   ] Fedora-Budgie-Live-x86_64-44.torrent 2026-04-28 06:42 230K +[   ] Fedora-Budgie-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 238K +[   ] Fedora-COSMIC-Atomic-ostree-aarch64-43.torrent 2025-10-29 17:20 189K +[   ] Fedora-COSMIC-Atomic-ostree-aarch64-44.torrent 2026-04-28 06:42 193K +[   ] Fedora-COSMIC-Atomic-ostree-x86_64-43.torrent 2025-10-29 17:20 199K +[   ] Fedora-COSMIC-Atomic-ostree-x86_64-44.torrent 2026-04-28 06:42 203K +[   ] Fedora-COSMIC-Atomic-ostree.torrent 2026-03-09 19:48 230 +[   ] Fedora-COSMIC-Live-aarch64-43.torrent 2025-10-29 17:20 213K +[   ] Fedora-COSMIC-Live-aarch64-44.torrent 2026-05-12 04:54 216K +[   ] Fedora-COSMIC-Live-aarch64-45_Beta.torrent 2026-09-14 16:52 234K +[   ] Fedora-COSMIC-Live-x86_64-43.torrent 2025-10-29 17:20 226K +[   ] Fedora-COSMIC-Live-x86_64-44.torrent 2026-05-12 04:54 236K +[   ] Fedora-COSMIC-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 248K +[   ] Fedora-Cinnamon-Live-x86_64-43.torrent 2025-10-29 17:20 222K +[   ] Fedora-Cinnamon-Live-x86_64-44.torrent 2026-04-28 06:42 244K +[   ] Fedora-Cinnamon-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 251K +[   ] Fedora-CosmicAtomic-Installer-aarch64-45_Beta.torrent 2026-09-14 16:52 211K +[   ] Fedora-CosmicAtomic-Installer-x86_64-45_Beta.torrent 2026-09-14 16:52 215K +[   ] Fedora-KDE-Desktop-Live-aarch64-43.torrent 2025-10-29 17:20 231K +[   ] Fedora-KDE-Desktop-Live-aarch64-44.torrent 2026-04-28 06:42 233K +[   ] Fedora-KDE-Desktop-Live-aarch64-45_Beta.torrent 2026-09-14 16:52 255K +[   ] Fedora-KDE-Desktop-Live-x86_64-43.torrent 2025-10-29 17:20 244K +[   ] Fedora-KDE-Desktop-Live-x86_64-44.torrent 2026-04-28 06:42 251K +[   ] Fedora-KDE-Desktop-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 267K +[   ] Fedora-KDE-Mobile-Live-aarch64-43.torrent 2025-10-29 17:20 186K +[   ] Fedora-KDE-Mobile-Live-aarch64-44.torrent 2026-04-28 06:42 181K +[   ] Fedora-KDE-Mobile-Live-aarch64-45_Beta.torrent 2026-09-14 16:52 197K +[   ] Fedora-KDE-Mobile-Live-x86_64-43.torrent 2025-10-29 17:20 197K +[   ] Fedora-KDE-Mobile-Live-x86_64-44.torrent 2026-04-28 06:42 198K +[   ] Fedora-KDE-Mobile-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 208K +[   ] Fedora-Kinoite-Installer-aarch64-45_Beta.torrent 2026-09-14 16:52 285K +[   ] Fedora-Kinoite-Installer-x86_64-45_Beta.torrent 2026-09-14 16:52 296K +[   ] Fedora-Kinoite-ostree-aarch64-43.torrent 2025-10-29 17:20 278K +[   ] Fedora-Kinoite-ostree-aarch64-44.torrent 2026-04-28 06:42 290K +[   ] Fedora-Kinoite-ostree-x86_64-43.torrent 2025-10-29 17:20 291K +[   ] Fedora-Kinoite-ostree-x86_64-44.torrent 2026-04-28 06:42 305K +[   ] Fedora-Kinoite-ostree.torrent 2026-03-09 19:48 224 +[   ] Fedora-LXDE-Live-x86_64-43.torrent 2025-10-29 17:20 163K +[   ] Fedora-LXDE-Live-x86_64-44.torrent 2026-04-28 06:42 192K +[   ] Fedora-LXDE-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 202K +[   ] Fedora-LXQt-Live-aarch64-43.torrent 2025-10-29 17:20 162K +[   ] Fedora-LXQt-Live-aarch64-44.torrent 2026-04-28 06:42 165K +[   ] Fedora-LXQt-Live-aarch64-45_Beta.torrent 2026-09-14 16:52 184K +[   ] Fedora-LXQt-Live-x86_64-43.torrent 2025-10-29 17:20 172K +[   ] Fedora-LXQt-Live-x86_64-44.torrent 2026-04-28 06:42 182K +[   ] Fedora-LXQt-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 194K +[   ] Fedora-MATE_Compiz-Live-x86_64-43.torrent 2025-10-29 17:20 221K +[   ] Fedora-MATE_Compiz-Live-x86_64-44.torrent 2026-04-28 06:42 243K +[   ] Fedora-MATE_Compiz-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 254K +[   ] Fedora-MiracleWM-Live-aarch64-43.torrent 2025-10-29 17:20 168K +[   ] Fedora-MiracleWM-Live-aarch64-44.torrent 2026-04-28 06:42 170K +[   ] Fedora-MiracleWM-Live-x86_64-43.torrent 2025-10-29 17:20 178K +[   ] Fedora-MiracleWM-Live-x86_64-44.torrent 2026-04-28 06:42 186K +[   ] Fedora-Onyx-ostree-x86_64-43.torrent 2025-10-29 17:20 205K +[   ] Fedora-Onyx-ostree-x86_64-44.torrent 2026-04-28 06:42 208K +[   ] Fedora-Onyx-ostree.torrent 2026-03-09 19:48 221 +[   ] Fedora-Python-Classroom-Live-aarch64-43.torrent 2025-10-29 17:20 203K +[   ] Fedora-Python-Classroom-Live-aarch64-44.torrent 2026-04-28 06:42 211K +[   ] Fedora-Python-Classroom-Live-aarch64-45_Beta.torrent 2026-09-14 16:52 229K +[   ] Fedora-Scientific_KDE-Live-aarch64-44.torrent 2026-04-28 06:42 423K +[   ] Fedora-Scientific_KDE-Live-aarch64-45_Beta.torrent 2026-09-14 16:52 472K +[   ] Fedora-Sericea-ostree-x86_64-43.torrent 2025-10-29 17:20 178K +[   ] Fedora-Sericea-ostree-x86_64-44.torrent 2026-04-28 06:42 181K +[   ] Fedora-Sericea-ostree.torrent 2026-03-09 19:48 224 +[   ] Fedora-Server-dvd-aarch64-43.torrent 2025-10-29 17:20 251K +[   ] Fedora-Server-dvd-aarch64-44.torrent 2026-04-28 06:42 273K +[   ] Fedora-Server-dvd-aarch64-45_Beta.torrent 2026-09-14 16:52 304K +[   ] Fedora-Server-dvd-x86_64-43.torrent 2025-10-29 17:20 260K +[   ] Fedora-Server-dvd-x86_64-44.torrent 2026-04-28 06:42 292K +[   ] Fedora-Server-dvd-x86_64-45_Beta.torrent 2026-09-14 16:52 320K +[   ] Fedora-Server-dvd.torrent 2026-03-09 19:48 220 +[   ] Fedora-Silverblue-Installer-aarch64-45_Beta.torrent 2026-09-14 16:52 235K +[   ] Fedora-Silverblue-Installer-x86_64-45_Beta.torrent 2026-09-14 16:52 243K +[   ] Fedora-Silverblue-ostree-aarch64-43.torrent 2025-10-29 17:20 226K +[   ] Fedora-Silverblue-ostree-aarch64-44.torrent 2026-04-28 06:42 236K +[   ] Fedora-Silverblue-ostree-x86_64-43.torrent 2025-10-30 12:59 237K +[   ] Fedora-Silverblue-ostree-x86_64-44.torrent 2026-04-28 06:42 249K +[   ] Fedora-Silverblue-ostree.torrent 2026-03-09 19:48 227 +[   ] Fedora-SoaS-Live-aarch64-43.torrent 2025-10-29 17:20 155K +[   ] Fedora-SoaS-Live-aarch64-44.torrent 2026-04-28 06:42 154K +[   ] Fedora-SoaS-Live-aarch64-45_Beta.torrent 2026-09-14 16:52 170K +[   ] Fedora-SoaS-Live-x86_64-43.torrent 2025-10-31 12:52 167K +[   ] Fedora-SoaS-Live-x86_64-44.torrent 2026-04-28 06:42 174K +[   ] Fedora-SoaS-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 180K +[   ] Fedora-Sway-Live-x86_64-43.torrent 2025-10-29 17:20 154K +[   ] Fedora-Sway-Live-x86_64-44.torrent 2026-04-28 06:42 179K +[   ] Fedora-Sway-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 188K +[   ] Fedora-Workstation-Live-aarch64-43.torrent 2025-10-29 17:20 192K +[   ] Fedora-Workstation-Live-aarch64-44.torrent 2026-04-28 06:42 201K +[   ] Fedora-Workstation-Live-aarch64-45_Beta.torrent 2026-09-14 16:52 217K +[   ] Fedora-Workstation-Live-x86_64-43.torrent 2025-10-29 17:20 205K +[   ] Fedora-Workstation-Live-x86_64-44.torrent 2026-04-28 06:42 213K +[   ] Fedora-Workstation-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 223K +[   ] Fedora-Xfce-Live-aarch64-43.torrent 2025-10-29 17:20 198K +[   ] Fedora-Xfce-Live-aarch64-44.torrent 2026-04-28 06:42 199K +[   ] Fedora-Xfce-Live-aarch64-45_Beta.torrent 2026-09-14 16:52 216K +[   ] Fedora-Xfce-Live-x86_64-43.torrent 2025-10-29 17:20 210K +[   ] Fedora-Xfce-Live-x86_64-44.torrent 2026-04-28 06:42 219K +[   ] Fedora-Xfce-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 227K +[   ] Fedora-i3-Live-aarch64-43.torrent 2025-10-29 17:20 165K +[   ] Fedora-i3-Live-aarch64-44.torrent 2026-05-12 04:54 165K +[   ] Fedora-i3-Live-aarch64-45_Beta.torrent 2026-09-14 16:52 180K +[   ] Fedora-i3-Live-x86_64-43.torrent 2025-10-29 17:20 175K +[   ] Fedora-i3-Live-x86_64-44.torrent 2026-04-28 06:42 183K +[   ] Fedora-i3-Live-x86_64-45_Beta.torrent 2026-09-14 16:52 189K +
+ diff --git a/tests/fixtures/pages/fedora/torrents.meta.json b/tests/fixtures/pages/fedora/torrents.meta.json new file mode 100644 index 0000000..ef7db85 --- /dev/null +++ b/tests/fixtures/pages/fedora/torrents.meta.json @@ -0,0 +1,5 @@ +{ + "url": "https://torrent.fedoraproject.org/torrents/", + "captured": "2026-10-19T10:54:57+00:00", + "content_type": "text/html;charset=ISO-8859-1" +} diff --git a/tests/fixtures/pages/kali/cdimage.html b/tests/fixtures/pages/kali/cdimage.html new file mode 100644 index 0000000..6653cd5 --- /dev/null +++ b/tests/fixtures/pages/kali/cdimage.html @@ -0,0 +1,22 @@ +Kali.org | +Mirror Status | +Mirror Traces | +Your Nearest Mirror | +Mirror Stats | +Setting up a Kali Linux Mirror | +Official Kali Linux Mirrors + + +Index of / +

Index of /

+ + + + + + + + +
File Name  ↓ File Size  ↓ Date  ↓ 
-2026-Jun-24 04:39
-2026-Mar-23 14:57
-2026-Jun-24 04:39
-2026-Oct-18 03:42
-2014-Dec-04 14:11
625 B2022-Mar-30 07:15
1.2 KiB2026-Oct-18 03:42
This is Kali Linux's base-images repository.
+
+Kali Linux, The Most Advanced Penetration Testing Distribution. Ever. diff --git a/tests/fixtures/pages/kali/cdimage.meta.json b/tests/fixtures/pages/kali/cdimage.meta.json new file mode 100644 index 0000000..bdec821 --- /dev/null +++ b/tests/fixtures/pages/kali/cdimage.meta.json @@ -0,0 +1,5 @@ +{ + "url": "https://cdimage.kali.org/", + "captured": "2026-10-19T10:54:57+00:00", + "content_type": "text/html" +} diff --git a/tests/fixtures/pages/kali/get-kali.html b/tests/fixtures/pages/kali/get-kali.html new file mode 100644 index 0000000..2192397 --- /dev/null +++ b/tests/fixtures/pages/kali/get-kali.html @@ -0,0 +1,330 @@ +Get Kali | Kali Linux +
get-kali page is up +

Choose your Platform|

LIGHTDARK

Installer Images

  • Direct access to hardware
  • Customized Kali kernel
  • No overhead

Single or multiple boot Kali, giving you complete control over the hardware access (perfect for in-built Wi-Fi and GPU), enabling the best performance.

Virtual Machines

  • Snapshots functionary
  • Isolated environment
  • Customized Kali kernel
  • Limited direct access to hardware
  • Higher system requirements

VMware & VirtualBox pre-built images. Allowing for a Kali install without altering the host OS with additional features such as snapshots. Vagrant images for quick spin-up also available.

ARM

  • Range of hardware from the leave-behind devices end to high-end modern servers
  • System architecture limits certain packages
  • Not always customized kernel

Works on relatively inexpensive & low powered Single Board Computers (SBCs) as well as modern ARM based laptops, which combine high speed with long battery life.

Mobile

  • Kali layered on Android
  • Kali in your pocket, on the go
  • Mobile interface (compact view)

A mobile penetration testing platform for Android devices, based on Kali Linux. Kali NetHunter consists of an NetHunter App, App Store, Kali Container, and KeX.

Cloud

  • Fast deployment
  • Can leverage provider's resources
  • Provider may become costly
  • Not always customized kernel

Hosting providers which have Kali Linux pre-installed, ready to go, without worrying about infrastructure maintenance.

Containers

  • Low overhead to access Kali toolset
  • Userland actions only
  • Not Kali customized kernel
  • No direct access to hardware

Using Docker or LXD, allows for extremely quick and easy access to Kali's tool set without the overhead of an isolated virtual machine.

Live Boot

  • Un-altered host system
  • Direct access to hardware
  • Customized Kali kernel
  • Performance decrease when heavy I/O

Quick and easy access to a full Kali install. Your Kali, always with you, without altering the host OS, plus allows you to benefit from hardware access.

WSL

  • Access to the Kali toolset through the WSL framework
  • Userland actions only
  • Not Kali customized kernel
  • No direct access to hardware

Windows Subsystem for Linux (WSL) is included out of the box with modern Windows. Use Kali (and Win-KeX) without installing additional software.


Build Scripts

Installer Images

Kali is a rolling Linux distribution, meaning as soon as we have an update, we ship it. Would-be users have a variety of images to choose from. For more information, please see Which Image Should I Download? and Kali Branches. For most users, we recommend the latest “point release” image below, except in cases when a user requires a specific bug patch, in which case the weekly build may be best.

We generate fresh Kali Linux image files every quarter. These become the official "point" releases. These images are tested and subsequently announced with a blog post.

Installation Documentation

Our previous Kali Linux's releases.


SHA256sum

1880b3f1a95d5119ab50c779cf681176c86f115e5bf0914153f625ad3762f534

Weekly

Untested images with the latest updates

SHA256sum

d32f929dacc48134a31461a09f2160d13ad1d26b820cee920446813ca979b39b

NetInstaller

All packages are downloaded during installation

SHA256sum

ccedeb4e72fdaf7e5ba676ccf361cabe0e6e37174d83e38df0efcb6734b0f20d

Everything

Image for air-gapped networks

SHA256sum

9221d7d7afe732c37da435a4a0fb98062f739a0f45d7f9d111d085f4254ffff7

Weekly

Untested images with the latest updates

SHA256sum

8ca1bdd6a85f37e380eca84c780256b6e634fc768426396f51967ec3be006ed1

NetInstaller

All packages are downloaded during installation



Q.) +What’s the differences between: Installer? NetInstaller? Everything? Weekly? Live?
Q.) +How to download Kali Linux images securely?
Q.) +Can I still download the mini image?


Want an Updated or Custom Kali Image?

Feeling a little more adventurous? Want to build the latest version of Kali? Want to customize your ISO by adding more tools or change the default settings? Looking for something other than Xfce, GNOME or KDE like Enlightenment, i3mw, LXDE or MATE? Then this is the option for you.

Read More On Custom Images




Kali Purple

Feeling red? Feeling blue? Kali Purple: You do You!

A movement to make enterprise grade security accessible to everyone.

Kali Purple Documentation

SHA256sum

af69091e2db59aea3ee8a444988ee98bff245900c766cf72d3afef0a3d5393ec

Weekly

Untested images with the latest updates

Pre-built Virtual Machines

Kali Linux VMware & VirtualBox images are available for users who prefer, or whose specific needs require a virtual machine installation.

These images have the default credentials "kali/kali".

Virtual Machines Documentation

SHA256sum

19949a733a7715e2e2d6069805db46c64da62d8a440affd0a1b4e7432192b135

VMware Weekly

SHA256sum

eadbb304f60a32baa3040164ace04f0fba242fac31698270b8414d457a071f2a

VirtualBox Weekly

SHA256sum

0148d6bccb0123d05a217b9be2bc9b87878c8ffb576a9729b3fed59dcb667ddf

Hyper-V Weekly

SHA256sum

cb14e12fcca29067f09003e0a49720389fc70dae100fa643396b9ba1ec549f50

QEMU Weekly


Vagrant

Vagrant is a tool for building and managing virtual machine environments. With a single configuration file, you can download a base “box” and apply additional configurations like adding an additional network interface, setting the number of CPU cores and memory, or running a script on first boot.

Kali-Vagrant Boxes

Want an Updated or Custom Kali Image?

Feeling a little more adventurous? Want to build the latest version of Kali? Want to customize your VM by selecting a different desktop environment, add more tools, change the default settings? Then this is the option for you.

Kali-VM Build Script

ARM

Are you looking for Kali Linux ARM images? We have generated flavours of Kali using the same build infrastructure as the official Kali releases for ARM architecture.

These images have a default credentials of "kali/kali".


Kali-ARM Documentation

raspberrypi

SHA256sum

9d04ba33ea068218ebbf599afa67fa1834e0cf1e315f51a0d849307153806d6f

Raspberry Pi 2, 3, 4 and 400 (32-bit)

SHA256sum

795495b65b31e567613304cdfcccf67e0e1e5d018a006fdee00a5b5d2d49d72c

Raspberry Pi 3, 4, 400, 5 and 500 (64-bit)

SHA256sum

cffa49572c930483da38238be35981a5737f18ab00e978b0fddaaa9a5d01eb10

Raspberry Pi Zero 2 W (Pi-Tail)

pine64

SHA256sum

90a4965a272c55441bd073e41c9e5ab0e2328061372c55c6f9c8988d06c620e9

Pinebook

SHA256sum

78735e1ea3e798e079ac73db99a1233b7463dc1950e2ffa9a3ce3482d4cef36f

Pinebook Pro

gateworks

SHA256sum

81e96e1bb3905f1e9f3b483090cb7a7ae844f4bb71a0a233be67056859413327

Gateworks Newport



I.) SHA256SUMS are for the compressed file (*.img.xz)

NetHunter Pro



Kali NetHunter Pro is the official Kali Linux build for mobile devices such as the Pine64 PinePhone and PinePhone Pro.

Installation:

  1. Install Tow-Boot bootloader on your device
  2. Write the image to your MicroSD card, e.g.
    sudo dd if=IMAGE.img of=/dev/[DEVICE] bs=1M status=progress conv=fsync
  3. Insert the MicroSD card into your device
  4. Boot your device from MicroSD card (hold Volume down key until the LED turns blue)
  5. Login with user "kali" and password "1234"

Kali NetHunter Pro Documentation

PinePhone

SHA256sum

be2e3ce7d148bfff661c063bbc49cbc20bbd17f6a34b5dd8c06b83f766fa5010

Kali NetHunterPro image with Phosh for Pine64 PinePhone

PinePhonePro

SHA256sum

65290b07979833336f902f946661a991e72a57be8b0156c9da1a5ac9ec36514a

Kali NetHunterPro image with Phosh for Pine64 PinePhone Pro

SDM845

SHA256sum

223992a5efcf43a657c0d646e58760c9b8cf80f3d67367c436424bd61c277112

Kali NetHunterPro image with Phosh for Qualcomm 'QCOM' SDM845

AMD64

SHA256sum

291a084b0a436bd4e5664af3041af690ee065f048b9e596f12d1148384c51750

Kali NetHunterPro AMD64 image with Phosh for Testing/Deployment






NetHunter

The Kali NetHunter project is the first Open-source Android penetration testing platform for Android devices, allowing for access to the Kali toolset from various supported Android devices. There are multiple unique features not possible on other hardware platforms.

The Kali NetHunter interface allows you to easily work with complex configuration files through a local web interface. This feature, together with a custom kernel that supports 802.11 wireless injection and preconfigured connect back VPN services, make the Kali NetHunter a formidable network security tool or discrete drop box - with Kali Linux at the tip of your fingers wherever you are!

Statistics

Want to see Kali NetHunter progress? Look at the stats page, to see if your device is supported yet.

Kali NetHunter provides:

    • A dedicated NetHunter App, providing a touch screen optimized GUI for common attack categories, such as:
    • One-click MANA Evil Access Point setups.
    • USB HID Keyboard attacks, much like the Teensy device is able to do.
    • BadUSB MITM attacks. Plug in your Kali NetHunter to a victim PC, and have your traffic relayed though it.
    • Bluetooth attacks.

  • Full Kali Linux toolset, with many tools available via a simple menu system.
  • Command line interface to the Kali Linux container.
  • Kali desktop EXperience (KeX)
  • Custom, device specific kernel with wireless injection support.
  • HDMI output of Kali desktop to external display for supported devices.
  • USB Y-cable with the Kali NetHunter kernel - use your OTG cable while still charging your device!
  • Software Defined Radio support (SDR). Use Kali NetHunter with your HackRF to explore the wireless radio space.
  • NetHunter app store allowing you to grow the potential of Kali NetHunter.

Kali NetHunter Documentation

NetHunter Generic

SHA256sum

b09756fab7939092bcf7c6242c360022426aa85b435a3285daee36a608bf8880

NetHunter Generic ARM64 (Full)

SHA256sum

54d0387bfd011a9d2a81286b2442dddb79464dee49260aeaabec2280f53b2578

NetHunter Generic ARM64 (Minimal)

SHA256sum

6560221639dcf4ef6c76788a36fd6d9dc67fd3317e3b25729a8a961179e2b670

NetHunter Generic ARMhf (Full)

SHA256sum

bb770de8c99178aae2a4aca0e29f9fa7f9dbdf3fadd1da6d0d3724e030a6fd91

NetHunter Generic ARMhf (Minimal)

OPPO

SHA256sum

393b9cfb17958d39d59b5c8989a36b180e84db86501a01a53e907b1f3dfc3cd2

OPPO A37 (2016) (LineageOS 17.1)

Samsung

SHA256sum

4e220d11ea7bddbe9c0a5c76dc3bcd5f2b299f8d41dc56d35442d5934f88d790

Samsung Galaxy A7 (2016) (LineageOS 18.1)

SHA256sum

21d068e7c5f1f605a57f0bbd91506a20eba408b3a98d8e03ae5d95b10bf97741

Samsung Galaxy S10e (LineageOS 23.2)

SHA256sum

64c37a303c1daf36750ba5508ce4e6bd7117c43e01656d83095980cf20a1ff6d

Samsung Galaxy S10 (LineageOS 23.2)

SHA256sum

62a15a4e566ac792bcd432deadbe902f8ea47901b9aedd31b14379e40c56f979

Samsung Galaxy S10+ (LineageOS 23.2)

SHA256sum

e5f123b6fe4d11c3741f8d76ef937962ef0a04fceb74a0e1c1a392fa96f829b5

Samsung Galaxy S10 5G (LineageOS 23.2)

SHA256sum

17ca5d44b7a3214b3e5fcb87c3be320661382bd41d5bb6c954a23212aa0dbc88

Samsung Galaxy Note20 Ultra (One UI 5.1 KernelSU)

SHA256sum

f23f2e56ce7c5b6b5805430e87663b548f4a78f4afe14780508b16c3c90d39ee

Samsung Galaxy Tab S4 LTE (Oreo)

SHA256sum

1c9d2b37b72a9ff8fb1537d5eb8c8488ef2ef4a4fb4f6293a959e24b04e1172e

Samsung Galaxy Tab S4 WiFi (Oreo)

SHA256sum

d562dc46e757edeca61622ee4bc67fce560c8829c295fb02bd1c9ed525455527

Samsung Galaxy S7 Edge (Pie)

SHA256sum

99f23b29b9dd6b6ffc2b51136df258c8a6e3a2917f33cb187530099ab307f2e6

Samsung Galaxy S7 Edge (LineageOS 16.0)

SHA256sum

0d4081317d2a5fc1784e9c3429345d1f0371a33a46435c2923378b4567137966

Samsung Galaxy S7 (Pie)

SHA256sum

27069936b4d52d2ed250c30b0344f0e190f920b9859de7366f1b4d6af583fbfe

Samsung Galaxy S7 (LineageOS 16.0)

SHA256sum

3f76ce89c0325aee6f81d0e317a8f4dd0b3eb51b394c8224a7c6f2f1f7c87733

Samsung Galaxy J7 Pro (Pie)

SHA256sum

74bb3c35a670cfa5d100840f84715851510633db4fd8c8de1fe773d46c19042b

Samsung Galaxy S5 Mini (LineageOS 16.0)

SHA256sum

476b083aefc070171bafb688e210e48d30a30fc9218f883981afad275380451b

Samsung Galaxy J7 Prime (Pie)

SHA256sum

bb19d946b7ae5bea15c12e062aa608e9918971a3473a46d40bb7055b000f9507

Samsung Galaxy S20 FE 5G (Thirteen)

SHA256sum

73aa5617be53b8e1a74e45f591735eafe893e774d44bc010016dd4c18b90748a

Samsung Galaxy S9 (LineageOS 20)

SHA256sum

0281bf79b2c65613cf1d27039d66b33888caee87f41a0b6c4c2e4fa39a1756e0

Samsung Galaxy S9+ (LineageOS 20)

SHA256sum

2cc1242e38a32ce96c6205ae51ba4c13201bde5b57f56cf0cff4ce84079d880a

Samsung Galaxy S6 Edge (LineageOS 14.1)

Xiaomi

SHA256sum

8f4e476f9c565a7843c07d8e039644b545133b1a56f26415ec2aa2026a2a5c23

Xiaomi Pocophone F3 (LineageOS 21)

SHA256sum

bfd9cd54371fc96190ccaa10e01f454a8a9195c9e1d04337533c05769ce6314d

Xiaomi Pocophone F1 (Eleven)

SHA256sum

dbfec2a002693a1b828cb65b5ac357ed669db59f4b5aeff1c75aac5aaaa5b682

Xiaomi Pocophone F1 (LineageOS 21)

SHA256sum

a541d06c2d47cb548ddee3ae19aaa7469c983ffa50dda8d27187004989a941d9

Xiaomi Mi 9 - cepheus (PixelOS 15)

SHA256sum

7ec657f168bfd1f3151128734b23e8ddb1f279edade682099a0b3010c18a2b98

Xiaomi Mi 9T (MIUI 11)

SHA256sum

6b926331d5a78abc43cfed0a3604be61eb9ae08ca112973a7156c9d93d3aeccb

Xiaomi Mi A2 (LineageOS 22.1)

SHA256sum

5b188a1562ed9c6902ce96518ec21beb1643cf6be6b0eedbf868be7bf86469b4

Xiaomi Mi A3 (LineageOS 22)

SHA256sum

65008f88bba20194e252a63fabc3108bbc0419b2987c03ff510ed21cd7879112

Xiaomi Mi A3 (LineageOS 23.2)

SHA256sum

72fce6741d1b994e6fdd975db790ce31323f002346a4b9c8dfbe6bf740b542b4

Xiaomi Redmi 4 Prime (LineageOS 16.0)

SHA256sum

c2b14d807ef889a517e801681b737eb6d1ff3993cf7a8d2a9d7f2ad79e78aacb

Xiaomi Mi 9 Lite / CC9 (LineageOS 21)

SHA256sum

7b03ee4f753bbe0859775bbe697a98ce3a7041decd0f67cc34c91cb7bdca5b84

Xiaomi Redmi 5A (crDroid Android 14)

SHA256sum

ad017e248bd32ec6977eaa459c5343a8920dfbe136091895ac15d06ea2361251

Xiaomi Redmi 6 Pro (LineageOS 18.1)

SHA256sum

5edbca49a3ec4d1651cab6bbf4c4009d2a051ab498da13429a8bad6230db32b8

Xiaomi Redmi Note 4X (Eleven)

SHA256sum

0207ea01411841ebed99585fed09ea2c64aba2d6094d7a7b5d78ab11b0e8a81e

Xiaomi Redmi Note 4X (Fourteen)

SHA256sum

3e7605f195ef6354fb81b2a0ba9ac00ac9fd81747be40f0edd31b6c52d25e894

Xiaomi Pocophone X3 NFC (LineageOS 20)

SHA256sum

2277282d0763a465725c59bde920ad5751e826c3349f7f6fffddcc7871eccbde

Xiaomi Redmi Note 6 Pro (tulip)

SHA256sum

5c5bd49c8dfc20c3711759b66a09346e5c8764eb187b770b475b40f2200bf42b

Xiaomi Pocophone X3 Pro (Eleven)

SHA256sum

644ea7f8d797520b04a76cdc5792105c81380ed2ad8142e674fab0dbbf2ccf4e

Xiaomi Poco X3 NFC (PixelOS Android 16)

Google/Huawei

SHA256sum

c459871bd5255901ddfbb565cabdcf9618719b86203f58e2b0e8f95311dc4114

Google/Huawei Nexus 6P (Oreo)

SHA256sum

4dee1f1d48d491620a8d0f373fb051123bd929e19c5ccafec5c3aea5f928c0a3

Google/Huawei Nexus 6P Nexus 6P (LineageOS 17.1)

Redmi

SHA256sum

e7d50a8f9a0d67ca3d1fa10fc9ead0e075f9eb385d2314da613451d510ccc5b3

Redmi Note 8 (A13)

SHA256sum

2c8d00d6cbff3143ef63186ead51e8a4b0decd4ebcdc2b0383fe4995da820baf

Redmi Note 8 (A16)

Google

SHA256sum

5f3897c8c4a5559bab0f4d18a1dddf551b049f5f96b1e311feed61063e8210b6

Google Pixel 3 (Thirteen)

SHA256sum

e4ef37e3df6df0e4a7ad8269692e2149b2579772d2cf7e26f17f722997c01a22

Google Pixel 4a 5G (Twelve)

SHA256sum

4a036ef008af44da676ece7ec35ee6e22ec3155dfdb8494a4bac8ca90fe50b1b

Google Pixel 6a (LineageOS 23.2)

SHA256sum

44099157ae6b468667bf374594e5e71738f59968c4d9e19e8ec548bd057aa0c3

Google Nexus 5X (Oreo)

SHA256sum

869168b6114f7bee401f9cb1e2736387db2789f2656d605899ae6a489880bfd3

Google Pixel 3 XL (Thirteen)

SHA256sum

df436d1fca7a581bef3e61e1456be1459e461b58d31b2648e77deeca7dbb474f

Google Nexus 9 (Nougat)

SHA256sum

e0d3f8bb320fd2dee96e555f16bc2aed6a2f8e5732535643ef3f5c6dff935a38

Google Nexus 5 (LineageOS 14.1)

SHA256sum

9076dc6ed85f7b82c4d641bf9ca2df66202389284830f4c96b6f048d06c3f762

Google Nexus 6 (LineageOS 16.0)

SHA256sum

d5d29f3afbc8ef8cd5bf4aefb65115772ed7819f1912062884d6a4938e0863de

Google Pixel 4a (Twelve)

Motorola

SHA256sum

54e5b33314b910e149bc4d17f8ec97796845261722a4c0374a319f3ba6f8d5c9

Motorola Moto G5 (LineageOS 15.1)

SHA256sum

b0c9355e209ac9ee93b413c5a59051d2fff6f9a25da2e6a7707e652b4549f7d1

Motorola Moto X4 (LineageOS 20)

SHA256sum

4a4dc0ee244e1a323d8a6625e80a3e1aec6b3c4f9a6486429191c530d4829397

Motorola Moto G5 Plus (LineageOS 16.0)

Nokia

SHA256sum

c234ece0bc808f268032ad9910cae5697d01463a04ad1e4da8acd0f2b579f1d3

Nokia 6.1 Plus (LineageOS 21)

SHA256sum

0de58d83b0c5a73708c697eb84a869f07eaae35690ff890c6015904a092d5464

Nokia 3.1 (Pie)

SHA256sum

976f13ff5fd3e8f23617c22b116eb90d7d8496fda4b6cb8cd861c1bd156c3b89

Nokia 6.1 (LineageOS 21)

Huawei

SHA256sum

e71d88452aebaac6a49d2cb5e1e48e8c4423de6bfcee29172bf3794491ffc7a5

Huawei P9 (LineageOS 16.0)

Gemini

SHA256sum

9ee1bc6610a3df7f5688824f19ecbb0c35fda9a6f77f5176fc0b1d8978cee45d

Gemini PDA (Nougat)

LG

SHA256sum

c45921a2c61c498e2a12839018a1d27d8e603dbaf4ae837e6e1c85bdcc5e7cb1

LG V20 International (LineageOS 21)

Sony

SHA256sum

923d41193debec1a631760043707b897c2bce1bbe37a3a3d2acd774c550ca4d1

Sony Xperia Z1 (LineageOS 16)

SHA256sum

c30b3f0d76be7cd5d711f51fc46be2f659a5a4e1d008dffeb2789e526ddc01ab

Sony Xperia 10 II (LineageOS 20)

OnePlus

SHA256sum

4addbffc2692e8dab8b168d9f30134a64951e61929c07136f1a651f7115a5907

OnePlus Nord AC2003 (OxygenOS 16)

SHA256sum

0f012562b8f430d89de1a4a7d1a00c031faa6e8a3222626ce6b46c6649665c48

OnePlus One (LineageOS 18.1)

SHA256sum

8300d5a1dbd08cba62ed3da22d85ec2ed5b3d48aad3c49cc6c9d65d552d6b2f6

OnePlus 2 (LineageOS 16.0)

SHA256sum

52e79542f427779b763783ce081a4d129ba497db97b58c26f6249f40f4208c9c

OnePlus 3 / 3T (LineageOS 17.1)

SHA256sum

7b9a545b7af5d5f7112b45b6f999be6cb28e2a083ad6262aa2ec74473bd1aa4a

OnePlus 5 / 5T (LineageOS 22)

SHA256sum

638c69b0029f0be576bbda7ae6403f2f1fdf8247685b9e75083be82fa36eebf0

OnePlus 6 / 6T (LineageOS 19.1 & LineageOS 22.2)

SHA256sum

a466503f3ffb237bb76de5fccf8414f8c77febce1df93ff6c1ce3603c5eead7b

OnePlus 7 / 7 Pro / 7T / 7T Pro (OxygenOS 11)

SHA256sum

a0025eb18fe5d5443f65cb1b31b1bc6cfa94a164c91808c9a3b2ffd17c4ceda0

OnePlus 7 / 7 Pro / 7T / 7T Pro (OxygenOS 11 / Custom A11)

SHA256sum

471c9a62d0965215cc365077d146807395f710b4c63be1342bef36a841d40731

OnePlus 7 / 7 Pro / 7T / 7T Pro for LineageOS 23.2

SHA256sum

4f387d446b9d27d14f084cbe0d308a55d3cf46fb2664e0a762f2acfda0a09c27

OnePlus 8 / 8T / 8 Pro (OxygenOS 12 / Custom A12)

SHA256sum

864d370cd5d8f7501d8a4589dc8330469a16d83e08fe21b44ae2c8dd4f39cb78

OnePlus 9 / 9 Pro (LineageOS 23.2, Evolution X, Crdroid on Android 16)

SHA256sum

2f2671768e9c4345cbf130b4571ff8f49d8b6df61f300dbb4b88a0a8c3f218ff

OnePlus 11 5G (OxygenOS 15)

SHA256sum

59b804cfeb36df128eb9ac5b9ade2b0b62e7a26a82d5ab7c2d143ff8658c1ee0

OnePlus 11 5G (OxygenOS 16)

Realme

SHA256sum

702b2833dcaa39924d0e45a3111e91a8383dfbf6b581b9c118cc45998749ef15

Realme 5 / 5i / 5s (Corvus OS 4.1)

SHA256sum

947a2548ddeaa39c6cbc695955502751d8580501e274e9a3ba227735a889efd3

Realme 5 Pro (Twelve)

SHA256sum

655e6cba79d74eeae41c3feb26cd2ed45066535c02ca5a158b3dbc970ffbdb28

Realme 5 Pro (RUI2)

SHA256sum

25c9e1ead28c71d2e2414944ae2c5f383e5e4aadf83b1bd58c8d0e2436d7a6e8

Realme C15 (LineageOS 17.1)

SHA256sum

d483a0c9ca089eda70db6d549acf0eea97636380e8dcf2494160bda401d8f43e

Realme X7 Max 5G (Thirteen)

Nothing

SHA256sum

44fe85f21159dbde8ab70369d890dc8c5619477b960ea779a089409d094bc7b6

Nothing Phone (1) (Fourteen)

TicWatch

SHA256sum

ef293895796018426f07997e157549f1ff6514504f1ae27f46e46caea144a3ef

TicWatch Pro / Pro 4G/LTE / Pro 2020 (Wear OS)

SHA256sum

b1167741c57bcaeacf20ece77584f77fd02305963f9a052257abd77375798c2b

TicWatch Pro 3 GPS/LTE / Pro 3 Ultra GPS/LTE (OneOS)

Cloud

Kali Cloud images can quickly be deployed in multiple different cloud provider’s infrastructures. This allows easy access to a Kali install in a modern environment with strong hardware resources. These are often used for short term Kali installs that are quick to deploy and quick to tear down.


Kali-Cloud Documentation

Containers

Using Container technology such as Docker and LXC/LXD, our Kali containers allow you access to the Kali toolset on your host operating system without the overhead of running an additional full operating system. This does come with limitations, as you won’t have direct hardware access and dealing with inbound connections to tools running in the Kali container can be complicated.


Kali-Containers Documentation

Live Boot

A Kali Linux Live image on a CD/DVD/USB/PXE can allow you to have access to a full bare metal Kali install without needing to alter an already-installed operating system. This allows for quick easy access to the Kali toolset with all the advantages of a bare metal install. There are some drawbacks, as disk operations may slow due to the utilized storage media.

For most users, we recommend the latest “point release” image below, except in cases when a user requires a specific bug patch, in which case the weekly build may be best.


Kali-USB Documentation

SHA256sum

89ba86d14e0287b28ba25a185a32ce2ca68ef2233c628e8995c26311c09ebae4

Everything

Image for air-gapped networks

WSL

Windows Subsystem for Linux (WSL) is a software package on modern Windows installs that allow you to run Linux alongside your Windows system in an optimized container. The Kali WSL package allows easy access to the Kali toolset. This comes with the same drawbacks of a standard VM, but allows for less overhead and tighter integration with your Windows systems.

WSL Documentation

Microsoft Store

Build Scripts

Experienced penetration testers and security professionals use and trust Kali Linux because we provide full transparency into the build scripts. Feel free to read, investigate, and change build scripts for any images below.


Build Scripts Repository

\ No newline at end of file diff --git a/tests/fixtures/pages/kali/get-kali.meta.json b/tests/fixtures/pages/kali/get-kali.meta.json new file mode 100644 index 0000000..d895c69 --- /dev/null +++ b/tests/fixtures/pages/kali/get-kali.meta.json @@ -0,0 +1,5 @@ +{ + "url": "https://www.kali.org/get-kali/", + "captured": "2026-10-19T10:54:57+00:00", + "content_type": "text/html; charset=utf-8" +} diff --git a/tests/fixtures/pages/mint/download.html b/tests/fixtures/pages/mint/download.html new file mode 100644 index 0000000..ef611af --- /dev/null +++ b/tests/fixtures/pages/mint/download.html @@ -0,0 +1,446 @@ + + + + + + Download Linux Mint 22.3 - Linux Mint + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+

Linux Mint 22.3 'Zena'

+
+
+
+
+ + + + + +
+
+
+
+

Ready to download?

+

Linux Mint is available in different flavors. Choose an edition below.

+

For more information read the installation instructions.

+
+ +
+
+
+ +
+
+
+ +
+ +
+ +
+ Sleek, modern, innovative +

Cinnamon Edition

+

The most popular version of Linux Mint is the Cinnamon edition. Cinnamon is primarily developed for and by Linux Mint. It is slick, beautiful, and full of new features.

+ +
+
+
+
+ +
+
+
+
+ Light, simple, efficient +

Xfce Edition

+

Xfce is a lightweight desktop environment. It doesn’t support as many features as Cinnamon but it’s lighter on resource usage.

+ +
+
+ +
+
+
+
+ +
+
+
+ +
+ +
+ +
+ Classic, traditional +

MATE Edition

+

MATE is a classic desktop environment. It is the continuation of GNOME 2, which was Linux Mint’s default desktop between 2006 and 2011.

+ +
+
+
+
+ + + + + + +
+
+
+ + + +
+
+
+ + + + + + + +
+ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/fixtures/pages/mint/download.meta.json b/tests/fixtures/pages/mint/download.meta.json new file mode 100644 index 0000000..cc62a59 --- /dev/null +++ b/tests/fixtures/pages/mint/download.meta.json @@ -0,0 +1,5 @@ +{ + "url": "https://www.linuxmint.com/download.php", + "captured": "2026-10-19T10:54:57+00:00", + "content_type": "text/html; charset=utf-8" +} diff --git a/tests/fixtures/pages/ubuntu/meta-release-lts.meta.json b/tests/fixtures/pages/ubuntu/meta-release-lts.meta.json new file mode 100644 index 0000000..b5cbc52 --- /dev/null +++ b/tests/fixtures/pages/ubuntu/meta-release-lts.meta.json @@ -0,0 +1,5 @@ +{ + "url": "https://changelogs.ubuntu.com/meta-release-lts", + "captured": "2026-10-19T10:54:57+00:00", + "content_type": null +} diff --git a/tests/fixtures/pages/ubuntu/meta-release-lts.txt b/tests/fixtures/pages/ubuntu/meta-release-lts.txt new file mode 100644 index 0000000..000973d --- /dev/null +++ b/tests/fixtures/pages/ubuntu/meta-release-lts.txt @@ -0,0 +1,127 @@ +Dist: dapper +Name: Dapper Drake +Version: 6.06 LTS +Date: Thu, 01 Jun 2006 9:00:00 UTC +Supported: 0 +Description: This is the Dapper Drake release +Release-File: http://old-releases.ubuntu.com/ubuntu/dists/dapper-updates/Release +ReleaseNotes: http://changelogs.ubuntu.com/EOLReleaseAnnouncement +UpgradeTool: http://old-releases.ubuntu.com/ubuntu/dists/dapper-updates/main/dist-upgrader-all/current/dapper.tar.gz +UpgradeToolSignature: http://old-releases.ubuntu.com/ubuntu/dists/dapper-updates/main/dist-upgrader-all/current/dapper.tar.gz.gpg + +Dist: hardy +Name: Hardy Heron +Version: 8.04.1 LTS +Date: Thu, 24 Apr 2008 12:00:00 UTC +Supported: 0 +Description: This is the 8.04.1 LTS release +Release-File: http://old-releases.ubuntu.com/ubuntu/dists/hardy-updates/Release +ReleaseNotes: http://changelogs.ubuntu.com/EOLReleaseAnnouncement +UpgradeTool: http://old-releases.ubuntu.com/ubuntu/dists/hardy-proposed/main/dist-upgrader-all/0.87.31/hardy.tar.gz +UpgradeToolSignature: http://old-releases.ubuntu.com/ubuntu/dists/hardy-proposed/main/dist-upgrader-all/0.87.31/hardy.tar.gz.gpg + +Dist: lucid +Name: Lucid Lynx +Version: 10.04.4 LTS +Date: Thu, 29 Apr 2010 12:00:00 UTC +Supported: 0 +Description: This is the 10.04.4 LTS release +Release-File: http://old-releases.ubuntu.com/ubuntu/dists/lucid-updates/Release +ReleaseNotes: http://changelogs.ubuntu.com/EOLReleaseAnnouncement +UpgradeTool: http://old-releases.ubuntu.com/ubuntu/dists/lucid-updates/main/dist-upgrader-all/current/lucid.tar.gz +UpgradeToolSignature: http://old-releases.ubuntu.com/ubuntu/dists/lucid-updates/main/dist-upgrader-all/current/lucid.tar.gz.gpg + +Dist: precise +Name: Precise Pangolin +Version: 12.04.5 LTS +Date: Thu, 26 Apr 2012 12:04:00 UTC +Supported: 0 +Description: This is the 12.04.5 LTS release +Release-File: http://old-releases.ubuntu.com/ubuntu/dists/precise-updates/Release +ReleaseNotes: http://changelogs.ubuntu.com/EOLReleaseAnnouncement +UpgradeTool: http://old-releases.ubuntu.com/ubuntu/dists/precise-updates/main/dist-upgrader-all/current/precise.tar.gz +UpgradeToolSignature: http://old-releases.ubuntu.com/ubuntu/dists/precise-updates/main/dist-upgrader-all/current/precise.tar.gz.gpg + +Dist: trusty +Name: Trusty Tahr +Version: 14.04.6 LTS +Date: Thu, 17 Apr 2014 14:04:00 UTC +Supported: 1 +Description: This is the 14.04.6 LTS release +Release-File: http://archive.ubuntu.com/ubuntu/dists/trusty-updates/Release +ReleaseNotes: http://archive.ubuntu.com/ubuntu/dists/trusty-updates/main/dist-upgrader-all/current/ReleaseAnnouncement +ReleaseNotesHtml: http://archive.ubuntu.com/ubuntu/dists/trusty-updates/main/dist-upgrader-all/current/ReleaseAnnouncement.html +UpgradeTool: http://archive.ubuntu.com/ubuntu/dists/trusty-updates/main/dist-upgrader-all/current/trusty.tar.gz +UpgradeToolSignature: http://archive.ubuntu.com/ubuntu/dists/trusty-updates/main/dist-upgrader-all/current/trusty.tar.gz.gpg + +Dist: xenial +Name: Xenial Xerus +Version: 16.04.7 LTS +Date: Thu, 21 April 2016 16:04:00 UTC +Supported: 1 +Description: This is the 16.04.7 LTS release +Release-File: http://archive.ubuntu.com/ubuntu/dists/xenial-updates/Release +ReleaseNotes: http://archive.ubuntu.com/ubuntu/dists/xenial-updates/main/dist-upgrader-all/current/ReleaseAnnouncement +ReleaseNotesHtml: http://archive.ubuntu.com/ubuntu/dists/xenial-updates/main/dist-upgrader-all/current/ReleaseAnnouncement.html +UpgradeTool: http://archive.ubuntu.com/ubuntu/dists/xenial-updates/main/dist-upgrader-all/current/xenial.tar.gz +UpgradeToolSignature: http://archive.ubuntu.com/ubuntu/dists/xenial-updates/main/dist-upgrader-all/current/xenial.tar.gz.gpg + +Dist: bionic +Name: Bionic Beaver +Version: 18.04.6 LTS +Date: Thu, 26 April 2018 18:04:00 UTC +Supported: 1 +Description: This is the 18.04.6 LTS release +Release-File: http://archive.ubuntu.com/ubuntu/dists/bionic-updates/Release +ReleaseNotes: http://archive.ubuntu.com/ubuntu/dists/bionic-updates/main/dist-upgrader-all/current/ReleaseAnnouncement +ReleaseNotesHtml: http://archive.ubuntu.com/ubuntu/dists/bionic-updates/main/dist-upgrader-all/current/ReleaseAnnouncement.html +UpgradeTool: http://archive.ubuntu.com/ubuntu/dists/bionic-updates/main/dist-upgrader-all/current/bionic.tar.gz +UpgradeToolSignature: http://archive.ubuntu.com/ubuntu/dists/bionic-updates/main/dist-upgrader-all/current/bionic.tar.gz.gpg + +Dist: focal +Name: Focal Fossa +Version: 20.04.6 LTS +Date: Thu, 23 April 2020 20:04:00 UTC +Supported: 1 +Description: This is the 20.04.6 LTS release +Release-File: http://archive.ubuntu.com/ubuntu/dists/focal-updates/Release +ReleaseNotes: http://archive.ubuntu.com/ubuntu/dists/focal-updates/main/dist-upgrader-all/current/ReleaseAnnouncement +ReleaseNotesHtml: http://archive.ubuntu.com/ubuntu/dists/focal-updates/main/dist-upgrader-all/current/ReleaseAnnouncement.html +UpgradeTool: http://archive.ubuntu.com/ubuntu/dists/focal-updates/main/dist-upgrader-all/current/focal.tar.gz +UpgradeToolSignature: http://archive.ubuntu.com/ubuntu/dists/focal-updates/main/dist-upgrader-all/current/focal.tar.gz.gpg + +Dist: jammy +Name: Jammy Jellyfish +Version: 22.04.5 LTS +Date: Thu, 21 April 2022 22:04:00 UTC +Supported: 1 +Description: This is the 22.04.5 LTS release +Release-File: http://archive.ubuntu.com/ubuntu/dists/jammy-updates/Release +ReleaseNotes: http://archive.ubuntu.com/ubuntu/dists/jammy-updates/main/dist-upgrader-all/current/ReleaseAnnouncement +ReleaseNotesHtml: http://archive.ubuntu.com/ubuntu/dists/jammy-updates/main/dist-upgrader-all/current/ReleaseAnnouncement.html +UpgradeTool: http://archive.ubuntu.com/ubuntu/dists/jammy-updates/main/dist-upgrader-all/current/jammy.tar.gz +UpgradeToolSignature: http://archive.ubuntu.com/ubuntu/dists/jammy-updates/main/dist-upgrader-all/current/jammy.tar.gz.gpg + +Dist: noble +Name: Noble Numbat +Version: 24.04.5 LTS +Date: Thu, 25 April 2024 00:24:04 UTC +Supported: 1 +Description: This is the 24.04.5 LTS release +Release-File: http://archive.ubuntu.com/ubuntu/dists/noble-updates/Release +ReleaseNotes: http://archive.ubuntu.com/ubuntu/dists/noble-updates/main/dist-upgrader-all/current/ReleaseAnnouncement +ReleaseNotesHtml: http://archive.ubuntu.com/ubuntu/dists/noble-updates/main/dist-upgrader-all/current/ReleaseAnnouncement.html +UpgradeTool: http://archive.ubuntu.com/ubuntu/dists/noble-updates/main/dist-upgrader-all/current/noble.tar.gz +UpgradeToolSignature: http://archive.ubuntu.com/ubuntu/dists/noble-updates/main/dist-upgrader-all/current/noble.tar.gz.gpg + +Dist: resolute +Name: Resolute Raccoon +Version: 26.04.1 LTS +Date: Thu, 23 April 2026 00:26:04 UTC +Supported: 1 +Description: This is the 26.04.1 LTS release +Release-File: http://archive.ubuntu.com/ubuntu/dists/resolute-updates/Release +ReleaseNotes: http://archive.ubuntu.com/ubuntu/dists/resolute-updates/main/dist-upgrader-all/current/ReleaseAnnouncement +ReleaseNotesHtml: http://archive.ubuntu.com/ubuntu/dists/resolute-updates/main/dist-upgrader-all/current/ReleaseAnnouncement.html +UpgradeTool: http://archive.ubuntu.com/ubuntu/dists/resolute-updates/main/dist-upgrader-all/current/resolute.tar.gz +UpgradeToolSignature: http://archive.ubuntu.com/ubuntu/dists/resolute-updates/main/dist-upgrader-all/current/resolute.tar.gz.gpg diff --git a/tests/helpers.py b/tests/helpers.py index a7bb565..dc3d74e 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -5,12 +5,17 @@ this as `import helpers`. """ +import glob import inspect +import json import os import shutil +import socket import tempfile import unittest.mock +from urllib.parse import urldefrag +import requests import transmission_rpc # Env vars production code reads at call time. A developer who happens to @@ -139,3 +144,83 @@ def __call__(self, path): self.sizes.get(t.id, 0) for t in self.transmission.torrents ) return (self.total, used, self.total - used) + + +PAGES_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "fixtures", "pages" +) + + +def fake_response(body=b"", status=200, url="", headers=None): + """A real requests.Response with the given body (str or bytes), so + .text, .content, .json() and raise_for_status() behave as for real.""" + response = requests.Response() + response.status_code = status + response.reason = {200: "OK", 404: "Not Found", 500: "Internal Server Error"}.get( + status, "" + ) + response._content = body.encode("utf-8") if isinstance(body, str) else body + response.encoding = "utf-8" + response.url = url + response.headers.update(headers or {}) + return response + + +class RecordedPages: + """A requests.get stand-in serving tests/fixtures/pages (captured by + scripts/record_fixtures.py), keyed by the URL in each .meta.json. + + `overrides` maps a URL to a response, or to an exception to raise. + Any other URL is recorded in `unexpected` and answered with a 404, so a + test can assert the scraper asked for nothing it wasn't expected to.""" + + def __init__(self, overrides=None): + self.pages = {} + for meta_path in glob.glob(os.path.join(PAGES_DIR, "*", "*.meta.json")): + with open(meta_path, encoding="utf-8") as f: + url = json.load(f)["url"] + stem = meta_path[: -len(".meta.json")] + (body_path,) = ( + path for path in glob.glob(stem + ".*") if path != meta_path + ) + self.pages[url] = body_path + self.overrides = dict(overrides or {}) + self.requested = [] + self.unexpected = [] + + def get(self, url, timeout=None, **kwargs): + # A fragment never reaches the server, so match without it. + url = urldefrag(url).url + self.requested.append(url) + if url in self.overrides: + override = self.overrides[url] + if isinstance(override, BaseException): + raise override + return override + if url in self.pages: + with open(self.pages[url], "rb") as f: + return fake_response(f.read(), url=url) + self.unexpected.append(url) + return fake_response(b"not recorded", status=404, url=url) + + +def _refuse_network(*args, **kwargs): + raise RuntimeError("tests must not open network connections") + + +def block_network(): + """Makes every real socket connection fail for the rest of the module + (call from setUpModule; returns the function that undoes it, for + unittest.addModuleCleanup). Mocked sockets are unaffected.""" + patchers = [ + unittest.mock.patch.object(socket.socket, "connect", _refuse_network), + unittest.mock.patch.object(socket.socket, "connect_ex", _refuse_network), + ] + for patcher in patchers: + patcher.start() + + def stop(): + for patcher in reversed(patchers): + patcher.stop() + + return stop diff --git a/tests/test_configure_transmission.py b/tests/test_configure_transmission.py index 9207246..721e021 100644 --- a/tests/test_configure_transmission.py +++ b/tests/test_configure_transmission.py @@ -21,6 +21,10 @@ import configure_transmission as ct +def setUpModule(): + unittest.addModuleCleanup(helpers.block_network()) + + class BuildRpcOverridesTests(unittest.TestCase): def test_no_relevant_env_vars_yields_no_overrides(self): self.assertEqual(ct.build_rpc_overrides({}), {}) diff --git a/tests/test_daily_run.py b/tests/test_daily_run.py index 6be9dd6..0fd6ad2 100644 --- a/tests/test_daily_run.py +++ b/tests/test_daily_run.py @@ -15,6 +15,7 @@ import contextlib import io +import json import logging import os import runpy @@ -35,6 +36,11 @@ import fetch_torrents as ft + +def setUpModule(): + unittest.addModuleCleanup(helpers.block_network()) + + UBUNTU_META = ( "Dist: jammy\nName: Jammy Jellyfish\nVersion: 22.04 LTS\nSupported: 0\n\n" "Dist: noble\nName: Noble Numbat\nVersion: 24.04 LTS\nSupported: 1\n" @@ -52,22 +58,6 @@ ARCH_TORRENT = "https://archlinux.org/releng/releases/2026.09.01/torrent/" -class FakeResponse: - def __init__(self, status_code=200, text="", content=b"", headers=None, body=None): - self.status_code = status_code - self.text = text - self.content = content - self.headers = headers or {} - self._body = body - - def raise_for_status(self): - if self.status_code >= 400: - raise ft.requests.HTTPError(f"HTTP {self.status_code}") - - def json(self): - return self._body - - def torrent(name, id_, ratio): return helpers.FakeTorrent(name, id_, ratio=ratio) @@ -88,18 +78,18 @@ def get(self, url, timeout=None): self.requested.append(url) route = self.routes.get(url) if route is None: - return FakeResponse(404) + return helpers.fake_response(status=404) if isinstance(route, Exception): raise route return route - def post(self, url, json=None, **kwargs): - if json is None: - return FakeResponse(headers={"X-Transmission-Session-Id": "sid"}) + def post(self, url, **kwargs): + if kwargs.get("json") is None: # the session-id handshake + return helpers.fake_response(headers={"X-Transmission-Session-Id": "sid"}) torrents = [ {"name": t.name, "uploadRatio": t.ratio} for t in self.transmission.torrents ] - return FakeResponse(body={"arguments": {"torrents": torrents}}) + return helpers.fake_response(json.dumps({"arguments": {"torrents": torrents}})) def connect(self): self.events.append("connect") @@ -128,21 +118,25 @@ def write_ratio_log(self, ratios): def route_ubuntu(self): self.routes.update( { - "https://changelogs.ubuntu.com/meta-release-lts": FakeResponse( - text=UBUNTU_META + "https://changelogs.ubuntu.com/meta-release-lts": helpers.fake_response( + UBUNTU_META ), - UBUNTU + "ubuntu-24.04-desktop-amd64.iso.torrent": FakeResponse( - content=b"d1" + UBUNTU + + "ubuntu-24.04-desktop-amd64.iso.torrent": helpers.fake_response( + b"d1" ), # live-server: not routed -> 404 - LUBUNTU + "lubuntu-24.04-desktop-amd64.iso.torrent": FakeResponse(500), + LUBUNTU + + "lubuntu-24.04-desktop-amd64.iso.torrent": helpers.fake_response( + status=500 + ), XUBUNTU - + "desktop/xubuntu-24.04-desktop-amd64.iso.torrent": FakeResponse( - content=b"x1" + + "desktop/xubuntu-24.04-desktop-amd64.iso.torrent": helpers.fake_response( + b"x1" ), XUBUNTU - + "minimal/xubuntu-24.04-minimal-amd64.iso.torrent": FakeResponse( - content=b"x2" + + "minimal/xubuntu-24.04-minimal-amd64.iso.torrent": helpers.fake_response( + b"x2" ), } ) @@ -155,8 +149,8 @@ def route_ubuntu(self): def route_arch(self): self.routes.update( { - ARCH_RELEASES: FakeResponse(text=ARCH_HTML), - ARCH_TORRENT: FakeResponse(content=b"a1"), + ARCH_RELEASES: helpers.fake_response(ARCH_HTML), + ARCH_TORRENT: helpers.fake_response(b"a1"), } ) diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py index cb50c0c..a83ec11 100644 --- a/tests/test_fetch_torrents.py +++ b/tests/test_fetch_torrents.py @@ -35,6 +35,7 @@ def setUpModule(): patcher = helpers.isolated_env() patcher.start() unittest.addModuleCleanup(patcher.stop) + unittest.addModuleCleanup(helpers.block_network()) FakeTorrent = helpers.FakeTorrent @@ -1095,12 +1096,9 @@ class FetchUbuntuLtsNamingTests(unittest.TestCase): def test_candidate_names_include_arch_and_iso_suffix(self): meta_release = "Dist: noble\nVersion: 24.04\nSupported: 1\n" - class FakeResponse: - text = meta_release + response = helpers.fake_response(meta_release) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_ubuntu_lts() self.assertTrue(results) @@ -1115,15 +1113,9 @@ class FetchDebianStableNamingTests(unittest.TestCase): def test_candidate_name_keeps_iso_suffix(self): html = 'link' - class FakeResponse: - text = html - - def raise_for_status(self): - pass + response = helpers.fake_response(html) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_debian_stable() self.assertIn("debian-12.5.0-amd64-DVD-1.iso", results) @@ -1138,12 +1130,9 @@ class FetchLinuxMintCinnamonNamingTests(unittest.TestCase): def test_candidate_name_matches_torrent_internal_name(self): html = "Download Linux Mint 22.3 - Linux Mint" - class FakeResponse: - text = html + response = helpers.fake_response(html) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_linuxmint_cinnamon() self.assertIn("linuxmint-22.3-cinnamon-64bit.iso", results) @@ -1170,15 +1159,9 @@ def test_picks_highest_version_per_arch(self): "" ) - class FakeResponse: - text = html - - def raise_for_status(self): - pass + response = helpers.fake_response(html) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_fedora_workstation() self.assertEqual( @@ -1206,15 +1189,9 @@ def test_picks_only_the_single_newest_release(self): "" ) - class FakeResponse: - text = html - - def raise_for_status(self): - pass + response = helpers.fake_response(html) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_arch_latest() self.assertEqual( @@ -1227,15 +1204,9 @@ def raise_for_status(self): def test_no_available_releases_returns_false(self): html = "
" - class FakeResponse: - text = html - - def raise_for_status(self): - pass + response = helpers.fake_response(html) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): result = ft.fetch_arch_latest() self.assertFalse(result) @@ -1255,12 +1226,9 @@ def test_extracts_current_release_torrent(self): "" ) - class FakeResponse: - text = html + response = helpers.fake_response(html) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_devuan_latest() self.assertEqual( @@ -1269,12 +1237,9 @@ class FakeResponse: ) def test_returns_false_when_no_torrent_link_found(self): - class FakeResponse: - text = "nothing here" + response = helpers.fake_response("nothing here") - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): result = ft.fetch_devuan_latest() self.assertFalse(result) @@ -1420,12 +1385,9 @@ def setUp(self): self.addCleanup(patcher.stop) def test_404_response_is_not_found_not_failed(self): - class FakeResponse: - status_code = 404 + response = helpers.fake_response(status=404) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): status = ft.download_torrent( "ubuntu-14.04.6-live-server-amd64.iso", "https://example.invalid/x.torrent", @@ -1438,15 +1400,9 @@ class FakeResponse: self.assertFalse(os.path.exists(dest)) def test_other_http_error_is_still_failed(self): - class FakeResponse: - status_code = 500 - - def raise_for_status(self): - raise ft.requests.HTTPError("500 Server Error") + response = helpers.fake_response(status=500) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): status = ft.download_torrent( "ubuntu-24.04-desktop-amd64.iso", "https://example.invalid/x.torrent" ) @@ -1454,16 +1410,9 @@ def raise_for_status(self): self.assertEqual(status, "failed") def test_successful_download_is_still_added(self): - class FakeResponse: - status_code = 200 - content = b"fake torrent bytes" - - def raise_for_status(self): - pass + response = helpers.fake_response(b"fake torrent bytes") - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): status = ft.download_torrent( "ubuntu-24.04-desktop-amd64.iso", "https://example.invalid/x.torrent" ) @@ -1958,12 +1907,9 @@ def test_returns_only_the_non_newest_versions(self): "" ) - class FakeResponse: - text = html + response = helpers.fake_response(html) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_kali_old_versions() self.assertTrue(results) @@ -1973,12 +1919,9 @@ class FakeResponse: def test_single_version_folder_returns_nothing(self): html = 'currentkali-2026.2' - class FakeResponse: - text = html + response = helpers.fake_response(html) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_kali_old_versions() self.assertEqual(results, {}) @@ -1998,15 +1941,9 @@ def test_returns_non_latest_versions_per_arch(self): "" ) - class FakeResponse: - text = html - - def raise_for_status(self): - pass + response = helpers.fake_response(html) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_fedora_workstation_old_versions() self.assertEqual( @@ -2020,15 +1957,9 @@ def raise_for_status(self): def test_single_version_per_arch_returns_nothing(self): html = 'a' - class FakeResponse: - text = html - - def raise_for_status(self): - pass + response = helpers.fake_response(html) - with unittest.mock.patch.object( - ft.requests, "get", return_value=FakeResponse() - ): + with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_fedora_workstation_old_versions() self.assertEqual(results, {}) @@ -2177,11 +2108,10 @@ def test_extreme_oversupply_still_fails_at_default_ratio(self): class EvaluateOldReleaseDemandTests(unittest.TestCase): def test_404_is_not_kept_and_not_scraped(self): - class FakeResponse: - status_code = 404 + response = helpers.fake_response(status=404) with ( - unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()), + unittest.mock.patch.object(ft.requests, "get", return_value=response), unittest.mock.patch.object( ft.torrent_scrape, "scrape_torrent" ) as mock_scrape, @@ -2216,15 +2146,10 @@ def test_download_failure_is_not_kept(self): self.assertIsNone(scrape_result) def test_scrape_below_threshold_is_not_kept(self): - class FakeResponse: - status_code = 200 - content = b"fake torrent bytes" - - def raise_for_status(self): - pass + response = helpers.fake_response(b"fake torrent bytes") with ( - unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()), + unittest.mock.patch.object(ft.requests, "get", return_value=response), unittest.mock.patch.object( ft.torrent_scrape, "scrape_torrent", @@ -2249,15 +2174,10 @@ def raise_for_status(self): self.assertEqual(scrape_result["leechers"], 0) def test_scrape_meeting_threshold_is_kept(self): - class FakeResponse: - status_code = 200 - content = b"fake torrent bytes" - - def raise_for_status(self): - pass + response = helpers.fake_response(b"fake torrent bytes") with ( - unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()), + unittest.mock.patch.object(ft.requests, "get", return_value=response), unittest.mock.patch.object( ft.torrent_scrape, "scrape_torrent", @@ -2282,17 +2202,12 @@ def raise_for_status(self): self.assertEqual(scrape_result["leechers"], 30) def test_meets_absolute_floor_but_already_well_seeded_is_not_kept(self): - class FakeResponse: - status_code = 200 - content = b"fake torrent bytes" - - def raise_for_status(self): - pass + response = helpers.fake_response(b"fake torrent bytes") # 1 leecher clears a min_leechers=1 floor, but 1000 seeders already # cover it - not unmet demand. with ( - unittest.mock.patch.object(ft.requests, "get", return_value=FakeResponse()), + unittest.mock.patch.object(ft.requests, "get", return_value=response), unittest.mock.patch.object( ft.torrent_scrape, "scrape_torrent", diff --git a/tests/test_scrapers.py b/tests/test_scrapers.py new file mode 100644 index 0000000..c54d967 --- /dev/null +++ b/tests/test_scrapers.py @@ -0,0 +1,418 @@ +"""Contract tests for fetch_torrents' upstream scrapers, replaying real +pages captured into tests/fixtures/pages by scripts/record_fixtures.py. + +The expected results below were read off the recorded pages by hand, not +produced by running the scrapers. When a re-recording changes a page, update +them to the new snapshot - after checking they still make sense. + +No test here touches the network (helpers.block_network()). + +Run with: python -m unittest discover -s tests +""" + +import os +import sys +import unittest +import unittest.mock + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + +import helpers +import requests + +import fetch_torrents as ft + + +def setUpModule(): + unittest.addModuleCleanup(helpers.block_network()) + patcher = helpers.isolated_env() + patcher.start() + unittest.addModuleCleanup(patcher.stop) + + +UBUNTU_SUPPORTED = ( # newest first, as fetch_ubuntu_lts() orders them + ("resolute", "26.04.1"), + ("noble", "24.04.5"), + ("jammy", "22.04.5"), + ("focal", "20.04.6"), + ("bionic", "18.04.6"), + ("xenial", "16.04.7"), + ("trusty", "14.04.6"), +) +UBUNTU_TEMPLATES = ( + "https://releases.ubuntu.com/{c}/ubuntu-{v}-desktop-amd64.iso.torrent", + "https://releases.ubuntu.com/{c}/ubuntu-{v}-live-server-amd64.iso.torrent", + "https://cdimage.ubuntu.com/lubuntu/releases/{c}/release/lubuntu-{v}-desktop-amd64.iso.torrent", + "https://torrent.ubuntu.com/xubuntu/releases/{c}/release/desktop/xubuntu-{v}-desktop-amd64.iso.torrent", + "https://torrent.ubuntu.com/xubuntu/releases/{c}/release/minimal/xubuntu-{v}-minimal-amd64.iso.torrent", +) +KALI_SUFFIXES = ( + "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-amd64.iso", + "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-netinst-amd64.iso", + "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-everything-amd64.iso", + "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-arm64.iso", + "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-netinst-arm64.iso", + "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-purple-amd64.iso", + "kali.download/arm-images/kali-{v}/kali-linux-{v}-raspberry-pi-armhf.img.xz", + "kali.download/arm-images/kali-{v}/kali-linux-{v}-raspberry-pi-zero-2-w-armhf.img.xz", + "kali.download/arm-images/kali-{v}/kali-linux-{v}-raspberry-pi-zero-w-armel.img.xz", + "kali.download/cloud-images/kali-{v}/kali-linux-{v}-cloud-genericcloud-amd64.tar.xz", + "kali.download/cloud-images/kali-{v}/kali-linux-{v}-cloud-genericcloud-arm64.tar.xz", +) +FEDORA = "https://torrent.fedoraproject.org/torrents/" +DEBIAN = "https://cdimage.debian.org/debian-cd/current/" + + +def kali_candidates(version): + return { + os.path.basename(suffix.format(v=version)): ( + "https://" + suffix.format(v=version) + ".torrent" + ) + for suffix in KALI_SUFFIXES + } + + +class ScraperTestCase(unittest.TestCase): + def fetch(self, fetcher, overrides=None): + """Runs fetcher against the recorded pages; returns (result, logs).""" + pages = helpers.RecordedPages(overrides) + with ( + unittest.mock.patch.object(ft.requests, "get", pages.get), + self.assertLogs(ft.logger, level="DEBUG") as logs, + ): + ft.logger.debug("fetch start") # assertLogs needs at least one + result = fetcher() + self.assertEqual(pages.unexpected, []) + self.pages = pages + return result, [r.getMessage() for r in logs.records[1:]] + + +class RecordedSnapshotTests(ScraperTestCase): + """Each fetcher, replayed against its captured pages, must produce + exactly the candidates the snapshot lists.""" + + def test_ubuntu_lists_every_supported_lts_newest_first(self): + result, _logs = self.fetch(ft.fetch_ubuntu_lts) + + expected = {} + for codename, version in UBUNTU_SUPPORTED: + for template in UBUNTU_TEMPLATES: + url = template.format(c=codename, v=version) + expected[os.path.basename(url)[: -len(".torrent")]] = url + self.assertEqual(result, expected) + self.assertEqual(list(result), list(expected)) + + def test_debian_takes_the_first_torrent_of_each_listing(self): + result, logs = self.fetch(ft.fetch_debian_stable) + + self.assertEqual( + result, + { + "debian-13.7.0-amd64-DVD-1.iso": DEBIAN + + "amd64/bt-dvd/debian-13.7.0-amd64-DVD-1.iso.torrent", + "debian-13.7.0-arm64-DVD-1.iso": DEBIAN + + "arm64/bt-dvd/debian-13.7.0-arm64-DVD-1.iso.torrent", + "debian-13.7.0-amd64-netinst.iso": DEBIAN + + "amd64/bt-cd/debian-13.7.0-amd64-netinst.iso.torrent", + "debian-13.7.0-arm64-netinst.iso": DEBIAN + + "arm64/bt-cd/debian-13.7.0-arm64-netinst.iso.torrent", + }, + ) + self.assertEqual(logs, []) + + def test_kali_latest_uses_the_newest_installer_version(self): + result, _logs = self.fetch(ft.fetch_kali_latest) + + self.assertEqual(result, kali_candidates("2026.2")) + self.assertEqual(self.pages.requested, ["https://www.kali.org/get-kali/"]) + + def test_kali_old_versions_lists_every_folder_but_the_newest(self): + result, _logs = self.fetch(ft.fetch_kali_old_versions) + + self.assertEqual(result, kali_candidates("2026.1")) + + def test_arch_takes_the_newest_available_release(self): + result, _logs = self.fetch(ft.fetch_arch_latest) + + self.assertEqual( + result, + { + "archlinux-2026.10.01": ( + "https://archlinux.org/releng/releases/2026.10.01/torrent/" + ) + }, + ) + + def test_devuan_takes_the_linked_release_torrent(self): + result, _logs = self.fetch(ft.fetch_devuan_latest) + + self.assertEqual( + result, + {"devuan_excalibur": "https://files.devuan.org/devuan_excalibur.torrent"}, + ) + + def test_mint_reads_the_version_from_the_title(self): + result, _logs = self.fetch(ft.fetch_linuxmint_cinnamon) + + self.assertEqual( + result, + { + "linuxmint-22.3-cinnamon-64bit.iso": ( + "https://www.linuxmint.com/torrents/" + "linuxmint-22.3-cinnamon-64bit.iso.torrent" + ) + }, + ) + + def test_fedora_takes_the_newest_final_release_per_arch(self): + # 45_Beta is listed too, but isn't a final release. + result, _logs = self.fetch(ft.fetch_fedora_workstation) + + self.assertEqual( + result, + { + "Fedora-Workstation-Live-aarch64-44": ( + FEDORA + "Fedora-Workstation-Live-aarch64-44.torrent" + ), + "Fedora-Workstation-Live-x86_64-44": ( + FEDORA + "Fedora-Workstation-Live-x86_64-44.torrent" + ), + }, + ) + + def test_fedora_old_versions_lists_the_older_final_releases(self): + result, _logs = self.fetch(ft.fetch_fedora_workstation_old_versions) + + self.assertEqual( + result, + { + "Fedora-Workstation-Live-aarch64-43": ( + FEDORA + "Fedora-Workstation-Live-aarch64-43.torrent" + ), + "Fedora-Workstation-Live-x86_64-43": ( + FEDORA + "Fedora-Workstation-Live-x86_64-43.torrent" + ), + }, + ) + + +# (fetcher, the URL it reads, the error it logs, what it returns on failure) +LATEST_FETCHERS = ( + ( + ft.fetch_ubuntu_lts, + "https://changelogs.ubuntu.com/meta-release-lts", + "Ubuntu fetch error", + False, + ), + (ft.fetch_kali_latest, "https://www.kali.org/get-kali/", "Kali fetch error", False), + ( + ft.fetch_kali_old_versions, + "https://cdimage.kali.org/", + "Kali old-release fetch error", + {}, + ), + ( + ft.fetch_arch_latest, + "https://archlinux.org/releng/releases/", + "Arch Linux fetch error", + False, + ), + ( + ft.fetch_devuan_latest, + "https://www.devuan.org/get-devuan", + "Devuan fetch error", + False, + ), + ( + ft.fetch_linuxmint_cinnamon, + "https://www.linuxmint.com/download.php", + "Linux Mint fetch error", + False, + ), + (ft.fetch_fedora_workstation, FEDORA, "Fedora fetch error", False), + ( + ft.fetch_fedora_workstation_old_versions, + FEDORA, + "Fedora old-release fetch error", + {}, + ), +) + + +class FailurePathTests(ScraperTestCase): + """An unreachable or erroring upstream must make a fetcher log what went + wrong and return its empty result - never raise into the daily run, and + never pass an error page off as "no releases".""" + + def _assert_fails_with(self, overrides, expected_in_log): + for fetcher, url, error, empty in LATEST_FETCHERS: + with self.subTest(fetcher=fetcher.__name__): + result, logs = self.fetch(fetcher, {url: overrides(url)}) + + self.assertIs(type(result), type(empty)) + self.assertEqual(result, empty) + self.assertTrue( + any(m.startswith(error) and expected_in_log in m for m in logs), + logs, + ) + + def test_connection_error(self): + self._assert_fails_with( + lambda url: requests.ConnectionError("connection refused"), + "connection refused", + ) + + def test_http_500(self): + self._assert_fails_with( + lambda url: helpers.fake_response("oops", status=500, url=url), + "500 Server Error", + ) + + def test_http_404(self): + self._assert_fails_with( + lambda url: helpers.fake_response("gone", status=404, url=url), + "404 Client Error", + ) + + def test_debian_logs_each_failing_listing_and_keeps_the_rest(self): + amd64_dvd = DEBIAN + "amd64/bt-dvd/" + arm64_cd = DEBIAN + "arm64/bt-cd/" + + result, logs = self.fetch( + ft.fetch_debian_stable, + { + amd64_dvd: requests.ConnectionError("refused"), + arm64_cd: helpers.fake_response("oops", status=500, url=arm64_cd), + }, + ) + + self.assertEqual( + sorted(result), + ["debian-13.7.0-amd64-netinst.iso", "debian-13.7.0-arm64-DVD-1.iso"], + ) + self.assertEqual([m.split(":")[0] for m in logs], ["Debian fetch error"] * 2) + self.assertIn("refused", logs[0]) + self.assertIn("500 Server Error", logs[1]) + + +class LayoutChangeTests(ScraperTestCase): + """A page that loads fine but no longer has the element a scraper looks + for must give an empty result and a warning, not an exception.""" + + def _page(self, url, body): + return {url: helpers.fake_response(body, url=url)} + + def test_kali_page_without_a_version(self): + url = "https://www.kali.org/get-kali/" + + result, logs = self.fetch( + ft.fetch_kali_latest, self._page(url, "Get Kali") + ) + + self.assertIs(result, False) + self.assertEqual( + logs, + [f"Could not detect a Kali release number on {url}#kali-installer-images"], + ) + + def test_kali_cdimage_with_a_single_version_has_no_old_ones(self): + result, logs = self.fetch( + ft.fetch_kali_old_versions, + self._page("https://cdimage.kali.org/", 'x'), + ) + + self.assertEqual((result, logs), ({}, [])) + + def test_ubuntu_block_without_a_version_line(self): + result, logs = self.fetch( + ft.fetch_ubuntu_lts, + self._page( + "https://changelogs.ubuntu.com/meta-release-lts", + "Dist: noble\nSupported: 1\n", + ), + ) + + self.assertIs(result, False) + self.assertEqual(len(logs), 1) + self.assertTrue(logs[0].startswith("Ubuntu fetch error")) + + def test_debian_listing_without_torrents(self): + overrides = {} + for listing in ( + "amd64/bt-dvd/", + "arm64/bt-dvd/", + "amd64/bt-cd/", + "arm64/bt-cd/", + ): + overrides.update(self._page(DEBIAN + listing, "x")) + + result, logs = self.fetch(ft.fetch_debian_stable, overrides) + + self.assertEqual(result, {}) + self.assertEqual(logs, ["No Debian DVD-1 torrent found."] * 4) + + def test_arch_page_without_the_release_table(self): + result, logs = self.fetch( + ft.fetch_arch_latest, + self._page("https://archlinux.org/releng/releases/", ""), + ) + + self.assertIs(result, False) + self.assertEqual(len(logs), 1) + self.assertTrue(logs[0].startswith("Arch Linux fetch error")) + + def test_arch_table_without_available_releases(self): + url = "https://archlinux.org/releng/releases/" + result, logs = self.fetch( + ft.fetch_arch_latest, + self._page( + url, + "" + "" + "
t
", + ), + ) + + self.assertIs(result, False) + self.assertEqual(logs, [f"No available Arch Linux releases found on {url}"]) + + def test_devuan_page_without_a_torrent_link(self): + url = "https://www.devuan.org/get-devuan" + + result, logs = self.fetch(ft.fetch_devuan_latest, self._page(url, "")) + + self.assertIs(result, False) + self.assertEqual( + logs, [f"Could not find a Devuan release torrent link on {url}"] + ) + + def test_mint_page_without_a_versioned_title(self): + url = "https://www.linuxmint.com/download.php" + + result, logs = self.fetch( + ft.fetch_linuxmint_cinnamon, self._page(url, "Linux Mint") + ) + + self.assertIs(result, False) + self.assertEqual(logs, [f"Could not detect a Linux Mint version on {url}"]) + + def test_fedora_listing_without_workstation_torrents(self): + result, logs = self.fetch( + ft.fetch_fedora_workstation, + self._page(FEDORA, "x"), + ) + + self.assertIs(result, False) + self.assertEqual(logs, [f"No Fedora Workstation torrents found on {FEDORA}"]) + + +class NetworkGuardTests(unittest.TestCase): + def test_real_connections_are_refused(self): + with self.assertRaisesRegex(RuntimeError, "must not open network"): + requests.get("http://192.0.2.1/", timeout=1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index c6d9ba6..b075656 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -1,8 +1,7 @@ """Tests for torrent_scrape.py: bencode parsing and tracker "scrape" support. -Unlike fetch_torrents.py, this module has no apk-only or container-specific -imports (just requests + stdlib), so it can be imported directly in a plain -test environment. +Socket-level tests replace socket.socket with mocks; any real connection +is refused (helpers.block_network()). Run with: python -m unittest discover -s tests """ @@ -20,9 +19,15 @@ if REPO_ROOT not in sys.path: sys.path.insert(0, REPO_ROOT) +import helpers + import torrent_scrape as ts +def setUpModule(): + unittest.addModuleCleanup(helpers.block_network()) + + def bstr(s): """Hand-rolled bencode byte-string, independent of ts.bencode(), so it serves as an oracle rather than testing the implementation against From ad49ba9de56822872697b473cdbaddc87c437213 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 13:03:36 +0200 Subject: [PATCH 21/46] Add a weekly live drift check (task 23) The merge check is deterministic, but the distro sites, the trackers and the published image (apk upgrade at start) keep moving. Nothing checked them until a user noticed missing torrents. .github/workflows/drift.yml runs on a weekly schedule and by hand, and is not part of the PR check: - scrapers: scripts/live_check.py runs every fetcher live. Each must return candidates whose names match the distro pattern with a parseable version and https .torrent URLs, at least one answering 200. On failure the pages are re-recorded and uploaded as an artifact. - tracker: scrapes a current Ubuntu torrent; counts must be integers. - transmission: scripts/transmission_roundtrip.py, run inside the published image against its own daemon. It adds a self-made tracker-less torrent via /watch, reads it like cleanup does, logs ratios over raw RPC and removes it with delete_data=True, all through fetch_torrents' own load_config/make_client_factory. - published-image: scripts/smoke_test.sh against :latest with network and apk upgrade. The script now takes SMOKE_NETWORK and SMOKE_SKIP_PACKAGE_UPDATES. - mutation: mutmut results uploaded for trend tracking; never fails. - Any failure opens, or comments on, a single issue labelled `drift`. All checks run locally on 2026-10-19 and pass. The round trip ran against an image built from this branch, since the current :latest predates load_config() and make_client_factory(). Co-Authored-By: Claude Opus 5.5 --- .github/workflows/drift.yml | 154 ++++++++++++++++++++++++++++++ CONTRIBUTING.md | 15 +++ scripts/live_check.py | 139 +++++++++++++++++++++++++++ scripts/smoke_test.sh | 14 ++- scripts/transmission_roundtrip.py | 87 +++++++++++++++++ tests/test_live_check.py | 126 ++++++++++++++++++++++++ 6 files changed, 531 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/drift.yml create mode 100644 scripts/live_check.py create mode 100644 scripts/transmission_roundtrip.py create mode 100644 tests/test_live_check.py diff --git a/.github/workflows/drift.yml b/.github/workflows/drift.yml new file mode 100644 index 0000000..37e782e --- /dev/null +++ b/.github/workflows/drift.yml @@ -0,0 +1,154 @@ +name: Live drift check + +# The merge check is deterministic by design (recorded pages, fakes, no +# network). This weekly job checks the moving parts it can't: the live +# distro sites, a live tracker, a real transmission-daemon, and the +# published image as deployed (network + apk upgrade at start). It never +# blocks anything; a failure opens (or comments on) one issue labelled +# `drift`. + +on: + schedule: + - cron: "17 5 * * 1" # Mondays 05:17 UTC + workflow_dispatch: + +permissions: + contents: read + +env: + IMAGE: ghcr.io/squazz/linux-iso-seeder:latest + +jobs: + scrapers: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.x" + + - run: pip install -r requirements-test.txt + + # Every fetcher must return well-formed candidates, at least one of + # which answers 200. + - name: Run fetchers against the live sites + run: python scripts/live_check.py scrapers + + # Fresh captures to diff against tests/fixtures/pages. + - name: Re-record fixtures + if: failure() + run: python scripts/record_fixtures.py + + - name: Upload re-recorded fixtures + if: failure() + uses: actions/upload-artifact@v4 + with: + name: rerecorded-fixtures + path: tests/fixtures/pages + + tracker: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.x" + + - run: pip install -r requirements-test.txt + + - name: Scrape a current Ubuntu torrent's tracker + run: python scripts/live_check.py tracker + + transmission: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + # Just the daemon (no daily fetch loop), offline: the round trip uses a + # self-made tracker-less torrent, so nothing is downloaded. + - name: Start the published image's transmission-daemon + run: | + docker run -d --name seeder --network none --entrypoint sh "$IMAGE" -c \ + 'python3 /usr/local/bin/configure_transmission.py && exec transmission-daemon --foreground --config-dir /config --download-dir /downloads --watch-dir /watch' + + # Inside the image, so it's the image's own apk py3-transmission-rpc + # and fetch_torrents.py talking to its own daemon. + - name: Add, read, log and remove a torrent through the RPC + run: | + docker cp scripts/transmission_roundtrip.py seeder:/tmp/ + docker exec -w /usr/local/bin seeder python3 /tmp/transmission_roundtrip.py + + - name: Daemon logs + if: always() + run: docker logs seeder + + published-image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Smoke-test the published image as deployed + run: sh scripts/smoke_test.sh "$IMAGE" + env: + SMOKE_NETWORK: bridge + SMOKE_SKIP_PACKAGE_UPDATES: "false" + SMOKE_TIMEOUT_SECONDS: "300" + + # Trend only: results are uploaded, and a low score never fails the run. + mutation: + runs-on: ubuntu-latest + timeout-minutes: 180 + continue-on-error: true + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.x" + + - run: pip install -r requirements-dev.txt + + - name: Run mutmut + run: | + mutmut run --no-progress || true + mutmut results > mutmut-results.txt || true + + - uses: actions/upload-artifact@v4 + with: + name: mutmut-results + path: | + mutmut-results.txt + .mutmut-cache + + report: + needs: [scrapers, tracker, transmission, published-image] + if: failure() + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - uses: actions/github-script@v7 + with: + script: | + const label = "drift"; + const run = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const body = `The weekly live drift check failed: ${run}\n\n` + + "Check which job failed. For scraper drift, the run has a " + + "`rerecorded-fixtures` artifact to diff against `tests/fixtures/pages`."; + try { + await github.rest.issues.createLabel({ ...context.repo, name: label, color: "d93f0b" }); + } catch (error) { + if (error.status !== 422) throw error; // 422: already exists + } + const { data: open } = await github.rest.issues.listForRepo({ + ...context.repo, labels: label, state: "open", + }); + if (open.length > 0) { + await github.rest.issues.createComment({ ...context.repo, issue_number: open[0].number, body }); + } else { + await github.rest.issues.create({ + ...context.repo, title: "Live drift check failed", labels: [label], body, + }); + } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bca1a9a..f70c781 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,6 +39,21 @@ applied its overrides, that `transmission-daemon` is running, and that every module imports. When `alpine:latest` moves to a new `py3-transmission-rpc`, bump the `transmission-rpc` pin in `requirements-test.txt` to match. +## Upstream drift + +The merge check never touches the network. `.github/workflows/drift.yml` +runs weekly (or by hand from the Actions tab) against the live distro sites, +a live tracker, a real `transmission-daemon` and the published image, and +opens or comments on an issue labelled `drift` when something has moved. To +fix scraper drift, re-record the pages and update the expected results in +`tests/test_scrapers.py` to the new snapshot: + +```sh +python scripts/live_check.py scrapers # what the weekly job checks +python scripts/record_fixtures.py [distro ...] +git diff tests/fixtures/pages +``` + ## Mutation testing Mutation testing checks that the tests actually pin behavior, not just diff --git a/scripts/live_check.py b/scripts/live_check.py new file mode 100644 index 0000000..5f31d2e --- /dev/null +++ b/scripts/live_check.py @@ -0,0 +1,139 @@ +"""Live drift checks for .github/workflows/drift.yml: the scrapers and the +tracker scrape, run against the real upstream sites. + +Usage: + python scripts/live_check.py scrapers every fetcher returns well-formed, + reachable candidates + python scripts/live_check.py tracker a current Ubuntu torrent's tracker + answers a scrape with integer counts + +Prints what it checked and exits 1 if anything looks wrong. Not part of the +merge check - upstream sites move independently of this repo. +""" + +import os +import sys +import tempfile + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + +import requests + +import fetch_torrents as ft +import torrent_scrape + +FETCHERS = ( + *ft.DEFAULT_FETCHERS, + ("kali-old", ft.fetch_kali_old_versions), + ("fedora-old", ft.fetch_fedora_workstation_old_versions), +) +# Old-release fetchers may legitimately find nothing (e.g. right after +# cdimage.kali.org drops the previous release's folder). +MAY_BE_EMPTY = {"kali-old", "fedora-old"} + + +def distro_of(fetcher_name): + return fetcher_name.removesuffix("-old") + + +def problems(distro, name, url): + """What's wrong with one candidate a fetcher returned, as messages + (empty if it looks like a real release of `distro`).""" + found = [] + pattern = ( + ft.DEVUAN_NAME_PATTERN if distro == "devuan" else ft.distro_patterns[distro] + ) + if not pattern.match(name): + found.append(f"name {name!r} doesn't look like a {distro} release") + elif distro != "devuan": + try: + version, _type = ft.parse_version_type(name, distro) + ft.version_to_tuple(version) + except Exception: + found.append(f"name {name!r} has no parseable version") + if not url.startswith("https://"): + found.append(f"URL {url!r} is not https") + if not ( + url.endswith(".torrent") or (distro == "arch" and url.endswith("/torrent/")) + ): + found.append(f"URL {url!r} is not a .torrent") + return found + + +def is_reachable(url): + try: + r = requests.head(url, allow_redirects=True, timeout=30) + if r.status_code == 405: # HEAD not allowed: ask for the body instead + with requests.get(url, stream=True, timeout=30) as r: + return r.status_code == 200 + return r.status_code == 200 + except requests.RequestException: + return False + + +def check_scrapers(): + failures = [] + for fetcher_name, fetcher in FETCHERS: + distro = distro_of(fetcher_name) + candidates = fetcher() or {} + print(f"{fetcher_name}: {len(candidates)} candidates") + if not candidates: + if fetcher_name not in MAY_BE_EMPTY: + failures.append(f"{fetcher_name}: no candidates") + continue + + well_formed = [] + for name, url in candidates.items(): + found = problems(distro, name, url) + failures.extend(f"{fetcher_name}: {problem}" for problem in found) + if not found: + well_formed.append(url) + + # ESM-only Ubuntu releases etc. are expected to 404 (download_torrent() + # counts those as "not found upstream"), so one reachable is enough. + reachable = next((url for url in well_formed if is_reachable(url)), None) + if reachable: + print(f" reachable: {reachable}") + elif well_formed: + failures.append(f"{fetcher_name}: none of its torrent URLs returned 200") + return failures + + +def check_tracker(): + candidates = ft.fetch_ubuntu_lts() or {} + desktop = [url for name, url in candidates.items() if "-desktop-" in name] + for url in desktop: # newest first; older ones may be gone (404) + r = requests.get(url, timeout=30) + if r.status_code == 200: + break + else: + return ["tracker: no current Ubuntu desktop torrent could be downloaded"] + + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "ubuntu.torrent") + with open(path, "wb") as f: + f.write(r.content) + result = torrent_scrape.scrape_torrent(path) + print(f"tracker: {url} -> {result}") + if result is None: + return ["tracker: no tracker answered the scrape"] + if not all(isinstance(result.get(key), int) for key in ("seeders", "leechers")): + return [f"tracker: non-integer seeders/leechers in {result}"] + return [] + + +def main(argv): + checks = {"scrapers": check_scrapers, "tracker": check_tracker} + if len(argv) != 1 or argv[0] not in checks: + print(__doc__, file=sys.stderr) + return 2 + failures = checks[argv[0]]() + for failure in failures: + print(f"DRIFT: {failure}", file=sys.stderr) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh index 8f6485c..09f393d 100755 --- a/scripts/smoke_test.sh +++ b/scripts/smoke_test.sh @@ -6,12 +6,18 @@ # # Usage: scripts/smoke_test.sh # -# --network none keeps it deterministic: the daily fetch inside the -# container logs network errors, which is expected - only startup is checked. +# By default it runs with --network none and SKIP_PACKAGE_UPDATES=true, which +# keeps it deterministic: the daily fetch inside the container logs network +# errors, which is expected - only startup is checked. The weekly drift job +# (.github/workflows/drift.yml) runs the published image as deployed instead: +# SMOKE_NETWORK=bridge SMOKE_SKIP_PACKAGE_UPDATES=false \ +# SMOKE_TIMEOUT_SECONDS=300 scripts/smoke_test.sh set -eu IMAGE=$1 TIMEOUT_SECONDS=${SMOKE_TIMEOUT_SECONDS:-30} +NETWORK=${SMOKE_NETWORK:-none} +SKIP_PACKAGE_UPDATES=${SMOKE_SKIP_PACKAGE_UPDATES:-true} # Git Bash on Windows would otherwise rewrite /config etc. into Windows paths. export MSYS_NO_PATHCONV=1 @@ -34,8 +40,8 @@ fail() { # Usage: start_container ; prints the container id. start_container() { - docker run -d --network none \ - -e SKIP_PACKAGE_UPDATES=true \ + docker run -d --network "$NETWORK" \ + -e SKIP_PACKAGE_UPDATES="$SKIP_PACKAGE_UPDATES" \ -e TRANSMISSION_PEER_LIMIT_GLOBAL=123 \ "$@" "$IMAGE" } diff --git a/scripts/transmission_roundtrip.py b/scripts/transmission_roundtrip.py new file mode 100644 index 0000000..eba0204 --- /dev/null +++ b/scripts/transmission_roundtrip.py @@ -0,0 +1,87 @@ +"""Drives a real transmission-daemon through every Transmission call the +daily run makes, using fetch_torrents' own code paths - for +.github/workflows/drift.yml, run *inside* the image so it also exercises +the image's apk py3-transmission-rpc against its transmission-daemon: + + docker cp scripts/transmission_roundtrip.py :/tmp/ + docker exec -w /usr/local/bin python3 /tmp/transmission_roundtrip.py + +Drops a tiny self-made .torrent (one 16-byte file, random name, no +trackers - so nothing is ever downloaded) into /watch, waits for the daemon +to pick it up, reads it the way cleanup does, logs seed ratios over raw RPC +the way the daily run does, then removes it with delete_data=True. +Exits 1 on any failure. +""" + +import hashlib +import os +import secrets +import sys +import time + +sys.path.insert(0, "/usr/local/bin") + +import fetch_torrents as ft +import torrent_scrape + +TIMEOUT_SECONDS = 90 + + +def make_torrent(name): + data = secrets.token_bytes(16) + info = { + "name": name, + "length": len(data), + "piece length": 16384, + "pieces": hashlib.sha1(data).digest(), + } + return torrent_scrape.bencode({"info": info}) + + +def wait_for(description, predicate): + deadline = time.time() + TIMEOUT_SECONDS + while time.time() < deadline: + result = predicate() + if result: + return result + time.sleep(2) + raise SystemExit(f"FAILED: timed out waiting for {description}") + + +def main(): + ft.configure_logging("/tmp") + if not ft.wait_for_transmission_rpc(timeout=TIMEOUT_SECONDS): + raise SystemExit("FAILED: Transmission RPC never came up") + + config = ft.load_config(os.environ) + client = ft.make_client_factory(config)() + + name = f"drift-check-{secrets.token_hex(4)}.bin" + with open(os.path.join(ft.watch_dir, f"{name}.torrent"), "wb") as f: + f.write(make_torrent(name)) + + def added(): + return next((t for t in client.get_torrents() if t.name == name), None) + + torrent = wait_for(f"{name} to be added from {ft.watch_dir}", added) + # The attributes cleanup and the stagnation history read. + print( + f"added: id={torrent.id} name={torrent.name} ratio={float(torrent.ratio)} " + f"added_date={torrent.added_date.timestamp()}" + ) + + ft.log_seed_ratios_via_http(auth=config.rpc_auth) + with open(os.path.join("/tmp", ft.RATIO_LOG_FILE_NAME), encoding="utf-8") as f: + if name not in f.read(): + raise SystemExit("FAILED: torrent missing from the RPC ratio log") + + client.remove_torrent(torrent.id, delete_data=True) + wait_for( + f"{name} to be removed", + lambda: all(t.name != name for t in client.get_torrents()), + ) + print("Transmission round trip passed.") + + +if __name__ == "__main__": + main() diff --git a/tests/test_live_check.py b/tests/test_live_check.py new file mode 100644 index 0000000..3634492 --- /dev/null +++ b/tests/test_live_check.py @@ -0,0 +1,126 @@ +"""Tests for the pure part of scripts/live_check.py (the weekly drift job): +deciding whether a live fetcher's candidate looks like a real release. + +Run with: python -m unittest discover -s tests +""" + +import os +import sys +import unittest + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +for path in (REPO_ROOT, os.path.join(REPO_ROOT, "scripts")): + if path not in sys.path: + sys.path.insert(0, path) + +import helpers +import live_check + + +def setUpModule(): + unittest.addModuleCleanup(helpers.block_network()) + + +class ProblemsTests(unittest.TestCase): + def test_well_formed_candidates_have_no_problems(self): + cases = ( + ( + "ubuntu", + "xubuntu-24.04.5-minimal-amd64.iso", + ( + "https://torrent.ubuntu.com/xubuntu/releases/noble/release/minimal/" + "xubuntu-24.04.5-minimal-amd64.iso.torrent" + ), + ), + ( + "debian", + "debian-13.7.0-amd64-DVD-1.iso", + ( + "https://cdimage.debian.org/debian-cd/current/amd64/bt-dvd/" + "debian-13.7.0-amd64-DVD-1.iso.torrent" + ), + ), + ( + "kali", + "kali-linux-2026.2-installer-amd64.iso", + ( + "https://cdimage.kali.org/kali-2026.2/" + "kali-linux-2026.2-installer-amd64.iso.torrent" + ), + ), + ( + "arch", + "archlinux-2026.10.01", + "https://archlinux.org/releng/releases/2026.10.01/torrent/", + ), + ( + "devuan", + "devuan_excalibur", + "https://files.devuan.org/devuan_excalibur.torrent", + ), + ( + "mint", + "linuxmint-22.3-cinnamon-64bit.iso", + ( + "https://www.linuxmint.com/torrents/" + "linuxmint-22.3-cinnamon-64bit.iso.torrent" + ), + ), + ( + "fedora", + "Fedora-Workstation-Live-x86_64-44", + ( + "https://torrent.fedoraproject.org/torrents/" + "Fedora-Workstation-Live-x86_64-44.torrent" + ), + ), + ) + for distro, name, url in cases: + with self.subTest(distro=distro): + self.assertEqual(live_check.problems(distro, name, url), []) + + def test_name_from_another_distro(self): + self.assertEqual( + live_check.problems( + "debian", "ubuntu-24.04.5-desktop-amd64.iso", "https://x/a.torrent" + ), + [ + "name 'ubuntu-24.04.5-desktop-amd64.iso' doesn't look like a debian release" + ], + ) + + def test_url_must_be_https_and_a_torrent(self): + self.assertEqual( + live_check.problems( + "devuan", "devuan_excalibur", "http://files.devuan.org/devuan.iso" + ), + [ + "URL 'http://files.devuan.org/devuan.iso' is not https", + "URL 'http://files.devuan.org/devuan.iso' is not a .torrent", + ], + ) + + def test_only_arch_may_use_a_torrent_path(self): + url = "https://cdimage.debian.org/debian-13.7.0-amd64-DVD-1/torrent/" + + self.assertEqual( + live_check.problems("debian", "debian-13.7.0-amd64-DVD-1.iso", url), + [f"URL {url!r} is not a .torrent"], + ) + + def test_name_without_a_parseable_version(self): + self.assertEqual( + live_check.problems( + "mint", "linuxmint-latest-cinnamon", "https://x/linuxmint.torrent" + ), + ["name 'linuxmint-latest-cinnamon' has no parseable version"], + ) + + def test_old_release_fetchers_are_checked_as_their_distro(self): + self.assertEqual(live_check.distro_of("kali-old"), "kali") + self.assertEqual(live_check.distro_of("fedora-old"), "fedora") + self.assertEqual(live_check.distro_of("mint"), "mint") + + +if __name__ == "__main__": + unittest.main() From 4025f273d494ad8586294b35bddacee2157c8a09 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 13:07:04 +0200 Subject: [PATCH 22/46] Split test_fetch_torrents.py by concern (task 26, part 1) The 3,400-line module becomes test_ratio_gate, test_fetchers, test_cleanup, test_persistence, test_logging and test_old_release_demand. The config, run-summary and client-factory tests move to test_daily_run. Shared pieces move to tests/helpers.py: DuckTorrent, fixed_date, names_of, the repo-root sys.path bootstrap (helpers.REPO_ROOT), and isolate_module(), which every module's setUpModule now calls (env scrubbing plus the network guard) in place of per-file boilerplate. Purely mechanical: the set of Class.method test IDs is identical before and after (378), and the suite and hostile-env suite pass. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 15 +- fetch_torrents.py | 2 +- tests/helpers.py | 61 +- tests/test_cleanup.py | 1129 +++++++++ tests/test_configure_transmission.py | 7 +- tests/test_daily_run.py | 245 +- tests/test_fetch_torrents.py | 3399 -------------------------- tests/test_fetchers.py | 497 ++++ tests/test_live_check.py | 10 +- tests/test_logging.py | 320 +++ tests/test_old_release_demand.py | 611 +++++ tests/test_persistence.py | 103 + tests/test_ratio_gate.py | 603 +++++ tests/test_scrapers.py | 10 +- tests/test_torrent_scrape.py | 7 +- 15 files changed, 3582 insertions(+), 3437 deletions(-) create mode 100644 tests/test_cleanup.py delete mode 100644 tests/test_fetch_torrents.py create mode 100644 tests/test_fetchers.py create mode 100644 tests/test_logging.py create mode 100644 tests/test_old_release_demand.py create mode 100644 tests/test_persistence.py create mode 100644 tests/test_ratio_gate.py diff --git a/CLAUDE.md b/CLAUDE.md index 028b9b6..6bb0f78 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,9 +49,18 @@ The daily run itself is `run(config, client_factory=..., today=...)` with its I/O (Transmission client, clock, disk usage) injected, so it is tested directly with fakes from `tests/helpers.py` (`FakeTransmission`, `FakeDisk`); `tests/test_daily_run.py` keeps one runpy test for the `__main__` wiring. -Env vars are read once by `load_config(env)`, and `tests/helpers.py` -`isolated_env()` scrubs the developer's own `CLEANUP_*`, `FETCH_TORRENTS_*` -etc. so they can't change test outcomes. +Env vars are read once by `load_config(env)`. + +Every test module starts with `import helpers` (which also puts the repo +root on `sys.path`) and a `setUpModule()` calling `helpers.isolate_module()`: +that scrubs the developer's own `CLEANUP_*`, `FETCH_TORRENTS_*` etc. so they +can't change test outcomes, and refuses any real network connection. HTTP is +faked with `helpers.fake_response()` (a real `requests.Response`) or +`helpers.RecordedPages` (pages captured by `scripts/record_fixtures.py`). +`fetch_torrents` tests are split by concern: `test_ratio_gate.py`, +`test_fetchers.py`, `test_scrapers.py`, `test_cleanup.py`, +`test_persistence.py`, `test_logging.py`, `test_old_release_demand.py` and +`test_daily_run.py` (config, `run()`, `__main__`). Business logic that needs to be unit-testable should be kept as pure functions (taking plain data in, returning plain data out) separate from diff --git a/fetch_torrents.py b/fetch_torrents.py index c047263..bbc585f 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -1213,7 +1213,7 @@ def _devuan_sort_key(torrent): # (should_fetch_torrent() only knows to skip a fetch once a *previous* # version's ratio is on record). Kept separate from the cleanup_*() # functions so the selection logic can be unit tested without a live -# Transmission RPC connection (see tests/test_fetch_torrents.py). +# Transmission RPC connection (see tests/test_cleanup.py). def _old_version_candidates(torrents): groups = {} for torrent in torrents: diff --git a/tests/helpers.py b/tests/helpers.py index dc3d74e..031cade 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -2,7 +2,18 @@ Not a test module itself (doesn't match the test*.py discovery pattern); `unittest discover -s tests` puts tests/ on sys.path, so test modules import -this as `import helpers`. +this as `import helpers`. Importing it also puts the repo root on sys.path, +so `import fetch_torrents` etc. work after it. + +A typical test module: + + import helpers + + import fetch_torrents as ft + + + def setUpModule(): + helpers.isolate_module() """ import glob @@ -11,13 +22,20 @@ import os import shutil import socket +import sys import tempfile +import unittest import unittest.mock +from datetime import date from urllib.parse import urldefrag import requests import transmission_rpc +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + # Env vars production code reads at call time. A developer who happens to # have any of these exported (e.g. SKIP_RATIO_CHECK=true) must not be able # to flip test outcomes, so isolated_env() removes them unless a test sets @@ -62,6 +80,32 @@ def use_isolated_env(testcase, **overrides): testcase.addCleanup(patcher.stop) +def isolate_module(): + """The baseline for a test module (call from setUpModule): the + developer's exported SKIP_RATIO_CHECK/CLEANUP_*/TRANSMISSION_*/ + FETCH_TORRENTS_* can't leak in (including into subprocesses, which copy + os.environ), and no real network connection can be opened.""" + patcher = isolated_env() + patcher.start() + unittest.addModuleCleanup(patcher.stop) + unittest.addModuleCleanup(block_network()) + + +def fixed_date(today): + """A drop-in for fetch_torrents' `date` whose today() is pinned.""" + + class FixedDate(date): + @classmethod + def today(cls): + return today + + return FixedDate + + +def names_of(torrents): + return sorted(t.name for t in torrents) + + def make_temp_dir(testcase, prefix): """mkdtemp() that is removed again when the test finishes.""" path = tempfile.mkdtemp(prefix=prefix) @@ -88,6 +132,21 @@ def FakeTorrent(name, id_, ratio=0.0, added_date=0): _REAL_CLIENT = transmission_rpc.Client +class DuckTorrent: + """A torrent-shaped object a real transmission_rpc.Torrent can't be - + ratio None, or added_date None/a float/a datetime - only for exercising + the defensive getattr()/`or 0.0` branches in production code.""" + + def __init__(self, name, id_, ratio=0.0, added_date=None): + self.name = name + self.id = id_ + self.ratio = ratio + self.added_date = added_date + + def __repr__(self): + return f"DuckTorrent({self.name!r}, ratio={self.ratio})" + + def _bind_to_client(method_name, args, kwargs): """The arguments of a call to transmission_rpc.Client., by parameter name with defaults filled in. Raises TypeError for a call the diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py new file mode 100644 index 0000000..7712ee6 --- /dev/null +++ b/tests/test_cleanup.py @@ -0,0 +1,1129 @@ +"""Tests for cleanup: the pure planners (plan_cleanup(), +plan_stagnation_cleanup(), plan_disk_pressure_cleanup(), +update_ratio_history()) and the functions that apply them to a +Transmission client (helpers.FakeTransmission). + +Run with: python -m unittest discover -s tests +""" + +import json +import os +import types +import unittest +import unittest.mock +from datetime import UTC, date, timedelta + +import helpers + +import fetch_torrents as ft + + +def setUpModule(): + helpers.isolate_module() + + +class PlanCleanupTests(unittest.TestCase): + def test_selects_older_version_per_distro_and_type(self): + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), + helpers.FakeTorrent("ubuntu-24.04.1-live-server-amd64.iso", 3, ratio=2.0), + helpers.FakeTorrent("ubuntu-22.04.1-live-server-amd64.iso", 4, ratio=2.0), + helpers.FakeTorrent("debian-12.5.0-amd64-DVD-1.iso", 5, ratio=2.0), + helpers.FakeTorrent("debian-12.4.0-amd64-DVD-1.iso", 6, ratio=2.0), + helpers.FakeTorrent("kali-linux-2024.1-installer-amd64.iso", 7, ratio=2.0), + helpers.FakeTorrent("kali-linux-2023.4-installer-amd64.iso", 8, ratio=2.0), + helpers.FakeTorrent("archlinux-2024.05.01-x86_64.iso", 9, ratio=2.0), + helpers.FakeTorrent("archlinux-2024.04.01-x86_64.iso", 10, ratio=2.0), + helpers.FakeTorrent("linuxmint-22.3-cinnamon-64bit.iso", 11, ratio=2.0), + helpers.FakeTorrent("linuxmint-22.2-cinnamon-64bit.iso", 12, ratio=2.0), + helpers.FakeTorrent("Fedora-Workstation-Live-x86_64-44", 13, ratio=2.0), + helpers.FakeTorrent("Fedora-Workstation-Live-x86_64-43", 14, ratio=2.0), + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) + + self.assertEqual( + helpers.names_of(to_remove), + [ + "Fedora-Workstation-Live-x86_64-43", + "archlinux-2024.04.01-x86_64.iso", + "debian-12.4.0-amd64-DVD-1.iso", + "kali-linux-2023.4-installer-amd64.iso", + "linuxmint-22.2-cinnamon-64bit.iso", + "ubuntu-22.04.1-live-server-amd64.iso", + "ubuntu-23.10-desktop-amd64.iso", + ], + ) + self.assertEqual(to_keep_low_ratio, []) + + # The newest torrent in each (distro, type) group must never be + # scheduled for removal. + removed_names = set(helpers.names_of(to_remove)) + for newest in ( + "ubuntu-24.04-desktop-amd64.iso", + "ubuntu-24.04.1-live-server-amd64.iso", + "debian-12.5.0-amd64-DVD-1.iso", + "kali-linux-2024.1-installer-amd64.iso", + "archlinux-2024.05.01-x86_64.iso", + "linuxmint-22.3-cinnamon-64bit.iso", + "Fedora-Workstation-Live-x86_64-44", + ): + self.assertNotIn(newest, removed_names) + + def test_single_torrent_per_group_is_left_alone(self): + torrents = [helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=0.0)] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) + + self.assertEqual(to_remove, []) + self.assertEqual(to_keep_low_ratio, []) + + def test_unrelated_torrents_are_ignored(self): + torrents = [ + helpers.FakeTorrent("some-random-linux-distro-1.0.iso", 1, ratio=5.0), + helpers.FakeTorrent("some-random-linux-distro-2.0.iso", 2, ratio=5.0), + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) + + self.assertEqual(to_remove, []) + self.assertEqual(to_keep_low_ratio, []) + + def test_old_version_below_ratio_threshold_is_kept_not_removed(self): + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.4), + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) + + self.assertEqual(to_remove, []) + self.assertEqual(len(to_keep_low_ratio), 1) + kept_torrent, kept_ratio = to_keep_low_ratio[0] + self.assertEqual(kept_torrent.name, "ubuntu-23.10-desktop-amd64.iso") + self.assertEqual(kept_ratio, 0.4) + + def test_skip_ratio_check_removes_regardless_of_ratio(self): + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.0), + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup( + torrents, skip_ratio_check=True, min_ratio=1.0 + ) + + self.assertEqual( + helpers.names_of(to_remove), ["ubuntu-23.10-desktop-amd64.iso"] + ) + self.assertEqual(to_keep_low_ratio, []) + + def test_different_types_within_same_distro_are_not_cross_removed(self): + # desktop and live-server are different "type_" groups for ubuntu; + # having only the desktop build twice must not affect live-server. + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), + helpers.FakeTorrent("ubuntu-24.04.1-live-server-amd64.iso", 3, ratio=2.0), + ] + + to_remove, _ = ft.plan_cleanup(torrents) + + self.assertEqual( + helpers.names_of(to_remove), ["ubuntu-23.10-desktop-amd64.iso"] + ) + + def test_unparseable_name_for_matched_distro_is_skipped_not_raised(self): + # "ubuntu-desktop" has no version segment at index 1 that parses as + # a version; parse_version_type/version_to_tuple should fail and + # plan_cleanup must skip it rather than raising. + torrents = [ + helpers.FakeTorrent("ubuntu-desktop", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 2, ratio=2.0), + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) + + self.assertEqual(to_remove, []) + self.assertEqual(to_keep_low_ratio, []) + + def test_ratio_exactly_at_min_ratio_is_removed(self): + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.0), + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) + + self.assertEqual( + helpers.names_of(to_remove), ["ubuntu-23.10-desktop-amd64.iso"] + ) + self.assertEqual(to_keep_low_ratio, []) + + def test_ratio_just_below_min_ratio_is_kept(self): + older = helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.999) + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + older, + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) + + self.assertEqual(to_remove, []) + self.assertEqual(to_keep_low_ratio, [(older, 0.999)]) + + def test_default_min_ratio_is_one(self): + older = helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5) + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + older, + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) + + self.assertEqual(to_remove, [older]) + self.assertEqual(to_keep_low_ratio, []) + + def test_missing_or_none_ratio_counts_as_zero(self): + no_attr = types.SimpleNamespace(name="ubuntu-23.04-desktop-amd64.iso", id=2) + none_ratio = helpers.DuckTorrent( + "ubuntu-23.10-desktop-amd64.iso", 3, ratio=None + ) + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + no_attr, + none_ratio, + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) + + self.assertEqual(to_remove, []) + self.assertEqual( + sorted(to_keep_low_ratio, key=lambda entry: entry[0].name), + [(no_attr, 0.0), (none_ratio, 0.0)], + ) + + def test_skipped_torrents_listed_first_do_not_stop_planning(self): + # Each kind of torrent _old_version_candidates() skips (or routes + # elsewhere) is placed *before* a valid pair, so a 'continue' that + # became 'break' would drop the pair. + leading = { + "devuan": helpers.FakeTorrent( + "devuan_excalibur", 10, ratio=2.0, added_date=1 + ), + "unknown distro": helpers.FakeTorrent("slackware-15.0-install-dvd.iso", 11), + "unparseable": helpers.FakeTorrent("ubuntu-noble-desktop-amd64.iso", 12), + } + for kind, first in leading.items(): + with self.subTest(kind): + older = helpers.FakeTorrent( + "ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0 + ) + torrents = [ + first, + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + older, + ] + + with self.assertNoLogs(ft.logger, level="ERROR"): + to_remove, _ = ft.plan_cleanup(torrents) + + self.assertEqual(to_remove, [older]) + + def test_unparseable_name_is_skipped_with_a_warning(self): + torrents = [ + helpers.FakeTorrent("ubuntu-noble-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-mantic-desktop-amd64.iso", 2, ratio=2.0), + ] + + with self.assertLogs(ft.logger, level="WARNING") as logs: + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) + + self.assertEqual((to_remove, to_keep_low_ratio), ([], [])) + self.assertEqual(len(logs.records), 2) + + +class PlanCleanupDevuanGroupingTests(unittest.TestCase): + """Devuan releases (e.g. "devuan_excalibur") carry no dotted version to + compare - see DEVUAN_NAME_PATTERN - so _old_version_candidates() groups + them separately and orders by added_date (when Transmission added the + torrent) instead of a parsed version, same idea as should_fetch_torrent()'s + ratio gate for Devuan.""" + + def test_older_devuan_release_is_a_cleanup_candidate(self): + torrents = [ + helpers.FakeTorrent("devuan_daedalus", 1, ratio=2.0, added_date=1000), + helpers.FakeTorrent("devuan_excalibur", 2, ratio=2.0, added_date=2000), + ] + + to_remove, _ = ft.plan_cleanup(torrents) + + self.assertEqual(helpers.names_of(to_remove), ["devuan_daedalus"]) + + def test_single_devuan_release_is_left_alone(self): + torrents = [ + helpers.FakeTorrent("devuan_excalibur", 1, ratio=0.0, added_date=1000) + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) + + self.assertEqual(to_remove, []) + self.assertEqual(to_keep_low_ratio, []) + + def test_older_devuan_release_below_ratio_threshold_is_kept(self): + torrents = [ + helpers.FakeTorrent("devuan_daedalus", 1, ratio=0.4, added_date=1000), + helpers.FakeTorrent("devuan_excalibur", 2, ratio=2.0, added_date=2000), + ] + + to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) + + self.assertEqual(to_remove, []) + self.assertEqual(len(to_keep_low_ratio), 1) + kept_torrent, kept_ratio = to_keep_low_ratio[0] + self.assertEqual(kept_torrent.name, "devuan_daedalus") + self.assertEqual(kept_ratio, 0.4) + + def test_stagnant_older_devuan_release_is_removed(self): + today = date(2026, 8, 29) + torrents = [ + helpers.FakeTorrent("devuan_daedalus", 1, ratio=1.5, added_date=1000), + helpers.FakeTorrent("devuan_excalibur", 2, ratio=2.0, added_date=2000), + ] + history = {"devuan_daedalus": [{"date": "2026-07-20", "ratio": 1.495}]} + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) + + self.assertEqual(helpers.names_of(to_remove), ["devuan_daedalus"]) + self.assertEqual(to_keep, []) + + def test_missing_added_date_does_not_crash(self): + # Real Transmission torrents always carry added_date; this just + # guards _devuan_sort_key() against a hypothetical missing attribute. + torrents = [ + helpers.DuckTorrent("devuan_daedalus", 1, ratio=2.0, added_date=None), + helpers.DuckTorrent("devuan_excalibur", 2, ratio=2.0, added_date=None), + ] + + to_remove, _ = ft.plan_cleanup(torrents) + + self.assertEqual(len(to_remove), 1) + + def test_release_without_added_date_sorts_as_oldest(self): + undated = helpers.DuckTorrent("devuan_daedalus", 1, ratio=2.0, added_date=None) + torrents = [ + undated, + helpers.DuckTorrent("devuan_excalibur", 2, ratio=2.0, added_date=0.5), + ] + + to_remove, _ = ft.plan_cleanup(torrents) + + self.assertEqual(to_remove, [undated]) + + def test_datetime_added_dates_are_ordered_by_timestamp(self): + from datetime import datetime + + older = helpers.DuckTorrent( + "devuan_daedalus", + 1, + ratio=2.0, + added_date=datetime(2026, 1, 1, tzinfo=UTC), + ) + torrents = [ + helpers.DuckTorrent( + "devuan_excalibur", + 2, + ratio=2.0, + added_date=datetime(2026, 6, 1, tzinfo=UTC), + ), + older, + ] + + to_remove, _ = ft.plan_cleanup(torrents) + + self.assertEqual(to_remove, [older]) + + +class UpdateRatioHistoryTests(unittest.TestCase): + """update_ratio_history() must keep the on-disk history bounded: drop + torrents that are no longer seeding, only add a new sample once per + sample_interval_days, and prune samples once they're older than + window_days + sample_interval_days - so it stays a fixed handful of + samples per torrent forever, not one entry per run.""" + + def test_new_torrent_gets_a_first_sample(self): + today = date(2026, 8, 29) + history = ft.update_ratio_history( + {}, {"archlinux-2026.08.01-x86_64.iso": 1.2}, today + ) + + self.assertEqual( + history, + {"archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-29", "ratio": 1.2}]}, + ) + + def test_recent_sample_is_not_duplicated(self): + today = date(2026, 8, 29) + history = { + "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-25", "ratio": 1.0}], + } + + updated = ft.update_ratio_history( + history, {"archlinux-2026.08.01-x86_64.iso": 1.1}, today + ) + + # Only 4 days since the last sample (< the 7-day interval) - no new + # sample yet, and the ratio value on the existing sample is untouched. + self.assertEqual( + updated, + {"archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-25", "ratio": 1.0}]}, + ) + + def test_new_sample_added_once_interval_elapses(self): + today = date(2026, 8, 29) + history = { + "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-20", "ratio": 1.0}], + } + + updated = ft.update_ratio_history( + history, {"archlinux-2026.08.01-x86_64.iso": 1.3}, today + ) + + self.assertEqual( + updated, + { + "archlinux-2026.08.01-x86_64.iso": [ + {"date": "2026-08-20", "ratio": 1.0}, + {"date": "2026-08-29", "ratio": 1.3}, + ] + }, + ) + + def test_samples_older_than_window_plus_interval_are_pruned(self): + today = date(2026, 8, 29) + history = { + "archlinux-2026.08.01-x86_64.iso": [ + {"date": "2026-06-01", "ratio": 0.5}, # ~89 days old - long stale + {"date": "2026-08-10", "ratio": 1.0}, # ~19 days old - kept + ], + } + + updated = ft.update_ratio_history( + history, {"archlinux-2026.08.01-x86_64.iso": 1.0}, today, window_days=30 + ) + + dates = [s["date"] for s in updated["archlinux-2026.08.01-x86_64.iso"]] + self.assertNotIn("2026-06-01", dates) + self.assertIn("2026-08-10", dates) + + def test_torrent_no_longer_seeding_is_dropped(self): + today = date(2026, 8, 29) + history = { + "archlinux-2026.07.01-x86_64.iso": [{"date": "2026-08-20", "ratio": 1.0}], + } + + updated = ft.update_ratio_history(history, {}, today) + + self.assertEqual(updated, {}) + + def _days_ago(self, today, days): + return (today - timedelta(days=days)).isoformat() + + def test_sample_exactly_one_interval_old_triggers_a_new_sample(self): + today = date(2026, 8, 29) + history = {"a.iso": [{"date": self._days_ago(today, 7), "ratio": 1.0}]} + + updated = ft.update_ratio_history( + history, {"a.iso": 1.2}, today, sample_interval_days=7, window_days=30 + ) + + self.assertEqual( + updated["a.iso"], + [ + {"date": self._days_ago(today, 7), "ratio": 1.0}, + {"date": today.isoformat(), "ratio": 1.2}, + ], + ) + + def test_sample_just_under_one_interval_old_adds_nothing(self): + today = date(2026, 8, 29) + history = {"a.iso": [{"date": self._days_ago(today, 6), "ratio": 1.0}]} + + updated = ft.update_ratio_history( + history, {"a.iso": 1.2}, today, sample_interval_days=7, window_days=30 + ) + + self.assertEqual( + updated["a.iso"], [{"date": self._days_ago(today, 6), "ratio": 1.0}] + ) + + def test_prune_horizon_defaults_to_window_plus_interval(self): + # Defaults: window 30 + interval 7 = 37 days. A sample exactly 37 + # days old is kept; 38 days old is pruned. + today = date(2026, 8, 29) + history = { + "a.iso": [ + {"date": self._days_ago(today, 38), "ratio": 0.5}, + {"date": self._days_ago(today, 37), "ratio": 0.6}, + {"date": self._days_ago(today, 1), "ratio": 0.9}, + ] + } + + updated = ft.update_ratio_history(history, {"a.iso": 1.0}, today) + + self.assertEqual( + updated["a.iso"], + [ + {"date": self._days_ago(today, 37), "ratio": 0.6}, + {"date": self._days_ago(today, 1), "ratio": 0.9}, + ], + ) + + +class PlanStagnationCleanupTests(unittest.TestCase): + """plan_stagnation_cleanup() removes only superseded (non-latest) + versions within a (distro, type) group once their ratio has stopped + growing - the newest/only version of a group is never a candidate, + since removing it would just get it re-fetched and re-downloaded the + next run (should_fetch_torrent() only skips a fetch when a *previous* + version's ratio is on record).""" + + def test_latest_or_only_version_is_never_removed_even_if_flat(self): + today = date(2026, 8, 29) + torrents = [ + helpers.FakeTorrent("archlinux-2026.08.01-x86_64.iso", 1, ratio=1.0) + ] + # 60 days of a perfectly flat ratio - as stagnant as it gets. + history = { + "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-06-30", "ratio": 1.0}], + } + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) + + self.assertEqual(to_remove, []) + self.assertEqual(to_keep, []) + + def test_old_version_with_no_ratio_growth_is_removed(self): + today = date(2026, 8, 29) + torrents = [ + helpers.FakeTorrent("ubuntu-24.04.1-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.500), + ] + history = { + "ubuntu-23.10-desktop-amd64.iso": [{"date": "2026-07-20", "ratio": 1.495}], + } + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) + + self.assertEqual( + helpers.names_of(to_remove), ["ubuntu-23.10-desktop-amd64.iso"] + ) + self.assertEqual(to_keep, []) + + def test_old_version_still_growing_is_kept(self): + today = date(2026, 8, 29) + torrents = [ + helpers.FakeTorrent("ubuntu-24.04.1-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.8), + ] + history = { + "ubuntu-23.10-desktop-amd64.iso": [{"date": "2026-07-20", "ratio": 1.2}], + } + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) + + self.assertEqual(to_remove, []) + self.assertEqual(len(to_keep), 1) + kept_torrent, delta = to_keep[0] + self.assertEqual(kept_torrent.name, "ubuntu-23.10-desktop-amd64.iso") + self.assertAlmostEqual(delta, 0.6) + + def test_old_version_without_enough_history_is_kept(self): + today = date(2026, 8, 29) + torrents = [ + helpers.FakeTorrent("ubuntu-24.04.1-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), + ] + # Only 5 days of history - too young to judge either way. + history = { + "ubuntu-23.10-desktop-amd64.iso": [{"date": "2026-08-24", "ratio": 0.1}], + } + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, window_days=30, min_ratio_delta=0.02 + ) + + self.assertEqual(to_remove, []) + self.assertEqual(len(to_keep), 1) + kept_torrent, delta = to_keep[0] + self.assertEqual(kept_torrent.name, "ubuntu-23.10-desktop-amd64.iso") + self.assertIsNone(delta) + + def test_old_version_with_no_history_at_all_is_kept(self): + today = date(2026, 8, 29) + torrents = [ + helpers.FakeTorrent("ubuntu-24.04.1-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), + ] + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, {}, today, window_days=30, min_ratio_delta=0.02 + ) + + self.assertEqual(to_remove, []) + self.assertEqual(len(to_keep), 1) + + def test_unrelated_single_torrent_groups_are_ignored(self): + today = date(2026, 8, 29) + torrents = [ + helpers.FakeTorrent("some-random-linux-distro-1.0.iso", 1, ratio=5.0) + ] + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, {}, today, window_days=30, min_ratio_delta=0.02 + ) + + self.assertEqual(to_remove, []) + self.assertEqual(to_keep, []) + + def _pair(self, older_ratio): + older = helpers.FakeTorrent( + "ubuntu-23.10-desktop-amd64.iso", 2, ratio=older_ratio + ) + return older, [helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), older] + + def _history(self, today, *samples): + return { + "ubuntu-23.10-desktop-amd64.iso": [ + {"date": (today - timedelta(days=days)).isoformat(), "ratio": ratio} + for days, ratio in samples + ] + } + + def test_anchor_exactly_window_days_old_is_used(self): + today = date(2026, 8, 29) + older, torrents = self._pair(1.0) + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, self._history(today, (30, 1.0)), today, 30, 0.02 + ) + + self.assertEqual((to_remove, to_keep), ([older], [])) + + def test_anchor_one_day_short_of_window_is_too_new_to_judge(self): + today = date(2026, 8, 29) + older, torrents = self._pair(1.0) + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, self._history(today, (29, 1.0)), today, 30, 0.02 + ) + + self.assertEqual((to_remove, to_keep), ([], [(older, None)])) + + def test_oldest_qualifying_sample_is_the_anchor(self): + # Listed newest-first, so picking the first old-enough sample + # instead of the oldest would give a different delta. + today = date(2026, 8, 29) + older, torrents = self._pair(1.0) + history = self._history(today, (35, 0.99), (45, 0.5)) + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, 30, 0.02 + ) + + self.assertEqual((to_remove, to_keep), ([], [(older, 0.5)])) + + def test_delta_exactly_at_min_ratio_delta_is_kept(self): + today = date(2026, 8, 29) + older, torrents = self._pair(1.5) + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, self._history(today, (40, 1.0)), today, 30, 0.5 + ) + + self.assertEqual((to_remove, to_keep), ([], [(older, 0.5)])) + + def test_delta_just_below_min_ratio_delta_is_removed(self): + today = date(2026, 8, 29) + older, torrents = self._pair(1.25) + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, self._history(today, (40, 1.0)), today, 30, 0.5 + ) + + self.assertEqual((to_remove, to_keep), ([older], [])) + + def test_defaults_are_a_30_day_window_and_0_02_delta(self): + today = date(2026, 8, 29) + growing, torrents = self._pair(1.5) + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, self._history(today, (30, 1.0)), today + ) + + self.assertEqual((to_remove, to_keep), ([], [(growing, 0.5)])) + + stagnant, torrents = self._pair(1.01) + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, self._history(today, (30, 1.0)), today + ) + + self.assertEqual((to_remove, to_keep), ([stagnant], [])) + + def test_missing_or_none_ratio_counts_as_zero(self): + today = date(2026, 8, 29) + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), + types.SimpleNamespace(name="ubuntu-23.04-desktop-amd64.iso", id=2), + helpers.DuckTorrent("ubuntu-23.10-desktop-amd64.iso", 3, ratio=None), + ] + history = { + name: [{"date": (today - timedelta(days=40)).isoformat(), "ratio": -0.5}] + for name in ( + "ubuntu-23.04-desktop-amd64.iso", + "ubuntu-23.10-desktop-amd64.iso", + ) + } + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, 30, 0.02 + ) + + self.assertEqual(to_remove, []) + self.assertEqual( + sorted((t.name, delta) for t, delta in to_keep), + [ + ("ubuntu-23.04-desktop-amd64.iso", 0.5), + ("ubuntu-23.10-desktop-amd64.iso", 0.5), + ], + ) + + def test_candidate_without_anchor_does_not_stop_later_ones(self): + today = date(2026, 8, 29) + no_history = helpers.FakeTorrent("ubuntu-23.04-desktop-amd64.iso", 3, ratio=1.0) + stagnant = helpers.FakeTorrent("ubuntu-22.10-desktop-amd64.iso", 4, ratio=1.0) + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), + no_history, + stagnant, + ] + history = { + "ubuntu-22.10-desktop-amd64.iso": [ + {"date": (today - timedelta(days=40)).isoformat(), "ratio": 1.0} + ] + } + + to_remove, to_keep = ft.plan_stagnation_cleanup( + torrents, history, today, 30, 0.02 + ) + + self.assertEqual((to_remove, to_keep), ([stagnant], [(no_history, None)])) + + +class RecordRemovalTests(unittest.TestCase): + def test_adds_entry_without_mutating_input(self): + history = {} + updated = ft.record_removal( + history, + "kali-linux-2026.1-installer-amd64.iso", + date(2026, 2, 1), + "stagnant", + 0.42, + ) + + self.assertEqual(history, {}) # input untouched + self.assertEqual( + updated["kali-linux-2026.1-installer-amd64.iso"], + {"removed_date": "2026-02-01", "reason": "stagnant", "ratio": 0.42}, + ) + + +class CleanupRemovesWatchFileTests(unittest.TestCase): + """When cleanup removes a torrent's downloaded data, it must also remove + the small .torrent file it left behind in /watch - otherwise these + accumulate forever, since nothing else (including Transmission itself, + by default) ever cleans them up.""" + + def setUp(self): + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_") + for attr, filename in ( + ("removed_history_file", "removed_history.json"), + ("ratio_history_file", "ratio_history.json"), + ): + patcher = unittest.mock.patch.object( + ft, attr, os.path.join(tmp_dir, filename) + ) + patcher.start() + self.addCleanup(patcher.stop) + + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) + patcher.start() + self.addCleanup(patcher.stop) + + def _touch_watch_file(self, name): + path = os.path.join(self.tmp_watch_dir, f"{name}.torrent") + with open(path, "w", encoding="utf-8") as f: + f.write("fake torrent bytes") + return path + + def _transmission(self): + return helpers.FakeTransmission( + [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), + ] + ) + + def test_cleanup_old_versions_removes_the_watch_file(self): + old_path = self._touch_watch_file("ubuntu-23.10-desktop-amd64.iso") + new_path = self._touch_watch_file("ubuntu-24.04-desktop-amd64.iso") + transmission = self._transmission() + + ft.cleanup_old_versions( + transmission, today=date(2026, 8, 1), skip_ratio_check=True + ) + + self.assertEqual(transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + self.assertIn(("remove_torrent", 2, True), transmission.calls) + self.assertFalse(os.path.exists(old_path)) + self.assertTrue(os.path.exists(new_path)) + self.assertEqual( + ft.load_removed_history(ft.removed_history_file), + { + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-08-01", + "reason": "keep_only_latest", + "ratio": 2.0, + } + }, + ) + + def test_missing_watch_file_does_not_raise(self): + # No .torrent file created for the superseded version - cleanup must + # still succeed and remove the torrent from Transmission. + transmission = self._transmission() + + ft.cleanup_old_versions( + transmission, today=date(2026, 8, 1), skip_ratio_check=True + ) + + self.assertEqual(transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + + def test_cleanup_old_versions_keeps_versions_below_min_ratio(self): + transmission = helpers.FakeTransmission( + [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), + ] + ) + + ft.cleanup_old_versions(transmission, today=date(2026, 8, 1), min_ratio=2.0) + + self.assertEqual(len(transmission.torrents), 2) + + def test_cleanup_stagnant_torrents_removes_the_watch_file(self): + old_path = self._touch_watch_file("ubuntu-23.10-desktop-amd64.iso") + self._touch_watch_file("ubuntu-24.04-desktop-amd64.iso") + transmission = self._transmission() + + # 32 days old: past the default 30-day stagnation window, but still + # within update_ratio_history()'s own 37-day (window + sample + # interval) prune horizon, since that prune runs - on this same + # history - before plan_stagnation_cleanup() ever sees it. Ratio + # unchanged since then, so plan_stagnation_cleanup() must select this + # torrent for removal. + today = date(2026, 8, 1) + anchor_date = (today - timedelta(days=32)).isoformat() + history = { + "ubuntu-23.10-desktop-amd64.iso": [{"date": anchor_date, "ratio": 2.0}] + } + with open(ft.ratio_history_file, "w", encoding="utf-8") as f: + json.dump(history, f) + + ft.cleanup_stagnant_torrents(transmission, today=today) + + self.assertEqual(transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + self.assertFalse(os.path.exists(old_path)) + removed = ft.load_removed_history(ft.removed_history_file) + self.assertEqual( + removed["ubuntu-23.10-desktop-amd64.iso"], + {"removed_date": "2026-08-01", "reason": "stagnant", "ratio": 2.0}, + ) + + def test_cleanup_stagnant_torrents_records_todays_sample(self): + transmission = self._transmission() + + ft.cleanup_stagnant_torrents( + transmission, today=date(2026, 8, 1), window_days=10 + ) + + self.assertEqual( + ft.load_ratio_history(ft.ratio_history_file), + { + "ubuntu-24.04-desktop-amd64.iso": [ + {"date": "2026-08-01", "ratio": 2.0} + ], + "ubuntu-23.10-desktop-amd64.iso": [ + {"date": "2026-08-01", "ratio": 2.0} + ], + }, + ) + + +class PlanDiskPressureCleanupTests(unittest.TestCase): + """plan_disk_pressure_cleanup() is the pure selection logic behind + CLEANUP_DISK_USAGE_THRESHOLD_PERCENT - the safety valve that kicks in + when normal ratio/stagnation-based cleanup isn't freeing space fast + enough for a forgotten, never-updated deployment.""" + + def test_below_threshold_returns_nothing(self): + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), + ] + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=80.0, threshold_percent=95.0 + ), + [], + ) + + def test_disabled_when_threshold_is_zero_or_negative(self): + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), + ] + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=99.0, threshold_percent=0 + ), + [], + ) + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=99.0, threshold_percent=-1 + ), + [], + ) + + def test_at_or_above_threshold_returns_candidates_lowest_ratio_first(self): + torrents = [ + helpers.FakeTorrent( + "ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0 + ), # latest - never a candidate + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), + helpers.FakeTorrent("ubuntu-22.04.1-live-server-amd64.iso", 3, ratio=0.2), + helpers.FakeTorrent( + "ubuntu-24.04.1-live-server-amd64.iso", 4, ratio=5.0 + ), # latest of this group + ] + result = ft.plan_disk_pressure_cleanup( + torrents, used_percent=95.0, threshold_percent=95.0 + ) + self.assertEqual( + helpers.names_of(result), + ["ubuntu-22.04.1-live-server-amd64.iso", "ubuntu-23.10-desktop-amd64.iso"], + ) + # lowest ratio (least useful) first + self.assertEqual(result[0].name, "ubuntu-22.04.1-live-server-amd64.iso") + + def test_no_superseded_candidates_returns_empty(self): + torrents = [helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=0.0)] + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=99.0, threshold_percent=95.0 + ), + [], + ) + + def test_any_positive_threshold_is_enabled(self): + older = helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1) + torrents = [helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), older] + + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=50.0, threshold_percent=1 + ), + [older], + ) + + def test_usage_exactly_at_threshold_triggers_cleanup(self): + older = helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1) + torrents = [helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), older] + + self.assertEqual( + ft.plan_disk_pressure_cleanup( + torrents, used_percent=95.0, threshold_percent=95.0 + ), + [older], + ) + + def test_missing_or_none_ratio_sorts_as_zero(self): + half = helpers.FakeTorrent("ubuntu-22.10-desktop-amd64.iso", 2, ratio=0.5) + no_attr = types.SimpleNamespace(name="ubuntu-23.04-desktop-amd64.iso", id=3) + none_ratio = helpers.DuckTorrent( + "ubuntu-23.10-desktop-amd64.iso", 4, ratio=None + ) + torrents = [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), + half, + no_attr, + none_ratio, + ] + + planned = ft.plan_disk_pressure_cleanup( + torrents, used_percent=99.0, threshold_percent=95.0 + ) + + self.assertEqual(planned[-1], half) + self.assertEqual( + sorted(t.name for t in planned[:2]), + ["ubuntu-23.04-desktop-amd64.iso", "ubuntu-23.10-desktop-amd64.iso"], + ) + + +class EnforceDiskUsageLimitTests(unittest.TestCase): + """enforce_disk_usage_limit() wires plan_disk_pressure_cleanup() to a + Transmission connection and real disk usage, re-checking usage between + removals since ratio data alone can't say how many bytes each torrent + holds.""" + + def setUp(self): + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_") + for attr, filename in ( + ("removed_history_file", "removed_history.json"), + ("ratio_history_file", "ratio_history.json"), + ): + patcher = unittest.mock.patch.object( + ft, attr, os.path.join(tmp_dir, filename) + ) + patcher.start() + self.addCleanup(patcher.stop) + + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) + patcher.start() + self.addCleanup(patcher.stop) + + self.transmission = helpers.FakeTransmission( + [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), + helpers.FakeTorrent("ubuntu-22.04-desktop-amd64.iso", 3, ratio=0.2), + ] + ) + self.connections = 0 + + def _connect(self): + self.connections += 1 + return self.transmission + + def _disk(self, base_used, sizes=None): + # 100 GB disk; each torrent's size in GB. + return helpers.FakeDisk( + self.transmission, 100, sizes or {1: 2, 2: 2, 3: 6}, base_used + ) + + def _enforce(self, threshold, disk): + ft.enforce_disk_usage_limit( + self._connect, + threshold, + today=date(2026, 8, 1), + disk_usage=disk, + download_path="/downloads", + ) + + def test_disabled_threshold_never_touches_disk_or_rpc(self): + disk = self._disk(base_used=99) + + self._enforce(0, disk) + + self.assertEqual((disk.paths, self.connections), ([], 0)) + + def test_below_threshold_does_not_connect_to_transmission(self): + disk = self._disk(base_used=40) # 50% + + self._enforce(95, disk) + + self.assertEqual(disk.paths, ["/downloads"]) + self.assertEqual(self.connections, 0) + + def test_removes_lowest_ratio_candidates_until_under_threshold(self): + # 86 + 10 = 96% used. Removing 22.04 (lowest ratio, 6 GB) gets to + # 90%, under the threshold, so 23.10 must be left alone. + disk = self._disk(base_used=86) + + self._enforce(95, disk) + + self.assertEqual( + self.transmission.names, + ["ubuntu-23.10-desktop-amd64.iso", "ubuntu-24.04-desktop-amd64.iso"], + ) + + def test_keeps_removing_while_still_over_threshold(self): + # 96%; removing 22.04 (1 GB) leaves 95% - still at the threshold - + # so 23.10 (4 GB) goes too. + disk = self._disk(base_used=90, sizes={1: 1, 2: 4, 3: 1}) + + self._enforce(95, disk) + + self.assertEqual(self.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + + def test_never_removes_the_latest_version(self): + disk = self._disk(base_used=99) + + self._enforce(95, disk) + + self.assertEqual(self.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + + def test_removes_the_watch_file_for_each_removal(self): + watch_path = os.path.join( + self.tmp_watch_dir, "ubuntu-22.04-desktop-amd64.iso.torrent" + ) + with open(watch_path, "w", encoding="utf-8") as f: + f.write("fake torrent bytes") + + self._enforce(95, self._disk(base_used=86)) + + self.assertFalse(os.path.exists(watch_path)) + + def test_no_candidates_logs_warning_and_does_not_raise(self): + self.transmission.torrents = self.transmission.torrents[:1] + + with self.assertLogs(ft.logger, level="WARNING") as logs: + self._enforce(95, self._disk(base_used=96)) + + self.assertEqual(self.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + self.assertIn("no superseded versions are available", logs.output[0]) + + def test_records_removal_reason_as_disk_pressure(self): + self._enforce(95, self._disk(base_used=86)) + + self.assertEqual( + ft.load_removed_history(ft.removed_history_file), + { + "ubuntu-22.04-desktop-amd64.iso": { + "removed_date": "2026-08-01", + "reason": "disk_pressure", + "ratio": 0.2, + } + }, + ) + + +if __name__ == "__main__": + unittest.main() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_configure_transmission.py b/tests/test_configure_transmission.py index 721e021..ee7a6dc 100644 --- a/tests/test_configure_transmission.py +++ b/tests/test_configure_transmission.py @@ -8,21 +8,16 @@ import json import os import runpy -import sys import unittest import unittest.mock -REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if REPO_ROOT not in sys.path: - sys.path.insert(0, REPO_ROOT) - import helpers import configure_transmission as ct def setUpModule(): - unittest.addModuleCleanup(helpers.block_network()) + helpers.isolate_module() class BuildRpcOverridesTests(unittest.TestCase): diff --git a/tests/test_daily_run.py b/tests/test_daily_run.py index 0fd6ad2..589ba22 100644 --- a/tests/test_daily_run.py +++ b/tests/test_daily_run.py @@ -27,18 +27,15 @@ from datetime import date from unittest.mock import MagicMock -REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SCRIPT = os.path.join(REPO_ROOT, "fetch_torrents.py") -if REPO_ROOT not in sys.path: - sys.path.insert(0, REPO_ROOT) - import helpers import fetch_torrents as ft +SCRIPT = os.path.join(helpers.REPO_ROOT, "fetch_torrents.py") + def setUpModule(): - unittest.addModuleCleanup(helpers.block_network()) + helpers.isolate_module() UBUNTU_META = ( @@ -557,5 +554,241 @@ def rpc_port_open(*args, **kwargs): self.assertTrue(os.path.exists(os.path.join(log_dir, "fetch_torrents.log"))) +class FormatRunSummaryTests(unittest.TestCase): + """format_run_summary() builds the single end-of-run log line - it must + surface how many old-release torrents FETCH_TORRENTS_CHECK_OLD_RELEASES + introduced, not just the counts from the normal latest-release fetch.""" + + def test_without_old_release_check(self): + summary = ft.format_run_summary( + elapsed=12.5, + success_count=3, + existing_count=1, + not_found_count=0, + failure_count=0, + ) + self.assertEqual( + summary, + "Run complete in 12.50 seconds. 3 added, 1 existing, 0 not found upstream, 0 failed.", + ) + + def test_with_old_release_check_appends_counts(self): + summary = ft.format_run_summary( + elapsed=12.5, + success_count=3, + existing_count=1, + not_found_count=0, + failure_count=0, + old_added=2, + old_skipped=6, + ) + self.assertEqual( + summary, + "Run complete in 12.50 seconds. 3 added, 1 existing, 0 not found upstream, 0 failed. " + "Old releases (FETCH_TORRENTS_CHECK_OLD_RELEASES): 2 added, 6 skipped.", + ) + + def test_still_starts_with_run_complete_for_always_log_filter(self): + summary = ft.format_run_summary( + elapsed=1.0, + success_count=0, + existing_count=0, + not_found_count=0, + failure_count=0, + old_added=0, + old_skipped=0, + ) + self.assertTrue(summary.startswith("Run complete in")) + + +class LoadConfigTests(unittest.TestCase): + """load_config() is the one place the daily run reads its environment: + each env var must land on the right field, and a bad number must be + deferred (recorded, then raised by require()) rather than crash the + whole run up front.""" + + def test_empty_env_gives_the_documented_defaults(self): + self.assertEqual(ft.load_config({}), ft.Config()) + self.assertEqual( + ft.Config(), + ft.Config( + distros=ft.DEFAULT_DISTROS, + include_low_demand=False, + skip_ratio_check=False, + check_old_releases=False, + old_release_min_leechers=10, + old_release_min_leecher_ratio=0.1, + old_release_allow_zero_seeders=False, + old_release_recheck_days=7, + old_release_skip_removed=True, + cleanup_keep_only_latest=False, + cleanup_skip_ratio_check=False, + cleanup_min_ratio=1.0, + stagnation_window_days=30, + stagnation_min_ratio_delta=0.02, + disk_usage_threshold_percent=95.0, + rpc_username=None, + rpc_password=None, + invalid={}, + ), + ) + + def test_each_env_var_sets_its_field(self): + config = ft.load_config( + { + "FETCH_TORRENTS_DISTROS": "devuan, arch", + "FETCH_TORRENTS_INCLUDE_LOW_DEMAND": "true", + "SKIP_RATIO_CHECK": "yes", + "FETCH_TORRENTS_CHECK_OLD_RELEASES": "1", + "FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS": "3", + "FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO": "0.5", + "FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS": "true", + "FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS": "2", + "FETCH_TORRENTS_OLD_RELEASE_SKIP_REMOVED": "false", + "CLEANUP_KEEP_ONLY_LATEST_VERSION": "true", + "CLEANUP_SKIP_RATIO_CHECK": "true", + "CLEANUP_MIN_RATIO": "1.5", + "CLEANUP_STAGNATION_WINDOW_DAYS": "14", + "CLEANUP_STAGNATION_MIN_RATIO_DELTA": "0.1", + "CLEANUP_DISK_USAGE_THRESHOLD_PERCENT": "80", + "TRANSMISSION_RPC_USERNAME": " alice ", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + } + ) + + self.assertEqual( + config, + ft.Config( + distros=tuple(d for d in ft.ALL_DISTROS if d in {"arch", "devuan"}), + include_low_demand=True, + skip_ratio_check=True, + check_old_releases=True, + old_release_min_leechers=3, + old_release_min_leecher_ratio=0.5, + old_release_allow_zero_seeders=True, + old_release_recheck_days=2, + old_release_skip_removed=False, + cleanup_keep_only_latest=True, + cleanup_skip_ratio_check=True, + cleanup_min_ratio=1.5, + stagnation_window_days=14, + stagnation_min_ratio_delta=0.1, + disk_usage_threshold_percent=80.0, + rpc_username="alice", + rpc_password="hunter2", + ), + ) + + def test_reads_only_the_env_it_is_given(self): + with helpers.isolated_env(SKIP_RATIO_CHECK="true", CLEANUP_MIN_RATIO="9"): + config = ft.load_config({}) + + self.assertFalse(config.skip_ratio_check) + self.assertEqual(config.cleanup_min_ratio, 1.0) + + def test_invalid_number_is_recorded_not_raised(self): + config = ft.load_config( + {"CLEANUP_MIN_RATIO": "high", "CLEANUP_STAGNATION_WINDOW_DAYS": "1.5"} + ) + + self.assertIsNone(config.cleanup_min_ratio) + self.assertIsNone(config.stagnation_window_days) + self.assertEqual( + config.invalid, + { + "cleanup_min_ratio": ( + "CLEANUP_MIN_RATIO: could not convert string to float: 'high'" + ), + "stagnation_window_days": ( + "CLEANUP_STAGNATION_WINDOW_DAYS: invalid literal for int() " + "with base 10: '1.5'" + ), + }, + ) + + def test_require_raises_the_first_invalid_field_named(self): + config = ft.load_config({"CLEANUP_MIN_RATIO": "high"}) + + config.require("stagnation_window_days") # valid: no error + with self.assertRaisesRegex(ValueError, "^CLEANUP_MIN_RATIO: "): + config.require("stagnation_window_days", "cleanup_min_ratio") + + def test_rpc_auth_needs_both_username_and_password(self): + cases = ( + ({}, None), + ({"TRANSMISSION_RPC_USERNAME": "alice"}, None), + ({"TRANSMISSION_RPC_PASSWORD": "hunter2"}, None), + ( + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": " ", + }, + None, + ), + ( + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, + ("alice", "hunter2"), + ), + ) + for env, expected in cases: + with self.subTest(env=env): + self.assertEqual(ft.load_config(env).rpc_auth, expected) + + +class MakeClientFactoryTests(unittest.TestCase): + """The daily run's own Transmission connections (cleanup, disk-usage + limit) must authenticate the same way configure_transmission.py + configured the daemon. Without this, a user who sets + TRANSMISSION_RPC_USERNAME/PASSWORD (as the README requires once + TRANSMISSION_RPC_WHITELIST is widened beyond localhost) finds cleanup + silently stops working forever: every RPC call fails with 401 and the + failure is only logged, never surfaced.""" + + def _connect(self, env): + config = ft.load_config(env) + with unittest.mock.patch.object(ft, "Client", autospec=True) as client_cls: + client = ft.make_client_factory(config)() + return client_cls, client + + def test_passes_configured_credentials(self): + client_cls, client = self._connect( + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + } + ) + + client_cls.assert_called_once_with( + host="localhost", port=9091, username="alice", password="hunter2" + ) + self.assertIs(client, client_cls.return_value) + + def test_passes_no_credentials_when_unset(self): + client_cls, _client = self._connect({}) + + client_cls.assert_called_once_with( + host="localhost", port=9091, username=None, password=None + ) + + def test_blank_credentials_count_as_unset(self): + client_cls, _client = self._connect( + {"TRANSMISSION_RPC_USERNAME": " ", "TRANSMISSION_RPC_PASSWORD": ""} + ) + + client_cls.assert_called_once_with( + host="localhost", port=9091, username=None, password=None + ) + + def test_factory_connects_only_when_called(self): + with unittest.mock.patch.object(ft, "Client", autospec=True) as client_cls: + factory = ft.make_client_factory(ft.load_config({})) + + client_cls.assert_not_called() + self.assertTrue(callable(factory)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_fetch_torrents.py b/tests/test_fetch_torrents.py deleted file mode 100644 index a83ec11..0000000 --- a/tests/test_fetch_torrents.py +++ /dev/null @@ -1,3399 +0,0 @@ -"""Tests for fetch_torrents.py. - -Importing fetch_torrents touches no files (logging is only set up by -configure_logging()). transmission_rpc is the real library, pinned in -requirements-test.txt to the version Alpine's py3-transmission-rpc ships. - -Run with: python -m unittest discover -s tests -""" - -import json -import logging -import logging.handlers -import os -import subprocess -import sys -import types -import unittest -import unittest.mock -from datetime import UTC, date, timedelta - -REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if REPO_ROOT not in sys.path: - sys.path.insert(0, REPO_ROOT) - -import helpers -import transmission_rpc.constants - -import fetch_torrents as ft - - -def setUpModule(): - # Baseline for every test here: the developer's exported - # SKIP_RATIO_CHECK/CLEANUP_*/TRANSMISSION_*/FETCH_TORRENTS_* can't leak in - # (including into the subprocess tests, which copy os.environ). - patcher = helpers.isolated_env() - patcher.start() - unittest.addModuleCleanup(patcher.stop) - unittest.addModuleCleanup(helpers.block_network()) - - -FakeTorrent = helpers.FakeTorrent - - -class DuckTorrent: - """A torrent-shaped object a real transmission_rpc.Torrent can't be - - ratio None, or added_date None/a float/a datetime - only for exercising - the defensive getattr()/`or 0.0` branches in production code.""" - - def __init__(self, name, id_, ratio=0.0, added_date=None): - self.name = name - self.id = id_ - self.ratio = ratio - self.added_date = added_date - - def __repr__(self): - return f"DuckTorrent({self.name!r}, ratio={self.ratio})" - - -def fixed_date(today): - """A drop-in for fetch_torrents' `date` whose today() is pinned.""" - - class FixedDate(date): - @classmethod - def today(cls): - return today - - return FixedDate - - -def names_of(torrents): - return sorted(t.name for t in torrents) - - -class PlanCleanupTests(unittest.TestCase): - def test_selects_older_version_per_distro_and_type(self): - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), - FakeTorrent("ubuntu-24.04.1-live-server-amd64.iso", 3, ratio=2.0), - FakeTorrent("ubuntu-22.04.1-live-server-amd64.iso", 4, ratio=2.0), - FakeTorrent("debian-12.5.0-amd64-DVD-1.iso", 5, ratio=2.0), - FakeTorrent("debian-12.4.0-amd64-DVD-1.iso", 6, ratio=2.0), - FakeTorrent("kali-linux-2024.1-installer-amd64.iso", 7, ratio=2.0), - FakeTorrent("kali-linux-2023.4-installer-amd64.iso", 8, ratio=2.0), - FakeTorrent("archlinux-2024.05.01-x86_64.iso", 9, ratio=2.0), - FakeTorrent("archlinux-2024.04.01-x86_64.iso", 10, ratio=2.0), - FakeTorrent("linuxmint-22.3-cinnamon-64bit.iso", 11, ratio=2.0), - FakeTorrent("linuxmint-22.2-cinnamon-64bit.iso", 12, ratio=2.0), - FakeTorrent("Fedora-Workstation-Live-x86_64-44", 13, ratio=2.0), - FakeTorrent("Fedora-Workstation-Live-x86_64-43", 14, ratio=2.0), - ] - - to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) - - self.assertEqual( - names_of(to_remove), - [ - "Fedora-Workstation-Live-x86_64-43", - "archlinux-2024.04.01-x86_64.iso", - "debian-12.4.0-amd64-DVD-1.iso", - "kali-linux-2023.4-installer-amd64.iso", - "linuxmint-22.2-cinnamon-64bit.iso", - "ubuntu-22.04.1-live-server-amd64.iso", - "ubuntu-23.10-desktop-amd64.iso", - ], - ) - self.assertEqual(to_keep_low_ratio, []) - - # The newest torrent in each (distro, type) group must never be - # scheduled for removal. - removed_names = set(names_of(to_remove)) - for newest in ( - "ubuntu-24.04-desktop-amd64.iso", - "ubuntu-24.04.1-live-server-amd64.iso", - "debian-12.5.0-amd64-DVD-1.iso", - "kali-linux-2024.1-installer-amd64.iso", - "archlinux-2024.05.01-x86_64.iso", - "linuxmint-22.3-cinnamon-64bit.iso", - "Fedora-Workstation-Live-x86_64-44", - ): - self.assertNotIn(newest, removed_names) - - def test_single_torrent_per_group_is_left_alone(self): - torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=0.0)] - - to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) - - self.assertEqual(to_remove, []) - self.assertEqual(to_keep_low_ratio, []) - - def test_unrelated_torrents_are_ignored(self): - torrents = [ - FakeTorrent("some-random-linux-distro-1.0.iso", 1, ratio=5.0), - FakeTorrent("some-random-linux-distro-2.0.iso", 2, ratio=5.0), - ] - - to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) - - self.assertEqual(to_remove, []) - self.assertEqual(to_keep_low_ratio, []) - - def test_old_version_below_ratio_threshold_is_kept_not_removed(self): - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.4), - ] - - to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) - - self.assertEqual(to_remove, []) - self.assertEqual(len(to_keep_low_ratio), 1) - kept_torrent, kept_ratio = to_keep_low_ratio[0] - self.assertEqual(kept_torrent.name, "ubuntu-23.10-desktop-amd64.iso") - self.assertEqual(kept_ratio, 0.4) - - def test_skip_ratio_check_removes_regardless_of_ratio(self): - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.0), - ] - - to_remove, to_keep_low_ratio = ft.plan_cleanup( - torrents, skip_ratio_check=True, min_ratio=1.0 - ) - - self.assertEqual(names_of(to_remove), ["ubuntu-23.10-desktop-amd64.iso"]) - self.assertEqual(to_keep_low_ratio, []) - - def test_different_types_within_same_distro_are_not_cross_removed(self): - # desktop and live-server are different "type_" groups for ubuntu; - # having only the desktop build twice must not affect live-server. - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), - FakeTorrent("ubuntu-24.04.1-live-server-amd64.iso", 3, ratio=2.0), - ] - - to_remove, _ = ft.plan_cleanup(torrents) - - self.assertEqual(names_of(to_remove), ["ubuntu-23.10-desktop-amd64.iso"]) - - def test_unparseable_name_for_matched_distro_is_skipped_not_raised(self): - # "ubuntu-desktop" has no version segment at index 1 that parses as - # a version; parse_version_type/version_to_tuple should fail and - # plan_cleanup must skip it rather than raising. - torrents = [ - FakeTorrent("ubuntu-desktop", 1, ratio=2.0), - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 2, ratio=2.0), - ] - - to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) - - self.assertEqual(to_remove, []) - self.assertEqual(to_keep_low_ratio, []) - - def test_ratio_exactly_at_min_ratio_is_removed(self): - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.0), - ] - - to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) - - self.assertEqual(names_of(to_remove), ["ubuntu-23.10-desktop-amd64.iso"]) - self.assertEqual(to_keep_low_ratio, []) - - def test_ratio_just_below_min_ratio_is_kept(self): - older = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.999) - torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), older] - - to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) - - self.assertEqual(to_remove, []) - self.assertEqual(to_keep_low_ratio, [(older, 0.999)]) - - def test_default_min_ratio_is_one(self): - older = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5) - torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), older] - - to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) - - self.assertEqual(to_remove, [older]) - self.assertEqual(to_keep_low_ratio, []) - - def test_missing_or_none_ratio_counts_as_zero(self): - no_attr = types.SimpleNamespace(name="ubuntu-23.04-desktop-amd64.iso", id=2) - none_ratio = DuckTorrent("ubuntu-23.10-desktop-amd64.iso", 3, ratio=None) - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - no_attr, - none_ratio, - ] - - to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) - - self.assertEqual(to_remove, []) - self.assertEqual( - sorted(to_keep_low_ratio, key=lambda entry: entry[0].name), - [(no_attr, 0.0), (none_ratio, 0.0)], - ) - - def test_skipped_torrents_listed_first_do_not_stop_planning(self): - # Each kind of torrent _old_version_candidates() skips (or routes - # elsewhere) is placed *before* a valid pair, so a 'continue' that - # became 'break' would drop the pair. - leading = { - "devuan": FakeTorrent("devuan_excalibur", 10, ratio=2.0, added_date=1), - "unknown distro": FakeTorrent("slackware-15.0-install-dvd.iso", 11), - "unparseable": FakeTorrent("ubuntu-noble-desktop-amd64.iso", 12), - } - for kind, first in leading.items(): - with self.subTest(kind): - older = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0) - torrents = [ - first, - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - older, - ] - - with self.assertNoLogs(ft.logger, level="ERROR"): - to_remove, _ = ft.plan_cleanup(torrents) - - self.assertEqual(to_remove, [older]) - - def test_unparseable_name_is_skipped_with_a_warning(self): - torrents = [ - FakeTorrent("ubuntu-noble-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-mantic-desktop-amd64.iso", 2, ratio=2.0), - ] - - with self.assertLogs(ft.logger, level="WARNING") as logs: - to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) - - self.assertEqual((to_remove, to_keep_low_ratio), ([], [])) - self.assertEqual(len(logs.records), 2) - - -class PlanCleanupDevuanGroupingTests(unittest.TestCase): - """Devuan releases (e.g. "devuan_excalibur") carry no dotted version to - compare - see DEVUAN_NAME_PATTERN - so _old_version_candidates() groups - them separately and orders by added_date (when Transmission added the - torrent) instead of a parsed version, same idea as should_fetch_torrent()'s - ratio gate for Devuan.""" - - def test_older_devuan_release_is_a_cleanup_candidate(self): - torrents = [ - FakeTorrent("devuan_daedalus", 1, ratio=2.0, added_date=1000), - FakeTorrent("devuan_excalibur", 2, ratio=2.0, added_date=2000), - ] - - to_remove, _ = ft.plan_cleanup(torrents) - - self.assertEqual(names_of(to_remove), ["devuan_daedalus"]) - - def test_single_devuan_release_is_left_alone(self): - torrents = [FakeTorrent("devuan_excalibur", 1, ratio=0.0, added_date=1000)] - - to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) - - self.assertEqual(to_remove, []) - self.assertEqual(to_keep_low_ratio, []) - - def test_older_devuan_release_below_ratio_threshold_is_kept(self): - torrents = [ - FakeTorrent("devuan_daedalus", 1, ratio=0.4, added_date=1000), - FakeTorrent("devuan_excalibur", 2, ratio=2.0, added_date=2000), - ] - - to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents, min_ratio=1.0) - - self.assertEqual(to_remove, []) - self.assertEqual(len(to_keep_low_ratio), 1) - kept_torrent, kept_ratio = to_keep_low_ratio[0] - self.assertEqual(kept_torrent.name, "devuan_daedalus") - self.assertEqual(kept_ratio, 0.4) - - def test_stagnant_older_devuan_release_is_removed(self): - today = date(2026, 8, 29) - torrents = [ - FakeTorrent("devuan_daedalus", 1, ratio=1.5, added_date=1000), - FakeTorrent("devuan_excalibur", 2, ratio=2.0, added_date=2000), - ] - history = {"devuan_daedalus": [{"date": "2026-07-20", "ratio": 1.495}]} - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, history, today, window_days=30, min_ratio_delta=0.02 - ) - - self.assertEqual(names_of(to_remove), ["devuan_daedalus"]) - self.assertEqual(to_keep, []) - - def test_missing_added_date_does_not_crash(self): - # Real Transmission torrents always carry added_date; this just - # guards _devuan_sort_key() against a hypothetical missing attribute. - torrents = [ - DuckTorrent("devuan_daedalus", 1, ratio=2.0, added_date=None), - DuckTorrent("devuan_excalibur", 2, ratio=2.0, added_date=None), - ] - - to_remove, _ = ft.plan_cleanup(torrents) - - self.assertEqual(len(to_remove), 1) - - def test_release_without_added_date_sorts_as_oldest(self): - undated = DuckTorrent("devuan_daedalus", 1, ratio=2.0, added_date=None) - torrents = [ - undated, - DuckTorrent("devuan_excalibur", 2, ratio=2.0, added_date=0.5), - ] - - to_remove, _ = ft.plan_cleanup(torrents) - - self.assertEqual(to_remove, [undated]) - - def test_datetime_added_dates_are_ordered_by_timestamp(self): - from datetime import datetime - - older = DuckTorrent( - "devuan_daedalus", - 1, - ratio=2.0, - added_date=datetime(2026, 1, 1, tzinfo=UTC), - ) - torrents = [ - DuckTorrent( - "devuan_excalibur", - 2, - ratio=2.0, - added_date=datetime(2026, 6, 1, tzinfo=UTC), - ), - older, - ] - - to_remove, _ = ft.plan_cleanup(torrents) - - self.assertEqual(to_remove, [older]) - - -class ParseVersionTypeTests(unittest.TestCase): - """(version, type_) is the grouping key for the ratio gate and cleanup: - two names share a ratio history only when their type_ is equal.""" - - CASES = ( - ( - "ubuntu-24.04-desktop-amd64.iso", - "ubuntu", - ("24.04", "ubuntu-desktop-amd64.iso"), - ), - ( - "ubuntu-24.04-desktop-arm64.iso", - "ubuntu", - ("24.04", "ubuntu-desktop-arm64.iso"), - ), - ( - "xubuntu-24.04-desktop-amd64.iso", - "ubuntu", - ("24.04", "xubuntu-desktop-amd64.iso"), - ), - ("debian-12.5.0-amd64-DVD-1.iso", "debian", ("12.5.0", "amd64-DVD-1.iso")), - ("debian-12.5.0-arm64-DVD-1.iso", "debian", ("12.5.0", "arm64-DVD-1.iso")), - ( - "kali-linux-2024.1-installer-amd64.iso", - "kali", - ("2024.1", "installer-amd64.iso"), - ), - ( - "kali-linux-2024.1-installer-arm64.iso", - "kali", - ("2024.1", "installer-arm64.iso"), - ), - ("archlinux-2024.05.01-x86_64.iso", "arch", ("2024.05.01", "")), - ("linuxmint-22.3-cinnamon-64bit.iso", "mint", ("22.3", "cinnamon-64bit.iso")), - ("Fedora-Workstation-Live-x86_64-44", "fedora", ("44", "x86_64")), - ("Fedora-Workstation-Live-aarch64-44", "fedora", ("44", "aarch64")), - ("anything-1.0", "not-a-distro", ("", "")), - ) - - def test_every_ubuntu_flavour_fetched_is_recognised_as_ubuntu(self): - # fetch_ubuntu_lts() fetches ubuntu, lubuntu and xubuntu images; a - # flavour get_distro() doesn't recognise skips the ratio gate and - # cleanup grouping entirely. - for name in ( - "ubuntu-24.04-desktop-amd64.iso", - "lubuntu-24.04-desktop-amd64.iso", - "xubuntu-24.04-desktop-amd64.iso", - ): - with self.subTest(name): - self.assertEqual(ft.get_distro(name), "ubuntu") - - def test_lubuntu_is_ratio_gated_on_its_own_previous_release(self): - ratios = { - "lubuntu-23.10-desktop-amd64.iso": 0.2, - "ubuntu-23.10-desktop-amd64.iso": 5.0, - } - self.assertFalse( - ft.should_fetch_torrent("lubuntu-24.04-desktop-amd64.iso", ratios) - ) - - def test_version_and_type_per_distro(self): - for name, distro, expected in self.CASES: - with self.subTest(name): - self.assertEqual(ft.parse_version_type(name, distro), expected) - - def test_architectures_of_the_same_release_get_different_types(self): - pairs = [ - ( - "ubuntu-24.04-desktop-amd64.iso", - "ubuntu-24.04-desktop-arm64.iso", - "ubuntu", - ), - ( - "debian-12.5.0-amd64-DVD-1.iso", - "debian-12.5.0-arm64-DVD-1.iso", - "debian", - ), - ( - "kali-linux-2024.1-installer-amd64.iso", - "kali-linux-2024.1-installer-arm64.iso", - "kali", - ), - ( - "Fedora-Workstation-Live-x86_64-44", - "Fedora-Workstation-Live-aarch64-44", - "fedora", - ), - ] - for a, b, distro in pairs: - with self.subTest(distro): - self.assertNotEqual( - ft.parse_version_type(a, distro)[1], - ft.parse_version_type(b, distro)[1], - ) - - -class ShouldFetchTorrentRatioKeyTests(unittest.TestCase): - """Candidate names must key into the same ratio-lookup bucket as the names Transmission - actually reports (which include the arch suffix and .iso extension), - for all supported distros.""" - - def test_finds_previous_version_and_honors_low_ratio_ubuntu(self): - ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.5} - self.assertFalse( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) - ) - - def test_finds_previous_version_and_honors_low_ratio_debian(self): - ratios = {"debian-12.4.0-amd64-DVD-1.iso": 0.5} - self.assertFalse( - ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios) - ) - - def test_finds_previous_version_and_honors_low_ratio_kali(self): - ratios = {"kali-linux-2023.4-installer-amd64.iso": 0.3} - self.assertFalse( - ft.should_fetch_torrent("kali-linux-2024.1-installer-amd64.iso", ratios) - ) - - def test_finds_previous_version_and_honors_low_ratio_arch(self): - ratios = {"archlinux-2024.04.01-x86_64.iso": 0.2} - self.assertFalse( - ft.should_fetch_torrent("archlinux-2024.05.01-x86_64.iso", ratios) - ) - - def test_finds_previous_version_and_allows_high_ratio_ubuntu(self): - ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.5} - self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) - ) - - def test_finds_previous_version_and_allows_high_ratio_debian(self): - ratios = {"debian-12.4.0-amd64-DVD-1.iso": 1.5} - self.assertTrue( - ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios) - ) - - def test_finds_previous_version_and_honors_low_ratio_mint(self): - ratios = {"linuxmint-22.2-cinnamon-64bit.iso": 0.4} - self.assertFalse( - ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios) - ) - - def test_finds_previous_version_and_allows_high_ratio_mint(self): - ratios = {"linuxmint-22.2-cinnamon-64bit.iso": 1.5} - self.assertTrue( - ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios) - ) - - def test_finds_previous_version_and_honors_low_ratio_fedora(self): - ratios = {"Fedora-Workstation-Live-x86_64-43": 0.4} - self.assertFalse( - ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios) - ) - - def test_finds_previous_version_and_allows_high_ratio_fedora(self): - ratios = {"Fedora-Workstation-Live-x86_64-43": 1.5} - self.assertTrue( - ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios) - ) - - -class ShouldFetchTorrentRemovedHistoryFallbackTests(unittest.TestCase): - """Once a previous version is removed by cleanup, Transmission stops - reporting it and it drops out of `ratios` (see get_previous_ratios()) - - without a fallback, should_fetch_torrent() would then hit its "no - previous version" branch and fetch unconditionally, silently bypassing - the ratio gate for exactly the versions cleanup already gave up on. - removed_history (record_removal()'s output) is the fallback source.""" - - def test_falls_back_to_removed_history_ratio_when_not_live(self): - removed_history = { - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.4, - }, - } - self.assertFalse( - ft.should_fetch_torrent( - "ubuntu-24.04-desktop-amd64.iso", - ratios={}, - removed_history=removed_history, - ) - ) - - def test_removed_history_ratio_above_floor_allows_fetch(self): - removed_history = { - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-01-01", - "reason": "keep_only_latest", - "ratio": 1.5, - }, - } - self.assertTrue( - ft.should_fetch_torrent( - "ubuntu-24.04-desktop-amd64.iso", - ratios={}, - removed_history=removed_history, - ) - ) - - def test_live_ratio_takes_priority_over_removed_history(self): - # Same name in both is not expected in practice (a torrent Transmission - # still reports hasn't been removed), but live data winning if it ever - # happens is the safer default. - removed_history = { - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.1, - }, - } - ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.5} - self.assertTrue( - ft.should_fetch_torrent( - "ubuntu-24.04-desktop-amd64.iso", - ratios, - removed_history=removed_history, - ) - ) - - def test_low_live_ratio_is_not_overridden_by_high_removed_history(self): - removed_history = { - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 1.5, - } - } - ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.4} - self.assertFalse( - ft.should_fetch_torrent( - "ubuntu-24.04-desktop-amd64.iso", - ratios, - removed_history=removed_history, - ) - ) - - def test_removed_history_entry_without_ratio_is_ignored(self): - # Legacy entries recorded before this field existed have no 'ratio' - # key - must not crash, and must not count as a usable prior version. - removed_history = { - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - }, - } - # Skipped up front, not by tripping over the missing key and logging - # a parse-error warning. - with self.assertNoLogs(ft.logger, level="WARNING"): - result = ft.should_fetch_torrent( - "ubuntu-24.04-desktop-amd64.iso", - ratios={}, - removed_history=removed_history, - ) - self.assertTrue(result) - - def test_skipped_removed_history_entry_does_not_hide_later_ones(self): - removed_history = { - "ubuntu-23.04-desktop-amd64.iso": { - "removed_date": "2025-06-01", - "reason": "stagnant", - }, - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.4, - }, - } - self.assertFalse( - ft.should_fetch_torrent( - "ubuntu-24.04-desktop-amd64.iso", - ratios={}, - removed_history=removed_history, - ) - ) - - def test_removed_history_from_a_different_distro_is_not_consulted(self): - removed_history = { - "debian-12.4.0-amd64-DVD-1.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.1, - }, - } - self.assertTrue( - ft.should_fetch_torrent( - "ubuntu-24.04-desktop-amd64.iso", - ratios={}, - removed_history=removed_history, - ) - ) - - -class ShouldFetchTorrentDevuanRatioGateTests(unittest.TestCase): - """Devuan has no per-name version to compare (get_distro() returns None - for it - see distro_patterns), so should_fetch_torrent() instead gates - fetching a new Devuan release on the ratio achieved by whichever Devuan - release we currently hold, using the same 1.0 floor as other distros.""" - - def test_no_previous_devuan_release_fetches(self): - self.assertTrue(ft.should_fetch_torrent("devuan_excalibur", ratios={})) - - def test_low_ratio_on_current_release_blocks_next_release(self): - ratios = {"devuan_daedalus": 0.4} - self.assertFalse(ft.should_fetch_torrent("devuan_excalibur", ratios)) - - def test_high_ratio_on_current_release_allows_next_release(self): - ratios = {"devuan_daedalus": 1.5} - self.assertTrue(ft.should_fetch_torrent("devuan_excalibur", ratios)) - - def test_already_holding_this_exact_release_is_always_fetched(self): - # download_torrent()'s own file-existence check is what actually - # dedupes a re-offered current release - not this ratio gate. - ratios = {"devuan_excalibur": 0.1} - self.assertTrue(ft.should_fetch_torrent("devuan_excalibur", ratios)) - - def test_falls_back_to_removed_history_when_no_live_ratio(self): - removed_history = { - "devuan_daedalus": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.4, - }, - } - self.assertFalse( - ft.should_fetch_torrent( - "devuan_excalibur", ratios={}, removed_history=removed_history - ) - ) - - def test_other_distros_removed_history_not_consulted_for_devuan(self): - removed_history = { - "debian-12.4.0-amd64-DVD-1.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.1, - }, - } - self.assertTrue( - ft.should_fetch_torrent( - "devuan_excalibur", ratios={}, removed_history=removed_history - ) - ) - - -class ShouldFetchTorrentBoundaryTests(unittest.TestCase): - """The 1.0 floor is inclusive, only strictly older versions count as the - "previous" one, and the branches that bypass the gate entirely.""" - - def test_previous_ratio_exactly_at_floor_allows_fetch(self): - ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.0} - self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) - ) - - def test_previous_ratio_just_below_floor_blocks_fetch(self): - ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.999} - self.assertFalse( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) - ) - - def test_devuan_previous_ratio_exactly_at_floor_allows_fetch(self): - self.assertTrue( - ft.should_fetch_torrent("devuan_excalibur", {"devuan_daedalus": 1.0}) - ) - - def test_devuan_previous_ratio_just_below_floor_blocks_fetch(self): - self.assertFalse( - ft.should_fetch_torrent("devuan_excalibur", {"devuan_daedalus": 0.999}) - ) - - def test_only_newer_versions_on_record_fetches(self): - # A newer release with a low ratio isn't a "previous" one, so there - # is nothing to gate on. - ratios = {"ubuntu-24.10-desktop-amd64.iso": 0.1} - self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) - ) - - def test_same_version_on_record_is_not_its_own_predecessor(self): - ratios = {"ubuntu-24.04-desktop-amd64.iso": 0.1} - self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) - ) - - def test_highest_older_version_decides_not_any_older_version(self): - ratios = { - "ubuntu-22.04-desktop-amd64.iso": 5.0, - "ubuntu-23.10-desktop-amd64.iso": 0.4, - } - self.assertFalse( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) - ) - - def test_other_architecture_ratio_is_not_consulted(self): - ratios = {"ubuntu-23.10-desktop-arm64.iso": 0.1} - self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) - ) - - def test_devuan_ignores_non_devuan_ratios(self): - ratios = {"debian-12.4.0-amd64-DVD-1.iso": 5.0, "devuan_daedalus": 0.4} - self.assertFalse(ft.should_fetch_torrent("devuan_excalibur", ratios)) - - def test_devuan_removed_history_for_the_same_release_is_not_a_predecessor(self): - removed_history = { - "devuan_excalibur": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.1, - } - } - self.assertTrue( - ft.should_fetch_torrent( - "devuan_excalibur", ratios={}, removed_history=removed_history - ) - ) - - def test_devuan_live_ratio_beats_removed_history(self): - removed_history = { - "devuan_chimaera": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.1, - } - } - self.assertTrue( - ft.should_fetch_torrent( - "devuan_excalibur", - {"devuan_daedalus": 1.5}, - removed_history=removed_history, - ) - ) - - def test_skip_ratio_check_fetches_despite_low_previous_ratio(self): - ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.1} - self.assertTrue( - ft.should_fetch_torrent( - "ubuntu-24.04-desktop-amd64.iso", ratios, skip_ratio_check=True - ) - ) - - def test_unknown_distro_fetches(self): - ratios = {"slackware-15.0-install-dvd.iso": 0.1} - self.assertTrue( - ft.should_fetch_torrent("slackware-15.1-install-dvd.iso", ratios) - ) - - def test_unparseable_name_of_known_distro_fetches_and_logs_error(self): - ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.1} - with self.assertLogs(ft.logger, level="ERROR") as logs: - result = ft.should_fetch_torrent("ubuntu-noble-desktop-amd64.iso", ratios) - - self.assertTrue(result) - self.assertEqual(len(logs.records), 1) - self.assertIn("ubuntu-noble-desktop-amd64.iso", logs.output[0]) - - def test_unparseable_stored_ratio_entry_is_skipped(self): - ratios = { - "ubuntu-mantic-desktop-amd64.iso": 0.1, - "ubuntu-23.10-desktop-amd64.iso": 1.5, - } - with self.assertLogs(ft.logger, level="WARNING"): - self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) - ) - - -def configure_logging_for_test(testcase, log_dir, **env): - """ft.configure_logging() with only `env`, undone when the test ends.""" - saved = ( - list(ft.logger.handlers), - list(ft._configured_handlers), - ft.logger.level, - ft.logger.propagate, - ) - - def restore(): - for handler in ft._configured_handlers: - handler.close() - ft.logger.handlers[:] = saved[0] - ft._configured_handlers[:] = saved[1] - ft.logger.setLevel(saved[2]) - ft.logger.propagate = saved[3] - - testcase.addCleanup(restore) - return ft.configure_logging(log_dir, env=env) - - -class RatioLogSurvivesConfigureLoggingTests(unittest.TestCase): - """Each container run is a fresh process: the ratio log from the - *previous* run is what should_fetch_torrent() needs in order to gate a - new version's download. configure_logging() opens that same file with - mode="w" for this run's ratios, so it must not truncate it before - get_previous_ratios() has read it - otherwise the low-ratio fetch gate - is silently defeated for every distro.""" - - def setUp(self): - self.log_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_restart_") - self.path = os.path.join(self.log_dir, "fetch_torrents_ratios.log") - with open(self.path, "w", encoding="utf-8") as f: - f.write( - "2026-08-01 00:00:00,000 INFO: [ratio] RATIOS START\n" - "2026-08-01 00:00:00,000 INFO: [ratio] archlinux-2026.04.01-x86_64.iso → 0.664\n" - "2026-08-01 00:00:00,000 INFO: [ratio] RATIOS END\n" - ) - - def test_previous_ratios_are_readable_after_configure_logging(self): - configure_logging_for_test(self, self.log_dir) - - self.assertEqual( - ft.get_previous_ratios(self.path), - {"archlinux-2026.04.01-x86_64.iso": 0.664}, - ) - - def test_first_ratio_line_of_the_run_replaces_the_previous_log(self): - _stream, _file, ratio_handler = configure_logging_for_test(self, self.log_dir) - - ft.logger.info("[ratio] %-50s → %.3f", "new.iso", 1.0) - ratio_handler.flush() - - self.assertEqual(ft.get_previous_ratios(self.path), {"new.iso": 1.0}) - - -class ImportOpensNoFilesTests(unittest.TestCase): - """Importing fetch_torrents (tests, other scripts) must not create or - truncate anything under FETCH_TORRENTS_LOG_DIR - only - configure_logging() does. Needs a fresh interpreter, since this process - imported the module long ago.""" - - def test_import_leaves_the_log_dir_untouched(self): - log_dir = helpers.make_temp_dir(self, "fetch_torrents_import_") - script = ( - "import sys\n" - f"sys.path.insert(0, {REPO_ROOT!r})\n" - "import fetch_torrents as ft\n" - "ft.logger.warning('logged before configure_logging()')\n" - ) - result = subprocess.run( - [sys.executable, "-c", script], - check=False, - env=dict(os.environ, FETCH_TORRENTS_LOG_DIR=log_dir), - capture_output=True, - text=True, - timeout=30, - ) - - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(os.listdir(log_dir), []) - self.assertEqual(result.stderr, "") - - -class GetPreviousRatiosTests(unittest.TestCase): - """get_previous_ratios() is what feeds should_fetch_torrent()'s ratios: - only the last START/END segment of the previous run's ratio log counts, - so a ratio from an older run can't satisfy (or block) the gate.""" - - def _write_log(self, *messages): - path = os.path.join( - helpers.make_temp_dir(self, "fetch_torrents_prev_ratios_"), - "fetch_torrents_ratios.log", - ) - with open(path, "w", encoding="utf-8") as f: - f.writelines( - f"2026-08-01 00:00:00,000 INFO: {message}\n" for message in messages - ) - return path - - def test_missing_file_returns_empty(self): - missing = os.path.join( - helpers.make_temp_dir(self, "fetch_torrents_prev_ratios_"), "nope.log" - ) - self.assertEqual(ft.get_previous_ratios(missing), {}) - - def test_without_markers_every_ratio_line_is_parsed(self): - path = self._write_log( - "[ratio] ubuntu-24.04-desktop-amd64.iso → 1.250", - "unrelated line", - "[ratio] debian-12.5.0-amd64-DVD-1.iso → 0.664", - ) - self.assertEqual( - ft.get_previous_ratios(path), - { - "ubuntu-24.04-desktop-amd64.iso": 1.25, - "debian-12.5.0-amd64-DVD-1.iso": 0.664, - }, - ) - - def test_only_the_last_complete_segment_is_returned(self): - path = self._write_log( - "[ratio] RATIOS START", - "[ratio] old-only.iso → 0.100", - "[ratio] shared.iso → 0.200", - "[ratio] RATIOS END", - "[ratio] RATIOS START", - "[ratio] shared.iso → 1.500", - "[ratio] RATIOS END", - "[ratio] after-end.iso → 0.300", - ) - self.assertEqual(ft.get_previous_ratios(path), {"shared.iso": 1.5}) - - def test_start_without_end_parses_to_end_of_file(self): - # A run that crashed mid-way never wrote its END marker. - path = self._write_log( - "[ratio] RATIOS START", - "[ratio] old.iso → 0.100", - "[ratio] RATIOS END", - "[ratio] RATIOS START", - "[ratio] crashed-run.iso → 0.700", - ) - self.assertEqual(ft.get_previous_ratios(path), {"crashed-run.iso": 0.7}) - - def test_end_before_the_last_start_is_ignored(self): - path = self._write_log( - "[ratio] RATIOS END", - "[ratio] before-start.iso → 0.100", - "[ratio] RATIOS START", - "[ratio] current.iso → 0.900", - ) - self.assertEqual(ft.get_previous_ratios(path), {"current.iso": 0.9}) - - def test_last_end_after_the_last_start_closes_the_segment(self): - # Not written by a normal run; pins current behavior. - path = self._write_log( - "[ratio] RATIOS START", - "[ratio] first.iso → 0.100", - "[ratio] RATIOS END", - "[ratio] second.iso → 0.200", - "[ratio] RATIOS END", - "[ratio] after.iso → 0.300", - ) - self.assertEqual( - ft.get_previous_ratios(path), {"first.iso": 0.1, "second.iso": 0.2} - ) - - def test_padded_names_are_stripped(self): - path = self._write_log( - "[ratio] archlinux-2026.04.01-x86_64.iso → 0.664", - ) - self.assertEqual( - ft.get_previous_ratios(path), {"archlinux-2026.04.01-x86_64.iso": 0.664} - ) - - def test_names_with_internal_spaces_keep_them(self): - path = self._write_log("[ratio] Some Torrent Name → 2.000") - self.assertEqual(ft.get_previous_ratios(path), {"Some Torrent Name": 2.0}) - - def test_ratio_without_decimal_point_is_not_matched(self): - # Pins current behavior: log_seed_ratios_via_http() always writes - # three decimals, so an integer ratio means a line it didn't write. - path = self._write_log("[ratio] whole.iso → 1") - self.assertEqual(ft.get_previous_ratios(path), {}) - - def test_duplicate_name_in_a_segment_last_one_wins(self): - path = self._write_log( - "[ratio] RATIOS START", - "[ratio] dup.iso → 0.100", - "[ratio] dup.iso → 1.100", - "[ratio] RATIOS END", - ) - self.assertEqual(ft.get_previous_ratios(path), {"dup.iso": 1.1}) - - -class ConfigureLoggingTests(unittest.TestCase): - """fetch_torrents.log must be bounded (RotatingFileHandler), not a plain - FileHandler appending forever for the life of a 'deploy and forget' - container.""" - - def _configure(self, **env): - log_dir = helpers.make_temp_dir(self, "fetch_torrents_log_rotation_") - return configure_logging_for_test(self, log_dir, **env) - - def test_defaults_to_a_bounded_rotating_handler(self): - _stream, file_handler, _ratio = self._configure() - self.assertIsInstance(file_handler, logging.handlers.RotatingFileHandler) - self.assertEqual(file_handler.maxBytes, 5 * 1024 * 1024) - self.assertEqual(file_handler.backupCount, 3) - - def test_env_vars_override_rotation_settings(self): - _stream, file_handler, _ratio = self._configure( - FETCH_TORRENTS_LOG_MAX_BYTES="1000", FETCH_TORRENTS_LOG_BACKUP_COUNT="1" - ) - self.assertEqual(file_handler.maxBytes, 1000) - self.assertEqual(file_handler.backupCount, 1) - - def test_invalid_env_vars_fall_back_to_defaults(self): - _stream, file_handler, _ratio = self._configure( - FETCH_TORRENTS_LOG_MAX_BYTES="not-a-number", - FETCH_TORRENTS_LOG_BACKUP_COUNT="also-bad", - ) - self.assertEqual(file_handler.maxBytes, 5 * 1024 * 1024) - self.assertEqual(file_handler.backupCount, 3) - - def test_calling_again_replaces_its_own_handlers_only(self): - other = logging.NullHandler() - ft.logger.addHandler(other) - self.addCleanup(ft.logger.removeHandler, other) - - first = self._configure() - second = self._configure() - - self.assertEqual( - [h for h in ft.logger.handlers if h is not other], list(second) - ) - self.assertIn(other, ft.logger.handlers) - self.assertTrue(all(h not in ft.logger.handlers for h in first)) - - -class FetchUbuntuLtsNamingTests(unittest.TestCase): - """The names fetch_ubuntu_lts() uses as dict keys are also what gets - passed to should_fetch_torrent() before download, so they must match the - shape of the names Transmission later reports for those same torrents - (real arch suffix + .iso extension) — see - tasks/02-ratio-key-mismatch-ubuntu-debian.md.""" - - def test_candidate_names_include_arch_and_iso_suffix(self): - meta_release = "Dist: noble\nVersion: 24.04\nSupported: 1\n" - - response = helpers.fake_response(meta_release) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - results = ft.fetch_ubuntu_lts() - - self.assertTrue(results) - self.assertIn("ubuntu-24.04-desktop-amd64.iso", results) - self.assertIn("ubuntu-24.04-live-server-amd64.iso", results) - - -class FetchDebianStableNamingTests(unittest.TestCase): - """Mirrors FetchUbuntuLtsNamingTests for Debian: the dict key must keep - the .iso extension so it matches Transmission's reported torrent name.""" - - def test_candidate_name_keeps_iso_suffix(self): - html = 'link' - - response = helpers.fake_response(html) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - results = ft.fetch_debian_stable() - - self.assertIn("debian-12.5.0-amd64-DVD-1.iso", results) - - -class FetchLinuxMintCinnamonNamingTests(unittest.TestCase): - """The dict key fetch_linuxmint_cinnamon() uses must match the 'name' - field embedded in the actual .torrent file (verified against a real - download from linuxmint.com: linuxmint-22.3-cinnamon-64bit.iso), so it - lines up with what Transmission later reports as torrent.name.""" - - def test_candidate_name_matches_torrent_internal_name(self): - html = "Download Linux Mint 22.3 - Linux Mint" - - response = helpers.fake_response(html) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - results = ft.fetch_linuxmint_cinnamon() - - self.assertIn("linuxmint-22.3-cinnamon-64bit.iso", results) - self.assertEqual( - results["linuxmint-22.3-cinnamon-64bit.iso"], - "https://www.linuxmint.com/torrents/linuxmint-22.3-cinnamon-64bit.iso.torrent", - ) - - -class FetchFedoraWorkstationNamingTests(unittest.TestCase): - """The dict key fetch_fedora_workstation() uses must match the 'name' - field embedded in the actual .torrent file (verified against a real - download from torrent.fedoraproject.org: Fedora-Workstation-Live-x86_64-44, - which has no .iso extension), so it lines up with what Transmission later - reports as torrent.name.""" - - def test_picks_highest_version_per_arch(self): - html = ( - "" - 'a' - 'b' - 'c' - 'd' - "" - ) - - response = helpers.fake_response(html) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - results = ft.fetch_fedora_workstation() - - self.assertEqual( - results, - { - "Fedora-Workstation-Live-x86_64-44": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-x86_64-44.torrent", - "Fedora-Workstation-Live-aarch64-44": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-aarch64-44.torrent", - }, - ) - - -class FetchArchLatestNamingTests(unittest.TestCase): - """archlinux.org/releng/releases/ lists every release still available - (typically the last 2-3 months) as available-yes. fetch_arch_latest() - must pick only the single newest one - fetching all of them fragments - whatever little torrent demand Arch has across several near-duplicate - monthly builds (see tasks/01-demand-checks-for-default-distros.md).""" - - def test_picks_only_the_single_newest_release(self): - html = ( - "" - "" - "" - "" - "
t
t
t
" - ) - - response = helpers.fake_response(html) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - results = ft.fetch_arch_latest() - - self.assertEqual( - results, - { - "archlinux-2026.09.01": "https://archlinux.org/releng/releases/2026.09.01/torrent/" - }, - ) - - def test_no_available_releases_returns_false(self): - html = "
" - - response = helpers.fake_response(html) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - result = ft.fetch_arch_latest() - - self.assertFalse(result) - - -class FetchDevuanLatestNamingTests(unittest.TestCase): - """Devuan's get-devuan page links a single release torrent whose - filename embeds the current release codename (e.g. - devuan_excalibur.torrent, verified against the real download) - the - dict key must match that basename since it's also what Transmission - later reports as torrent.name.""" - - def test_extracts_current_release_torrent(self): - html = ( - '" - ) - - response = helpers.fake_response(html) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - results = ft.fetch_devuan_latest() - - self.assertEqual( - results, - {"devuan_excalibur": "https://files.devuan.org/devuan_excalibur.torrent"}, - ) - - def test_returns_false_when_no_torrent_link_found(self): - response = helpers.fake_response("nothing here") - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - result = ft.fetch_devuan_latest() - - self.assertFalse(result) - - -class ParseSupportedDistrosTests(unittest.TestCase): - """Arch and Devuan are opt-in only: neither can be pre-verified for real - demand (Arch publishes no tracker at all; Devuan bundles every edition - into one multi-file torrent, so a demand reading only ever covers the - whole bundle, never a specific file within it) - see - tasks/01-demand-checks-for-default-distros.md. They must never be - fetched just because FETCH_TORRENTS_DISTROS is unset.""" - - def test_default_excludes_opt_in_distros(self): - with helpers.isolated_env(): - result = ft.parse_supported_distros() - self.assertNotIn("arch", result) - self.assertNotIn("devuan", result) - for expected in ("ubuntu", "debian", "kali", "mint", "fedora"): - self.assertIn(expected, result) - - def test_explicit_opt_in_distros_are_honored(self): - with unittest.mock.patch.dict( - os.environ, {"FETCH_TORRENTS_DISTROS": "arch,devuan"} - ): - result = ft.parse_supported_distros() - self.assertEqual(sorted(result), ["arch", "devuan"]) - - def test_can_mix_default_and_opt_in_distros(self): - with unittest.mock.patch.dict( - os.environ, {"FETCH_TORRENTS_DISTROS": "ubuntu,arch"} - ): - result = ft.parse_supported_distros() - self.assertEqual(sorted(result), ["arch", "ubuntu"]) - - def test_unknown_entries_are_dropped_but_valid_ones_kept(self): - with unittest.mock.patch.dict( - os.environ, {"FETCH_TORRENTS_DISTROS": "ubuntu,nonsense"} - ): - result = ft.parse_supported_distros() - self.assertEqual(result, ["ubuntu"]) - - def test_all_invalid_falls_back_to_default_not_all_distros(self): - with unittest.mock.patch.dict( - os.environ, {"FETCH_TORRENTS_DISTROS": "nonsense"} - ): - result = ft.parse_supported_distros() - self.assertEqual(sorted(result), sorted(ft.DEFAULT_DISTROS)) - self.assertNotIn("arch", result) - self.assertNotIn("devuan", result) - - -class LowDemandVariantTests(unittest.TestCase): - """cloud-genericcloud images and Kali's netinst installer are - chronically low-ratio in fetch_torrents_ratios.log regardless of - architecture or how recent the release is — these are the 'image - families' FETCH_TORRENTS_INCLUDE_LOW_DEMAND gates. - - arm64/aarch64/armhf/armel builds are deliberately NOT low-demand: current- - version arm64/armhf builds (e.g. kali-linux-2026.2-raspberry-pi-armhf-img-xz - at 2.351, debian-13.6.0-arm64-netinst.iso at 1.571) have solid ratios in - the log — the low arm64/armhf numbers there are almost all older, - superseded versions, already handled by should_fetch_torrent().""" - - def test_cloud_images_are_low_demand_regardless_of_arch(self): - for name in ( - "kali-linux-2026.2-cloud-genericcloud-amd64-tar-xz", - "kali-linux-2026.1-cloud-genericcloud-arm64-tar-xz", - ): - self.assertTrue(ft.is_low_demand_variant(name), name) - - def test_kali_netinst_is_low_demand_regardless_of_arch(self): - for name in ( - "kali-linux-2026.1-installer-netinst-amd64.iso", - "kali-linux-2025.2-installer-netinst-arm64.iso", - ): - self.assertTrue(ft.is_low_demand_variant(name), name) - - def test_debian_netinst_is_not_low_demand(self): - # Debian's netinst is the highest-ratio image in the whole log — - # only Kali's netinst installer is excluded, not the substring. - for name in ( - "debian-13.6.0-amd64-netinst.iso", - "debian-13.6.0-arm64-netinst.iso", - ): - self.assertFalse(ft.is_low_demand_variant(name), name) - - def test_current_arm64_and_armhf_builds_are_not_low_demand(self): - for name in ( - "debian-13.6.0-arm64-DVD-1.iso", - "kali-linux-2026.2-installer-arm64.iso", - "Fedora-Workstation-Live-aarch64-44", - "kali-linux-2026.2-raspberry-pi-armhf-img-xz", - "kali-linux-2025.2-raspberry-pi-zero-w-armel-img-xz", - ): - self.assertFalse(ft.is_low_demand_variant(name), name) - - def test_mainstream_amd64_variants_are_not_low_demand(self): - for name in ( - "ubuntu-24.04.4-desktop-amd64.iso", - "kali-linux-2026.2-installer-everything-amd64.iso", - "kali-linux-2026.2-installer-purple-amd64.iso", - "kali-linux-2026.2-installer-amd64.iso", - "archlinux-2026.04.01-x86_64.iso", - "Fedora-Workstation-Live-x86_64-44", - ): - self.assertFalse(ft.is_low_demand_variant(name), name) - - def test_filter_low_demand_excludes_by_default(self): - torrents = { - "kali-linux-2026.2-installer-amd64.iso": "http://x/a", - "kali-linux-2026.2-installer-netinst-amd64.iso": "http://x/b", - } - - filtered = ft.filter_low_demand(torrents, include_low_demand=False) - - self.assertEqual( - filtered, {"kali-linux-2026.2-installer-amd64.iso": "http://x/a"} - ) - - def test_filter_low_demand_includes_when_enabled(self): - torrents = { - "kali-linux-2026.2-installer-amd64.iso": "http://x/a", - "kali-linux-2026.2-installer-netinst-amd64.iso": "http://x/b", - } - - filtered = ft.filter_low_demand(torrents, include_low_demand=True) - - self.assertEqual(filtered, torrents) - - -class DownloadTorrent404Tests(unittest.TestCase): - """Torrent URLs for releases no longer hosted upstream (e.g. old Ubuntu - LTS releases that changelogs.ubuntu.com still marks 'Supported: 1' for - ESM purposes, long after their installer media was pulled) 404. That's - an expected, non-actionable outcome and shouldn't be logged/counted the - same way as a genuine download failure.""" - - def setUp(self): - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) - - def test_404_response_is_not_found_not_failed(self): - response = helpers.fake_response(status=404) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - status = ft.download_torrent( - "ubuntu-14.04.6-live-server-amd64.iso", - "https://example.invalid/x.torrent", - ) - - self.assertEqual(status, "not_found") - dest = os.path.join( - self.tmp_watch_dir, "ubuntu-14.04.6-live-server-amd64.iso.torrent" - ) - self.assertFalse(os.path.exists(dest)) - - def test_other_http_error_is_still_failed(self): - response = helpers.fake_response(status=500) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - status = ft.download_torrent( - "ubuntu-24.04-desktop-amd64.iso", "https://example.invalid/x.torrent" - ) - - self.assertEqual(status, "failed") - - def test_successful_download_is_still_added(self): - response = helpers.fake_response(b"fake torrent bytes") - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - status = ft.download_torrent( - "ubuntu-24.04-desktop-amd64.iso", "https://example.invalid/x.torrent" - ) - - self.assertEqual(status, "added") - dest = os.path.join( - self.tmp_watch_dir, "ubuntu-24.04-desktop-amd64.iso.torrent" - ) - with open(dest, "rb") as f: - self.assertEqual(f.read(), b"fake torrent bytes") - - -class SafeWatchPathTests(unittest.TestCase): - """_safe_watch_path() is the shared guard download_torrent(), - _remove_watch_file(), and check_old_releases_for_demand() all rely on to - keep a torrent name confined to a single file directly under watch_dir - - defense in depth against a future fetch_*() parser turning scraped - upstream content into a path escape.""" - - def setUp(self): - patcher = unittest.mock.patch.object(ft, "watch_dir", "/watch") - patcher.start() - self.addCleanup(patcher.stop) - - def test_normal_name_resolves_under_watch_dir(self): - self.assertEqual( - ft._safe_watch_path("ubuntu-24.04-desktop-amd64.iso"), - os.path.join("/watch", "ubuntu-24.04-desktop-amd64.iso.torrent"), - ) - - def test_rejects_path_separators(self): - self.assertIsNone(ft._safe_watch_path("archlinux-../../../etc/cron.d/evil")) - self.assertIsNone(ft._safe_watch_path("foo/bar")) - self.assertIsNone(ft._safe_watch_path("foo\\bar")) - - def test_rejects_empty_or_dot_names(self): - self.assertIsNone(ft._safe_watch_path("")) - self.assertIsNone(ft._safe_watch_path(None)) - self.assertIsNone(ft._safe_watch_path(".")) - self.assertIsNone(ft._safe_watch_path("..")) - - -class DownloadTorrentUnsafeNameTests(unittest.TestCase): - def setUp(self): - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) - - def test_download_torrent_refuses_unsafe_name_without_any_request(self): - with unittest.mock.patch.object(ft.requests, "get") as mock_get: - status = ft.download_torrent("../evil", "https://example.invalid/x.torrent") - - mock_get.assert_not_called() - self.assertEqual(status, "failed") - self.assertEqual(os.listdir(self.tmp_watch_dir), []) - - def test_remove_watch_file_refuses_unsafe_name_without_touching_disk(self): - # Asserted on disk rather than by patching os.remove: that patch is - # process-wide, so a log rotation triggered by the warning would - # trip it too. - parent = helpers.make_temp_dir(self, "fetch_torrents_test_escape_") - watch_dir = os.path.join(parent, "watch") - os.mkdir(watch_dir) - outside = os.path.join(parent, "evil.torrent") - with open(outside, "w", encoding="utf-8") as f: - f.write("must survive") - - with unittest.mock.patch.object(ft, "watch_dir", watch_dir): - ft._remove_watch_file("../evil") - - self.assertTrue(os.path.exists(outside)) - - -class UpdateRatioHistoryTests(unittest.TestCase): - """update_ratio_history() must keep the on-disk history bounded: drop - torrents that are no longer seeding, only add a new sample once per - sample_interval_days, and prune samples once they're older than - window_days + sample_interval_days - so it stays a fixed handful of - samples per torrent forever, not one entry per run.""" - - def test_new_torrent_gets_a_first_sample(self): - today = date(2026, 8, 29) - history = ft.update_ratio_history( - {}, {"archlinux-2026.08.01-x86_64.iso": 1.2}, today - ) - - self.assertEqual( - history, - {"archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-29", "ratio": 1.2}]}, - ) - - def test_recent_sample_is_not_duplicated(self): - today = date(2026, 8, 29) - history = { - "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-25", "ratio": 1.0}], - } - - updated = ft.update_ratio_history( - history, {"archlinux-2026.08.01-x86_64.iso": 1.1}, today - ) - - # Only 4 days since the last sample (< the 7-day interval) - no new - # sample yet, and the ratio value on the existing sample is untouched. - self.assertEqual( - updated, - {"archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-25", "ratio": 1.0}]}, - ) - - def test_new_sample_added_once_interval_elapses(self): - today = date(2026, 8, 29) - history = { - "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-20", "ratio": 1.0}], - } - - updated = ft.update_ratio_history( - history, {"archlinux-2026.08.01-x86_64.iso": 1.3}, today - ) - - self.assertEqual( - updated, - { - "archlinux-2026.08.01-x86_64.iso": [ - {"date": "2026-08-20", "ratio": 1.0}, - {"date": "2026-08-29", "ratio": 1.3}, - ] - }, - ) - - def test_samples_older_than_window_plus_interval_are_pruned(self): - today = date(2026, 8, 29) - history = { - "archlinux-2026.08.01-x86_64.iso": [ - {"date": "2026-06-01", "ratio": 0.5}, # ~89 days old - long stale - {"date": "2026-08-10", "ratio": 1.0}, # ~19 days old - kept - ], - } - - updated = ft.update_ratio_history( - history, {"archlinux-2026.08.01-x86_64.iso": 1.0}, today, window_days=30 - ) - - dates = [s["date"] for s in updated["archlinux-2026.08.01-x86_64.iso"]] - self.assertNotIn("2026-06-01", dates) - self.assertIn("2026-08-10", dates) - - def test_torrent_no_longer_seeding_is_dropped(self): - today = date(2026, 8, 29) - history = { - "archlinux-2026.07.01-x86_64.iso": [{"date": "2026-08-20", "ratio": 1.0}], - } - - updated = ft.update_ratio_history(history, {}, today) - - self.assertEqual(updated, {}) - - def _days_ago(self, today, days): - return (today - timedelta(days=days)).isoformat() - - def test_sample_exactly_one_interval_old_triggers_a_new_sample(self): - today = date(2026, 8, 29) - history = {"a.iso": [{"date": self._days_ago(today, 7), "ratio": 1.0}]} - - updated = ft.update_ratio_history( - history, {"a.iso": 1.2}, today, sample_interval_days=7, window_days=30 - ) - - self.assertEqual( - updated["a.iso"], - [ - {"date": self._days_ago(today, 7), "ratio": 1.0}, - {"date": today.isoformat(), "ratio": 1.2}, - ], - ) - - def test_sample_just_under_one_interval_old_adds_nothing(self): - today = date(2026, 8, 29) - history = {"a.iso": [{"date": self._days_ago(today, 6), "ratio": 1.0}]} - - updated = ft.update_ratio_history( - history, {"a.iso": 1.2}, today, sample_interval_days=7, window_days=30 - ) - - self.assertEqual( - updated["a.iso"], [{"date": self._days_ago(today, 6), "ratio": 1.0}] - ) - - def test_prune_horizon_defaults_to_window_plus_interval(self): - # Defaults: window 30 + interval 7 = 37 days. A sample exactly 37 - # days old is kept; 38 days old is pruned. - today = date(2026, 8, 29) - history = { - "a.iso": [ - {"date": self._days_ago(today, 38), "ratio": 0.5}, - {"date": self._days_ago(today, 37), "ratio": 0.6}, - {"date": self._days_ago(today, 1), "ratio": 0.9}, - ] - } - - updated = ft.update_ratio_history(history, {"a.iso": 1.0}, today) - - self.assertEqual( - updated["a.iso"], - [ - {"date": self._days_ago(today, 37), "ratio": 0.6}, - {"date": self._days_ago(today, 1), "ratio": 0.9}, - ], - ) - - -class PlanStagnationCleanupTests(unittest.TestCase): - """plan_stagnation_cleanup() removes only superseded (non-latest) - versions within a (distro, type) group once their ratio has stopped - growing - the newest/only version of a group is never a candidate, - since removing it would just get it re-fetched and re-downloaded the - next run (should_fetch_torrent() only skips a fetch when a *previous* - version's ratio is on record).""" - - def test_latest_or_only_version_is_never_removed_even_if_flat(self): - today = date(2026, 8, 29) - torrents = [FakeTorrent("archlinux-2026.08.01-x86_64.iso", 1, ratio=1.0)] - # 60 days of a perfectly flat ratio - as stagnant as it gets. - history = { - "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-06-30", "ratio": 1.0}], - } - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, history, today, window_days=30, min_ratio_delta=0.02 - ) - - self.assertEqual(to_remove, []) - self.assertEqual(to_keep, []) - - def test_old_version_with_no_ratio_growth_is_removed(self): - today = date(2026, 8, 29) - torrents = [ - FakeTorrent("ubuntu-24.04.1-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.500), - ] - history = { - "ubuntu-23.10-desktop-amd64.iso": [{"date": "2026-07-20", "ratio": 1.495}], - } - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, history, today, window_days=30, min_ratio_delta=0.02 - ) - - self.assertEqual(names_of(to_remove), ["ubuntu-23.10-desktop-amd64.iso"]) - self.assertEqual(to_keep, []) - - def test_old_version_still_growing_is_kept(self): - today = date(2026, 8, 29) - torrents = [ - FakeTorrent("ubuntu-24.04.1-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.8), - ] - history = { - "ubuntu-23.10-desktop-amd64.iso": [{"date": "2026-07-20", "ratio": 1.2}], - } - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, history, today, window_days=30, min_ratio_delta=0.02 - ) - - self.assertEqual(to_remove, []) - self.assertEqual(len(to_keep), 1) - kept_torrent, delta = to_keep[0] - self.assertEqual(kept_torrent.name, "ubuntu-23.10-desktop-amd64.iso") - self.assertAlmostEqual(delta, 0.6) - - def test_old_version_without_enough_history_is_kept(self): - today = date(2026, 8, 29) - torrents = [ - FakeTorrent("ubuntu-24.04.1-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), - ] - # Only 5 days of history - too young to judge either way. - history = { - "ubuntu-23.10-desktop-amd64.iso": [{"date": "2026-08-24", "ratio": 0.1}], - } - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, history, today, window_days=30, min_ratio_delta=0.02 - ) - - self.assertEqual(to_remove, []) - self.assertEqual(len(to_keep), 1) - kept_torrent, delta = to_keep[0] - self.assertEqual(kept_torrent.name, "ubuntu-23.10-desktop-amd64.iso") - self.assertIsNone(delta) - - def test_old_version_with_no_history_at_all_is_kept(self): - today = date(2026, 8, 29) - torrents = [ - FakeTorrent("ubuntu-24.04.1-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), - ] - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, {}, today, window_days=30, min_ratio_delta=0.02 - ) - - self.assertEqual(to_remove, []) - self.assertEqual(len(to_keep), 1) - - def test_unrelated_single_torrent_groups_are_ignored(self): - today = date(2026, 8, 29) - torrents = [FakeTorrent("some-random-linux-distro-1.0.iso", 1, ratio=5.0)] - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, {}, today, window_days=30, min_ratio_delta=0.02 - ) - - self.assertEqual(to_remove, []) - self.assertEqual(to_keep, []) - - def _pair(self, older_ratio): - older = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=older_ratio) - return older, [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), older] - - def _history(self, today, *samples): - return { - "ubuntu-23.10-desktop-amd64.iso": [ - {"date": (today - timedelta(days=days)).isoformat(), "ratio": ratio} - for days, ratio in samples - ] - } - - def test_anchor_exactly_window_days_old_is_used(self): - today = date(2026, 8, 29) - older, torrents = self._pair(1.0) - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, self._history(today, (30, 1.0)), today, 30, 0.02 - ) - - self.assertEqual((to_remove, to_keep), ([older], [])) - - def test_anchor_one_day_short_of_window_is_too_new_to_judge(self): - today = date(2026, 8, 29) - older, torrents = self._pair(1.0) - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, self._history(today, (29, 1.0)), today, 30, 0.02 - ) - - self.assertEqual((to_remove, to_keep), ([], [(older, None)])) - - def test_oldest_qualifying_sample_is_the_anchor(self): - # Listed newest-first, so picking the first old-enough sample - # instead of the oldest would give a different delta. - today = date(2026, 8, 29) - older, torrents = self._pair(1.0) - history = self._history(today, (35, 0.99), (45, 0.5)) - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, history, today, 30, 0.02 - ) - - self.assertEqual((to_remove, to_keep), ([], [(older, 0.5)])) - - def test_delta_exactly_at_min_ratio_delta_is_kept(self): - today = date(2026, 8, 29) - older, torrents = self._pair(1.5) - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, self._history(today, (40, 1.0)), today, 30, 0.5 - ) - - self.assertEqual((to_remove, to_keep), ([], [(older, 0.5)])) - - def test_delta_just_below_min_ratio_delta_is_removed(self): - today = date(2026, 8, 29) - older, torrents = self._pair(1.25) - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, self._history(today, (40, 1.0)), today, 30, 0.5 - ) - - self.assertEqual((to_remove, to_keep), ([older], [])) - - def test_defaults_are_a_30_day_window_and_0_02_delta(self): - today = date(2026, 8, 29) - growing, torrents = self._pair(1.5) - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, self._history(today, (30, 1.0)), today - ) - - self.assertEqual((to_remove, to_keep), ([], [(growing, 0.5)])) - - stagnant, torrents = self._pair(1.01) - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, self._history(today, (30, 1.0)), today - ) - - self.assertEqual((to_remove, to_keep), ([stagnant], [])) - - def test_missing_or_none_ratio_counts_as_zero(self): - today = date(2026, 8, 29) - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), - types.SimpleNamespace(name="ubuntu-23.04-desktop-amd64.iso", id=2), - DuckTorrent("ubuntu-23.10-desktop-amd64.iso", 3, ratio=None), - ] - history = { - name: [{"date": (today - timedelta(days=40)).isoformat(), "ratio": -0.5}] - for name in ( - "ubuntu-23.04-desktop-amd64.iso", - "ubuntu-23.10-desktop-amd64.iso", - ) - } - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, history, today, 30, 0.02 - ) - - self.assertEqual(to_remove, []) - self.assertEqual( - sorted((t.name, delta) for t, delta in to_keep), - [ - ("ubuntu-23.04-desktop-amd64.iso", 0.5), - ("ubuntu-23.10-desktop-amd64.iso", 0.5), - ], - ) - - def test_candidate_without_anchor_does_not_stop_later_ones(self): - today = date(2026, 8, 29) - no_history = FakeTorrent("ubuntu-23.04-desktop-amd64.iso", 3, ratio=1.0) - stagnant = FakeTorrent("ubuntu-22.10-desktop-amd64.iso", 4, ratio=1.0) - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), - no_history, - stagnant, - ] - history = { - "ubuntu-22.10-desktop-amd64.iso": [ - {"date": (today - timedelta(days=40)).isoformat(), "ratio": 1.0} - ] - } - - to_remove, to_keep = ft.plan_stagnation_cleanup( - torrents, history, today, 30, 0.02 - ) - - self.assertEqual((to_remove, to_keep), ([stagnant], [(no_history, None)])) - - -class RatioHistoryPersistenceTests(unittest.TestCase): - """load_ratio_history()/save_ratio_history() must round-trip cleanly and - tolerate a missing or corrupt file rather than crashing a run.""" - - def test_round_trips_through_disk(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") - path = os.path.join(tmp_dir, "fetch_torrents_ratio_history.json") - history = { - "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-29", "ratio": 1.2}] - } - - ft.save_ratio_history(path, history) - loaded = ft.load_ratio_history(path) - - self.assertEqual(loaded, history) - - def test_missing_file_returns_empty_history(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") - path = os.path.join(tmp_dir, "does-not-exist.json") - - self.assertEqual(ft.load_ratio_history(path), {}) - - def test_corrupt_file_returns_empty_history_instead_of_raising(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") - path = os.path.join(tmp_dir, "fetch_torrents_ratio_history.json") - with open(path, "w", encoding="utf-8") as f: - f.write("{not valid json") - - self.assertEqual(ft.load_ratio_history(path), {}) - - -class FetchKaliOldVersionsTests(unittest.TestCase): - """fetch_kali_old_versions() discovers the prior release(s) still listed - on cdimage.kali.org (a 'current/' symlink plus a small number of - 'kali-X.Y/' folders), for FETCH_TORRENTS_CHECK_OLD_RELEASES. Only ever - used when that flag is enabled - see CheckOldReleasesForDemandTests.""" - - def test_returns_only_the_non_newest_versions(self): - html = ( - "" - 'current' - 'kali-2026.1' - 'kali-2026.2' - "" - ) - - response = helpers.fake_response(html) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - results = ft.fetch_kali_old_versions() - - self.assertTrue(results) - self.assertTrue(any("2026.1" in name for name in results)) - self.assertFalse(any("2026.2" in name for name in results)) - - def test_single_version_folder_returns_nothing(self): - html = 'currentkali-2026.2' - - response = helpers.fake_response(html) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - results = ft.fetch_kali_old_versions() - - self.assertEqual(results, {}) - - -class FetchFedoraWorkstationOldVersionsTests(unittest.TestCase): - """Mirrors FetchFedoraWorkstationNamingTests, but for the versions - fetch_fedora_workstation() discards as not-latest.""" - - def test_returns_non_latest_versions_per_arch(self): - html = ( - "" - 'a' - 'b' - 'c' - 'd' - "" - ) - - response = helpers.fake_response(html) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - results = ft.fetch_fedora_workstation_old_versions() - - self.assertEqual( - results, - { - "Fedora-Workstation-Live-x86_64-43": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-x86_64-43.torrent", - "Fedora-Workstation-Live-aarch64-43": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-aarch64-43.torrent", - }, - ) - - def test_single_version_per_arch_returns_nothing(self): - html = 'a' - - response = helpers.fake_response(html) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - results = ft.fetch_fedora_workstation_old_versions() - - self.assertEqual(results, {}) - - -class HasUnmetDemandTests(unittest.TestCase): - """Demand requires an absolute floor (enough leechers to be worth the - disk/bandwidth), a relative one (leechers reach some fraction of the - seeder count - i.e. the swarm isn't already vastly over-served), and by - default a real, currently-seeded copy to exist at all. A single leecher - on a 1000-seeder swarm must never pass just because leechers>0, and a - swarm with plenty of leechers but even more seeders (already well - served) shouldn't either.""" - - def test_none_scrape_result_is_no_demand(self): - self.assertFalse( - ft.has_unmet_demand( - None, min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False - ) - ) - - def test_below_absolute_threshold_is_no_demand(self): - self.assertFalse( - ft.has_unmet_demand( - {"leechers": 0, "seeders": 0}, - min_leechers=1, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - ) - - def test_single_leecher_on_huge_swarm_is_no_demand(self): - self.assertFalse( - ft.has_unmet_demand( - {"leechers": 1, "seeders": 1000}, - min_leechers=1, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - ) - - def test_meets_absolute_but_not_ratio_is_no_demand(self): - # 11 leechers clears a min_leechers=10 floor, but 11 < 1.2x the 10 seeders. - self.assertFalse( - ft.has_unmet_demand( - {"leechers": 11, "seeders": 10}, - min_leechers=10, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - ) - - def test_meets_both_absolute_and_ratio_thresholds_is_demand(self): - self.assertTrue( - ft.has_unmet_demand( - {"leechers": 25, "seeders": 10}, - min_leechers=10, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - ) - - def test_exactly_at_ratio_threshold_is_demand(self): - self.assertTrue( - ft.has_unmet_demand( - {"leechers": 12, "seeders": 10}, - min_leechers=10, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - ) - - def test_zero_seeders_is_excluded_by_default(self): - # A scrape can't tell us whether the leechers collectively hold every - # piece - with zero seeders there's no verified-complete copy in the - # swarm at all, so we could end up leeching something we can never - # finish and therefore never seed back. Excluded unless the operator - # opts into the gamble via allow_zero_seeders. - self.assertFalse( - ft.has_unmet_demand( - {"leechers": 10, "seeders": 0}, - min_leechers=10, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - ) - - def test_zero_seeders_counts_as_demand_when_explicitly_allowed(self): - self.assertTrue( - ft.has_unmet_demand( - {"leechers": 10, "seeders": 0}, - min_leechers=10, - min_leecher_ratio=1.2, - allow_zero_seeders=True, - ) - ) - - def test_zero_seeders_still_needs_the_absolute_floor_when_allowed(self): - self.assertFalse( - ft.has_unmet_demand( - {"leechers": 5, "seeders": 0}, - min_leechers=10, - min_leecher_ratio=1.2, - allow_zero_seeders=True, - ) - ) - - def test_lopsided_but_substantial_swarms_still_count_as_demand_at_default_ratio( - self, - ): - # FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO defaults to 0.1: we - # have no way of knowing whether existing seeders have spare upload - # capacity or are bandwidth-limited, so leechers don't need to - # approach or outnumber seeders to represent real demand - 10:100 - # and 100:1000 both clearly indicate room for another seeder. - self.assertTrue( - ft.has_unmet_demand( - {"leechers": 10, "seeders": 100}, - min_leechers=1, - min_leecher_ratio=0.1, - allow_zero_seeders=False, - ) - ) - self.assertTrue( - ft.has_unmet_demand( - {"leechers": 100, "seeders": 1000}, - min_leechers=1, - min_leecher_ratio=0.1, - allow_zero_seeders=False, - ) - ) - - def test_extreme_oversupply_still_fails_at_default_ratio(self): - # 3 leechers on 673 seeders (an actual old Kali release scraped - # during development) is still not demand, even at the more - # permissive 0.1 default - it needs at least 67.3 leechers. - self.assertFalse( - ft.has_unmet_demand( - {"leechers": 3, "seeders": 673}, - min_leechers=1, - min_leecher_ratio=0.1, - allow_zero_seeders=False, - ) - ) - - -class EvaluateOldReleaseDemandTests(unittest.TestCase): - def test_404_is_not_kept_and_not_scraped(self): - response = helpers.fake_response(status=404) - - with ( - unittest.mock.patch.object(ft.requests, "get", return_value=response), - unittest.mock.patch.object( - ft.torrent_scrape, "scrape_torrent" - ) as mock_scrape, - ): - keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - "ubuntu-14.04.6-desktop-amd64.iso", - "https://example.invalid/x.torrent", - min_leechers=1, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - - mock_scrape.assert_not_called() - self.assertFalse(keep) - self.assertIsNone(torrent_bytes) - self.assertIsNone(scrape_result) - - def test_download_failure_is_not_kept(self): - with unittest.mock.patch.object( - ft.requests, "get", side_effect=ConnectionError("nope") - ): - keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - "ubuntu-14.04.6-desktop-amd64.iso", - "https://example.invalid/x.torrent", - min_leechers=1, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - - self.assertFalse(keep) - self.assertIsNone(torrent_bytes) - self.assertIsNone(scrape_result) - - def test_scrape_below_threshold_is_not_kept(self): - response = helpers.fake_response(b"fake torrent bytes") - - with ( - unittest.mock.patch.object(ft.requests, "get", return_value=response), - unittest.mock.patch.object( - ft.torrent_scrape, - "scrape_torrent", - return_value={ - "tracker": "http://t.example/scrape", - "seeders": 10, - "leechers": 0, - "completed": 0, - }, - ), - ): - keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - "ubuntu-14.04.6-desktop-amd64.iso", - "https://example.invalid/x.torrent", - min_leechers=1, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - - self.assertFalse(keep) - self.assertEqual(torrent_bytes, b"fake torrent bytes") - self.assertEqual(scrape_result["leechers"], 0) - - def test_scrape_meeting_threshold_is_kept(self): - response = helpers.fake_response(b"fake torrent bytes") - - with ( - unittest.mock.patch.object(ft.requests, "get", return_value=response), - unittest.mock.patch.object( - ft.torrent_scrape, - "scrape_torrent", - return_value={ - "tracker": "http://t.example/scrape", - "seeders": 10, - "leechers": 30, - "completed": 0, - }, - ), - ): - keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - "ubuntu-14.04.6-desktop-amd64.iso", - "https://example.invalid/x.torrent", - min_leechers=1, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - - self.assertTrue(keep) - self.assertEqual(torrent_bytes, b"fake torrent bytes") - self.assertEqual(scrape_result["leechers"], 30) - - def test_meets_absolute_floor_but_already_well_seeded_is_not_kept(self): - response = helpers.fake_response(b"fake torrent bytes") - - # 1 leecher clears a min_leechers=1 floor, but 1000 seeders already - # cover it - not unmet demand. - with ( - unittest.mock.patch.object(ft.requests, "get", return_value=response), - unittest.mock.patch.object( - ft.torrent_scrape, - "scrape_torrent", - return_value={ - "tracker": "http://t.example/scrape", - "seeders": 1000, - "leechers": 1, - "completed": 0, - }, - ), - ): - keep, _torrent_bytes, scrape_result = ft.evaluate_old_release_demand( - "ubuntu-14.04.6-desktop-amd64.iso", - "https://example.invalid/x.torrent", - min_leechers=1, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - - self.assertFalse(keep) - self.assertEqual(scrape_result["leechers"], 1) - - -class CheckOldReleasesForDemandTests(unittest.TestCase): - def setUp(self): - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) - - def test_adds_only_torrents_with_unmet_demand(self): - # Neither name matches LOW_DEMAND_PATTERN - that filtering is - # covered separately by test_low_demand_variant_is_filtered_before_evaluation. - candidates = { - "kali-linux-2026.1-installer-purple-amd64.iso": "https://example.invalid/dead.torrent", - "kali-linux-2026.1-installer-amd64.iso": "https://example.invalid/alive.torrent", - } - - def fake_evaluate( - name, url, min_leechers, min_leecher_ratio, allow_zero_seeders, timeout=15 - ): - if "purple" in name: - return ( - False, - b"dead bytes", - {"tracker": "t", "seeders": 5, "leechers": 0, "completed": 0}, - ) - return ( - True, - b"alive bytes", - {"tracker": "t", "seeders": 5, "leechers": 30, "completed": 0}, - ) - - with ( - unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True - ), - unittest.mock.patch.object( - ft, "evaluate_old_release_demand", side_effect=fake_evaluate - ), - ): - added, skipped, _ = ft.check_old_releases_for_demand( - ["kali"], - include_low_demand=False, - min_leechers=1, - min_leecher_ratio=0.0, - allow_zero_seeders=False, - ) - - self.assertEqual(added, 1) - self.assertEqual(skipped, 1) - alive_path = os.path.join( - self.tmp_watch_dir, "kali-linux-2026.1-installer-amd64.iso.torrent" - ) - dead_path = os.path.join( - self.tmp_watch_dir, "kali-linux-2026.1-installer-purple-amd64.iso.torrent" - ) - with open(alive_path, "rb") as f: - self.assertEqual(f.read(), b"alive bytes") - self.assertFalse(os.path.exists(dead_path)) - - def test_skips_distros_without_old_release_discovery(self): - with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {}, clear=True): - added, skipped, _ = ft.check_old_releases_for_demand( - ["ubuntu"], - include_low_demand=False, - min_leechers=1, - min_leecher_ratio=0.0, - allow_zero_seeders=False, - ) - - self.assertEqual((added, skipped), (0, 0)) - - def test_already_present_torrent_is_not_re_evaluated(self): - name = "kali-linux-2026.1-installer-amd64.iso" - existing_path = os.path.join(self.tmp_watch_dir, f"{name}.torrent") - with open(existing_path, "wb") as f: - f.write(b"already here") - - with ( - unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, - {"kali": lambda: {name: "https://example.invalid/x.torrent"}}, - clear=True, - ), - unittest.mock.patch.object( - ft, "evaluate_old_release_demand" - ) as mock_evaluate, - ): - added, skipped, _ = ft.check_old_releases_for_demand( - ["kali"], - include_low_demand=False, - min_leechers=1, - min_leecher_ratio=0.0, - allow_zero_seeders=False, - ) - - mock_evaluate.assert_not_called() - self.assertEqual((added, skipped), (0, 0)) - - def test_unsafe_candidate_name_is_skipped_without_evaluation(self): - name = "kali-linux-2026.1-installer-../../etc/cron.d/evil" - - with ( - unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, - {"kali": lambda: {name: "https://example.invalid/x.torrent"}}, - clear=True, - ), - unittest.mock.patch.object( - ft, "evaluate_old_release_demand" - ) as mock_evaluate, - ): - added, skipped, _ = ft.check_old_releases_for_demand( - ["kali"], - include_low_demand=False, - min_leechers=1, - min_leecher_ratio=0.0, - allow_zero_seeders=False, - ) - - mock_evaluate.assert_not_called() - self.assertEqual((added, skipped), (0, 1)) - - def test_low_demand_variant_is_filtered_before_evaluation(self): - name = "kali-linux-2026.1-installer-netinst-amd64.iso" - - with ( - unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, - {"kali": lambda: {name: "https://example.invalid/x.torrent"}}, - clear=True, - ), - unittest.mock.patch.object( - ft, "evaluate_old_release_demand" - ) as mock_evaluate, - ): - added, skipped, _ = ft.check_old_releases_for_demand( - ["kali"], - include_low_demand=False, - min_leechers=1, - min_leecher_ratio=0.0, - allow_zero_seeders=False, - ) - - mock_evaluate.assert_not_called() - self.assertEqual((added, skipped), (0, 0)) - - -class FormatRunSummaryTests(unittest.TestCase): - """format_run_summary() builds the single end-of-run log line - it must - surface how many old-release torrents FETCH_TORRENTS_CHECK_OLD_RELEASES - introduced, not just the counts from the normal latest-release fetch.""" - - def test_without_old_release_check(self): - summary = ft.format_run_summary( - elapsed=12.5, - success_count=3, - existing_count=1, - not_found_count=0, - failure_count=0, - ) - self.assertEqual( - summary, - "Run complete in 12.50 seconds. 3 added, 1 existing, 0 not found upstream, 0 failed.", - ) - - def test_with_old_release_check_appends_counts(self): - summary = ft.format_run_summary( - elapsed=12.5, - success_count=3, - existing_count=1, - not_found_count=0, - failure_count=0, - old_added=2, - old_skipped=6, - ) - self.assertEqual( - summary, - "Run complete in 12.50 seconds. 3 added, 1 existing, 0 not found upstream, 0 failed. " - "Old releases (FETCH_TORRENTS_CHECK_OLD_RELEASES): 2 added, 6 skipped.", - ) - - def test_still_starts_with_run_complete_for_always_log_filter(self): - summary = ft.format_run_summary( - elapsed=1.0, - success_count=0, - existing_count=0, - not_found_count=0, - failure_count=0, - old_added=0, - old_skipped=0, - ) - self.assertTrue(summary.startswith("Run complete in")) - - -class ShouldRecheckOldReleaseTests(unittest.TestCase): - """Without this gate, check_old_releases_for_demand() would re-download - and re-scrape every not-yet-wanted candidate on every daily run - forever - this is what lets a recent "no demand" result be trusted for - a while instead.""" - - def test_never_checked_is_due(self): - self.assertTrue( - ft.should_recheck_old_release( - "x", {}, date(2026, 1, 8), recheck_interval_days=7 - ) - ) - - def test_checked_recently_is_not_due(self): - state = {"x": {"last_checked": "2026-01-05", "leechers": 0}} - self.assertFalse( - ft.should_recheck_old_release( - "x", state, date(2026, 1, 8), recheck_interval_days=7 - ) - ) - - def test_checked_exactly_at_interval_is_due(self): - state = {"x": {"last_checked": "2026-01-01", "leechers": 0}} - self.assertTrue( - ft.should_recheck_old_release( - "x", state, date(2026, 1, 8), recheck_interval_days=7 - ) - ) - - def test_checked_past_interval_is_due(self): - state = {"x": {"last_checked": "2025-12-01", "leechers": 0}} - self.assertTrue( - ft.should_recheck_old_release( - "x", state, date(2026, 1, 8), recheck_interval_days=7 - ) - ) - - -class OldReleaseCheckStatePersistenceTests(unittest.TestCase): - def test_round_trips_through_disk(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") - path = os.path.join(tmp_dir, "fetch_torrents_old_release_check_state.json") - state = {"x": {"last_checked": "2026-01-08", "leechers": 0}} - - ft.save_old_release_check_state(path, state) - loaded = ft.load_old_release_check_state(path) - - self.assertEqual(loaded, state) - - def test_missing_file_returns_empty_state(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") - path = os.path.join(tmp_dir, "does-not-exist.json") - - self.assertEqual(ft.load_old_release_check_state(path), {}) - - def test_corrupt_file_returns_empty_state_instead_of_raising(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") - path = os.path.join(tmp_dir, "fetch_torrents_old_release_check_state.json") - with open(path, "w", encoding="utf-8") as f: - f.write("{not valid json") - - self.assertEqual(ft.load_old_release_check_state(path), {}) - - -class RecordRemovalTests(unittest.TestCase): - def test_adds_entry_without_mutating_input(self): - history = {} - updated = ft.record_removal( - history, - "kali-linux-2026.1-installer-amd64.iso", - date(2026, 2, 1), - "stagnant", - 0.42, - ) - - self.assertEqual(history, {}) # input untouched - self.assertEqual( - updated["kali-linux-2026.1-installer-amd64.iso"], - {"removed_date": "2026-02-01", "reason": "stagnant", "ratio": 0.42}, - ) - - -class RemovedHistoryPersistenceTests(unittest.TestCase): - def test_round_trips_through_disk(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") - path = os.path.join(tmp_dir, "fetch_torrents_removed_history.json") - history = {"x": {"removed_date": "2026-02-01", "reason": "stagnant"}} - - ft.save_removed_history(path, history) - loaded = ft.load_removed_history(path) - - self.assertEqual(loaded, history) - - def test_missing_file_returns_empty_history(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") - path = os.path.join(tmp_dir, "does-not-exist.json") - - self.assertEqual(ft.load_removed_history(path), {}) - - def test_corrupt_file_returns_empty_history_instead_of_raising(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") - path = os.path.join(tmp_dir, "fetch_torrents_removed_history.json") - with open(path, "w", encoding="utf-8") as f: - f.write("{not valid json") - - self.assertEqual(ft.load_removed_history(path), {}) - - -class CheckOldReleasesForDemandStateTests(unittest.TestCase): - """check_old_releases_for_demand() must consult both the recheck-state - (skip candidates checked too recently) and the removed-history (skip - candidates cleanup has already given up on) before ever hitting the - network, and must record a fresh check for anything it does evaluate.""" - - def setUp(self): - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) - - def test_recently_checked_candidate_is_skipped_without_network_call(self): - name = "kali-linux-2026.1-installer-amd64.iso" - candidates = {name: "https://example.invalid/x.torrent"} - check_state = {name: {"last_checked": "2026-01-05", "leechers": 0}} - - with ( - unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True - ), - unittest.mock.patch.object( - ft, "evaluate_old_release_demand" - ) as mock_evaluate, - ): - added, skipped, new_state = ft.check_old_releases_for_demand( - ["kali"], - include_low_demand=False, - min_leechers=1, - min_leecher_ratio=0.0, - allow_zero_seeders=False, - check_state=check_state, - removed_history={}, - today=date(2026, 1, 8), - recheck_interval_days=7, - ) - - mock_evaluate.assert_not_called() - self.assertEqual((added, skipped), (0, 1)) - self.assertEqual(new_state, check_state) - - def test_previously_removed_candidate_is_skipped_without_network_call(self): - name = "kali-linux-2026.1-installer-amd64.iso" - candidates = {name: "https://example.invalid/x.torrent"} - removed_history = {name: {"removed_date": "2026-01-01", "reason": "stagnant"}} - - with ( - unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True - ), - unittest.mock.patch.object( - ft, "evaluate_old_release_demand" - ) as mock_evaluate, - ): - added, skipped, _new_state = ft.check_old_releases_for_demand( - ["kali"], - include_low_demand=False, - min_leechers=1, - min_leecher_ratio=0.0, - allow_zero_seeders=False, - check_state={}, - removed_history=removed_history, - today=date(2026, 1, 8), - recheck_interval_days=7, - ) - - mock_evaluate.assert_not_called() - self.assertEqual((added, skipped), (0, 1)) - - def test_due_candidate_is_evaluated_and_recorded(self): - name = "kali-linux-2026.1-installer-amd64.iso" - candidates = {name: "https://example.invalid/x.torrent"} - - with ( - unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True - ), - unittest.mock.patch.object( - ft, - "evaluate_old_release_demand", - return_value=( - True, - b"bytes", - {"tracker": "t", "seeders": 1, "leechers": 30, "completed": 0}, - ), - ), - ): - added, skipped, new_state = ft.check_old_releases_for_demand( - ["kali"], - include_low_demand=False, - min_leechers=1, - min_leecher_ratio=0.0, - allow_zero_seeders=False, - check_state={}, - removed_history={}, - today=date(2026, 1, 8), - recheck_interval_days=7, - ) - - self.assertEqual((added, skipped), (1, 0)) - self.assertEqual( - new_state[name], {"last_checked": "2026-01-08", "leechers": 30} - ) - - def test_no_demand_result_is_still_recorded_for_future_recheck_gating(self): - name = "kali-linux-2026.1-installer-arm64.iso" - candidates = {name: "https://example.invalid/x.torrent"} - - with ( - unittest.mock.patch.dict( - ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True - ), - unittest.mock.patch.object( - ft, - "evaluate_old_release_demand", - return_value=( - False, - b"bytes", - {"tracker": "t", "seeders": 1, "leechers": 0, "completed": 0}, - ), - ), - ): - added, skipped, new_state = ft.check_old_releases_for_demand( - ["kali"], - include_low_demand=False, - min_leechers=1, - min_leecher_ratio=0.0, - allow_zero_seeders=False, - check_state={}, - removed_history={}, - today=date(2026, 1, 8), - recheck_interval_days=7, - ) - - self.assertEqual((added, skipped), (0, 1)) - self.assertEqual(new_state[name], {"last_checked": "2026-01-08", "leechers": 0}) - - -class LoadConfigTests(unittest.TestCase): - """load_config() is the one place the daily run reads its environment: - each env var must land on the right field, and a bad number must be - deferred (recorded, then raised by require()) rather than crash the - whole run up front.""" - - def test_empty_env_gives_the_documented_defaults(self): - self.assertEqual(ft.load_config({}), ft.Config()) - self.assertEqual( - ft.Config(), - ft.Config( - distros=ft.DEFAULT_DISTROS, - include_low_demand=False, - skip_ratio_check=False, - check_old_releases=False, - old_release_min_leechers=10, - old_release_min_leecher_ratio=0.1, - old_release_allow_zero_seeders=False, - old_release_recheck_days=7, - old_release_skip_removed=True, - cleanup_keep_only_latest=False, - cleanup_skip_ratio_check=False, - cleanup_min_ratio=1.0, - stagnation_window_days=30, - stagnation_min_ratio_delta=0.02, - disk_usage_threshold_percent=95.0, - rpc_username=None, - rpc_password=None, - invalid={}, - ), - ) - - def test_each_env_var_sets_its_field(self): - config = ft.load_config( - { - "FETCH_TORRENTS_DISTROS": "devuan, arch", - "FETCH_TORRENTS_INCLUDE_LOW_DEMAND": "true", - "SKIP_RATIO_CHECK": "yes", - "FETCH_TORRENTS_CHECK_OLD_RELEASES": "1", - "FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS": "3", - "FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO": "0.5", - "FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS": "true", - "FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS": "2", - "FETCH_TORRENTS_OLD_RELEASE_SKIP_REMOVED": "false", - "CLEANUP_KEEP_ONLY_LATEST_VERSION": "true", - "CLEANUP_SKIP_RATIO_CHECK": "true", - "CLEANUP_MIN_RATIO": "1.5", - "CLEANUP_STAGNATION_WINDOW_DAYS": "14", - "CLEANUP_STAGNATION_MIN_RATIO_DELTA": "0.1", - "CLEANUP_DISK_USAGE_THRESHOLD_PERCENT": "80", - "TRANSMISSION_RPC_USERNAME": " alice ", - "TRANSMISSION_RPC_PASSWORD": "hunter2", - } - ) - - self.assertEqual( - config, - ft.Config( - distros=tuple(d for d in ft.ALL_DISTROS if d in {"arch", "devuan"}), - include_low_demand=True, - skip_ratio_check=True, - check_old_releases=True, - old_release_min_leechers=3, - old_release_min_leecher_ratio=0.5, - old_release_allow_zero_seeders=True, - old_release_recheck_days=2, - old_release_skip_removed=False, - cleanup_keep_only_latest=True, - cleanup_skip_ratio_check=True, - cleanup_min_ratio=1.5, - stagnation_window_days=14, - stagnation_min_ratio_delta=0.1, - disk_usage_threshold_percent=80.0, - rpc_username="alice", - rpc_password="hunter2", - ), - ) - - def test_reads_only_the_env_it_is_given(self): - with helpers.isolated_env(SKIP_RATIO_CHECK="true", CLEANUP_MIN_RATIO="9"): - config = ft.load_config({}) - - self.assertFalse(config.skip_ratio_check) - self.assertEqual(config.cleanup_min_ratio, 1.0) - - def test_invalid_number_is_recorded_not_raised(self): - config = ft.load_config( - {"CLEANUP_MIN_RATIO": "high", "CLEANUP_STAGNATION_WINDOW_DAYS": "1.5"} - ) - - self.assertIsNone(config.cleanup_min_ratio) - self.assertIsNone(config.stagnation_window_days) - self.assertEqual( - config.invalid, - { - "cleanup_min_ratio": ( - "CLEANUP_MIN_RATIO: could not convert string to float: 'high'" - ), - "stagnation_window_days": ( - "CLEANUP_STAGNATION_WINDOW_DAYS: invalid literal for int() " - "with base 10: '1.5'" - ), - }, - ) - - def test_require_raises_the_first_invalid_field_named(self): - config = ft.load_config({"CLEANUP_MIN_RATIO": "high"}) - - config.require("stagnation_window_days") # valid: no error - with self.assertRaisesRegex(ValueError, "^CLEANUP_MIN_RATIO: "): - config.require("stagnation_window_days", "cleanup_min_ratio") - - def test_rpc_auth_needs_both_username_and_password(self): - cases = ( - ({}, None), - ({"TRANSMISSION_RPC_USERNAME": "alice"}, None), - ({"TRANSMISSION_RPC_PASSWORD": "hunter2"}, None), - ( - { - "TRANSMISSION_RPC_USERNAME": "alice", - "TRANSMISSION_RPC_PASSWORD": " ", - }, - None, - ), - ( - { - "TRANSMISSION_RPC_USERNAME": "alice", - "TRANSMISSION_RPC_PASSWORD": "hunter2", - }, - ("alice", "hunter2"), - ), - ) - for env, expected in cases: - with self.subTest(env=env): - self.assertEqual(ft.load_config(env).rpc_auth, expected) - - -class MakeClientFactoryTests(unittest.TestCase): - """The daily run's own Transmission connections (cleanup, disk-usage - limit) must authenticate the same way configure_transmission.py - configured the daemon. Without this, a user who sets - TRANSMISSION_RPC_USERNAME/PASSWORD (as the README requires once - TRANSMISSION_RPC_WHITELIST is widened beyond localhost) finds cleanup - silently stops working forever: every RPC call fails with 401 and the - failure is only logged, never surfaced.""" - - def _connect(self, env): - config = ft.load_config(env) - with unittest.mock.patch.object(ft, "Client", autospec=True) as client_cls: - client = ft.make_client_factory(config)() - return client_cls, client - - def test_passes_configured_credentials(self): - client_cls, client = self._connect( - { - "TRANSMISSION_RPC_USERNAME": "alice", - "TRANSMISSION_RPC_PASSWORD": "hunter2", - } - ) - - client_cls.assert_called_once_with( - host="localhost", port=9091, username="alice", password="hunter2" - ) - self.assertIs(client, client_cls.return_value) - - def test_passes_no_credentials_when_unset(self): - client_cls, _client = self._connect({}) - - client_cls.assert_called_once_with( - host="localhost", port=9091, username=None, password=None - ) - - def test_blank_credentials_count_as_unset(self): - client_cls, _client = self._connect( - {"TRANSMISSION_RPC_USERNAME": " ", "TRANSMISSION_RPC_PASSWORD": ""} - ) - - client_cls.assert_called_once_with( - host="localhost", port=9091, username=None, password=None - ) - - def test_factory_connects_only_when_called(self): - with unittest.mock.patch.object(ft, "Client", autospec=True) as client_cls: - factory = ft.make_client_factory(ft.load_config({})) - - client_cls.assert_not_called() - self.assertTrue(callable(factory)) - - -class LogSeedRatiosViaHttpTests(unittest.TestCase): - """log_seed_ratios_via_http() must not hang the daily run forever on a - stalled Transmission response, and must authenticate with whatever - credentials configure_transmission.py configured the daemon with when - the caller doesn't explicitly override auth.""" - - class FakeSessionIdResponse: - def __init__(self): - self.headers = {"X-Transmission-Session-Id": "abc"} - - class FakeTorrentGetResponse: - def raise_for_status(self): - pass - - def json(self): - return {"arguments": {"torrents": []}} - - def test_requests_follow_the_transmission_rpc_protocol(self): - # The pinned transmission_rpc's own tables of methods and torrent-get - # fields stand in for the RPC spec, so a renamed field fails here. - with unittest.mock.patch.object( - ft.requests, - "post", - side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], - ) as mock_post: - ft.log_seed_ratios_via_http(auth=("alice", "hunter2")) - - session_call, get_call = mock_post.call_args_list - self.assertEqual( - session_call, - unittest.mock.call("http://localhost:9091/transmission/rpc", timeout=15), - ) - self.assertEqual( - get_call, - unittest.mock.call( - "http://localhost:9091/transmission/rpc", - json={ - "method": "torrent-get", - "arguments": {"fields": ["name", "uploadRatio"]}, - }, - headers={"X-Transmission-Session-Id": "abc"}, - auth=("alice", "hunter2"), - timeout=15, - ), - ) - payload = get_call.kwargs["json"] - self.assertEqual( - payload["method"], transmission_rpc.constants.RpcMethod.TorrentGet.value - ) - self.assertLessEqual( - set(payload["arguments"]["fields"]), - set(transmission_rpc.constants.TORRENT_GET_ARGS), - ) - - def test_logs_ratios_highest_first_with_none_as_zero(self): - response = self.FakeTorrentGetResponse() - response.json = lambda: { - "arguments": { - "torrents": [ - {"name": "low.iso", "uploadRatio": 0.5}, - {"name": "unset.iso", "uploadRatio": None}, - {"name": "high.iso", "uploadRatio": 2.25}, - ] - } - } - with ( - unittest.mock.patch.object( - ft.requests, - "post", - side_effect=[self.FakeSessionIdResponse(), response], - ), - self.assertLogs(ft.logger, level="INFO") as logs, - ): - ft.log_seed_ratios_via_http(auth=None) - - ratio_lines = [ - r.getMessage() for r in logs.records if r.getMessage().startswith("[ratio]") - ] - self.assertEqual( - ratio_lines, - [ - "[ratio] RATIOS START", - f"[ratio] {'high.iso':<50} → 2.250", - f"[ratio] {'low.iso':<50} → 0.500", - f"[ratio] {'unset.iso':<50} → 0.000", - "[ratio] RATIOS END", - ], - ) - - def test_session_id_request_has_a_timeout(self): - with unittest.mock.patch.object( - ft.requests, - "post", - side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], - ) as mock_post: - ft.log_seed_ratios_via_http() - - first_call_kwargs = mock_post.call_args_list[0].kwargs - self.assertIn("timeout", first_call_kwargs) - - def test_uses_configured_credentials_when_auth_not_explicitly_passed(self): - with ( - unittest.mock.patch.dict( - os.environ, - { - "TRANSMISSION_RPC_USERNAME": "alice", - "TRANSMISSION_RPC_PASSWORD": "hunter2", - }, - ), - unittest.mock.patch.object( - ft.requests, - "post", - side_effect=[ - self.FakeSessionIdResponse(), - self.FakeTorrentGetResponse(), - ], - ) as mock_post, - ): - ft.log_seed_ratios_via_http() - - second_call_kwargs = mock_post.call_args_list[1].kwargs - self.assertEqual(second_call_kwargs.get("auth"), ("alice", "hunter2")) - - def test_no_auth_when_credentials_unset(self): - with ( - helpers.isolated_env(), - unittest.mock.patch.object( - ft.requests, - "post", - side_effect=[ - self.FakeSessionIdResponse(), - self.FakeTorrentGetResponse(), - ], - ) as mock_post, - ): - ft.log_seed_ratios_via_http() - - second_call_kwargs = mock_post.call_args_list[1].kwargs - self.assertIsNone(second_call_kwargs.get("auth")) - - def test_explicit_auth_argument_still_wins(self): - with ( - unittest.mock.patch.dict( - os.environ, - { - "TRANSMISSION_RPC_USERNAME": "alice", - "TRANSMISSION_RPC_PASSWORD": "hunter2", - }, - ), - unittest.mock.patch.object( - ft.requests, - "post", - side_effect=[ - self.FakeSessionIdResponse(), - self.FakeTorrentGetResponse(), - ], - ) as mock_post, - ): - ft.log_seed_ratios_via_http(auth=("bob", "other")) - - second_call_kwargs = mock_post.call_args_list[1].kwargs - self.assertEqual(second_call_kwargs.get("auth"), ("bob", "other")) - - -class CleanupRemovesWatchFileTests(unittest.TestCase): - """When cleanup removes a torrent's downloaded data, it must also remove - the small .torrent file it left behind in /watch - otherwise these - accumulate forever, since nothing else (including Transmission itself, - by default) ever cleans them up.""" - - def setUp(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_") - for attr, filename in ( - ("removed_history_file", "removed_history.json"), - ("ratio_history_file", "ratio_history.json"), - ): - patcher = unittest.mock.patch.object( - ft, attr, os.path.join(tmp_dir, filename) - ) - patcher.start() - self.addCleanup(patcher.stop) - - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) - - def _touch_watch_file(self, name): - path = os.path.join(self.tmp_watch_dir, f"{name}.torrent") - with open(path, "w", encoding="utf-8") as f: - f.write("fake torrent bytes") - return path - - def _transmission(self): - return helpers.FakeTransmission( - [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), - ] - ) - - def test_cleanup_old_versions_removes_the_watch_file(self): - old_path = self._touch_watch_file("ubuntu-23.10-desktop-amd64.iso") - new_path = self._touch_watch_file("ubuntu-24.04-desktop-amd64.iso") - transmission = self._transmission() - - ft.cleanup_old_versions( - transmission, today=date(2026, 8, 1), skip_ratio_check=True - ) - - self.assertEqual(transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) - self.assertIn(("remove_torrent", 2, True), transmission.calls) - self.assertFalse(os.path.exists(old_path)) - self.assertTrue(os.path.exists(new_path)) - self.assertEqual( - ft.load_removed_history(ft.removed_history_file), - { - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-08-01", - "reason": "keep_only_latest", - "ratio": 2.0, - } - }, - ) - - def test_missing_watch_file_does_not_raise(self): - # No .torrent file created for the superseded version - cleanup must - # still succeed and remove the torrent from Transmission. - transmission = self._transmission() - - ft.cleanup_old_versions( - transmission, today=date(2026, 8, 1), skip_ratio_check=True - ) - - self.assertEqual(transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) - - def test_cleanup_old_versions_keeps_versions_below_min_ratio(self): - transmission = helpers.FakeTransmission( - [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), - ] - ) - - ft.cleanup_old_versions(transmission, today=date(2026, 8, 1), min_ratio=2.0) - - self.assertEqual(len(transmission.torrents), 2) - - def test_cleanup_stagnant_torrents_removes_the_watch_file(self): - old_path = self._touch_watch_file("ubuntu-23.10-desktop-amd64.iso") - self._touch_watch_file("ubuntu-24.04-desktop-amd64.iso") - transmission = self._transmission() - - # 32 days old: past the default 30-day stagnation window, but still - # within update_ratio_history()'s own 37-day (window + sample - # interval) prune horizon, since that prune runs - on this same - # history - before plan_stagnation_cleanup() ever sees it. Ratio - # unchanged since then, so plan_stagnation_cleanup() must select this - # torrent for removal. - today = date(2026, 8, 1) - anchor_date = (today - timedelta(days=32)).isoformat() - history = { - "ubuntu-23.10-desktop-amd64.iso": [{"date": anchor_date, "ratio": 2.0}] - } - with open(ft.ratio_history_file, "w", encoding="utf-8") as f: - json.dump(history, f) - - ft.cleanup_stagnant_torrents(transmission, today=today) - - self.assertEqual(transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) - self.assertFalse(os.path.exists(old_path)) - removed = ft.load_removed_history(ft.removed_history_file) - self.assertEqual( - removed["ubuntu-23.10-desktop-amd64.iso"], - {"removed_date": "2026-08-01", "reason": "stagnant", "ratio": 2.0}, - ) - - def test_cleanup_stagnant_torrents_records_todays_sample(self): - transmission = self._transmission() - - ft.cleanup_stagnant_torrents( - transmission, today=date(2026, 8, 1), window_days=10 - ) - - self.assertEqual( - ft.load_ratio_history(ft.ratio_history_file), - { - "ubuntu-24.04-desktop-amd64.iso": [ - {"date": "2026-08-01", "ratio": 2.0} - ], - "ubuntu-23.10-desktop-amd64.iso": [ - {"date": "2026-08-01", "ratio": 2.0} - ], - }, - ) - - -class PlanDiskPressureCleanupTests(unittest.TestCase): - """plan_disk_pressure_cleanup() is the pure selection logic behind - CLEANUP_DISK_USAGE_THRESHOLD_PERCENT - the safety valve that kicks in - when normal ratio/stagnation-based cleanup isn't freeing space fast - enough for a forgotten, never-updated deployment.""" - - def test_below_threshold_returns_nothing(self): - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), - ] - self.assertEqual( - ft.plan_disk_pressure_cleanup( - torrents, used_percent=80.0, threshold_percent=95.0 - ), - [], - ) - - def test_disabled_when_threshold_is_zero_or_negative(self): - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1), - ] - self.assertEqual( - ft.plan_disk_pressure_cleanup( - torrents, used_percent=99.0, threshold_percent=0 - ), - [], - ) - self.assertEqual( - ft.plan_disk_pressure_cleanup( - torrents, used_percent=99.0, threshold_percent=-1 - ), - [], - ) - - def test_at_or_above_threshold_returns_candidates_lowest_ratio_first(self): - torrents = [ - FakeTorrent( - "ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0 - ), # latest - never a candidate - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), - FakeTorrent("ubuntu-22.04.1-live-server-amd64.iso", 3, ratio=0.2), - FakeTorrent( - "ubuntu-24.04.1-live-server-amd64.iso", 4, ratio=5.0 - ), # latest of this group - ] - result = ft.plan_disk_pressure_cleanup( - torrents, used_percent=95.0, threshold_percent=95.0 - ) - self.assertEqual( - names_of(result), - ["ubuntu-22.04.1-live-server-amd64.iso", "ubuntu-23.10-desktop-amd64.iso"], - ) - # lowest ratio (least useful) first - self.assertEqual(result[0].name, "ubuntu-22.04.1-live-server-amd64.iso") - - def test_no_superseded_candidates_returns_empty(self): - torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=0.0)] - self.assertEqual( - ft.plan_disk_pressure_cleanup( - torrents, used_percent=99.0, threshold_percent=95.0 - ), - [], - ) - - def test_any_positive_threshold_is_enabled(self): - older = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1) - torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), older] - - self.assertEqual( - ft.plan_disk_pressure_cleanup( - torrents, used_percent=50.0, threshold_percent=1 - ), - [older], - ) - - def test_usage_exactly_at_threshold_triggers_cleanup(self): - older = FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=0.1) - torrents = [FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), older] - - self.assertEqual( - ft.plan_disk_pressure_cleanup( - torrents, used_percent=95.0, threshold_percent=95.0 - ), - [older], - ) - - def test_missing_or_none_ratio_sorts_as_zero(self): - half = FakeTorrent("ubuntu-22.10-desktop-amd64.iso", 2, ratio=0.5) - no_attr = types.SimpleNamespace(name="ubuntu-23.04-desktop-amd64.iso", id=3) - none_ratio = DuckTorrent("ubuntu-23.10-desktop-amd64.iso", 4, ratio=None) - torrents = [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), - half, - no_attr, - none_ratio, - ] - - planned = ft.plan_disk_pressure_cleanup( - torrents, used_percent=99.0, threshold_percent=95.0 - ) - - self.assertEqual(planned[-1], half) - self.assertEqual( - sorted(t.name for t in planned[:2]), - ["ubuntu-23.04-desktop-amd64.iso", "ubuntu-23.10-desktop-amd64.iso"], - ) - - -class EnforceDiskUsageLimitTests(unittest.TestCase): - """enforce_disk_usage_limit() wires plan_disk_pressure_cleanup() to a - Transmission connection and real disk usage, re-checking usage between - removals since ratio data alone can't say how many bytes each torrent - holds.""" - - def setUp(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_") - for attr, filename in ( - ("removed_history_file", "removed_history.json"), - ("ratio_history_file", "ratio_history.json"), - ): - patcher = unittest.mock.patch.object( - ft, attr, os.path.join(tmp_dir, filename) - ) - patcher.start() - self.addCleanup(patcher.stop) - - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) - - self.transmission = helpers.FakeTransmission( - [ - FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=5.0), - FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.5), - FakeTorrent("ubuntu-22.04-desktop-amd64.iso", 3, ratio=0.2), - ] - ) - self.connections = 0 - - def _connect(self): - self.connections += 1 - return self.transmission - - def _disk(self, base_used, sizes=None): - # 100 GB disk; each torrent's size in GB. - return helpers.FakeDisk( - self.transmission, 100, sizes or {1: 2, 2: 2, 3: 6}, base_used - ) - - def _enforce(self, threshold, disk): - ft.enforce_disk_usage_limit( - self._connect, - threshold, - today=date(2026, 8, 1), - disk_usage=disk, - download_path="/downloads", - ) - - def test_disabled_threshold_never_touches_disk_or_rpc(self): - disk = self._disk(base_used=99) - - self._enforce(0, disk) - - self.assertEqual((disk.paths, self.connections), ([], 0)) - - def test_below_threshold_does_not_connect_to_transmission(self): - disk = self._disk(base_used=40) # 50% - - self._enforce(95, disk) - - self.assertEqual(disk.paths, ["/downloads"]) - self.assertEqual(self.connections, 0) - - def test_removes_lowest_ratio_candidates_until_under_threshold(self): - # 86 + 10 = 96% used. Removing 22.04 (lowest ratio, 6 GB) gets to - # 90%, under the threshold, so 23.10 must be left alone. - disk = self._disk(base_used=86) - - self._enforce(95, disk) - - self.assertEqual( - self.transmission.names, - ["ubuntu-23.10-desktop-amd64.iso", "ubuntu-24.04-desktop-amd64.iso"], - ) - - def test_keeps_removing_while_still_over_threshold(self): - # 96%; removing 22.04 (1 GB) leaves 95% - still at the threshold - - # so 23.10 (4 GB) goes too. - disk = self._disk(base_used=90, sizes={1: 1, 2: 4, 3: 1}) - - self._enforce(95, disk) - - self.assertEqual(self.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) - - def test_never_removes_the_latest_version(self): - disk = self._disk(base_used=99) - - self._enforce(95, disk) - - self.assertEqual(self.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) - - def test_removes_the_watch_file_for_each_removal(self): - watch_path = os.path.join( - self.tmp_watch_dir, "ubuntu-22.04-desktop-amd64.iso.torrent" - ) - with open(watch_path, "w", encoding="utf-8") as f: - f.write("fake torrent bytes") - - self._enforce(95, self._disk(base_used=86)) - - self.assertFalse(os.path.exists(watch_path)) - - def test_no_candidates_logs_warning_and_does_not_raise(self): - self.transmission.torrents = self.transmission.torrents[:1] - - with self.assertLogs(ft.logger, level="WARNING") as logs: - self._enforce(95, self._disk(base_used=96)) - - self.assertEqual(self.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) - self.assertIn("no superseded versions are available", logs.output[0]) - - def test_records_removal_reason_as_disk_pressure(self): - self._enforce(95, self._disk(base_used=86)) - - self.assertEqual( - ft.load_removed_history(ft.removed_history_file), - { - "ubuntu-22.04-desktop-amd64.iso": { - "removed_date": "2026-08-01", - "reason": "disk_pressure", - "ratio": 0.2, - } - }, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_fetchers.py b/tests/test_fetchers.py new file mode 100644 index 0000000..ac34e41 --- /dev/null +++ b/tests/test_fetchers.py @@ -0,0 +1,497 @@ +"""Tests for choosing and downloading releases: each fetch_* scraper's +candidate naming (hand-written pages; tests/test_scrapers.py replays +recorded ones), FETCH_TORRENTS_DISTROS, the low-demand filter and +download_torrent(). + +Run with: python -m unittest discover -s tests +""" + +import os +import unittest +import unittest.mock + +import helpers + +import fetch_torrents as ft + + +def setUpModule(): + helpers.isolate_module() + + +class FetchUbuntuLtsNamingTests(unittest.TestCase): + """The names fetch_ubuntu_lts() uses as dict keys are also what gets + passed to should_fetch_torrent() before download, so they must match the + shape of the names Transmission later reports for those same torrents + (real arch suffix + .iso extension) — see + tasks/02-ratio-key-mismatch-ubuntu-debian.md.""" + + def test_candidate_names_include_arch_and_iso_suffix(self): + meta_release = "Dist: noble\nVersion: 24.04\nSupported: 1\n" + + response = helpers.fake_response(meta_release) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + results = ft.fetch_ubuntu_lts() + + self.assertTrue(results) + self.assertIn("ubuntu-24.04-desktop-amd64.iso", results) + self.assertIn("ubuntu-24.04-live-server-amd64.iso", results) + + +class FetchDebianStableNamingTests(unittest.TestCase): + """Mirrors FetchUbuntuLtsNamingTests for Debian: the dict key must keep + the .iso extension so it matches Transmission's reported torrent name.""" + + def test_candidate_name_keeps_iso_suffix(self): + html = 'link' + + response = helpers.fake_response(html) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + results = ft.fetch_debian_stable() + + self.assertIn("debian-12.5.0-amd64-DVD-1.iso", results) + + +class FetchLinuxMintCinnamonNamingTests(unittest.TestCase): + """The dict key fetch_linuxmint_cinnamon() uses must match the 'name' + field embedded in the actual .torrent file (verified against a real + download from linuxmint.com: linuxmint-22.3-cinnamon-64bit.iso), so it + lines up with what Transmission later reports as torrent.name.""" + + def test_candidate_name_matches_torrent_internal_name(self): + html = "Download Linux Mint 22.3 - Linux Mint" + + response = helpers.fake_response(html) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + results = ft.fetch_linuxmint_cinnamon() + + self.assertIn("linuxmint-22.3-cinnamon-64bit.iso", results) + self.assertEqual( + results["linuxmint-22.3-cinnamon-64bit.iso"], + "https://www.linuxmint.com/torrents/linuxmint-22.3-cinnamon-64bit.iso.torrent", + ) + + +class FetchFedoraWorkstationNamingTests(unittest.TestCase): + """The dict key fetch_fedora_workstation() uses must match the 'name' + field embedded in the actual .torrent file (verified against a real + download from torrent.fedoraproject.org: Fedora-Workstation-Live-x86_64-44, + which has no .iso extension), so it lines up with what Transmission later + reports as torrent.name.""" + + def test_picks_highest_version_per_arch(self): + html = ( + "" + 'a' + 'b' + 'c' + 'd' + "" + ) + + response = helpers.fake_response(html) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + results = ft.fetch_fedora_workstation() + + self.assertEqual( + results, + { + "Fedora-Workstation-Live-x86_64-44": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-x86_64-44.torrent", + "Fedora-Workstation-Live-aarch64-44": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-aarch64-44.torrent", + }, + ) + + +class FetchArchLatestNamingTests(unittest.TestCase): + """archlinux.org/releng/releases/ lists every release still available + (typically the last 2-3 months) as available-yes. fetch_arch_latest() + must pick only the single newest one - fetching all of them fragments + whatever little torrent demand Arch has across several near-duplicate + monthly builds (see tasks/01-demand-checks-for-default-distros.md).""" + + def test_picks_only_the_single_newest_release(self): + html = ( + "" + "" + "" + "" + "
t
t
t
" + ) + + response = helpers.fake_response(html) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + results = ft.fetch_arch_latest() + + self.assertEqual( + results, + { + "archlinux-2026.09.01": "https://archlinux.org/releng/releases/2026.09.01/torrent/" + }, + ) + + def test_no_available_releases_returns_false(self): + html = "
" + + response = helpers.fake_response(html) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + result = ft.fetch_arch_latest() + + self.assertFalse(result) + + +class FetchDevuanLatestNamingTests(unittest.TestCase): + """Devuan's get-devuan page links a single release torrent whose + filename embeds the current release codename (e.g. + devuan_excalibur.torrent, verified against the real download) - the + dict key must match that basename since it's also what Transmission + later reports as torrent.name.""" + + def test_extracts_current_release_torrent(self): + html = ( + '" + ) + + response = helpers.fake_response(html) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + results = ft.fetch_devuan_latest() + + self.assertEqual( + results, + {"devuan_excalibur": "https://files.devuan.org/devuan_excalibur.torrent"}, + ) + + def test_returns_false_when_no_torrent_link_found(self): + response = helpers.fake_response("nothing here") + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + result = ft.fetch_devuan_latest() + + self.assertFalse(result) + + +class FetchKaliOldVersionsTests(unittest.TestCase): + """fetch_kali_old_versions() discovers the prior release(s) still listed + on cdimage.kali.org (a 'current/' symlink plus a small number of + 'kali-X.Y/' folders), for FETCH_TORRENTS_CHECK_OLD_RELEASES. Only ever + used when that flag is enabled - see CheckOldReleasesForDemandTests.""" + + def test_returns_only_the_non_newest_versions(self): + html = ( + "" + 'current' + 'kali-2026.1' + 'kali-2026.2' + "" + ) + + response = helpers.fake_response(html) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + results = ft.fetch_kali_old_versions() + + self.assertTrue(results) + self.assertTrue(any("2026.1" in name for name in results)) + self.assertFalse(any("2026.2" in name for name in results)) + + def test_single_version_folder_returns_nothing(self): + html = 'currentkali-2026.2' + + response = helpers.fake_response(html) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + results = ft.fetch_kali_old_versions() + + self.assertEqual(results, {}) + + +class FetchFedoraWorkstationOldVersionsTests(unittest.TestCase): + """Mirrors FetchFedoraWorkstationNamingTests, but for the versions + fetch_fedora_workstation() discards as not-latest.""" + + def test_returns_non_latest_versions_per_arch(self): + html = ( + "" + 'a' + 'b' + 'c' + 'd' + "" + ) + + response = helpers.fake_response(html) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + results = ft.fetch_fedora_workstation_old_versions() + + self.assertEqual( + results, + { + "Fedora-Workstation-Live-x86_64-43": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-x86_64-43.torrent", + "Fedora-Workstation-Live-aarch64-43": "https://torrent.fedoraproject.org/torrents/Fedora-Workstation-Live-aarch64-43.torrent", + }, + ) + + def test_single_version_per_arch_returns_nothing(self): + html = 'a' + + response = helpers.fake_response(html) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + results = ft.fetch_fedora_workstation_old_versions() + + self.assertEqual(results, {}) + + +class ParseSupportedDistrosTests(unittest.TestCase): + """Arch and Devuan are opt-in only: neither can be pre-verified for real + demand (Arch publishes no tracker at all; Devuan bundles every edition + into one multi-file torrent, so a demand reading only ever covers the + whole bundle, never a specific file within it) - see + tasks/01-demand-checks-for-default-distros.md. They must never be + fetched just because FETCH_TORRENTS_DISTROS is unset.""" + + def test_default_excludes_opt_in_distros(self): + with helpers.isolated_env(): + result = ft.parse_supported_distros() + self.assertNotIn("arch", result) + self.assertNotIn("devuan", result) + for expected in ("ubuntu", "debian", "kali", "mint", "fedora"): + self.assertIn(expected, result) + + def test_explicit_opt_in_distros_are_honored(self): + with unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "arch,devuan"} + ): + result = ft.parse_supported_distros() + self.assertEqual(sorted(result), ["arch", "devuan"]) + + def test_can_mix_default_and_opt_in_distros(self): + with unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "ubuntu,arch"} + ): + result = ft.parse_supported_distros() + self.assertEqual(sorted(result), ["arch", "ubuntu"]) + + def test_unknown_entries_are_dropped_but_valid_ones_kept(self): + with unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "ubuntu,nonsense"} + ): + result = ft.parse_supported_distros() + self.assertEqual(result, ["ubuntu"]) + + def test_all_invalid_falls_back_to_default_not_all_distros(self): + with unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "nonsense"} + ): + result = ft.parse_supported_distros() + self.assertEqual(sorted(result), sorted(ft.DEFAULT_DISTROS)) + self.assertNotIn("arch", result) + self.assertNotIn("devuan", result) + + +class LowDemandVariantTests(unittest.TestCase): + """cloud-genericcloud images and Kali's netinst installer are + chronically low-ratio in fetch_torrents_ratios.log regardless of + architecture or how recent the release is — these are the 'image + families' FETCH_TORRENTS_INCLUDE_LOW_DEMAND gates. + + arm64/aarch64/armhf/armel builds are deliberately NOT low-demand: current- + version arm64/armhf builds (e.g. kali-linux-2026.2-raspberry-pi-armhf-img-xz + at 2.351, debian-13.6.0-arm64-netinst.iso at 1.571) have solid ratios in + the log — the low arm64/armhf numbers there are almost all older, + superseded versions, already handled by should_fetch_torrent().""" + + def test_cloud_images_are_low_demand_regardless_of_arch(self): + for name in ( + "kali-linux-2026.2-cloud-genericcloud-amd64-tar-xz", + "kali-linux-2026.1-cloud-genericcloud-arm64-tar-xz", + ): + self.assertTrue(ft.is_low_demand_variant(name), name) + + def test_kali_netinst_is_low_demand_regardless_of_arch(self): + for name in ( + "kali-linux-2026.1-installer-netinst-amd64.iso", + "kali-linux-2025.2-installer-netinst-arm64.iso", + ): + self.assertTrue(ft.is_low_demand_variant(name), name) + + def test_debian_netinst_is_not_low_demand(self): + # Debian's netinst is the highest-ratio image in the whole log — + # only Kali's netinst installer is excluded, not the substring. + for name in ( + "debian-13.6.0-amd64-netinst.iso", + "debian-13.6.0-arm64-netinst.iso", + ): + self.assertFalse(ft.is_low_demand_variant(name), name) + + def test_current_arm64_and_armhf_builds_are_not_low_demand(self): + for name in ( + "debian-13.6.0-arm64-DVD-1.iso", + "kali-linux-2026.2-installer-arm64.iso", + "Fedora-Workstation-Live-aarch64-44", + "kali-linux-2026.2-raspberry-pi-armhf-img-xz", + "kali-linux-2025.2-raspberry-pi-zero-w-armel-img-xz", + ): + self.assertFalse(ft.is_low_demand_variant(name), name) + + def test_mainstream_amd64_variants_are_not_low_demand(self): + for name in ( + "ubuntu-24.04.4-desktop-amd64.iso", + "kali-linux-2026.2-installer-everything-amd64.iso", + "kali-linux-2026.2-installer-purple-amd64.iso", + "kali-linux-2026.2-installer-amd64.iso", + "archlinux-2026.04.01-x86_64.iso", + "Fedora-Workstation-Live-x86_64-44", + ): + self.assertFalse(ft.is_low_demand_variant(name), name) + + def test_filter_low_demand_excludes_by_default(self): + torrents = { + "kali-linux-2026.2-installer-amd64.iso": "http://x/a", + "kali-linux-2026.2-installer-netinst-amd64.iso": "http://x/b", + } + + filtered = ft.filter_low_demand(torrents, include_low_demand=False) + + self.assertEqual( + filtered, {"kali-linux-2026.2-installer-amd64.iso": "http://x/a"} + ) + + def test_filter_low_demand_includes_when_enabled(self): + torrents = { + "kali-linux-2026.2-installer-amd64.iso": "http://x/a", + "kali-linux-2026.2-installer-netinst-amd64.iso": "http://x/b", + } + + filtered = ft.filter_low_demand(torrents, include_low_demand=True) + + self.assertEqual(filtered, torrents) + + +class DownloadTorrent404Tests(unittest.TestCase): + """Torrent URLs for releases no longer hosted upstream (e.g. old Ubuntu + LTS releases that changelogs.ubuntu.com still marks 'Supported: 1' for + ESM purposes, long after their installer media was pulled) 404. That's + an expected, non-actionable outcome and shouldn't be logged/counted the + same way as a genuine download failure.""" + + def setUp(self): + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) + patcher.start() + self.addCleanup(patcher.stop) + + def test_404_response_is_not_found_not_failed(self): + response = helpers.fake_response(status=404) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + status = ft.download_torrent( + "ubuntu-14.04.6-live-server-amd64.iso", + "https://example.invalid/x.torrent", + ) + + self.assertEqual(status, "not_found") + dest = os.path.join( + self.tmp_watch_dir, "ubuntu-14.04.6-live-server-amd64.iso.torrent" + ) + self.assertFalse(os.path.exists(dest)) + + def test_other_http_error_is_still_failed(self): + response = helpers.fake_response(status=500) + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + status = ft.download_torrent( + "ubuntu-24.04-desktop-amd64.iso", "https://example.invalid/x.torrent" + ) + + self.assertEqual(status, "failed") + + def test_successful_download_is_still_added(self): + response = helpers.fake_response(b"fake torrent bytes") + + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + status = ft.download_torrent( + "ubuntu-24.04-desktop-amd64.iso", "https://example.invalid/x.torrent" + ) + + self.assertEqual(status, "added") + dest = os.path.join( + self.tmp_watch_dir, "ubuntu-24.04-desktop-amd64.iso.torrent" + ) + with open(dest, "rb") as f: + self.assertEqual(f.read(), b"fake torrent bytes") + + +class SafeWatchPathTests(unittest.TestCase): + """_safe_watch_path() is the shared guard download_torrent(), + _remove_watch_file(), and check_old_releases_for_demand() all rely on to + keep a torrent name confined to a single file directly under watch_dir - + defense in depth against a future fetch_*() parser turning scraped + upstream content into a path escape.""" + + def setUp(self): + patcher = unittest.mock.patch.object(ft, "watch_dir", "/watch") + patcher.start() + self.addCleanup(patcher.stop) + + def test_normal_name_resolves_under_watch_dir(self): + self.assertEqual( + ft._safe_watch_path("ubuntu-24.04-desktop-amd64.iso"), + os.path.join("/watch", "ubuntu-24.04-desktop-amd64.iso.torrent"), + ) + + def test_rejects_path_separators(self): + self.assertIsNone(ft._safe_watch_path("archlinux-../../../etc/cron.d/evil")) + self.assertIsNone(ft._safe_watch_path("foo/bar")) + self.assertIsNone(ft._safe_watch_path("foo\\bar")) + + def test_rejects_empty_or_dot_names(self): + self.assertIsNone(ft._safe_watch_path("")) + self.assertIsNone(ft._safe_watch_path(None)) + self.assertIsNone(ft._safe_watch_path(".")) + self.assertIsNone(ft._safe_watch_path("..")) + + +class DownloadTorrentUnsafeNameTests(unittest.TestCase): + def setUp(self): + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) + patcher.start() + self.addCleanup(patcher.stop) + + def test_download_torrent_refuses_unsafe_name_without_any_request(self): + with unittest.mock.patch.object(ft.requests, "get") as mock_get: + status = ft.download_torrent("../evil", "https://example.invalid/x.torrent") + + mock_get.assert_not_called() + self.assertEqual(status, "failed") + self.assertEqual(os.listdir(self.tmp_watch_dir), []) + + def test_remove_watch_file_refuses_unsafe_name_without_touching_disk(self): + # Asserted on disk rather than by patching os.remove: that patch is + # process-wide, so a log rotation triggered by the warning would + # trip it too. + parent = helpers.make_temp_dir(self, "fetch_torrents_test_escape_") + watch_dir = os.path.join(parent, "watch") + os.mkdir(watch_dir) + outside = os.path.join(parent, "evil.torrent") + with open(outside, "w", encoding="utf-8") as f: + f.write("must survive") + + with unittest.mock.patch.object(ft, "watch_dir", watch_dir): + ft._remove_watch_file("../evil") + + self.assertTrue(os.path.exists(outside)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_live_check.py b/tests/test_live_check.py index 3634492..1cc3036 100644 --- a/tests/test_live_check.py +++ b/tests/test_live_check.py @@ -8,17 +8,15 @@ import sys import unittest -REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -for path in (REPO_ROOT, os.path.join(REPO_ROOT, "scripts")): - if path not in sys.path: - sys.path.insert(0, path) - import helpers + +sys.path.insert(0, os.path.join(helpers.REPO_ROOT, "scripts")) + import live_check def setUpModule(): - unittest.addModuleCleanup(helpers.block_network()) + helpers.isolate_module() class ProblemsTests(unittest.TestCase): diff --git a/tests/test_logging.py b/tests/test_logging.py new file mode 100644 index 0000000..b6286f9 --- /dev/null +++ b/tests/test_logging.py @@ -0,0 +1,320 @@ +"""Tests for logging: configure_logging(), importing without touching +files, and the seed-ratio log written over the Transmission RPC for the +next run's ratio gate. + +Run with: python -m unittest discover -s tests +""" + +import logging +import logging.handlers +import os +import subprocess +import sys +import unittest +import unittest.mock + +import helpers +import transmission_rpc.constants + +import fetch_torrents as ft + + +def setUpModule(): + helpers.isolate_module() + + +def configure_logging_for_test(testcase, log_dir, **env): + """ft.configure_logging() with only `env`, undone when the test ends.""" + saved = ( + list(ft.logger.handlers), + list(ft._configured_handlers), + ft.logger.level, + ft.logger.propagate, + ) + + def restore(): + for handler in ft._configured_handlers: + handler.close() + ft.logger.handlers[:] = saved[0] + ft._configured_handlers[:] = saved[1] + ft.logger.setLevel(saved[2]) + ft.logger.propagate = saved[3] + + testcase.addCleanup(restore) + return ft.configure_logging(log_dir, env=env) + + +class RatioLogSurvivesConfigureLoggingTests(unittest.TestCase): + """Each container run is a fresh process: the ratio log from the + *previous* run is what should_fetch_torrent() needs in order to gate a + new version's download. configure_logging() opens that same file with + mode="w" for this run's ratios, so it must not truncate it before + get_previous_ratios() has read it - otherwise the low-ratio fetch gate + is silently defeated for every distro.""" + + def setUp(self): + self.log_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_restart_") + self.path = os.path.join(self.log_dir, "fetch_torrents_ratios.log") + with open(self.path, "w", encoding="utf-8") as f: + f.write( + "2026-08-01 00:00:00,000 INFO: [ratio] RATIOS START\n" + "2026-08-01 00:00:00,000 INFO: [ratio] archlinux-2026.04.01-x86_64.iso → 0.664\n" + "2026-08-01 00:00:00,000 INFO: [ratio] RATIOS END\n" + ) + + def test_previous_ratios_are_readable_after_configure_logging(self): + configure_logging_for_test(self, self.log_dir) + + self.assertEqual( + ft.get_previous_ratios(self.path), + {"archlinux-2026.04.01-x86_64.iso": 0.664}, + ) + + def test_first_ratio_line_of_the_run_replaces_the_previous_log(self): + _stream, _file, ratio_handler = configure_logging_for_test(self, self.log_dir) + + ft.logger.info("[ratio] %-50s → %.3f", "new.iso", 1.0) + ratio_handler.flush() + + self.assertEqual(ft.get_previous_ratios(self.path), {"new.iso": 1.0}) + + +class ImportOpensNoFilesTests(unittest.TestCase): + """Importing fetch_torrents (tests, other scripts) must not create or + truncate anything under FETCH_TORRENTS_LOG_DIR - only + configure_logging() does. Needs a fresh interpreter, since this process + imported the module long ago.""" + + def test_import_leaves_the_log_dir_untouched(self): + log_dir = helpers.make_temp_dir(self, "fetch_torrents_import_") + script = ( + "import sys\n" + f"sys.path.insert(0, {helpers.REPO_ROOT!r})\n" + "import fetch_torrents as ft\n" + "ft.logger.warning('logged before configure_logging()')\n" + ) + result = subprocess.run( + [sys.executable, "-c", script], + check=False, + env=dict(os.environ, FETCH_TORRENTS_LOG_DIR=log_dir), + capture_output=True, + text=True, + timeout=30, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(os.listdir(log_dir), []) + self.assertEqual(result.stderr, "") + + +class ConfigureLoggingTests(unittest.TestCase): + """fetch_torrents.log must be bounded (RotatingFileHandler), not a plain + FileHandler appending forever for the life of a 'deploy and forget' + container.""" + + def _configure(self, **env): + log_dir = helpers.make_temp_dir(self, "fetch_torrents_log_rotation_") + return configure_logging_for_test(self, log_dir, **env) + + def test_defaults_to_a_bounded_rotating_handler(self): + _stream, file_handler, _ratio = self._configure() + self.assertIsInstance(file_handler, logging.handlers.RotatingFileHandler) + self.assertEqual(file_handler.maxBytes, 5 * 1024 * 1024) + self.assertEqual(file_handler.backupCount, 3) + + def test_env_vars_override_rotation_settings(self): + _stream, file_handler, _ratio = self._configure( + FETCH_TORRENTS_LOG_MAX_BYTES="1000", FETCH_TORRENTS_LOG_BACKUP_COUNT="1" + ) + self.assertEqual(file_handler.maxBytes, 1000) + self.assertEqual(file_handler.backupCount, 1) + + def test_invalid_env_vars_fall_back_to_defaults(self): + _stream, file_handler, _ratio = self._configure( + FETCH_TORRENTS_LOG_MAX_BYTES="not-a-number", + FETCH_TORRENTS_LOG_BACKUP_COUNT="also-bad", + ) + self.assertEqual(file_handler.maxBytes, 5 * 1024 * 1024) + self.assertEqual(file_handler.backupCount, 3) + + def test_calling_again_replaces_its_own_handlers_only(self): + other = logging.NullHandler() + ft.logger.addHandler(other) + self.addCleanup(ft.logger.removeHandler, other) + + first = self._configure() + second = self._configure() + + self.assertEqual( + [h for h in ft.logger.handlers if h is not other], list(second) + ) + self.assertIn(other, ft.logger.handlers) + self.assertTrue(all(h not in ft.logger.handlers for h in first)) + + +class LogSeedRatiosViaHttpTests(unittest.TestCase): + """log_seed_ratios_via_http() must not hang the daily run forever on a + stalled Transmission response, and must authenticate with whatever + credentials configure_transmission.py configured the daemon with when + the caller doesn't explicitly override auth.""" + + class FakeSessionIdResponse: + def __init__(self): + self.headers = {"X-Transmission-Session-Id": "abc"} + + class FakeTorrentGetResponse: + def raise_for_status(self): + pass + + def json(self): + return {"arguments": {"torrents": []}} + + def test_requests_follow_the_transmission_rpc_protocol(self): + # The pinned transmission_rpc's own tables of methods and torrent-get + # fields stand in for the RPC spec, so a renamed field fails here. + with unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], + ) as mock_post: + ft.log_seed_ratios_via_http(auth=("alice", "hunter2")) + + session_call, get_call = mock_post.call_args_list + self.assertEqual( + session_call, + unittest.mock.call("http://localhost:9091/transmission/rpc", timeout=15), + ) + self.assertEqual( + get_call, + unittest.mock.call( + "http://localhost:9091/transmission/rpc", + json={ + "method": "torrent-get", + "arguments": {"fields": ["name", "uploadRatio"]}, + }, + headers={"X-Transmission-Session-Id": "abc"}, + auth=("alice", "hunter2"), + timeout=15, + ), + ) + payload = get_call.kwargs["json"] + self.assertEqual( + payload["method"], transmission_rpc.constants.RpcMethod.TorrentGet.value + ) + self.assertLessEqual( + set(payload["arguments"]["fields"]), + set(transmission_rpc.constants.TORRENT_GET_ARGS), + ) + + def test_logs_ratios_highest_first_with_none_as_zero(self): + response = self.FakeTorrentGetResponse() + response.json = lambda: { + "arguments": { + "torrents": [ + {"name": "low.iso", "uploadRatio": 0.5}, + {"name": "unset.iso", "uploadRatio": None}, + {"name": "high.iso", "uploadRatio": 2.25}, + ] + } + } + with ( + unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[self.FakeSessionIdResponse(), response], + ), + self.assertLogs(ft.logger, level="INFO") as logs, + ): + ft.log_seed_ratios_via_http(auth=None) + + ratio_lines = [ + r.getMessage() for r in logs.records if r.getMessage().startswith("[ratio]") + ] + self.assertEqual( + ratio_lines, + [ + "[ratio] RATIOS START", + f"[ratio] {'high.iso':<50} → 2.250", + f"[ratio] {'low.iso':<50} → 0.500", + f"[ratio] {'unset.iso':<50} → 0.000", + "[ratio] RATIOS END", + ], + ) + + def test_session_id_request_has_a_timeout(self): + with unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], + ) as mock_post: + ft.log_seed_ratios_via_http() + + first_call_kwargs = mock_post.call_args_list[0].kwargs + self.assertIn("timeout", first_call_kwargs) + + def test_uses_configured_credentials_when_auth_not_explicitly_passed(self): + with ( + unittest.mock.patch.dict( + os.environ, + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, + ), + unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[ + self.FakeSessionIdResponse(), + self.FakeTorrentGetResponse(), + ], + ) as mock_post, + ): + ft.log_seed_ratios_via_http() + + second_call_kwargs = mock_post.call_args_list[1].kwargs + self.assertEqual(second_call_kwargs.get("auth"), ("alice", "hunter2")) + + def test_no_auth_when_credentials_unset(self): + with ( + helpers.isolated_env(), + unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[ + self.FakeSessionIdResponse(), + self.FakeTorrentGetResponse(), + ], + ) as mock_post, + ): + ft.log_seed_ratios_via_http() + + second_call_kwargs = mock_post.call_args_list[1].kwargs + self.assertIsNone(second_call_kwargs.get("auth")) + + def test_explicit_auth_argument_still_wins(self): + with ( + unittest.mock.patch.dict( + os.environ, + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, + ), + unittest.mock.patch.object( + ft.requests, + "post", + side_effect=[ + self.FakeSessionIdResponse(), + self.FakeTorrentGetResponse(), + ], + ) as mock_post, + ): + ft.log_seed_ratios_via_http(auth=("bob", "other")) + + second_call_kwargs = mock_post.call_args_list[1].kwargs + self.assertEqual(second_call_kwargs.get("auth"), ("bob", "other")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_old_release_demand.py b/tests/test_old_release_demand.py new file mode 100644 index 0000000..0003a08 --- /dev/null +++ b/tests/test_old_release_demand.py @@ -0,0 +1,611 @@ +"""Tests for the old-release demand check +(FETCH_TORRENTS_CHECK_OLD_RELEASES): has_unmet_demand(), the +per-release evaluation and the recheck bookkeeping. + +Run with: python -m unittest discover -s tests +""" + +import os +import unittest +import unittest.mock +from datetime import date + +import helpers + +import fetch_torrents as ft + + +def setUpModule(): + helpers.isolate_module() + + +class HasUnmetDemandTests(unittest.TestCase): + """Demand requires an absolute floor (enough leechers to be worth the + disk/bandwidth), a relative one (leechers reach some fraction of the + seeder count - i.e. the swarm isn't already vastly over-served), and by + default a real, currently-seeded copy to exist at all. A single leecher + on a 1000-seeder swarm must never pass just because leechers>0, and a + swarm with plenty of leechers but even more seeders (already well + served) shouldn't either.""" + + def test_none_scrape_result_is_no_demand(self): + self.assertFalse( + ft.has_unmet_demand( + None, min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False + ) + ) + + def test_below_absolute_threshold_is_no_demand(self): + self.assertFalse( + ft.has_unmet_demand( + {"leechers": 0, "seeders": 0}, + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, + ) + ) + + def test_single_leecher_on_huge_swarm_is_no_demand(self): + self.assertFalse( + ft.has_unmet_demand( + {"leechers": 1, "seeders": 1000}, + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, + ) + ) + + def test_meets_absolute_but_not_ratio_is_no_demand(self): + # 11 leechers clears a min_leechers=10 floor, but 11 < 1.2x the 10 seeders. + self.assertFalse( + ft.has_unmet_demand( + {"leechers": 11, "seeders": 10}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=False, + ) + ) + + def test_meets_both_absolute_and_ratio_thresholds_is_demand(self): + self.assertTrue( + ft.has_unmet_demand( + {"leechers": 25, "seeders": 10}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=False, + ) + ) + + def test_exactly_at_ratio_threshold_is_demand(self): + self.assertTrue( + ft.has_unmet_demand( + {"leechers": 12, "seeders": 10}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=False, + ) + ) + + def test_zero_seeders_is_excluded_by_default(self): + # A scrape can't tell us whether the leechers collectively hold every + # piece - with zero seeders there's no verified-complete copy in the + # swarm at all, so we could end up leeching something we can never + # finish and therefore never seed back. Excluded unless the operator + # opts into the gamble via allow_zero_seeders. + self.assertFalse( + ft.has_unmet_demand( + {"leechers": 10, "seeders": 0}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=False, + ) + ) + + def test_zero_seeders_counts_as_demand_when_explicitly_allowed(self): + self.assertTrue( + ft.has_unmet_demand( + {"leechers": 10, "seeders": 0}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=True, + ) + ) + + def test_zero_seeders_still_needs_the_absolute_floor_when_allowed(self): + self.assertFalse( + ft.has_unmet_demand( + {"leechers": 5, "seeders": 0}, + min_leechers=10, + min_leecher_ratio=1.2, + allow_zero_seeders=True, + ) + ) + + def test_lopsided_but_substantial_swarms_still_count_as_demand_at_default_ratio( + self, + ): + # FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO defaults to 0.1: we + # have no way of knowing whether existing seeders have spare upload + # capacity or are bandwidth-limited, so leechers don't need to + # approach or outnumber seeders to represent real demand - 10:100 + # and 100:1000 both clearly indicate room for another seeder. + self.assertTrue( + ft.has_unmet_demand( + {"leechers": 10, "seeders": 100}, + min_leechers=1, + min_leecher_ratio=0.1, + allow_zero_seeders=False, + ) + ) + self.assertTrue( + ft.has_unmet_demand( + {"leechers": 100, "seeders": 1000}, + min_leechers=1, + min_leecher_ratio=0.1, + allow_zero_seeders=False, + ) + ) + + def test_extreme_oversupply_still_fails_at_default_ratio(self): + # 3 leechers on 673 seeders (an actual old Kali release scraped + # during development) is still not demand, even at the more + # permissive 0.1 default - it needs at least 67.3 leechers. + self.assertFalse( + ft.has_unmet_demand( + {"leechers": 3, "seeders": 673}, + min_leechers=1, + min_leecher_ratio=0.1, + allow_zero_seeders=False, + ) + ) + + +class EvaluateOldReleaseDemandTests(unittest.TestCase): + def test_404_is_not_kept_and_not_scraped(self): + response = helpers.fake_response(status=404) + + with ( + unittest.mock.patch.object(ft.requests, "get", return_value=response), + unittest.mock.patch.object( + ft.torrent_scrape, "scrape_torrent" + ) as mock_scrape, + ): + keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, + ) + + mock_scrape.assert_not_called() + self.assertFalse(keep) + self.assertIsNone(torrent_bytes) + self.assertIsNone(scrape_result) + + def test_download_failure_is_not_kept(self): + with unittest.mock.patch.object( + ft.requests, "get", side_effect=ConnectionError("nope") + ): + keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, + ) + + self.assertFalse(keep) + self.assertIsNone(torrent_bytes) + self.assertIsNone(scrape_result) + + def test_scrape_below_threshold_is_not_kept(self): + response = helpers.fake_response(b"fake torrent bytes") + + with ( + unittest.mock.patch.object(ft.requests, "get", return_value=response), + unittest.mock.patch.object( + ft.torrent_scrape, + "scrape_torrent", + return_value={ + "tracker": "http://t.example/scrape", + "seeders": 10, + "leechers": 0, + "completed": 0, + }, + ), + ): + keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, + ) + + self.assertFalse(keep) + self.assertEqual(torrent_bytes, b"fake torrent bytes") + self.assertEqual(scrape_result["leechers"], 0) + + def test_scrape_meeting_threshold_is_kept(self): + response = helpers.fake_response(b"fake torrent bytes") + + with ( + unittest.mock.patch.object(ft.requests, "get", return_value=response), + unittest.mock.patch.object( + ft.torrent_scrape, + "scrape_torrent", + return_value={ + "tracker": "http://t.example/scrape", + "seeders": 10, + "leechers": 30, + "completed": 0, + }, + ), + ): + keep, torrent_bytes, scrape_result = ft.evaluate_old_release_demand( + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, + ) + + self.assertTrue(keep) + self.assertEqual(torrent_bytes, b"fake torrent bytes") + self.assertEqual(scrape_result["leechers"], 30) + + def test_meets_absolute_floor_but_already_well_seeded_is_not_kept(self): + response = helpers.fake_response(b"fake torrent bytes") + + # 1 leecher clears a min_leechers=1 floor, but 1000 seeders already + # cover it - not unmet demand. + with ( + unittest.mock.patch.object(ft.requests, "get", return_value=response), + unittest.mock.patch.object( + ft.torrent_scrape, + "scrape_torrent", + return_value={ + "tracker": "http://t.example/scrape", + "seeders": 1000, + "leechers": 1, + "completed": 0, + }, + ), + ): + keep, _torrent_bytes, scrape_result = ft.evaluate_old_release_demand( + "ubuntu-14.04.6-desktop-amd64.iso", + "https://example.invalid/x.torrent", + min_leechers=1, + min_leecher_ratio=1.2, + allow_zero_seeders=False, + ) + + self.assertFalse(keep) + self.assertEqual(scrape_result["leechers"], 1) + + +class CheckOldReleasesForDemandTests(unittest.TestCase): + def setUp(self): + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) + patcher.start() + self.addCleanup(patcher.stop) + + def test_adds_only_torrents_with_unmet_demand(self): + # Neither name matches LOW_DEMAND_PATTERN - that filtering is + # covered separately by test_low_demand_variant_is_filtered_before_evaluation. + candidates = { + "kali-linux-2026.1-installer-purple-amd64.iso": "https://example.invalid/dead.torrent", + "kali-linux-2026.1-installer-amd64.iso": "https://example.invalid/alive.torrent", + } + + def fake_evaluate( + name, url, min_leechers, min_leecher_ratio, allow_zero_seeders, timeout=15 + ): + if "purple" in name: + return ( + False, + b"dead bytes", + {"tracker": "t", "seeders": 5, "leechers": 0, "completed": 0}, + ) + return ( + True, + b"alive bytes", + {"tracker": "t", "seeders": 5, "leechers": 30, "completed": 0}, + ) + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand", side_effect=fake_evaluate + ), + ): + added, skipped, _ = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) + + self.assertEqual(added, 1) + self.assertEqual(skipped, 1) + alive_path = os.path.join( + self.tmp_watch_dir, "kali-linux-2026.1-installer-amd64.iso.torrent" + ) + dead_path = os.path.join( + self.tmp_watch_dir, "kali-linux-2026.1-installer-purple-amd64.iso.torrent" + ) + with open(alive_path, "rb") as f: + self.assertEqual(f.read(), b"alive bytes") + self.assertFalse(os.path.exists(dead_path)) + + def test_skips_distros_without_old_release_discovery(self): + with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, {}, clear=True): + added, skipped, _ = ft.check_old_releases_for_demand( + ["ubuntu"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) + + self.assertEqual((added, skipped), (0, 0)) + + def test_already_present_torrent_is_not_re_evaluated(self): + name = "kali-linux-2026.1-installer-amd64.iso" + existing_path = os.path.join(self.tmp_watch_dir, f"{name}.torrent") + with open(existing_path, "wb") as f: + f.write(b"already here") + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, + {"kali": lambda: {name: "https://example.invalid/x.torrent"}}, + clear=True, + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): + added, skipped, _ = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) + + mock_evaluate.assert_not_called() + self.assertEqual((added, skipped), (0, 0)) + + def test_unsafe_candidate_name_is_skipped_without_evaluation(self): + name = "kali-linux-2026.1-installer-../../etc/cron.d/evil" + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, + {"kali": lambda: {name: "https://example.invalid/x.torrent"}}, + clear=True, + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): + added, skipped, _ = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) + + mock_evaluate.assert_not_called() + self.assertEqual((added, skipped), (0, 1)) + + def test_low_demand_variant_is_filtered_before_evaluation(self): + name = "kali-linux-2026.1-installer-netinst-amd64.iso" + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, + {"kali": lambda: {name: "https://example.invalid/x.torrent"}}, + clear=True, + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): + added, skipped, _ = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) + + mock_evaluate.assert_not_called() + self.assertEqual((added, skipped), (0, 0)) + + +class ShouldRecheckOldReleaseTests(unittest.TestCase): + """Without this gate, check_old_releases_for_demand() would re-download + and re-scrape every not-yet-wanted candidate on every daily run + forever - this is what lets a recent "no demand" result be trusted for + a while instead.""" + + def test_never_checked_is_due(self): + self.assertTrue( + ft.should_recheck_old_release( + "x", {}, date(2026, 1, 8), recheck_interval_days=7 + ) + ) + + def test_checked_recently_is_not_due(self): + state = {"x": {"last_checked": "2026-01-05", "leechers": 0}} + self.assertFalse( + ft.should_recheck_old_release( + "x", state, date(2026, 1, 8), recheck_interval_days=7 + ) + ) + + def test_checked_exactly_at_interval_is_due(self): + state = {"x": {"last_checked": "2026-01-01", "leechers": 0}} + self.assertTrue( + ft.should_recheck_old_release( + "x", state, date(2026, 1, 8), recheck_interval_days=7 + ) + ) + + def test_checked_past_interval_is_due(self): + state = {"x": {"last_checked": "2025-12-01", "leechers": 0}} + self.assertTrue( + ft.should_recheck_old_release( + "x", state, date(2026, 1, 8), recheck_interval_days=7 + ) + ) + + +class CheckOldReleasesForDemandStateTests(unittest.TestCase): + """check_old_releases_for_demand() must consult both the recheck-state + (skip candidates checked too recently) and the removed-history (skip + candidates cleanup has already given up on) before ever hitting the + network, and must record a fresh check for anything it does evaluate.""" + + def setUp(self): + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) + patcher.start() + self.addCleanup(patcher.stop) + + def test_recently_checked_candidate_is_skipped_without_network_call(self): + name = "kali-linux-2026.1-installer-amd64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + check_state = {name: {"last_checked": "2026-01-05", "leechers": 0}} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): + added, skipped, new_state = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state=check_state, + removed_history={}, + today=date(2026, 1, 8), + recheck_interval_days=7, + ) + + mock_evaluate.assert_not_called() + self.assertEqual((added, skipped), (0, 1)) + self.assertEqual(new_state, check_state) + + def test_previously_removed_candidate_is_skipped_without_network_call(self): + name = "kali-linux-2026.1-installer-amd64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + removed_history = {name: {"removed_date": "2026-01-01", "reason": "stagnant"}} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, "evaluate_old_release_demand" + ) as mock_evaluate, + ): + added, skipped, _new_state = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state={}, + removed_history=removed_history, + today=date(2026, 1, 8), + recheck_interval_days=7, + ) + + mock_evaluate.assert_not_called() + self.assertEqual((added, skipped), (0, 1)) + + def test_due_candidate_is_evaluated_and_recorded(self): + name = "kali-linux-2026.1-installer-amd64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, + "evaluate_old_release_demand", + return_value=( + True, + b"bytes", + {"tracker": "t", "seeders": 1, "leechers": 30, "completed": 0}, + ), + ), + ): + added, skipped, new_state = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state={}, + removed_history={}, + today=date(2026, 1, 8), + recheck_interval_days=7, + ) + + self.assertEqual((added, skipped), (1, 0)) + self.assertEqual( + new_state[name], {"last_checked": "2026-01-08", "leechers": 30} + ) + + def test_no_demand_result_is_still_recorded_for_future_recheck_gating(self): + name = "kali-linux-2026.1-installer-arm64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, + "evaluate_old_release_demand", + return_value=( + False, + b"bytes", + {"tracker": "t", "seeders": 1, "leechers": 0, "completed": 0}, + ), + ), + ): + added, skipped, new_state = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state={}, + removed_history={}, + today=date(2026, 1, 8), + recheck_interval_days=7, + ) + + self.assertEqual((added, skipped), (0, 1)) + self.assertEqual(new_state[name], {"last_checked": "2026-01-08", "leechers": 0}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_persistence.py b/tests/test_persistence.py new file mode 100644 index 0000000..f6e2584 --- /dev/null +++ b/tests/test_persistence.py @@ -0,0 +1,103 @@ +"""Tests for the JSON state files the daily run keeps between runs. + +Run with: python -m unittest discover -s tests +""" + +import os +import unittest +import unittest.mock + +import helpers + +import fetch_torrents as ft + + +def setUpModule(): + helpers.isolate_module() + + +class RatioHistoryPersistenceTests(unittest.TestCase): + """load_ratio_history()/save_ratio_history() must round-trip cleanly and + tolerate a missing or corrupt file rather than crashing a run.""" + + def test_round_trips_through_disk(self): + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") + path = os.path.join(tmp_dir, "fetch_torrents_ratio_history.json") + history = { + "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-29", "ratio": 1.2}] + } + + ft.save_ratio_history(path, history) + loaded = ft.load_ratio_history(path) + + self.assertEqual(loaded, history) + + def test_missing_file_returns_empty_history(self): + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") + path = os.path.join(tmp_dir, "does-not-exist.json") + + self.assertEqual(ft.load_ratio_history(path), {}) + + def test_corrupt_file_returns_empty_history_instead_of_raising(self): + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") + path = os.path.join(tmp_dir, "fetch_torrents_ratio_history.json") + with open(path, "w", encoding="utf-8") as f: + f.write("{not valid json") + + self.assertEqual(ft.load_ratio_history(path), {}) + + +class OldReleaseCheckStatePersistenceTests(unittest.TestCase): + def test_round_trips_through_disk(self): + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") + path = os.path.join(tmp_dir, "fetch_torrents_old_release_check_state.json") + state = {"x": {"last_checked": "2026-01-08", "leechers": 0}} + + ft.save_old_release_check_state(path, state) + loaded = ft.load_old_release_check_state(path) + + self.assertEqual(loaded, state) + + def test_missing_file_returns_empty_state(self): + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") + path = os.path.join(tmp_dir, "does-not-exist.json") + + self.assertEqual(ft.load_old_release_check_state(path), {}) + + def test_corrupt_file_returns_empty_state_instead_of_raising(self): + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") + path = os.path.join(tmp_dir, "fetch_torrents_old_release_check_state.json") + with open(path, "w", encoding="utf-8") as f: + f.write("{not valid json") + + self.assertEqual(ft.load_old_release_check_state(path), {}) + + +class RemovedHistoryPersistenceTests(unittest.TestCase): + def test_round_trips_through_disk(self): + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") + path = os.path.join(tmp_dir, "fetch_torrents_removed_history.json") + history = {"x": {"removed_date": "2026-02-01", "reason": "stagnant"}} + + ft.save_removed_history(path, history) + loaded = ft.load_removed_history(path) + + self.assertEqual(loaded, history) + + def test_missing_file_returns_empty_history(self): + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") + path = os.path.join(tmp_dir, "does-not-exist.json") + + self.assertEqual(ft.load_removed_history(path), {}) + + def test_corrupt_file_returns_empty_history_instead_of_raising(self): + tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") + path = os.path.join(tmp_dir, "fetch_torrents_removed_history.json") + with open(path, "w", encoding="utf-8") as f: + f.write("{not valid json") + + self.assertEqual(ft.load_removed_history(path), {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_ratio_gate.py b/tests/test_ratio_gate.py new file mode 100644 index 0000000..2acf4ac --- /dev/null +++ b/tests/test_ratio_gate.py @@ -0,0 +1,603 @@ +"""Tests for the ratio gate: parse_version_type(), should_fetch_torrent() +(and its removed-history / Devuan variants) and reading the previous +run's ratio log with get_previous_ratios(). + +Run with: python -m unittest discover -s tests +""" + +import os +import unittest +import unittest.mock + +import helpers + +import fetch_torrents as ft + + +def setUpModule(): + helpers.isolate_module() + + +class ParseVersionTypeTests(unittest.TestCase): + """(version, type_) is the grouping key for the ratio gate and cleanup: + two names share a ratio history only when their type_ is equal.""" + + CASES = ( + ( + "ubuntu-24.04-desktop-amd64.iso", + "ubuntu", + ("24.04", "ubuntu-desktop-amd64.iso"), + ), + ( + "ubuntu-24.04-desktop-arm64.iso", + "ubuntu", + ("24.04", "ubuntu-desktop-arm64.iso"), + ), + ( + "xubuntu-24.04-desktop-amd64.iso", + "ubuntu", + ("24.04", "xubuntu-desktop-amd64.iso"), + ), + ("debian-12.5.0-amd64-DVD-1.iso", "debian", ("12.5.0", "amd64-DVD-1.iso")), + ("debian-12.5.0-arm64-DVD-1.iso", "debian", ("12.5.0", "arm64-DVD-1.iso")), + ( + "kali-linux-2024.1-installer-amd64.iso", + "kali", + ("2024.1", "installer-amd64.iso"), + ), + ( + "kali-linux-2024.1-installer-arm64.iso", + "kali", + ("2024.1", "installer-arm64.iso"), + ), + ("archlinux-2024.05.01-x86_64.iso", "arch", ("2024.05.01", "")), + ("linuxmint-22.3-cinnamon-64bit.iso", "mint", ("22.3", "cinnamon-64bit.iso")), + ("Fedora-Workstation-Live-x86_64-44", "fedora", ("44", "x86_64")), + ("Fedora-Workstation-Live-aarch64-44", "fedora", ("44", "aarch64")), + ("anything-1.0", "not-a-distro", ("", "")), + ) + + def test_every_ubuntu_flavour_fetched_is_recognised_as_ubuntu(self): + # fetch_ubuntu_lts() fetches ubuntu, lubuntu and xubuntu images; a + # flavour get_distro() doesn't recognise skips the ratio gate and + # cleanup grouping entirely. + for name in ( + "ubuntu-24.04-desktop-amd64.iso", + "lubuntu-24.04-desktop-amd64.iso", + "xubuntu-24.04-desktop-amd64.iso", + ): + with self.subTest(name): + self.assertEqual(ft.get_distro(name), "ubuntu") + + def test_lubuntu_is_ratio_gated_on_its_own_previous_release(self): + ratios = { + "lubuntu-23.10-desktop-amd64.iso": 0.2, + "ubuntu-23.10-desktop-amd64.iso": 5.0, + } + self.assertFalse( + ft.should_fetch_torrent("lubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_version_and_type_per_distro(self): + for name, distro, expected in self.CASES: + with self.subTest(name): + self.assertEqual(ft.parse_version_type(name, distro), expected) + + def test_architectures_of_the_same_release_get_different_types(self): + pairs = [ + ( + "ubuntu-24.04-desktop-amd64.iso", + "ubuntu-24.04-desktop-arm64.iso", + "ubuntu", + ), + ( + "debian-12.5.0-amd64-DVD-1.iso", + "debian-12.5.0-arm64-DVD-1.iso", + "debian", + ), + ( + "kali-linux-2024.1-installer-amd64.iso", + "kali-linux-2024.1-installer-arm64.iso", + "kali", + ), + ( + "Fedora-Workstation-Live-x86_64-44", + "Fedora-Workstation-Live-aarch64-44", + "fedora", + ), + ] + for a, b, distro in pairs: + with self.subTest(distro): + self.assertNotEqual( + ft.parse_version_type(a, distro)[1], + ft.parse_version_type(b, distro)[1], + ) + + +class ShouldFetchTorrentRatioKeyTests(unittest.TestCase): + """Candidate names must key into the same ratio-lookup bucket as the names Transmission + actually reports (which include the arch suffix and .iso extension), + for all supported distros.""" + + def test_finds_previous_version_and_honors_low_ratio_ubuntu(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.5} + self.assertFalse( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_finds_previous_version_and_honors_low_ratio_debian(self): + ratios = {"debian-12.4.0-amd64-DVD-1.iso": 0.5} + self.assertFalse( + ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios) + ) + + def test_finds_previous_version_and_honors_low_ratio_kali(self): + ratios = {"kali-linux-2023.4-installer-amd64.iso": 0.3} + self.assertFalse( + ft.should_fetch_torrent("kali-linux-2024.1-installer-amd64.iso", ratios) + ) + + def test_finds_previous_version_and_honors_low_ratio_arch(self): + ratios = {"archlinux-2024.04.01-x86_64.iso": 0.2} + self.assertFalse( + ft.should_fetch_torrent("archlinux-2024.05.01-x86_64.iso", ratios) + ) + + def test_finds_previous_version_and_allows_high_ratio_ubuntu(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.5} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_finds_previous_version_and_allows_high_ratio_debian(self): + ratios = {"debian-12.4.0-amd64-DVD-1.iso": 1.5} + self.assertTrue( + ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios) + ) + + def test_finds_previous_version_and_honors_low_ratio_mint(self): + ratios = {"linuxmint-22.2-cinnamon-64bit.iso": 0.4} + self.assertFalse( + ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios) + ) + + def test_finds_previous_version_and_allows_high_ratio_mint(self): + ratios = {"linuxmint-22.2-cinnamon-64bit.iso": 1.5} + self.assertTrue( + ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios) + ) + + def test_finds_previous_version_and_honors_low_ratio_fedora(self): + ratios = {"Fedora-Workstation-Live-x86_64-43": 0.4} + self.assertFalse( + ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios) + ) + + def test_finds_previous_version_and_allows_high_ratio_fedora(self): + ratios = {"Fedora-Workstation-Live-x86_64-43": 1.5} + self.assertTrue( + ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios) + ) + + +class ShouldFetchTorrentRemovedHistoryFallbackTests(unittest.TestCase): + """Once a previous version is removed by cleanup, Transmission stops + reporting it and it drops out of `ratios` (see get_previous_ratios()) - + without a fallback, should_fetch_torrent() would then hit its "no + previous version" branch and fetch unconditionally, silently bypassing + the ratio gate for exactly the versions cleanup already gave up on. + removed_history (record_removal()'s output) is the fallback source.""" + + def test_falls_back_to_removed_history_ratio_when_not_live(self): + removed_history = { + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.4, + }, + } + self.assertFalse( + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) + ) + + def test_removed_history_ratio_above_floor_allows_fetch(self): + removed_history = { + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "keep_only_latest", + "ratio": 1.5, + }, + } + self.assertTrue( + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) + ) + + def test_live_ratio_takes_priority_over_removed_history(self): + # Same name in both is not expected in practice (a torrent Transmission + # still reports hasn't been removed), but live data winning if it ever + # happens is the safer default. + removed_history = { + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + }, + } + ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.5} + self.assertTrue( + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios, + removed_history=removed_history, + ) + ) + + def test_low_live_ratio_is_not_overridden_by_high_removed_history(self): + removed_history = { + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 1.5, + } + } + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.4} + self.assertFalse( + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios, + removed_history=removed_history, + ) + ) + + def test_removed_history_entry_without_ratio_is_ignored(self): + # Legacy entries recorded before this field existed have no 'ratio' + # key - must not crash, and must not count as a usable prior version. + removed_history = { + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + }, + } + # Skipped up front, not by tripping over the missing key and logging + # a parse-error warning. + with self.assertNoLogs(ft.logger, level="WARNING"): + result = ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) + self.assertTrue(result) + + def test_skipped_removed_history_entry_does_not_hide_later_ones(self): + removed_history = { + "ubuntu-23.04-desktop-amd64.iso": { + "removed_date": "2025-06-01", + "reason": "stagnant", + }, + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.4, + }, + } + self.assertFalse( + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) + ) + + def test_removed_history_from_a_different_distro_is_not_consulted(self): + removed_history = { + "debian-12.4.0-amd64-DVD-1.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + }, + } + self.assertTrue( + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", + ratios={}, + removed_history=removed_history, + ) + ) + + +class ShouldFetchTorrentDevuanRatioGateTests(unittest.TestCase): + """Devuan has no per-name version to compare (get_distro() returns None + for it - see distro_patterns), so should_fetch_torrent() instead gates + fetching a new Devuan release on the ratio achieved by whichever Devuan + release we currently hold, using the same 1.0 floor as other distros.""" + + def test_no_previous_devuan_release_fetches(self): + self.assertTrue(ft.should_fetch_torrent("devuan_excalibur", ratios={})) + + def test_low_ratio_on_current_release_blocks_next_release(self): + ratios = {"devuan_daedalus": 0.4} + self.assertFalse(ft.should_fetch_torrent("devuan_excalibur", ratios)) + + def test_high_ratio_on_current_release_allows_next_release(self): + ratios = {"devuan_daedalus": 1.5} + self.assertTrue(ft.should_fetch_torrent("devuan_excalibur", ratios)) + + def test_already_holding_this_exact_release_is_always_fetched(self): + # download_torrent()'s own file-existence check is what actually + # dedupes a re-offered current release - not this ratio gate. + ratios = {"devuan_excalibur": 0.1} + self.assertTrue(ft.should_fetch_torrent("devuan_excalibur", ratios)) + + def test_falls_back_to_removed_history_when_no_live_ratio(self): + removed_history = { + "devuan_daedalus": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.4, + }, + } + self.assertFalse( + ft.should_fetch_torrent( + "devuan_excalibur", ratios={}, removed_history=removed_history + ) + ) + + def test_other_distros_removed_history_not_consulted_for_devuan(self): + removed_history = { + "debian-12.4.0-amd64-DVD-1.iso": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + }, + } + self.assertTrue( + ft.should_fetch_torrent( + "devuan_excalibur", ratios={}, removed_history=removed_history + ) + ) + + +class ShouldFetchTorrentBoundaryTests(unittest.TestCase): + """The 1.0 floor is inclusive, only strictly older versions count as the + "previous" one, and the branches that bypass the gate entirely.""" + + def test_previous_ratio_exactly_at_floor_allows_fetch(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.0} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_previous_ratio_just_below_floor_blocks_fetch(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.999} + self.assertFalse( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_devuan_previous_ratio_exactly_at_floor_allows_fetch(self): + self.assertTrue( + ft.should_fetch_torrent("devuan_excalibur", {"devuan_daedalus": 1.0}) + ) + + def test_devuan_previous_ratio_just_below_floor_blocks_fetch(self): + self.assertFalse( + ft.should_fetch_torrent("devuan_excalibur", {"devuan_daedalus": 0.999}) + ) + + def test_only_newer_versions_on_record_fetches(self): + # A newer release with a low ratio isn't a "previous" one, so there + # is nothing to gate on. + ratios = {"ubuntu-24.10-desktop-amd64.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_same_version_on_record_is_not_its_own_predecessor(self): + ratios = {"ubuntu-24.04-desktop-amd64.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_highest_older_version_decides_not_any_older_version(self): + ratios = { + "ubuntu-22.04-desktop-amd64.iso": 5.0, + "ubuntu-23.10-desktop-amd64.iso": 0.4, + } + self.assertFalse( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_other_architecture_ratio_is_not_consulted(self): + ratios = {"ubuntu-23.10-desktop-arm64.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + def test_devuan_ignores_non_devuan_ratios(self): + ratios = {"debian-12.4.0-amd64-DVD-1.iso": 5.0, "devuan_daedalus": 0.4} + self.assertFalse(ft.should_fetch_torrent("devuan_excalibur", ratios)) + + def test_devuan_removed_history_for_the_same_release_is_not_a_predecessor(self): + removed_history = { + "devuan_excalibur": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + } + } + self.assertTrue( + ft.should_fetch_torrent( + "devuan_excalibur", ratios={}, removed_history=removed_history + ) + ) + + def test_devuan_live_ratio_beats_removed_history(self): + removed_history = { + "devuan_chimaera": { + "removed_date": "2026-01-01", + "reason": "stagnant", + "ratio": 0.1, + } + } + self.assertTrue( + ft.should_fetch_torrent( + "devuan_excalibur", + {"devuan_daedalus": 1.5}, + removed_history=removed_history, + ) + ) + + def test_skip_ratio_check_fetches_despite_low_previous_ratio(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent( + "ubuntu-24.04-desktop-amd64.iso", ratios, skip_ratio_check=True + ) + ) + + def test_unknown_distro_fetches(self): + ratios = {"slackware-15.0-install-dvd.iso": 0.1} + self.assertTrue( + ft.should_fetch_torrent("slackware-15.1-install-dvd.iso", ratios) + ) + + def test_unparseable_name_of_known_distro_fetches_and_logs_error(self): + ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.1} + with self.assertLogs(ft.logger, level="ERROR") as logs: + result = ft.should_fetch_torrent("ubuntu-noble-desktop-amd64.iso", ratios) + + self.assertTrue(result) + self.assertEqual(len(logs.records), 1) + self.assertIn("ubuntu-noble-desktop-amd64.iso", logs.output[0]) + + def test_unparseable_stored_ratio_entry_is_skipped(self): + ratios = { + "ubuntu-mantic-desktop-amd64.iso": 0.1, + "ubuntu-23.10-desktop-amd64.iso": 1.5, + } + with self.assertLogs(ft.logger, level="WARNING"): + self.assertTrue( + ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) + ) + + +class GetPreviousRatiosTests(unittest.TestCase): + """get_previous_ratios() is what feeds should_fetch_torrent()'s ratios: + only the last START/END segment of the previous run's ratio log counts, + so a ratio from an older run can't satisfy (or block) the gate.""" + + def _write_log(self, *messages): + path = os.path.join( + helpers.make_temp_dir(self, "fetch_torrents_prev_ratios_"), + "fetch_torrents_ratios.log", + ) + with open(path, "w", encoding="utf-8") as f: + f.writelines( + f"2026-08-01 00:00:00,000 INFO: {message}\n" for message in messages + ) + return path + + def test_missing_file_returns_empty(self): + missing = os.path.join( + helpers.make_temp_dir(self, "fetch_torrents_prev_ratios_"), "nope.log" + ) + self.assertEqual(ft.get_previous_ratios(missing), {}) + + def test_without_markers_every_ratio_line_is_parsed(self): + path = self._write_log( + "[ratio] ubuntu-24.04-desktop-amd64.iso → 1.250", + "unrelated line", + "[ratio] debian-12.5.0-amd64-DVD-1.iso → 0.664", + ) + self.assertEqual( + ft.get_previous_ratios(path), + { + "ubuntu-24.04-desktop-amd64.iso": 1.25, + "debian-12.5.0-amd64-DVD-1.iso": 0.664, + }, + ) + + def test_only_the_last_complete_segment_is_returned(self): + path = self._write_log( + "[ratio] RATIOS START", + "[ratio] old-only.iso → 0.100", + "[ratio] shared.iso → 0.200", + "[ratio] RATIOS END", + "[ratio] RATIOS START", + "[ratio] shared.iso → 1.500", + "[ratio] RATIOS END", + "[ratio] after-end.iso → 0.300", + ) + self.assertEqual(ft.get_previous_ratios(path), {"shared.iso": 1.5}) + + def test_start_without_end_parses_to_end_of_file(self): + # A run that crashed mid-way never wrote its END marker. + path = self._write_log( + "[ratio] RATIOS START", + "[ratio] old.iso → 0.100", + "[ratio] RATIOS END", + "[ratio] RATIOS START", + "[ratio] crashed-run.iso → 0.700", + ) + self.assertEqual(ft.get_previous_ratios(path), {"crashed-run.iso": 0.7}) + + def test_end_before_the_last_start_is_ignored(self): + path = self._write_log( + "[ratio] RATIOS END", + "[ratio] before-start.iso → 0.100", + "[ratio] RATIOS START", + "[ratio] current.iso → 0.900", + ) + self.assertEqual(ft.get_previous_ratios(path), {"current.iso": 0.9}) + + def test_last_end_after_the_last_start_closes_the_segment(self): + # Not written by a normal run; pins current behavior. + path = self._write_log( + "[ratio] RATIOS START", + "[ratio] first.iso → 0.100", + "[ratio] RATIOS END", + "[ratio] second.iso → 0.200", + "[ratio] RATIOS END", + "[ratio] after.iso → 0.300", + ) + self.assertEqual( + ft.get_previous_ratios(path), {"first.iso": 0.1, "second.iso": 0.2} + ) + + def test_padded_names_are_stripped(self): + path = self._write_log( + "[ratio] archlinux-2026.04.01-x86_64.iso → 0.664", + ) + self.assertEqual( + ft.get_previous_ratios(path), {"archlinux-2026.04.01-x86_64.iso": 0.664} + ) + + def test_names_with_internal_spaces_keep_them(self): + path = self._write_log("[ratio] Some Torrent Name → 2.000") + self.assertEqual(ft.get_previous_ratios(path), {"Some Torrent Name": 2.0}) + + def test_ratio_without_decimal_point_is_not_matched(self): + # Pins current behavior: log_seed_ratios_via_http() always writes + # three decimals, so an integer ratio means a line it didn't write. + path = self._write_log("[ratio] whole.iso → 1") + self.assertEqual(ft.get_previous_ratios(path), {}) + + def test_duplicate_name_in_a_segment_last_one_wins(self): + path = self._write_log( + "[ratio] RATIOS START", + "[ratio] dup.iso → 0.100", + "[ratio] dup.iso → 1.100", + "[ratio] RATIOS END", + ) + self.assertEqual(ft.get_previous_ratios(path), {"dup.iso": 1.1}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scrapers.py b/tests/test_scrapers.py index c54d967..4fecb71 100644 --- a/tests/test_scrapers.py +++ b/tests/test_scrapers.py @@ -11,14 +11,9 @@ """ import os -import sys import unittest import unittest.mock -REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if REPO_ROOT not in sys.path: - sys.path.insert(0, REPO_ROOT) - import helpers import requests @@ -26,10 +21,7 @@ def setUpModule(): - unittest.addModuleCleanup(helpers.block_network()) - patcher = helpers.isolated_env() - patcher.start() - unittest.addModuleCleanup(patcher.stop) + helpers.isolate_module() UBUNTU_SUPPORTED = ( # newest first, as fetch_ubuntu_lts() orders them diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index b075656..ba89021 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -10,22 +10,17 @@ import os import socket import struct -import sys import tempfile import unittest import unittest.mock -REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -if REPO_ROOT not in sys.path: - sys.path.insert(0, REPO_ROOT) - import helpers import torrent_scrape as ts def setUpModule(): - unittest.addModuleCleanup(helpers.block_network()) + helpers.isolate_module() def bstr(s): From 1542f88c3271b9ceebb365e0cb0e34d68a3d7925 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 13:09:53 +0200 Subject: [PATCH 23/46] Copy scripts/ into the Dockerfile test stage tests/test_live_check.py (task 23) imports scripts/live_check.py, so the in-image unit suite failed to import it. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 7bde249..c633d20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,6 +44,7 @@ ENTRYPOINT ["/entrypoint.sh"] FROM runtime AS test WORKDIR /src COPY fetch_torrents.py torrent_scrape.py configure_transmission.py ./ +COPY scripts ./scripts COPY tests ./tests RUN python3 -m unittest discover -s tests From 98444f4ce0dfc894ad7fc8bfd1cd3c78a7a1ed8d Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 13:10:01 +0200 Subject: [PATCH 24/46] Tighten weak assertions and share one persistence contract (task 26, part 2) - Presence-only checks become exact values: fetch_ubuntu_lts' full candidate dict, fetch_kali_old_versions' full result, warn_if_open_without_auth's message, the cleanup parse-error warnings, the session-id request's timeout (== 15), and assertIs(..., True) for should_fetch_torrent. - test_logging's two hand-rolled RPC response classes become helpers.fake_response(); no ad-hoc response class is left in tests/. - The load/save round-trip, missing-file and corrupt-file tests, written four times (ratio history, removed history, old-release check state, settings.json), are now one JsonStoreContract mixin. It also checks that a corrupt file is reported (logger or stderr) and that saving again leaves only the file. 12 -> 16 tests. Suite: 382 tests, green; hostile-env run and in-image run also green. Co-Authored-By: Claude Opus 5.5 --- tests/test_cleanup.py | 14 ++- tests/test_configure_transmission.py | 35 ++----- tests/test_fetchers.py | 39 +++++-- tests/test_logging.py | 57 +++++----- tests/test_persistence.py | 151 ++++++++++++++++----------- tests/test_ratio_gate.py | 4 +- 6 files changed, 176 insertions(+), 124 deletions(-) diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index 7712ee6..ce8fe78 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -241,7 +241,19 @@ def test_unparseable_name_is_skipped_with_a_warning(self): to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) self.assertEqual((to_remove, to_keep_low_ratio), ([], [])) - self.assertEqual(len(logs.records), 2) + self.assertEqual( + [r.getMessage() for r in logs.records], + [ + ( + "Skipping ubuntu-noble-desktop-amd64.iso during cleanup due to " + "parse error: Invalid version string 'noble'" + ), + ( + "Skipping ubuntu-mantic-desktop-amd64.iso during cleanup due to " + "parse error: Invalid version string 'mantic'" + ), + ], + ) class PlanCleanupDevuanGroupingTests(unittest.TestCase): diff --git a/tests/test_configure_transmission.py b/tests/test_configure_transmission.py index ee7a6dc..97744e8 100644 --- a/tests/test_configure_transmission.py +++ b/tests/test_configure_transmission.py @@ -196,7 +196,14 @@ def test_warns_when_whitelist_broadened_without_auth(self): {"rpc-whitelist-enabled": True, "rpc-whitelist": "*"} ) - self.assertIsNotNone(warning) + self.assertEqual( + warning, + "TRANSMISSION_RPC_WHITELIST is set but TRANSMISSION_RPC_USERNAME/" + "TRANSMISSION_RPC_PASSWORD are not - the web UI/RPC endpoint is " + "reachable from that whitelist with no login required, and " + "Transmission has no read-only RPC mode. Set both env vars to " + "require authentication.", + ) def test_no_warning_when_auth_also_required(self): warning = ct.warn_if_open_without_auth( @@ -231,32 +238,6 @@ def test_overrides_take_precedence_but_other_keys_survive(self): self.assertTrue(merged["rpc-whitelist-enabled"]) -class SettingsFilePersistenceTests(unittest.TestCase): - def test_round_trips_through_disk(self): - tmp_dir = helpers.make_temp_dir(self, "configure_transmission_") - path = os.path.join(tmp_dir, "settings.json") - settings = {"rpc-whitelist": "10.0.0.*"} - - ct.save_settings(path, settings) - loaded = ct.load_settings(path) - - self.assertEqual(loaded, settings) - - def test_missing_file_returns_empty_settings(self): - tmp_dir = helpers.make_temp_dir(self, "configure_transmission_") - path = os.path.join(tmp_dir, "does-not-exist.json") - - self.assertEqual(ct.load_settings(path), {}) - - def test_corrupt_file_returns_empty_settings_instead_of_raising(self): - tmp_dir = helpers.make_temp_dir(self, "configure_transmission_") - path = os.path.join(tmp_dir, "settings.json") - with open(path, "w", encoding="utf-8") as f: - f.write("{not valid json") - - self.assertEqual(ct.load_settings(path), {}) - - class MainTests(unittest.TestCase): """main() runs at every container start, before the daemon: env vars in, settings.json out.""" diff --git a/tests/test_fetchers.py b/tests/test_fetchers.py index ac34e41..646d0e6 100644 --- a/tests/test_fetchers.py +++ b/tests/test_fetchers.py @@ -34,9 +34,30 @@ def test_candidate_names_include_arch_and_iso_suffix(self): with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_ubuntu_lts() - self.assertTrue(results) - self.assertIn("ubuntu-24.04-desktop-amd64.iso", results) - self.assertIn("ubuntu-24.04-live-server-amd64.iso", results) + self.assertEqual( + results, + { + "ubuntu-24.04-desktop-amd64.iso": ( + "https://releases.ubuntu.com/noble/ubuntu-24.04-desktop-amd64.iso.torrent" + ), + "ubuntu-24.04-live-server-amd64.iso": ( + "https://releases.ubuntu.com/noble/" + "ubuntu-24.04-live-server-amd64.iso.torrent" + ), + "lubuntu-24.04-desktop-amd64.iso": ( + "https://cdimage.ubuntu.com/lubuntu/releases/noble/release/" + "lubuntu-24.04-desktop-amd64.iso.torrent" + ), + "xubuntu-24.04-desktop-amd64.iso": ( + "https://torrent.ubuntu.com/xubuntu/releases/noble/release/desktop/" + "xubuntu-24.04-desktop-amd64.iso.torrent" + ), + "xubuntu-24.04-minimal-amd64.iso": ( + "https://torrent.ubuntu.com/xubuntu/releases/noble/release/minimal/" + "xubuntu-24.04-minimal-amd64.iso.torrent" + ), + }, + ) class FetchDebianStableNamingTests(unittest.TestCase): @@ -198,9 +219,15 @@ def test_returns_only_the_non_newest_versions(self): with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_kali_old_versions() - self.assertTrue(results) - self.assertTrue(any("2026.1" in name for name in results)) - self.assertFalse(any("2026.2" in name for name in results)) + # The URL templates themselves are pinned in tests/test_scrapers.py; + # what matters here is which version they're built for. + self.assertEqual( + results, + { + os.path.basename(url).removesuffix(".torrent"): url + for url in ft._kali_torrent_urls("2026.1") + }, + ) def test_single_version_folder_returns_nothing(self): html = 'currentkali-2026.2' diff --git a/tests/test_logging.py b/tests/test_logging.py index b6286f9..1da8708 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -5,6 +5,7 @@ Run with: python -m unittest discover -s tests """ +import json import logging import logging.handlers import os @@ -158,16 +159,17 @@ class LogSeedRatiosViaHttpTests(unittest.TestCase): credentials configure_transmission.py configured the daemon with when the caller doesn't explicitly override auth.""" - class FakeSessionIdResponse: - def __init__(self): - self.headers = {"X-Transmission-Session-Id": "abc"} - - class FakeTorrentGetResponse: - def raise_for_status(self): - pass + @staticmethod + def session_id_response(): + return helpers.fake_response( + status=409, headers={"X-Transmission-Session-Id": "abc"} + ) - def json(self): - return {"arguments": {"torrents": []}} + @staticmethod + def torrent_get_response(torrents=()): + return helpers.fake_response( + json.dumps({"arguments": {"torrents": list(torrents)}}) + ) def test_requests_follow_the_transmission_rpc_protocol(self): # The pinned transmission_rpc's own tables of methods and torrent-get @@ -175,7 +177,7 @@ def test_requests_follow_the_transmission_rpc_protocol(self): with unittest.mock.patch.object( ft.requests, "post", - side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], + side_effect=[self.session_id_response(), self.torrent_get_response()], ) as mock_post: ft.log_seed_ratios_via_http(auth=("alice", "hunter2")) @@ -207,21 +209,18 @@ def test_requests_follow_the_transmission_rpc_protocol(self): ) def test_logs_ratios_highest_first_with_none_as_zero(self): - response = self.FakeTorrentGetResponse() - response.json = lambda: { - "arguments": { - "torrents": [ - {"name": "low.iso", "uploadRatio": 0.5}, - {"name": "unset.iso", "uploadRatio": None}, - {"name": "high.iso", "uploadRatio": 2.25}, - ] - } - } + response = self.torrent_get_response( + [ + {"name": "low.iso", "uploadRatio": 0.5}, + {"name": "unset.iso", "uploadRatio": None}, + {"name": "high.iso", "uploadRatio": 2.25}, + ] + ) with ( unittest.mock.patch.object( ft.requests, "post", - side_effect=[self.FakeSessionIdResponse(), response], + side_effect=[self.session_id_response(), response], ), self.assertLogs(ft.logger, level="INFO") as logs, ): @@ -245,12 +244,12 @@ def test_session_id_request_has_a_timeout(self): with unittest.mock.patch.object( ft.requests, "post", - side_effect=[self.FakeSessionIdResponse(), self.FakeTorrentGetResponse()], + side_effect=[self.session_id_response(), self.torrent_get_response()], ) as mock_post: ft.log_seed_ratios_via_http() first_call_kwargs = mock_post.call_args_list[0].kwargs - self.assertIn("timeout", first_call_kwargs) + self.assertEqual(first_call_kwargs, {"timeout": 15}) def test_uses_configured_credentials_when_auth_not_explicitly_passed(self): with ( @@ -265,8 +264,8 @@ def test_uses_configured_credentials_when_auth_not_explicitly_passed(self): ft.requests, "post", side_effect=[ - self.FakeSessionIdResponse(), - self.FakeTorrentGetResponse(), + self.session_id_response(), + self.torrent_get_response(), ], ) as mock_post, ): @@ -282,8 +281,8 @@ def test_no_auth_when_credentials_unset(self): ft.requests, "post", side_effect=[ - self.FakeSessionIdResponse(), - self.FakeTorrentGetResponse(), + self.session_id_response(), + self.torrent_get_response(), ], ) as mock_post, ): @@ -305,8 +304,8 @@ def test_explicit_auth_argument_still_wins(self): ft.requests, "post", side_effect=[ - self.FakeSessionIdResponse(), - self.FakeTorrentGetResponse(), + self.session_id_response(), + self.torrent_get_response(), ], ) as mock_post, ): diff --git a/tests/test_persistence.py b/tests/test_persistence.py index f6e2584..e306c25 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -1,14 +1,20 @@ -"""Tests for the JSON state files the daily run keeps between runs. +"""Tests for the JSON state files kept between runs: fetch_torrents' ratio +history, removed-torrent history and old-release check state, and +configure_transmission's settings.json. All four share one contract +(JsonStoreContract), so a store can't quietly drift from the others. Run with: python -m unittest discover -s tests """ +import contextlib +import io import os import unittest -import unittest.mock +from typing import ClassVar import helpers +import configure_transmission as ct import fetch_torrents as ft @@ -16,87 +22,114 @@ def setUpModule(): helpers.isolate_module() -class RatioHistoryPersistenceTests(unittest.TestCase): - """load_ratio_history()/save_ratio_history() must round-trip cleanly and - tolerate a missing or corrupt file rather than crashing a run.""" +class JsonStoreContract: + """Mixed into one TestCase per store. A store must round-trip, treat a + missing file as empty, and treat a corrupt one as empty *and say so* + (rather than crash the run), and saving must leave only the file.""" - def test_round_trips_through_disk(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") - path = os.path.join(tmp_dir, "fetch_torrents_ratio_history.json") - history = { - "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-29", "ratio": 1.2}] - } + filename = "" + sample: ClassVar[dict] = {} + corrupt_report = "" # start of what a corrupt file makes the store report + + def load(self, path): + raise NotImplementedError - ft.save_ratio_history(path, history) - loaded = ft.load_ratio_history(path) + def save(self, path, data): + raise NotImplementedError - self.assertEqual(loaded, history) + def load_reporting(self, path): + """load(path), plus whatever it reported, as text.""" + with self.assertLogs(ft.logger, level="WARNING") as logs: + data = self.load(path) + return data, "\n".join(r.getMessage() for r in logs.records) - def test_missing_file_returns_empty_history(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") - path = os.path.join(tmp_dir, "does-not-exist.json") + def setUp(self): + self.dir = helpers.make_temp_dir(self, "persistence_") + self.path = os.path.join(self.dir, self.filename) + + def test_round_trips_through_disk(self): + self.save(self.path, self.sample) - self.assertEqual(ft.load_ratio_history(path), {}) + self.assertEqual(self.load(self.path), self.sample) - def test_corrupt_file_returns_empty_history_instead_of_raising(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_ratio_history_") - path = os.path.join(tmp_dir, "fetch_torrents_ratio_history.json") - with open(path, "w", encoding="utf-8") as f: + def test_missing_file_loads_as_empty(self): + self.assertEqual(self.load(os.path.join(self.dir, "does-not-exist.json")), {}) + + def test_corrupt_file_loads_as_empty_and_says_so(self): + with open(self.path, "w", encoding="utf-8") as f: f.write("{not valid json") - self.assertEqual(ft.load_ratio_history(path), {}) + data, report = self.load_reporting(self.path) + self.assertEqual(data, {}) + self.assertTrue( + report.startswith(f"{self.corrupt_report} {self.path} ("), report + ) + self.assertIn("starting fresh", report) -class OldReleaseCheckStatePersistenceTests(unittest.TestCase): - def test_round_trips_through_disk(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") - path = os.path.join(tmp_dir, "fetch_torrents_old_release_check_state.json") - state = {"x": {"last_checked": "2026-01-08", "leechers": 0}} + def test_saving_again_replaces_the_file_and_leaves_nothing_else(self): + self.save(self.path, {"stale": 1}) + self.save(self.path, self.sample) - ft.save_old_release_check_state(path, state) - loaded = ft.load_old_release_check_state(path) + self.assertEqual(self.load(self.path), self.sample) + self.assertEqual(os.listdir(self.dir), [self.filename]) - self.assertEqual(loaded, state) - def test_missing_file_returns_empty_state(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") - path = os.path.join(tmp_dir, "does-not-exist.json") +class RatioHistoryPersistenceTests(JsonStoreContract, unittest.TestCase): + filename = "fetch_torrents_ratio_history.json" + sample: ClassVar[dict] = { + "archlinux-2026.08.01-x86_64.iso": [{"date": "2026-08-29", "ratio": 1.2}] + } + corrupt_report = "Could not read ratio history at" - self.assertEqual(ft.load_old_release_check_state(path), {}) + def load(self, path): + return ft.load_ratio_history(path) - def test_corrupt_file_returns_empty_state_instead_of_raising(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_old_release_state_") - path = os.path.join(tmp_dir, "fetch_torrents_old_release_check_state.json") - with open(path, "w", encoding="utf-8") as f: - f.write("{not valid json") + def save(self, path, data): + ft.save_ratio_history(path, data) - self.assertEqual(ft.load_old_release_check_state(path), {}) +class RemovedHistoryPersistenceTests(JsonStoreContract, unittest.TestCase): + filename = "fetch_torrents_removed_history.json" + sample: ClassVar[dict] = {"x": {"removed_date": "2026-02-01", "reason": "stagnant"}} + corrupt_report = "Could not read removed-torrent history at" -class RemovedHistoryPersistenceTests(unittest.TestCase): - def test_round_trips_through_disk(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") - path = os.path.join(tmp_dir, "fetch_torrents_removed_history.json") - history = {"x": {"removed_date": "2026-02-01", "reason": "stagnant"}} + def load(self, path): + return ft.load_removed_history(path) - ft.save_removed_history(path, history) - loaded = ft.load_removed_history(path) + def save(self, path, data): + ft.save_removed_history(path, data) - self.assertEqual(loaded, history) - def test_missing_file_returns_empty_history(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") - path = os.path.join(tmp_dir, "does-not-exist.json") +class OldReleaseCheckStatePersistenceTests(JsonStoreContract, unittest.TestCase): + filename = "fetch_torrents_old_release_check_state.json" + sample: ClassVar[dict] = {"x": {"last_checked": "2026-01-08", "leechers": 0}} + corrupt_report = "Could not read old-release check state at" - self.assertEqual(ft.load_removed_history(path), {}) + def load(self, path): + return ft.load_old_release_check_state(path) - def test_corrupt_file_returns_empty_history_instead_of_raising(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_removed_history_") - path = os.path.join(tmp_dir, "fetch_torrents_removed_history.json") - with open(path, "w", encoding="utf-8") as f: - f.write("{not valid json") + def save(self, path, data): + ft.save_old_release_check_state(path, data) + + +class SettingsFilePersistenceTests(JsonStoreContract, unittest.TestCase): + filename = "settings.json" + sample: ClassVar[dict] = {"rpc-whitelist": "10.0.0.*", "peer-limit-global": 200} + corrupt_report = "configure_transmission: could not read" + + def load(self, path): + return ct.load_settings(path) + + def save(self, path, data): + ct.save_settings(path, data) - self.assertEqual(ft.load_removed_history(path), {}) + def load_reporting(self, path): + # Runs before logging exists in the container, so it reports on stderr. + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + data = self.load(path) + return data, stderr.getvalue().strip() if __name__ == "__main__": diff --git a/tests/test_ratio_gate.py b/tests/test_ratio_gate.py index 2acf4ac..12ea07c 100644 --- a/tests/test_ratio_gate.py +++ b/tests/test_ratio_gate.py @@ -274,7 +274,7 @@ def test_removed_history_entry_without_ratio_is_ignored(self): ratios={}, removed_history=removed_history, ) - self.assertTrue(result) + self.assertIs(result, True) def test_skipped_removed_history_entry_does_not_hide_later_ones(self): removed_history = { @@ -473,7 +473,7 @@ def test_unparseable_name_of_known_distro_fetches_and_logs_error(self): with self.assertLogs(ft.logger, level="ERROR") as logs: result = ft.should_fetch_torrent("ubuntu-noble-desktop-amd64.iso", ratios) - self.assertTrue(result) + self.assertIs(result, True) self.assertEqual(len(logs.records), 1) self.assertIn("ubuntu-noble-desktop-amd64.iso", logs.output[0]) From 7080bbd36f1176046c5000e47fb7489cfd748ce8 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 13:31:20 +0200 Subject: [PATCH 25/46] Record the new test-health baseline and ratchet coverage (task 27) - Full mutmut runs (whole suite as the runner): - critical fetch_torrents logic: 70.9% -> 99.2% (256/258) - torrent_scrape: 82.6% -> 100% - configure_transmission: 89.8% -> 96.7% The two remaining critical survivors (has_unmet_demand's defaults for a missing seeder/leecher count) are now killed by test_missing_counts_mean_zero, checked by hand-applying each mutant, so 258/258. - coverage: [tool.coverage.report] fail_under = 94 (measured total 96.78% minus about 2), which CI's `coverage report` step now enforces. - tasks/README.md: targets table updated with the new numbers, every task marked done, and the remaining non-critical survivors (I/O and orchestration) listed as follow-ups. Co-Authored-By: Claude Opus 5.5 --- .claude/CLAUDE.md | 2 +- .github/workflows/docker-image.yml | 4 +- pyproject.toml | 4 ++ tasks/README.md | 86 +++++++++++++++++++----------- tests/test_old_release_demand.py | 22 ++++++++ 5 files changed, 84 insertions(+), 34 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 47710a2..2a8e783 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -12,7 +12,7 @@ Testing rules and the import-stubbing pattern live in the root [CLAUDE.md](../CL - Lint/format: `ruff check .` / `ruff format .` - Typecheck: `mypy` (files and settings in `pyproject.toml`) - CI's `test` job runs ruff check, ruff format --check and mypy before the tests; versions are pinned in `requirements-dev.txt` -- Coverage: `coverage run -m unittest discover -s tests` then `coverage report` (branch coverage; config in `pyproject.toml`) +- Coverage: `coverage run -m unittest discover -s tests` then `coverage report` (branch coverage; config in `pyproject.toml`, which also sets the `fail_under` ratchet CI enforces) - Mutation: `mutmut run` (mutmut<3; config in `pyproject.toml`, see CONTRIBUTING.md) - Dev deps: `pip install -r requirements-dev.txt` (includes `requirements-test.txt`) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 89afc40..33df55c 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -54,8 +54,8 @@ jobs: - name: Run tests run: coverage run -m unittest discover -s tests -v - # Report only for now; settings (branch coverage, which modules) are in - # pyproject.toml. + # Fails below [tool.coverage.report] fail_under; that and the other + # settings (branch coverage, which modules) are in pyproject.toml. - name: Coverage report run: coverage report diff --git a/pyproject.toml b/pyproject.toml index e8f3ee0..c177620 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,10 @@ omit = ["tests/*"] [tool.coverage.report] show_missing = true +# Ratchet: about 2 points under the measured total (96.78% on 2026-10-19), +# so coverage can only go up. Raise it as coverage improves; never lower it +# to get a change through. +fail_under = 94 [tool.mutmut] # mutmut<3 (see requirements-dev.txt). It edits these files in place while diff --git a/tasks/README.md b/tasks/README.md index 0ab5732..fda95c7 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -13,37 +13,61 @@ so search for the function name if they have drifted. ## Targets ("a better place") -| Metric | Now | Target | -| --- | --- | --- | -| Mutation score, critical `fetch_torrents.py` functions | 70.9% | ≥ 90% | -| Mutation score, critical `torrent_scrape.py` functions | 82.6% | ≥ 90% | -| Branch coverage, `fetch_torrents.py` | 75% | ≥ 90% (once `run()` exists) | -| CI gates | unittest only | unittest + ruff + mypy + container smoke | -| Scraper drift detection | none | recorded pages in the merge check + a scheduled live job | +Re-measured on 2026-10-19 after tasks 10–26 (task 27): 383 tests green. +Mutation scores are honest (string-literal-only survivors excluded). They +come from full mutmut 2.5.1 runs with the whole suite as the runner. + +| Metric | Baseline (84761d3) | Target | Now | +| --- | --- | --- | --- | +| Mutation score, critical `fetch_torrents.py` functions¹ | 70.9% | ≥ 90% | **100%** (258/258) | +| Mutation score, critical `torrent_scrape.py` functions | 82.6% | ≥ 90% | **100%** (247/247, every function) | +| Mutation score, `configure_transmission.py` | 89.8% | — | 96.7% (87/90) | +| Mutation score, all of `fetch_torrents.py` | — | — | 88.6% (915/1033) | +| Branch coverage, `fetch_torrents.py` | 75% | ≥ 90% | **96%** (total 96.8%; CI `fail_under = 94`) | +| CI gates | unittest only | unittest + ruff + mypy + container smoke | **met**: also the in-image unit suite, hostile env, coverage ratchet | +| Scraper drift detection | none | recorded pages in the merge check + a scheduled live job | **met**: `tests/test_scrapers.py` + `.github/workflows/drift.yml` | + +¹ The ratio gate, the cleanup planners and the parsers they rely on: +`should_fetch_torrent`, `_should_fetch_devuan`, `parse_version_type`, +`version_to_tuple`, `get_previous_ratios`, `plan_cleanup`, +`_old_version_candidates`, `_devuan_sort_key`, `plan_stagnation_cleanup`, +`update_ratio_history`, `plan_disk_pressure_cleanup`, `has_unmet_demand`, +`should_recheck_old_release`, `_safe_watch_path`, `filter_low_demand`. + +Remaining survivors (not critical logic, left as follow-ups) are mostly in +the I/O and orchestration layer: +- `run` (9), `_fetch_latest` (5), `_check_old_releases` (2) +- `check_old_releases_for_demand` (21), `evaluate_old_release_demand` (7) +- `enforce_disk_usage_limit` (10), `cleanup_stagnant_torrents` (5), + `cleanup_old_versions` (3) +- `configure_logging` (6), `parse_log_level` (3), `wait_for_transmission_rpc` (6) +- the `save_*` temp-file handling (2 each) + +The list is in a scratch run, not the repo: rerun `mutmut run` and +`mutmut results` to see the current set. ## Order -| # | Task | Priority | Class | Depends on | -| --- | --- | --- | --- | --- | -| 10 | [Ratio gate boundaries and branches](10-ratio-gate-boundaries.md) | P1 | NO_REFACTOR | — | -| 11 | [Test `get_previous_ratios` in-process](11-get-previous-ratios-tests.md) | P1 | NO_REFACTOR | — | -| 12 | [Isolate tests from env vars and the clock](12-test-env-clock-isolation.md) | P1 | NO_REFACTOR | — | -| 13 | [Fix `bdecode` accepting/mis-reporting truncated input](13-bdecode-truncated-input.md) | P1 (bug) | NO_REFACTOR | — | -| 14 | [Cleanup planning boundary tests](14-cleanup-boundaries.md) | P2 | NO_REFACTOR | 12 | -| 15 | [Add ruff + mypy to CI](15-ci-static-gates.md) | P2 | CI | — | -| 16 | [Dev tooling: coverage + mutmut in requirements-dev](16-dev-tooling-coverage-mutation.md) | P2 | CI | — | -| 17 | [Lock in daily-run behavior with a `runpy` test](17-daily-run-characterization.md) | P2 | NO_REFACTOR | 12 | -| 18 | [Extract `run()` and `configure_logging()`](18-extract-run-and-logging.md) | P2 | REFACTOR_REQUIRED | 17 | -| 19 | [Inject `today` / client / config into the `cleanup_*` functions](19-inject-cleanup-seams.md) | P2 | REFACTOR_REQUIRED | 17 | -| 20 | [Transmission client double that can't drift](20-transmission-double-fidelity.md) | P3 | REFACTOR_REQUIRED | 19 | -| 21 | [Recorded-page scraper contract tests (+ `fetch_kali_latest`)](21-scraper-contract-tests.md) | P2 | NO_REFACTOR | — | -| 22 | [Container smoke test in the build job](22-container-smoke-test.md) | P2 | CI | — | -| 23 | [Scheduled live-drift job](23-scheduled-live-drift-job.md) | P3 | CI | 21, 22 | -| 24 | [Parser gaps: bencode str, UDP parsers, `parse_version_type`](24-parser-gaps.md) | P3 | NO_REFACTOR | 13 | -| 25 | [`configure_transmission.main` end-to-end test](25-configure-transmission-main.md) | P3 | NO_REFACTOR | — | -| 26 | [Tighten weak assertions and consolidate fixtures](26-assertions-and-fixtures.md) | P3 | NO_REFACTOR | 10, 14 | -| 27 | [Re-measure and record the new baseline](27-remeasure-baseline.md) | P3 | — | all | - -Tasks 10–13 are independent and can run in parallel. Any task that touches -`tests/test_fetch_torrents.py` can conflict with task 26's file split, so do 26 after the -test-adding tasks rather than before. +| # | Task | Priority | Class | Depends on | Status | +| --- | --- | --- | --- | --- | --- | +| 10 | [Ratio gate boundaries and branches](10-ratio-gate-boundaries.md) | P1 | NO_REFACTOR | — | done | +| 11 | [Test `get_previous_ratios` in-process](11-get-previous-ratios-tests.md) | P1 | NO_REFACTOR | — | done | +| 12 | [Isolate tests from env vars and the clock](12-test-env-clock-isolation.md) | P1 | NO_REFACTOR | — | done | +| 13 | [Fix `bdecode` accepting/mis-reporting truncated input](13-bdecode-truncated-input.md) | P1 (bug) | NO_REFACTOR | — | done | +| 14 | [Cleanup planning boundary tests](14-cleanup-boundaries.md) | P2 | NO_REFACTOR | 12 | done | +| 15 | [Add ruff + mypy to CI](15-ci-static-gates.md) | P2 | CI | — | done | +| 16 | [Dev tooling: coverage + mutmut in requirements-dev](16-dev-tooling-coverage-mutation.md) | P2 | CI | — | done | +| 17 | [Lock in daily-run behavior with a `runpy` test](17-daily-run-characterization.md) | P2 | NO_REFACTOR | 12 | done | +| 18 | [Extract `run()` and `configure_logging()`](18-extract-run-and-logging.md) | P2 | REFACTOR_REQUIRED | 17 | done | +| 19 | [Inject `today` / client / config into the `cleanup_*` functions](19-inject-cleanup-seams.md) | P2 | REFACTOR_REQUIRED | 17 | done | +| 20 | [Transmission client double that can't drift](20-transmission-double-fidelity.md) | P3 | REFACTOR_REQUIRED | 19 | done | +| 21 | [Recorded-page scraper contract tests (+ `fetch_kali_latest`)](21-scraper-contract-tests.md) | P2 | NO_REFACTOR | — | done | +| 22 | [Container smoke test in the build job](22-container-smoke-test.md) | P2 | CI | — | done | +| 23 | [Scheduled live-drift job](23-scheduled-live-drift-job.md) | P3 | CI | 21, 22 | done | +| 24 | [Parser gaps: bencode str, UDP parsers, `parse_version_type`](24-parser-gaps.md) | P3 | NO_REFACTOR | 13 | done | +| 25 | [`configure_transmission.main` end-to-end test](25-configure-transmission-main.md) | P3 | NO_REFACTOR | — | done | +| 26 | [Tighten weak assertions and consolidate fixtures](26-assertions-and-fixtures.md) | P3 | NO_REFACTOR | 10, 14 | done | +| 27 | [Re-measure and record the new baseline](27-remeasure-baseline.md) | P3 | — | all | done | + +All tasks are done. `tests/test_fetch_torrents.py`, which the task files refer to, +was split by concern in task 26 (see CLAUDE.md for the new module list). diff --git a/tests/test_old_release_demand.py b/tests/test_old_release_demand.py index 0003a08..c0db9f1 100644 --- a/tests/test_old_release_demand.py +++ b/tests/test_old_release_demand.py @@ -35,6 +35,28 @@ def test_none_scrape_result_is_no_demand(self): ) ) + def test_missing_counts_mean_zero(self): + # A tracker's scrape can omit a count; missing must read as 0, not + # as some peer. + self.assertIs( + ft.has_unmet_demand( + {"seeders": 0}, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=True, + ), + False, + ) + self.assertIs( + ft.has_unmet_demand( + {"leechers": 5}, + min_leechers=1, + min_leecher_ratio=0.1, + allow_zero_seeders=False, + ), + False, + ) + def test_below_absolute_threshold_is_no_demand(self): self.assertFalse( ft.has_unmet_demand( From 90c7f8f2f85087866fd84af4bb5722be1626e160 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 14:53:42 +0200 Subject: [PATCH 26/46] Make FakeDisk honour delete_data and pin the cleanup safety valves FakeTransmission now keeps a torrent removed without delete_data (the real Client's default) in data_left_on_disk, and FakeDisk counts it as used, so a removal only frees space the way it would for real. Before this, flipping delete_data=True to False in enforce_disk_usage_limit or cleanup_stagnant_torrents passed the whole suite. New tests pin: - delete_data=True on disk-pressure and stagnation removals - the disk-pressure entry gate at exactly the threshold, just below it, a 1% threshold, a zero-size disk, a custom download path with a single connection, the per-removal warning, and a zero ratio recorded as 0.0 - stagnation's default 30-day window and 0.02 delta, custom window_days/min_ratio_delta wiring (including the history prune), the kept/removed log lines, and a zero ratio sampled as 0.0 Kills mutants #839 #843 #844 #846 #857 #864 #868 #924 #925 #930 #940 plus an argument swap and a dropped window_days in the stagnation path. Left as equivalent: #858 (disk total turning 0 mid-loop), #861 (break vs continue once under threshold), #863/#929 (a real Torrent always has .ratio). Co-Authored-By: Claude Opus 5.5 --- tests/helpers.py | 15 ++- tests/test_cleanup.py | 217 ++++++++++++++++++++++++++++++ tests/test_transmission_double.py | 24 ++++ 3 files changed, 251 insertions(+), 5 deletions(-) diff --git a/tests/helpers.py b/tests/helpers.py index 031cade..8b95ca7 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -161,6 +161,8 @@ class FakeTransmission: """In-memory stand-in for transmission_rpc.Client: get_torrents() returns what it currently holds, remove_torrent() drops that torrent. `calls` records both, in order, for tests that care about sequencing. + A torrent removed without delete_data (the real Client's default) moves + to `data_left_on_disk`, since Transmission leaves its files behind. Every call is bound against the real Client's signature first, so an argument the pinned library wouldn't accept raises TypeError here too @@ -168,6 +170,7 @@ class FakeTransmission: def __init__(self, torrents=()): self.torrents = list(torrents) + self.data_left_on_disk = [] self.calls = [] def get_torrents(self, *args, **kwargs): @@ -178,6 +181,8 @@ def get_torrents(self, *args, **kwargs): def remove_torrent(self, *args, **kwargs): call = _bind_to_client("remove_torrent", args, kwargs) self.calls.append(("remove_torrent", call["ids"], call["delete_data"])) + if not call["delete_data"]: + self.data_left_on_disk += [t for t in self.torrents if t.id == call["ids"]] self.torrents = [t for t in self.torrents if t.id != call["ids"]] @property @@ -187,8 +192,9 @@ def names(self): class FakeDisk: """shutil.disk_usage() stand-in whose used space is `base_used` plus the - size of every torrent `transmission` still holds - so removals free - space the way they would for real.""" + size of every torrent whose data is still on disk - those `transmission` + holds, plus any it removed without delete_data - so removals free space + only the way they would for real.""" def __init__(self, transmission, total, sizes, base_used=0): self.transmission = transmission @@ -199,9 +205,8 @@ def __init__(self, transmission, total, sizes, base_used=0): def __call__(self, path): self.paths.append(path) - used = self.base_used + sum( - self.sizes.get(t.id, 0) for t in self.transmission.torrents - ) + on_disk = self.transmission.torrents + self.transmission.data_left_on_disk + used = self.base_used + sum(self.sizes.get(t.id, 0) for t in on_disk) return (self.total, used, self.total - used) diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index ce8fe78..e385fab 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -872,6 +872,131 @@ def test_cleanup_stagnant_torrents_removes_the_watch_file(self): {"removed_date": "2026-08-01", "reason": "stagnant", "ratio": 2.0}, ) + def _write_ratio_history(self, history): + with open(ft.ratio_history_file, "w", encoding="utf-8") as f: + json.dump(history, f) + + def test_cleanup_stagnant_torrents_deletes_the_downloaded_data(self): + today = date(2026, 8, 1) + self._write_ratio_history( + { + "ubuntu-23.10-desktop-amd64.iso": [ + {"date": (today - timedelta(days=32)).isoformat(), "ratio": 2.0} + ] + } + ) + transmission = self._transmission() + + ft.cleanup_stagnant_torrents(transmission, today=today) + + self.assertEqual( + transmission.calls, ["get_torrents", ("remove_torrent", 2, True)] + ) + + def test_cleanup_stagnant_torrents_judges_over_a_30_day_window_by_default(self): + # Exactly 30 days of flat ratio: stagnant under the default window, + # though a day short of it under any longer one. + today = date(2026, 8, 1) + self._write_ratio_history( + { + "ubuntu-23.10-desktop-amd64.iso": [ + {"date": (today - timedelta(days=30)).isoformat(), "ratio": 2.0} + ] + } + ) + transmission = self._transmission() + + ft.cleanup_stagnant_torrents(transmission, today=today) + + self.assertEqual(transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + + def test_cleanup_stagnant_torrents_keeps_growth_above_the_default_delta(self): + # Ratio grew 0.5 over the window: well above the 0.02 default. + today = date(2026, 8, 1) + self._write_ratio_history( + { + "ubuntu-23.10-desktop-amd64.iso": [ + {"date": (today - timedelta(days=32)).isoformat(), "ratio": 1.5} + ] + } + ) + transmission = self._transmission() + + with self.assertLogs(ft.logger, level="INFO") as logs: + ft.cleanup_stagnant_torrents(transmission, today=today) + + self.assertEqual(len(transmission.torrents), 2) + self.assertIn( + "Keeping old version ubuntu-23.10-desktop-amd64.iso – ratio grew by " + "0.500 over the last 30 days.", + [r.getMessage() for r in logs.records], + ) + + def test_cleanup_stagnant_torrents_honours_custom_window_and_delta(self): + # window_days=10, min_ratio_delta=0.5. 23.10 grew 0.6 in 12 days + # (kept); 22.04 grew 0.3 (removed). Swapping the two settings would + # remove both. The 20-day-old 23.10 sample is past the 10 + 7 day + # prune horizon, so it must be dropped from the saved history. + today = date(2026, 8, 1) + + def days_ago(n): + return (today - timedelta(days=n)).isoformat() + + self._write_ratio_history( + { + "ubuntu-23.10-desktop-amd64.iso": [ + {"date": days_ago(20), "ratio": 0.9}, + {"date": days_ago(12), "ratio": 1.0}, + ], + "ubuntu-22.04-desktop-amd64.iso": [ + {"date": days_ago(12), "ratio": 1.0} + ], + } + ) + transmission = helpers.FakeTransmission( + [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=1.6), + helpers.FakeTorrent("ubuntu-22.04-desktop-amd64.iso", 3, ratio=1.3), + ] + ) + + with self.assertLogs(ft.logger, level="INFO") as logs: + ft.cleanup_stagnant_torrents( + transmission, today=today, window_days=10, min_ratio_delta=0.5 + ) + + self.assertEqual( + transmission.names, + ["ubuntu-23.10-desktop-amd64.iso", "ubuntu-24.04-desktop-amd64.iso"], + ) + self.assertIn( + "Removing stagnant torrent: ubuntu-22.04-desktop-amd64.iso " + "(no meaningful ratio growth in 10 days)", + [r.getMessage() for r in logs.records], + ) + self.assertEqual( + ft.load_ratio_history(ft.ratio_history_file)[ + "ubuntu-23.10-desktop-amd64.iso" + ], + [ + {"date": days_ago(12), "ratio": 1.0}, + {"date": "2026-08-01", "ratio": 1.6}, + ], + ) + + def test_cleanup_stagnant_torrents_records_a_zero_ratio_as_zero(self): + transmission = helpers.FakeTransmission( + [helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=0.0)] + ) + + ft.cleanup_stagnant_torrents(transmission, today=date(2026, 8, 1)) + + self.assertEqual( + ft.load_ratio_history(ft.ratio_history_file), + {"ubuntu-24.04-desktop-amd64.iso": [{"date": "2026-08-01", "ratio": 0.0}]}, + ) + def test_cleanup_stagnant_torrents_records_todays_sample(self): transmission = self._transmission() @@ -1091,6 +1216,98 @@ def test_keeps_removing_while_still_over_threshold(self): self.assertEqual(self.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + def test_removal_deletes_the_downloaded_data(self): + self._enforce(95, self._disk(base_used=86)) + + self.assertEqual( + self.transmission.calls, ["get_torrents", ("remove_torrent", 3, True)] + ) + + def test_usage_exactly_at_threshold_triggers_cleanup(self): + # 85 + 10 = 95% used: at the threshold counts as exceeded. + disk = self._disk(base_used=85) + + self._enforce(95, disk) + + self.assertEqual( + self.transmission.names, + ["ubuntu-23.10-desktop-amd64.iso", "ubuntu-24.04-desktop-amd64.iso"], + ) + + def test_usage_just_below_threshold_does_not_connect(self): + disk = self._disk(base_used=84.5) # 94.5% + + self._enforce(95, disk) + + self.assertEqual(self.connections, 0) + + def test_stops_once_usage_is_just_below_threshold(self): + # 95.5% used; removing 22.04 (1 GB) leaves 94.5%, so 23.10 stays. + disk = self._disk(base_used=89.5, sizes={1: 1, 2: 4, 3: 1}) + + self._enforce(95, disk) + + self.assertEqual( + self.transmission.names, + ["ubuntu-23.10-desktop-amd64.iso", "ubuntu-24.04-desktop-amd64.iso"], + ) + + def test_a_1_percent_threshold_is_enforced(self): + disk = self._disk(base_used=40) # 50% + + self._enforce(1, disk) + + self.assertEqual(self.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) + + def test_zero_size_disk_does_not_connect_or_raise(self): + disk = helpers.FakeDisk(self.transmission, 0, {}) + + self._enforce(1, disk) + + self.assertEqual((disk.paths, self.connections), (["/downloads"], 0)) + + def test_checks_the_given_download_path_and_connects_once(self): + disk = self._disk(base_used=99) + + ft.enforce_disk_usage_limit( + self._connect, + 95, + today=date(2026, 8, 1), + disk_usage=disk, + download_path="/mnt/seeds", + ) + + self.assertEqual(set(disk.paths), {"/mnt/seeds"}) + self.assertEqual(self.connections, 1) + + def test_logs_each_removal_with_usage_and_ratio(self): + with self.assertLogs(ft.logger, level="WARNING") as logs: + self._enforce(95, self._disk(base_used=86)) + + self.assertEqual( + [r.getMessage() for r in logs.records], + [ + ( + "Downloads folder usage 96.0% >= 95.0% threshold - removing " + "ubuntu-22.04-desktop-amd64.iso (ratio 0.200) to free space." + ) + ], + ) + + def test_records_a_zero_ratio_removal_as_zero(self): + self.transmission.torrents[2] = helpers.FakeTorrent( + "ubuntu-22.04-desktop-amd64.iso", 3, ratio=0.0 + ) + + self._enforce(95, self._disk(base_used=86)) + + self.assertEqual( + ft.load_removed_history(ft.removed_history_file)[ + "ubuntu-22.04-desktop-amd64.iso" + ]["ratio"], + 0.0, + ) + def test_never_removes_the_latest_version(self): disk = self._disk(base_used=99) diff --git a/tests/test_transmission_double.py b/tests/test_transmission_double.py index 2db5c2e..cc5083a 100644 --- a/tests/test_transmission_double.py +++ b/tests/test_transmission_double.py @@ -41,6 +41,30 @@ def test_every_public_method_exists_on_the_real_client(self): self.assertEqual(own - set(dir(transmission_rpc.Client)), set()) +class FakeDiskTests(unittest.TestCase): + """The real Client's remove_torrent() defaults to delete_data=False, + which takes the torrent out of Transmission but leaves its downloaded + data on disk - so FakeDisk must only count a removal as freed space + when the data was deleted too.""" + + def setUp(self): + self.transmission = helpers.FakeTransmission( + [helpers.FakeTorrent("ubuntu-22.04-desktop-amd64.iso", 1)] + ) + self.disk = helpers.FakeDisk(self.transmission, 100, {1: 6}, base_used=10) + + def test_removal_with_data_frees_its_space(self): + self.transmission.remove_torrent(1, delete_data=True) + + self.assertEqual(self.disk("/downloads"), (100, 10, 90)) + + def test_removal_without_data_frees_nothing(self): + self.transmission.remove_torrent(1) + + self.assertEqual(self.transmission.torrents, []) + self.assertEqual(self.disk("/downloads"), (100, 16, 84)) + + class FakeTorrentTests(unittest.TestCase): def test_is_a_real_torrent_reading_real_rpc_field_names(self): torrent = helpers.FakeTorrent( From db38abe82731d65196152d558b04e68e7819bdfd Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 15:06:58 +0200 Subject: [PATCH 27/46] Test the old-release check sociably instead of mocking its internals check_old_releases_for_demand() was tested with evaluate_old_release_demand() patched out, and the daily run with check_old_releases_for_demand() patched out, so the thresholds, skip paths and counters were never really exercised: 21 mutants survived in the check, 7 in the evaluation and 2 in the run's wiring. - OldReleaseTracker fakes only the two network boundaries (the .torrent download and the tracker scrape, which reads the temp file it is given), so the real evaluation runs in between. - One multi-candidate test covers every outcome in a single pass (removed x2, unsafe name, already present, recently checked, below floor, zero seeders, unknown demand, 404, wanted x2) behind a distro with no discovery and one that finds nothing. It asserts the counts, watch files, state, request order, log lines and that the caller's state isn't mutated. - evaluate_old_release_demand(): the temp copy is scraped and removed even when the scrape fails, temp-file failure, scrape timeout pass-through, 404 vs 500 logging, and the download timeout. - Daily run: the mocked test_old_release_check_runs_only_when_enabled is replaced by an off-by-default test and OldReleaseRunTests, which drive the env thresholds, recheck interval, saved check state and removed history (including SKIP_REMOVED=false) through the real check. RunTestCase now holds the shared setUp, and _run() takes fetchers. - test_below_absolute_threshold_is_no_demand used a zero-seeder swarm, so it passed with the leecher floor deleted. It now uses a seeded swarm and no ratio requirement. All targeted mutants are killed except #715/#729 (the .get("seeders", 0) default, which can't apply because scrape results always carry seeders). Co-Authored-By: Claude Opus 5.5 --- tests/test_daily_run.py | 146 +++++++++++-- tests/test_old_release_demand.py | 344 ++++++++++++++++++++++++++++++- 2 files changed, 471 insertions(+), 19 deletions(-) diff --git a/tests/test_daily_run.py b/tests/test_daily_run.py index 589ba22..af318df 100644 --- a/tests/test_daily_run.py +++ b/tests/test_daily_run.py @@ -152,7 +152,9 @@ def route_arch(self): ) -class RunTests(unittest.TestCase): +class RunTestCase(unittest.TestCase): + """Base for tests of ft.run() inside a FakeWorld; defines no tests.""" + def setUp(self): self.world = FakeWorld(self) patches = [ @@ -184,7 +186,7 @@ def setUp(self): def _rpc_port_open(*args, **kwargs): yield MagicMock() - def _run(self, **env): + def _run(self, fetchers=ft.DEFAULT_FETCHERS, **env): """Runs ft.run() with load_config(env); returns (summary, messages).""" with self.assertLogs(ft.logger, level="DEBUG") as logs: summary = ft.run( @@ -192,10 +194,13 @@ def _run(self, **env): client_factory=self.world.connect, today=date(2026, 9, 2), now=lambda: 1000.0, + fetchers=fetchers, disk_usage=self.world.disk_usage, ) return summary, [record.getMessage() for record in logs.records] + +class RunTests(RunTestCase): def test_counts_each_download_outcome(self): self.world.route_ubuntu() @@ -441,33 +446,140 @@ def test_rpc_credentials_from_config_are_used_for_ratio_logging(self): log_ratios.assert_called_once_with(auth=("alice", "hunter2")) - def test_old_release_check_runs_only_when_enabled(self): - with unittest.mock.patch.object( - ft, "check_old_releases_for_demand", return_value=(2, 3, {"x": "y"}) - ) as check: - disabled, _ = self._run(FETCH_TORRENTS_DISTROS="arch") - check.assert_not_called() + def test_old_release_check_is_off_by_default(self): + discovered = [] + with unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, + {"kali": lambda: discovered.append("kali") or {}}, + clear=True, + ): + summary, _ = self._run(FETCH_TORRENTS_DISTROS="kali", fetchers=()) - enabled, messages = self._run( - FETCH_TORRENTS_DISTROS="arch", + self.assertEqual(discovered, []) + self.assertEqual((summary.old_added, summary.old_skipped), (None, None)) + self.assertFalse(os.path.exists(ft.old_release_check_state_file)) + + +class OldReleaseRunTests(RunTestCase): + """The daily run's old-release check end to end: config thresholds, the + saved check state and removed history all reach the real + check_old_releases_for_demand(), with only the downloads (FakeWorld + routes) and the tracker scrape faked.""" + + TODAY = "2026-09-02" + + def setUp(self): + super().setUp() + self.swarms = {} + patcher = unittest.mock.patch.object( + ft.torrent_scrape, "scrape_torrent", side_effect=self._scrape + ) + patcher.start() + self.addCleanup(patcher.stop) + + def _scrape(self, path, timeout=15): + with open(path, "rb") as f: + return self.swarms[f.read()] + + def _candidate(self, version, variant, leechers, seeders): + name = f"kali-linux-2025.{version}-{variant}-amd64.iso" + url = f"https://cdimage.kali.org/{name}.torrent" + body = f"{name} metadata".encode() + self.world.routes[url] = helpers.fake_response(body) + self.swarms[body] = { + "tracker": "udp://tracker.example:6969", + "seeders": seeders, + "leechers": leechers, + "completed": 0, + } + return name, url + + def _write_state(self, filename, data): + with open(self.world.state_path(filename), "w", encoding="utf-8") as f: + json.dump(data, f) + + def _check(self, candidates, **env): + with unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: dict(candidates)}, clear=True + ): + return self._run( + fetchers=(), + FETCH_TORRENTS_DISTROS="kali", FETCH_TORRENTS_CHECK_OLD_RELEASES="true", FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS="4", + FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO="0.5", + FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS="true", + FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS="10", + **env, ) - self.assertEqual((disabled.old_added, disabled.old_skipped), (None, None)) - self.assertEqual((enabled.old_added, enabled.old_skipped), (2, 3)) - self.assertEqual(check.call_args.args[2], 4) - self.assertEqual(check.call_args.args[7], date(2026, 9, 2)) + def test_config_and_saved_state_drive_the_check(self): + recent = self._candidate(1, "installer", 50, 1) + removed = self._candidate(1, "live", 50, 1) + thin = self._candidate(2, "installer", 3, 1) # under the 4-leecher floor + crowded = self._candidate(2, "live", 5, 20) # under 0.5x the seeders + wanted = self._candidate(3, "installer", 6, 10) + unseeded = self._candidate(3, "live", 5, 0) # zero seeders allowed + netinst = self._candidate(3, "installer-netinst", 50, 1) # low demand + # Checked 8 days ago: due under the default 7-day interval, not 10. + self._write_state( + "fetch_torrents_old_release_check_state.json", + {recent[0]: {"last_checked": "2026-08-25", "leechers": 0}}, + ) + self._write_state( + "fetch_torrents_removed_history.json", + {removed[0]: {"removed_date": "2026-08-01", "reason": "stagnant"}}, + ) + + summary, messages = self._check( + [recent, removed, thin, crowded, wanted, unseeded, netinst] + ) + + self.assertEqual((summary.old_added, summary.old_skipped), (2, 4)) + self.assertEqual( + sorted(os.listdir(self.world.watch_dir)), + [f"{wanted[0]}.torrent", f"{unseeded[0]}.torrent"], + ) + self.assertEqual( + self.world.requested, [thin[1], crowded[1], wanted[1], unseeded[1]] + ) self.assertEqual( ft.load_old_release_check_state(ft.old_release_check_state_file), - {"x": "y"}, + { + recent[0]: {"last_checked": "2026-08-25", "leechers": 0}, + thin[0]: {"last_checked": self.TODAY, "leechers": 3}, + crowded[0]: {"last_checked": self.TODAY, "leechers": 5}, + wanted[0]: {"last_checked": self.TODAY, "leechers": 6}, + unseeded[0]: {"last_checked": self.TODAY, "leechers": 5}, + }, + ) + self.assertIn( + "Checking old releases for unmet demand (min leechers: 4, min " + "leecher/seeder ratio: 0.5x, allow zero-seeder swarms: True, " + "recheck interval: 10 days)...", + messages, ) self.assertTrue( messages[-1].endswith( - "Old releases (FETCH_TORRENTS_CHECK_OLD_RELEASES): 2 added, 3 skipped." - ) + "Old releases (FETCH_TORRENTS_CHECK_OLD_RELEASES): 2 added, 4 skipped." + ), + messages[-1], + ) + + def test_previously_removed_release_is_rechecked_when_skip_removed_is_off(self): + removed = self._candidate(1, "live", 50, 1) + self._write_state( + "fetch_torrents_removed_history.json", + {removed[0]: {"removed_date": "2026-08-01", "reason": "stagnant"}}, + ) + + summary, _ = self._check( + [removed], FETCH_TORRENTS_OLD_RELEASE_SKIP_REMOVED="false" ) + self.assertEqual((summary.old_added, summary.old_skipped), (1, 0)) + self.assertEqual(os.listdir(self.world.watch_dir), [f"{removed[0]}.torrent"]) + def test_invalid_old_release_setting_aborts_the_run(self): # Unlike cleanup and the disk limit, the old-release check has never # been guarded - a bad value fails the run at that point. diff --git a/tests/test_old_release_demand.py b/tests/test_old_release_demand.py index c0db9f1..0a6cefd 100644 --- a/tests/test_old_release_demand.py +++ b/tests/test_old_release_demand.py @@ -58,11 +58,13 @@ def test_missing_counts_mean_zero(self): ) def test_below_absolute_threshold_is_no_demand(self): + # Seeded swarm and no ratio requirement, so only the leecher floor + # can turn this down. self.assertFalse( ft.has_unmet_demand( - {"leechers": 0, "seeders": 0}, + {"leechers": 0, "seeders": 5}, min_leechers=1, - min_leecher_ratio=1.2, + min_leecher_ratio=0.0, allow_zero_seeders=False, ) ) @@ -629,5 +631,343 @@ def test_no_demand_result_is_still_recorded_for_future_recheck_gating(self): self.assertEqual(new_state[name], {"last_checked": "2026-01-08", "leechers": 0}) +class OldReleaseTracker: + """Plays both network boundaries of the old-release check: requests.get + for the candidate's .torrent (`routes`: url -> response, or an exception + to raise; anything else is a 404) and torrent_scrape.scrape_torrent, + which answers from `swarms` (torrent bytes -> scrape result, or an + exception to raise) by reading the file it was handed - so the real + evaluate_old_release_demand() runs in between. Records every request, + and every scrape's (bytes, path, timeout).""" + + def __init__(self): + self.routes = {} + self.swarms = {} + self.requests = [] + self.scrapes = [] + + def candidate(self, name, swarm): + """Serves name's .torrent and gives it `swarm` (a scrape result, + None for "no tracker answered", or an exception); returns its url.""" + url = f"https://cdimage.example/{name}.torrent" + body = f"{name} metadata".encode() + self.routes[url] = helpers.fake_response(body) + self.swarms[body] = swarm + return url + + def get(self, url, timeout=None): + self.requests.append((url, timeout)) + answer = self.routes.get(url, helpers.fake_response(status=404)) + if isinstance(answer, Exception): + raise answer + return answer + + def scrape_torrent(self, path, timeout=15): + with open(path, "rb") as f: + body = f.read() + self.scrapes.append((body, path, timeout)) + swarm = self.swarms[body] + if isinstance(swarm, Exception): + raise swarm + return swarm + + def patch(self, testcase): + for target, name, fake in ( + (ft.requests, "get", self.get), + (ft.torrent_scrape, "scrape_torrent", self.scrape_torrent), + ): + patcher = unittest.mock.patch.object(target, name, side_effect=fake) + patcher.start() + testcase.addCleanup(patcher.stop) + + +def swarm(leechers, seeders): + return { + "tracker": "udp://tracker.example:6969", + "seeders": seeders, + "leechers": leechers, + "completed": 0, + } + + +class EvaluateOldReleaseDemandBoundaryTests(unittest.TestCase): + """What evaluate_old_release_demand() hands the network: the download + request, and a temporary on-disk copy of the .torrent for the tracker + scrape that is gone again afterwards, whatever the scrape did.""" + + NAME = "kali-linux-2025.4-installer-amd64.iso" + BODY = b"kali-linux-2025.4-installer-amd64.iso metadata" + + def setUp(self): + self.tracker = OldReleaseTracker() + self.tracker.patch(self) + + def _evaluate(self, url, **kwargs): + return ft.evaluate_old_release_demand( + self.NAME, + url, + min_leechers=4, + min_leecher_ratio=0.5, + allow_zero_seeders=False, + **kwargs, + ) + + def test_scrapes_a_temporary_copy_that_is_removed_afterwards(self): + url = self.tracker.candidate(self.NAME, swarm(6, 10)) + + result = self._evaluate(url) + + self.assertEqual(result, (True, self.BODY, swarm(6, 10))) + self.assertEqual(self.tracker.requests, [(url, 30)]) + [(body, path, timeout)] = self.tracker.scrapes + self.assertEqual((body, timeout), (self.BODY, 15)) + self.assertFalse(os.path.exists(path)) + + def test_scrape_timeout_is_passed_through(self): + url = self.tracker.candidate(self.NAME, swarm(6, 10)) + + self._evaluate(url, timeout=3) + + self.assertEqual([t for _, _, t in self.tracker.scrapes], [3]) + + def test_scrape_error_counts_as_unknown_demand_and_cleans_up(self): + url = self.tracker.candidate(self.NAME, OSError("tracker went away")) + + with self.assertLogs(ft.logger, level="WARNING") as logs: + result = self._evaluate(url) + + self.assertEqual(result, (False, self.BODY, None)) + self.assertEqual( + [r.getMessage() for r in logs.records], + [f"Could not scrape {self.NAME}: tracker went away"], + ) + [(_, path, _)] = self.tracker.scrapes + self.assertFalse(os.path.exists(path)) + + def test_temp_file_that_cannot_be_created_counts_as_unknown_demand(self): + url = self.tracker.candidate(self.NAME, swarm(6, 10)) + + with ( + unittest.mock.patch.object( + ft.tempfile, "NamedTemporaryFile", side_effect=OSError("disk full") + ), + self.assertLogs(ft.logger, level="WARNING"), + ): + result = self._evaluate(url) + + self.assertEqual(result, (False, self.BODY, None)) + self.assertEqual(self.tracker.scrapes, []) + + def test_404_is_logged_as_not_found_not_as_a_failure(self): + url = "https://cdimage.example/gone.torrent" + + with self.assertLogs(ft.logger, level="INFO") as logs: + result = self._evaluate(url) + + self.assertEqual(result, (False, None, None)) + self.assertEqual( + [(r.levelname, r.getMessage()) for r in logs.records], + [("INFO", f"{url} not found upstream (404) during old-release check.")], + ) + + def test_server_error_is_logged_as_a_failed_download(self): + url = "https://cdimage.example/broken.torrent" + self.tracker.routes[url] = helpers.fake_response(status=500) + + with self.assertLogs(ft.logger, level="INFO") as logs: + result = self._evaluate(url) + + self.assertEqual(result, (False, None, None)) + [record] = logs.records + self.assertEqual(record.levelname, "ERROR") + self.assertRegex( + record.getMessage(), + f"^Failed to download {url} for old-release check: 500 Server Error", + ) + self.assertEqual(self.tracker.scrapes, []) + + +def kali(version, variant="installer"): + return f"kali-linux-2025.{version}-{variant}-amd64.iso" + + +class CheckOldReleasesSociableTests(unittest.TestCase): + """check_old_releases_for_demand() over the real + evaluate_old_release_demand(), with only the download and the tracker + scrape faked (OldReleaseTracker). Tests run several candidates, so one + skipped candidate can't stop the rest from being checked, and every + counter has to add up past 1.""" + + TRACKER = "udp://tracker.example:6969" + + def setUp(self): + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) + patcher.start() + self.addCleanup(patcher.stop) + self.tracker = OldReleaseTracker() + self.tracker.patch(self) + + def _check(self, discovery, distros=("kali",), **overrides): + """Thresholds: >= 4 leechers and >= 0.5x as many leechers as + seeders, zero-seeder swarms excluded.""" + kwargs = { + "include_low_demand": False, + "min_leechers": 4, + "min_leecher_ratio": 0.5, + "allow_zero_seeders": False, + "check_state": {}, + "removed_history": {}, + "today": date(2026, 1, 8), + } + kwargs.update(overrides) + with unittest.mock.patch.dict(ft.OLD_RELEASE_DISCOVERY, discovery, clear=True): + return ft.check_old_releases_for_demand(list(distros), **kwargs) + + def _watch_files(self): + return sorted(os.listdir(self.tmp_watch_dir)) + + def test_every_outcome_is_counted_and_later_candidates_still_checked(self): + removed_a, removed_b = kali(1), kali(1, "live") + unsafe = "kali-linux-../../etc/cron.d/evil" + present, recent = kali(2, "live"), kali(2) + thin, zero, unknown = kali(3), kali(3, "live"), kali(3, "installer-purple") + missing = kali(3, "everything") + wanted_a, wanted_b = kali(4), kali(4, "live") + candidates = { + removed_a: self.tracker.candidate(removed_a, swarm(50, 1)), + removed_b: self.tracker.candidate(removed_b, swarm(50, 1)), + unsafe: "https://cdimage.example/evil.torrent", + present: self.tracker.candidate(present, swarm(50, 1)), + recent: self.tracker.candidate(recent, swarm(50, 1)), + thin: self.tracker.candidate(thin, swarm(5, 20)), + zero: self.tracker.candidate(zero, swarm(9, 0)), + unknown: self.tracker.candidate(unknown, None), + missing: "https://cdimage.example/missing.torrent", + wanted_a: self.tracker.candidate(wanted_a, swarm(6, 10)), + wanted_b: self.tracker.candidate(wanted_b, swarm(30, 2)), + } + with open(os.path.join(self.tmp_watch_dir, f"{present}.torrent"), "wb") as f: + f.write(b"already seeding") + check_state = {recent: {"last_checked": "2026-01-05", "leechers": 0}} + removed_history = { + name: {"removed_date": "2025-12-01", "reason": "stagnant"} + for name in (removed_a, removed_b) + } + + with self.assertLogs(ft.logger, level="DEBUG") as logs: + added, skipped, new_state = self._check( + # ubuntu has no discovery and fedora finds nothing: neither + # may stop kali from being checked. + {"fedora": dict, "kali": lambda: candidates}, + distros=("ubuntu", "fedora", "kali"), + check_state=check_state, + removed_history=removed_history, + ) + + # Skipped: removed x2, unsafe, recent, thin, zero, unknown, missing. + # Already present isn't counted either way. + self.assertEqual((added, skipped), (2, 8)) + self.assertEqual( + self._watch_files(), + sorted(f"{n}.torrent" for n in (present, wanted_a, wanted_b)), + ) + with open(os.path.join(self.tmp_watch_dir, f"{wanted_a}.torrent"), "rb") as f: + self.assertEqual(f.read(), f"{wanted_a} metadata".encode()) + self.assertEqual( + new_state, + { + recent: {"last_checked": "2026-01-05", "leechers": 0}, + thin: {"last_checked": "2026-01-08", "leechers": 5}, + zero: {"last_checked": "2026-01-08", "leechers": 9}, + unknown: {"last_checked": "2026-01-08", "leechers": None}, + missing: {"last_checked": "2026-01-08", "leechers": None}, + wanted_a: {"last_checked": "2026-01-08", "leechers": 6}, + wanted_b: {"last_checked": "2026-01-08", "leechers": 30}, + }, + ) + self.assertEqual( + check_state, {recent: {"last_checked": "2026-01-05", "leechers": 0}} + ) + self.assertEqual( + [url for url, _ in self.tracker.requests], + [candidates[n] for n in (thin, zero, unknown, missing, wanted_a, wanted_b)], + ) + messages = [r.getMessage() for r in logs.records] + for expected in ( + ( + f"Skipping old release {removed_a} - previously removed on 2025-12-01, " + "not re-adding automatically." + ), + ( + f"Skipping old release {removed_b} - previously removed on 2025-12-01, " + "not re-adding automatically." + ), + f"Refusing to save old release with unsafe name: {unsafe!r}", + f"Skipping old release {recent} - checked recently, not due for recheck.", + ( + f"Skipping old release {thin} - 5 leechers / 20 seeders via " + f"{self.TRACKER} (need >= 4 leechers and >= 0.5x as many leechers " + "as seeders)." + ), + ( + f"Skipping old release {zero} - 9 leechers but 0 seeders via " + f"{self.TRACKER} (no verified-complete copy in the swarm; set " + "FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS=true to gamble on it)." + ), + ( + f"Skipping old release {unknown} - could not determine live demand " + "from any tracker." + ), + ( + f"Found unmet demand on old release {wanted_a} (6 leechers via " + f"{self.TRACKER}) - added." + ), + ( + f"Found unmet demand on old release {wanted_b} (30 leechers via " + f"{self.TRACKER}) - added." + ), + ): + self.assertIn(expected, messages) + self.assertEqual( + [m for m in messages if m.startswith(f"Skipping old release {missing}")], + [], + ) + + def test_below_floor_zero_seeder_swarm_is_logged_against_the_thresholds(self): + # allow_zero_seeders=True: a zero-seeder swarm can count as demand, + # but 2 leechers is still under the 4-leecher floor. + name = kali(3) + candidates = {name: self.tracker.candidate(name, swarm(2, 0))} + + with self.assertLogs(ft.logger, level="INFO") as logs: + added, skipped, _ = self._check( + {"kali": lambda: candidates}, allow_zero_seeders=True + ) + + self.assertEqual((added, skipped), (0, 1)) + self.assertEqual( + [r.getMessage() for r in logs.records], + [ + ( + f"Skipping old release {name} - 2 leechers / 0 seeders via " + f"{self.TRACKER} (need >= 4 leechers and >= 0.5x as many " + "leechers as seeders)." + ) + ], + ) + + def test_rechecks_after_7_days_by_default(self): + name = kali(3) + candidates = {name: self.tracker.candidate(name, swarm(6, 10))} + check_state = {name: {"last_checked": "2026-01-01", "leechers": 0}} + + # Checked exactly 7 days ago; _check() leaves recheck_interval_days + # at its default. + added, _, _ = self._check({"kali": lambda: candidates}, check_state=check_state) + + self.assertEqual(added, 1) + + if __name__ == "__main__": unittest.main() From b4d010a5bac5fe7f8aa5f5d7c9512fed9b39c32d Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 15:09:39 +0200 Subject: [PATCH 28/46] Pin the daily run's counters, timing, disk report and RPC credentials run() and _fetch_latest() had 14 surviving mutants: no test produced two of the same download outcome, the clock never advanced, the downloads-folder usage line was never asserted, and RPC credentials were checked by mocking log_seed_ratios_via_http. - FakeWorld.post now behaves like Transmission's RPC: a request without the session id gets a 409 carrying it, and the torrent-get records its auth. FakeWorld.disk_usage records the path it was asked about. - The credentials test asserts the auth the RPC endpoint actually received, plus a no-credentials case, instead of mocking the internal function. - New run tests: - two of every outcome (added, existing, not found, failed, and fetchers that found nothing), with a removed-history ratio-blocked candidate first that must not stop the rest - low-demand images skipped by default and fetched with FETCH_TORRENTS_INCLUDE_LOW_DEMAND=true - elapsed time from a clock that advances - the exact "Downloads folder usage" line and the /downloads path - _run() also takes `now`. All 17 targeted mutants in run()/_fetch_latest() are killed. Co-Authored-By: Claude Opus 5.5 --- tests/test_daily_run.py | 145 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 135 insertions(+), 10 deletions(-) diff --git a/tests/test_daily_run.py b/tests/test_daily_run.py index af318df..3f27592 100644 --- a/tests/test_daily_run.py +++ b/tests/test_daily_run.py @@ -69,7 +69,9 @@ def __init__(self, testcase): self.requested = [] self.events = [] self.transmission = helpers.FakeTransmission() + self.rpc_auths = [] self.disk_used_percent = 10 + self.disk_paths = [] def get(self, url, timeout=None): self.requested.append(url) @@ -81,8 +83,15 @@ def get(self, url, timeout=None): return route def post(self, url, **kwargs): - if kwargs.get("json") is None: # the session-id handshake - return helpers.fake_response(headers={"X-Transmission-Session-Id": "sid"}) + """Transmission's RPC: a request without the session id is refused + with a 409 carrying it; a torrent-get with it records its auth and + lists what `transmission` holds.""" + headers = kwargs.get("headers") or {} + if headers.get("X-Transmission-Session-Id") != "sid": + return helpers.fake_response( + status=409, headers={"X-Transmission-Session-Id": "sid"} + ) + self.rpc_auths.append(kwargs.get("auth")) torrents = [ {"name": t.name, "uploadRatio": t.ratio} for t in self.transmission.torrents ] @@ -94,6 +103,7 @@ def connect(self): def disk_usage(self, path): self.events.append("disk_usage") + self.disk_paths.append(path) total = 100 * 2**30 used = total * self.disk_used_percent // 100 return (total, used, total - used) @@ -186,14 +196,14 @@ def setUp(self): def _rpc_port_open(*args, **kwargs): yield MagicMock() - def _run(self, fetchers=ft.DEFAULT_FETCHERS, **env): + def _run(self, fetchers=ft.DEFAULT_FETCHERS, now=lambda: 1000.0, **env): """Runs ft.run() with load_config(env); returns (summary, messages).""" with self.assertLogs(ft.logger, level="DEBUG") as logs: summary = ft.run( ft.load_config(env), client_factory=self.world.connect, today=date(2026, 9, 2), - now=lambda: 1000.0, + now=now, fetchers=fetchers, disk_usage=self.world.disk_usage, ) @@ -437,14 +447,129 @@ def test_seed_ratios_are_logged_for_the_next_run(self): ) def test_rpc_credentials_from_config_are_used_for_ratio_logging(self): - with unittest.mock.patch.object(ft, "log_seed_ratios_via_http") as log_ratios: - self._run( - FETCH_TORRENTS_DISTROS="arch", - TRANSMISSION_RPC_USERNAME="alice", - TRANSMISSION_RPC_PASSWORD="hunter2", + self._run( + FETCH_TORRENTS_DISTROS="arch", + TRANSMISSION_RPC_USERNAME="alice", + TRANSMISSION_RPC_PASSWORD="hunter2", + ) + + self.assertEqual(self.world.rpc_auths, [("alice", "hunter2")]) + + def test_ratio_logging_sends_no_credentials_when_none_are_configured(self): + self._run(FETCH_TORRENTS_DISTROS="arch") + + self.assertEqual(self.world.rpc_auths, [None]) + + def test_every_download_outcome_is_counted_each_time(self): + # Two of each outcome (four failures: two downloads, two fetchers + # that found nothing), so no counter can get away with "= 1". The + # first candidate is ratio-blocked via the removed history, and must + # not stop the rest of its fetcher's candidates. + def url(name): + return f"https://mirror.example/{name}.torrent" + + for name in ("release-a.iso", "release-b.iso"): + self.world.routes[url(name)] = helpers.fake_response(b"torrent") + for name in ("broken-a.iso", "broken-b.iso"): + self.world.routes[url(name)] = helpers.fake_response(status=500) + for name in ("present-a.iso", "present-b.iso"): + with open(os.path.join(self.world.watch_dir, f"{name}.torrent"), "wb") as f: + f.write(b"already here") + with open( + self.world.state_path("fetch_torrents_removed_history.json"), + "w", + encoding="utf-8", + ) as f: + json.dump( + { + "archlinux-2026.08.01": { + "removed_date": "2026-08-20", + "reason": "stagnant", + "ratio": 0.4, + } + }, + f, ) - log_ratios.assert_called_once_with(auth=("alice", "hunter2")) + def found(*names): + return lambda: {name: url(name) for name in names} + + summary, messages = self._run( + fetchers=( + ( + "arch", + found( + "archlinux-2026.09.01", + "release-a.iso", + "present-a.iso", + "missing-a.iso", + "broken-a.iso", + ), + ), + ("arch", dict), + ( + "arch", + found( + "release-b.iso", + "present-b.iso", + "missing-b.iso", + "broken-b.iso", + ), + ), + ("arch", lambda: False), + ), + FETCH_TORRENTS_DISTROS="arch", + ) + + self.assertEqual( + summary, + ft.RunSummary(elapsed=0.0, added=2, existing=2, not_found=2, failed=4), + ) + self.assertNotIn(url("archlinux-2026.09.01"), self.world.requested) + self.assertIn( + "Skipping archlinux-2026.09.01 due to low ratio on previous version.", + messages, + ) + + def test_low_demand_images_are_skipped_by_default(self): + netinst = "kali-linux-2026.3-installer-netinst-amd64.iso" + fetchers = (("kali", lambda: {netinst: "https://mirror.example/n.torrent"}),) + + self._run(fetchers=fetchers, FETCH_TORRENTS_DISTROS="kali") + + self.assertEqual(self.world.requested, []) + + def test_include_low_demand_fetches_low_demand_images(self): + netinst = "kali-linux-2026.3-installer-netinst-amd64.iso" + fetchers = (("kali", lambda: {netinst: "https://mirror.example/n.torrent"}),) + + self._run( + fetchers=fetchers, + FETCH_TORRENTS_DISTROS="kali", + FETCH_TORRENTS_INCLUDE_LOW_DEMAND="true", + ) + + self.assertEqual(self.world.requested, ["https://mirror.example/n.torrent"]) + + def test_elapsed_time_is_measured_from_start_to_finish(self): + clock = iter([1000.0, 1012.5]) + + summary, messages = self._run( + fetchers=(), now=clock.__next__, FETCH_TORRENTS_DISTROS="arch" + ) + + self.assertEqual(summary.elapsed, 12.5) + self.assertTrue( + messages[-1].startswith("Run complete in 12.50 seconds."), messages[-1] + ) + + def test_reports_downloads_folder_usage_in_whole_gigabytes(self): + self.world.disk_used_percent = 37 + + _summary, messages = self._run(fetchers=(), FETCH_TORRENTS_DISTROS="arch") + + self.assertIn("Downloads folder usage: 37 GB used / 100 GB total", messages) + self.assertEqual(set(self.world.disk_paths), {"/downloads"}) def test_old_release_check_is_off_by_default(self): discovered = [] From aa81ef3d6ac415ece415d30b9866c89d879dc92a Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 15:21:05 +0200 Subject: [PATCH 29/46] Test where log records actually go, and the always-log prefix contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit configure_logging() was tested only for which handlers it builds. No record was ever sent through them, so the [ratio] split, FETCH_TORRENTS_ALWAYS_LOG, the log level and the formatter were unasserted. parse_log_level() had no direct test, and the always-log prefixes were a silent string contract with the run's message wording. 17 mutants survived across these. - ParseLogLevelTests: unset, blank, digits, names (case and whitespace), unknown, and a non-int digit ('²') falling back to the default. - LogRoutingTests emit debug/info/warning, an always-log progress line and [ratio] lines, then assert exactly what lands in stderr, fetch_torrents.log and the ratio log for: - the defaults - a higher FETCH_TORRENTS_LOG_LEVEL - ALWAYS_LOG off with the LOG_LEVEL fallback - FETCH_TORRENTS_LOG_LEVEL beating LOG_LEVEL It also checks the timestamped format and that the logger doesn't propagate. - A run-level test checks every get_always_log_prefixes() entry matches a message the daily run really emits, so rewording one now fails. - test_still_starts_with_run_complete_for_always_log_filter never touched the filter. Renamed to test_is_let_through_by_the_always_log_filter, it now runs the summary through ImportantMessageFilter at ERROR level. All targeted mutants are killed except #82 (propagate = None), which logging treats the same as False. Co-Authored-By: Claude Opus 5.5 --- tests/test_daily_run.py | 33 +++++++++++-- tests/test_logging.py | 107 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 4 deletions(-) diff --git a/tests/test_daily_run.py b/tests/test_daily_run.py index 3f27592..d1ee1a8 100644 --- a/tests/test_daily_run.py +++ b/tests/test_daily_run.py @@ -623,12 +623,12 @@ def _write_state(self, filename, data): with open(self.world.state_path(filename), "w", encoding="utf-8") as f: json.dump(data, f) - def _check(self, candidates, **env): + def _check(self, candidates, fetchers=(), **env): with unittest.mock.patch.dict( ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: dict(candidates)}, clear=True ): return self._run( - fetchers=(), + fetchers=fetchers, FETCH_TORRENTS_DISTROS="kali", FETCH_TORRENTS_CHECK_OLD_RELEASES="true", FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS="4", @@ -638,6 +638,24 @@ def _check(self, candidates, **env): **env, ) + def test_every_always_log_prefix_matches_a_message_the_run_emits(self): + # FETCH_TORRENTS_ALWAYS_LOG lets lines through by prefix, so a + # reworded message would silently stop showing at the default level. + # This run hits every prefixed message: a distro skipped and one + # fetched, an old release added, and the ratio query. + wanted = self._candidate(3, "installer", 6, 10) + + _summary, messages = self._check( + [wanted], fetchers=(("arch", dict), ("kali", dict)) + ) + + for prefix in ft.get_always_log_prefixes(): + with self.subTest(prefix=prefix): + self.assertTrue( + any(m.startswith(prefix) for m in messages), + f"no message starts with {prefix!r}", + ) + def test_config_and_saved_state_drive_the_check(self): recent = self._candidate(1, "installer", 50, 1) removed = self._candidate(1, "live", 50, 1) @@ -825,7 +843,7 @@ def test_with_old_release_check_appends_counts(self): "Old releases (FETCH_TORRENTS_CHECK_OLD_RELEASES): 2 added, 6 skipped.", ) - def test_still_starts_with_run_complete_for_always_log_filter(self): + def test_is_let_through_by_the_always_log_filter(self): summary = ft.format_run_summary( elapsed=1.0, success_count=0, @@ -835,7 +853,14 @@ def test_still_starts_with_run_complete_for_always_log_filter(self): old_added=0, old_skipped=0, ) - self.assertTrue(summary.startswith("Run complete in")) + record = logging.LogRecord( + "fetch_torrents", logging.INFO, __file__, 0, summary, None, None + ) + always_log = ft.ImportantMessageFilter( + logging.ERROR, ft.get_always_log_prefixes() + ) + + self.assertIs(always_log.filter(record), True) class LoadConfigTests(unittest.TestCase): diff --git a/tests/test_logging.py b/tests/test_logging.py index 1da8708..7bc5ade 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -5,6 +5,7 @@ Run with: python -m unittest discover -s tests """ +import io import json import logging import logging.handlers @@ -153,6 +154,112 @@ def test_calling_again_replaces_its_own_handlers_only(self): self.assertTrue(all(h not in ft.logger.handlers for h in first)) +class ParseLogLevelTests(unittest.TestCase): + def test_values(self): + for value, expected in ( + (None, logging.WARNING), # unset + ("", logging.WARNING), + (" ", logging.WARNING), + ("10", 10), + ("debug", logging.DEBUG), + (" Error ", logging.ERROR), + ("bogus", logging.WARNING), + ("²", logging.WARNING), # isdigit(), but not an int + ): + env = {} if value is None else {"LEVEL": value} + with self.subTest(value=value): + self.assertEqual( + ft.parse_log_level("LEVEL", logging.WARNING, env=env), expected + ) + + +class LogRoutingTests(unittest.TestCase): + """Where configure_logging() sends each record: "[ratio]" lines only to + the ratio log; everything else to stderr and fetch_torrents.log at the + configured level, except that with FETCH_TORRENTS_ALWAYS_LOG (on by + default) progress lines starting with an always-log prefix get through + whatever the level.""" + + def _route(self, **env): + """Configures logging with env, emits one record of each kind and + returns (stderr lines, fetch_torrents.log lines, ratio log lines), + each without the timestamp prefix.""" + log_dir = helpers.make_temp_dir(self, "fetch_torrents_routing_") + stream_handler, file_handler, ratio_handler = configure_logging_for_test( + self, log_dir, **env + ) + stderr = io.StringIO() + stream_handler.setStream(stderr) + + ft.logger.debug("plain debug") + ft.logger.info("plain info") + ft.logger.warning("plain warning") + ft.logger.info("Fetching latest arch torrents...") + ft.logger.info("[ratio] RATIOS START") + ft.logger.debug("[ratio] debug ratio") + for handler in (stream_handler, file_handler, ratio_handler): + handler.flush() + + def lines(text): + return [line.split(" ", 2)[2] for line in text.splitlines()] + + with open(os.path.join(log_dir, "fetch_torrents.log"), encoding="utf-8") as f: + main_log = lines(f.read()) + with open( + os.path.join(log_dir, "fetch_torrents_ratios.log"), encoding="utf-8" + ) as f: + ratio_log = lines(f.read()) + return lines(stderr.getvalue()), main_log, ratio_log + + def test_defaults_log_info_and_up_plus_always_log_lines(self): + stderr, main_log, ratio_log = self._route() + + expected = [ + "INFO: plain info", + "WARNING: plain warning", + "INFO: Fetching latest arch torrents...", + ] + self.assertEqual(stderr, expected) + self.assertEqual(main_log, expected) + self.assertEqual(ratio_log, ["INFO: [ratio] RATIOS START"]) + + def test_always_log_lines_get_through_a_higher_level(self): + stderr, main_log, _ = self._route(FETCH_TORRENTS_LOG_LEVEL="WARNING") + + expected = ["WARNING: plain warning", "INFO: Fetching latest arch torrents..."] + self.assertEqual(stderr, expected) + self.assertEqual(main_log, expected) + + def test_always_log_off_applies_the_level_to_everything(self): + stderr, main_log, _ = self._route( + LOG_LEVEL="WARNING", FETCH_TORRENTS_ALWAYS_LOG="false" + ) + + self.assertEqual(stderr, ["WARNING: plain warning"]) + self.assertEqual(main_log, ["WARNING: plain warning"]) + + def test_fetch_torrents_log_level_takes_precedence_over_log_level(self): + _, main_log, _ = self._route( + FETCH_TORRENTS_LOG_LEVEL="DEBUG", LOG_LEVEL="ERROR" + ) + + self.assertEqual(main_log[0], "DEBUG: plain debug") + + def test_lines_carry_a_timestamp_and_the_logger_does_not_propagate(self): + log_dir = helpers.make_temp_dir(self, "fetch_torrents_routing_") + _, file_handler, _ = configure_logging_for_test(self, log_dir) + + ft.logger.warning("plain warning") + file_handler.flush() + + with open(os.path.join(log_dir, "fetch_torrents.log"), encoding="utf-8") as f: + self.assertRegex( + f.read(), + r"^\d{4}-\d\d-\d\d \d\d:\d\d:\d\d,\d{3} WARNING: plain warning\n$", + ) + self.assertFalse(ft.logger.propagate) + + class LogSeedRatiosViaHttpTests(unittest.TestCase): """log_seed_ratios_via_http() must not hang the daily run forever on a stalled Transmission response, and must authenticate with whatever From 0ba4382b7ab6f91759657780b5114d9865a10625 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 15:40:34 +0200 Subject: [PATCH 30/46] Add round-2 test-health tasks (28-38) and record the new baseline The 2026-10-19 test-health review's top five items are done (90c7f8f, db38abe, b4d010a, aa81ef3). The rest of its findings become tasks 28-38: production seams for the RPC wait, UDP transaction ids and live_check; atomic .torrent writes; scrape_torrent fall-through and SSRF edges; configure_transmission gaps; the remaining mutation survivors; mechanical tidy-up; fixture consolidation; toolchain hardening; and a final re-measure with a coverage ratchet. The README records the new baseline at aa81ef3: 423 tests, 99% branch coverage, fetch_torrents.py mutation 95.4% (1027/1076). Co-Authored-By: Claude Opus 5.5 --- tasks/28-rpc-wait-seam.md | 17 +++++++ tasks/29-udp-transaction-id-seam.md | 15 ++++++ tasks/30-live-check-tests.md | 25 ++++++++++ tasks/31-configure-transmission-gaps.md | 19 +++++++ tasks/32-atomic-torrent-writes.md | 15 ++++++ tasks/33-scrape-torrent-branches-and-ssrf.md | 23 +++++++++ tasks/34-mechanical-test-tidy-up.md | 40 +++++++++++++++ tasks/35-fixture-consolidation.md | 31 ++++++++++++ tasks/36-remaining-mutation-survivors.md | 52 ++++++++++++++++++++ tasks/37-toolchain-and-q4.md | 17 +++++++ tasks/38-remeasure-and-ratchet.md | 12 +++++ tasks/README.md | 37 ++++++++++++++ 12 files changed, 303 insertions(+) create mode 100644 tasks/28-rpc-wait-seam.md create mode 100644 tasks/29-udp-transaction-id-seam.md create mode 100644 tasks/30-live-check-tests.md create mode 100644 tasks/31-configure-transmission-gaps.md create mode 100644 tasks/32-atomic-torrent-writes.md create mode 100644 tasks/33-scrape-torrent-branches-and-ssrf.md create mode 100644 tasks/34-mechanical-test-tidy-up.md create mode 100644 tasks/35-fixture-consolidation.md create mode 100644 tasks/36-remaining-mutation-survivors.md create mode 100644 tasks/37-toolchain-and-q4.md create mode 100644 tasks/38-remeasure-and-ratchet.md diff --git a/tasks/28-rpc-wait-seam.md b/tasks/28-rpc-wait-seam.md new file mode 100644 index 0000000..0727c6d --- /dev/null +++ b/tasks/28-rpc-wait-seam.md @@ -0,0 +1,17 @@ +# 28 — Make `wait_for_transmission_rpc` testable and test it + +**Priority**: P2 · **Class**: REFACTOR_REQUIRED · **Depends on**: — + +## Evidence +- `fetch_torrents.py:185` `wait_for_transmission_rpc` calls `time.time()`, `time.sleep(1)` and `socket.create_connection` directly. No test calls it: 3/9 mutants killed (port, timeout, sleep and `return False → True` all survive). +- `tests/test_daily_run.py` `test_rpc_unavailable_skips_ratio_logging_but_still_cleans_up` patches the whole function because the real one would sleep for up to 30 s. The `create_connection` → `refuse()` patch next to it never runs. +- `time.time()` is wall-clock, so a clock change during the wait skews it. `time.monotonic()` is the right clock for elapsed time. + +## Acceptance criteria +- `wait_for_transmission_rpc(host, port, timeout, *, connect=socket.create_connection, clock=time.monotonic, sleep=time.sleep)`. Existing callers are unchanged. +- Tests (write them first; they fail until the seam exists), all with a fake clock and a recording `sleep`, and no real sleeping: + - connects on the first try → `True`, no sleep, `connect` called with `(("localhost", 9091), timeout=3)` + - refused N times, then succeeds → `True`, sleeps recorded as `[1] * N` + - always refused → `False` once the fake clock passes `timeout`; the loop condition is `< timeout` (pin the boundary) +- `run()` gets the readiness check through an injectable seam (e.g. `run(..., wait_for_rpc=wait_for_transmission_rpc)`). The daily-run test drives the "RPC unavailable" path through it and drops both patches. +- Re-run mutmut on the function: no survivors, or each one documented as equivalent. diff --git a/tasks/29-udp-transaction-id-seam.md b/tasks/29-udp-transaction-id-seam.md new file mode 100644 index 0000000..413fa40 --- /dev/null +++ b/tasks/29-udp-transaction-id-seam.md @@ -0,0 +1,15 @@ +# 29 — Inject the UDP transaction id and assert what goes on the wire + +**Priority**: P3 · **Class**: REFACTOR_REQUIRED · **Depends on**: — + +## Evidence +- `torrent_scrape.py:256` `_random_transaction_id()` reads `os.urandom` directly. `ScrapeUdpTrackerTests` (`tests/test_torrent_scrape.py`, about lines 505–570) patches `ts.os.urandom` process-wide with a finite `side_effect` list, so any other `urandom` call in that window raises `StopIteration`. +- `test_full_round_trip_returns_stats` never inspects the first `sendto` payload, or the scrape request's transaction id and info_hash. It only reads `call_args_list[1][0][0]`. The socket is a bare `Mock()`, so a mistyped socket method would also pass. +- `torrent_scrape.py` is at 100% mutation, so this is about clarity and robustness (Farley Maintainable 5), not killing survivors. + +## Acceptance criteria +- `scrape_udp_tracker(host, port, info_hash, timeout=15, *, new_transaction_id=_random_transaction_id)`. `scrape_torrent` keeps its signature. +- Tests pass a deterministic id source instead of patching `os.urandom`. The socket is `unittest.mock.create_autospec(socket.socket, instance=True)`, or a small `FakeUdpTracker` that records datagrams. +- The round-trip test asserts both payloads exactly: `struct.pack(">QII", 0x41727101980, 0, tid1)`, and `struct.pack(">QII", connection_id, 2, tid2) + info_hash`. +- The test named `test_scrape_response_echoing_the_connect_id_raises` says what it guards (the scrape uses a fresh transaction id). Use `assertRaisesRegex` on the mismatch message. +- Bare `assertRaises` in `test_torrent_scrape.py` (about 12) become `assertRaisesRegex` on a message fragment. Fold the duplicate short-response tests together (`BuildParseUdpConnectRequestTests.test_short_response_raises` vs `test_short_response_reports_its_length`; the same pair in the scrape-request tests). diff --git a/tasks/30-live-check-tests.md b/tasks/30-live-check-tests.md new file mode 100644 index 0000000..09e252f --- /dev/null +++ b/tasks/30-live-check-tests.md @@ -0,0 +1,25 @@ +# 30 — Test `scripts/live_check.py` beyond `problems()` + +**Priority**: P2 · **Class**: REFACTOR_REQUIRED · **Depends on**: — + +## Evidence +- `tests/test_live_check.py` covers only `problems()` and `distro_of()`. Untested: + - `is_reachable` (`scripts/live_check.py:65`): HEAD 405 → GET fallback, status handling + - `check_scrapers` (`:76`): failure aggregation, `MAY_BE_EMPTY`, "one reachable is enough" + - `check_tracker` (`:104`): the for/else, integer validation + - `main` (`:127`): exit codes 0/1/2 and usage +- The weekly drift job (`.github/workflows/drift.yml`) trusts these, so a bug here gives false OKs (or false alarms) while the suite stays green. +- `main()` builds `checks` locally, and the check functions call module globals (`requests`, `ft.fetch_*`, `torrent_scrape`), so they can't be tested without patching module names. + +## Acceptance criteria +- Inject the collaborators with defaults: + - `check_scrapers(fetchers=FETCHERS, reachable=is_reachable)` + - `check_tracker(fetch=ft.fetch_ubuntu_lts, get=requests.get, scrape=torrent_scrape.scrape_torrent)` + - `main(argv, checks=CHECKS)` +- Tests (using `helpers.fake_response`) for each branch: + - HEAD 405 then GET 200 → reachable + - a fetcher returning nothing (and the `MAY_BE_EMPTY` exception) + - every candidate unreachable vs one reachable + - a tracker with no integer seeders/leechers + - `main` exit codes: all OK → 0, a problem → 1, bad usage → 2 +- The `sys.path.insert` in `test_live_check.py:13` is undone after the module (or comes from `helpers.REPO_ROOT`). diff --git a/tasks/31-configure-transmission-gaps.md b/tasks/31-configure-transmission-gaps.md new file mode 100644 index 0000000..ae6571a --- /dev/null +++ b/tasks/31-configure-transmission-gaps.md @@ -0,0 +1,19 @@ +# 31 — `configure_transmission`: hermetic default path, boundaries, whole-result asserts + +**Priority**: P3 · **Class**: NO_REFACTOR (plus one small seam) · **Depends on**: — + +## Evidence +- `test_configure_transmission.py` `test_defaults_to_config_settings_path` (about line 355): `main()` calls the real `os.path.exists('/config/settings.json')` (`configure_transmission.py:140`). On a host or container where that file exists, `main()` returns early and the test fails. It also only asserts on patched `load_settings`/`save_settings`. +- Mutation survivor #30: `_parse_positive_int` `parsed > 0` → `> 1` survives, because a limit of exactly 1 is untested. +- `_parse_bool`'s case and whitespace handling (`'TRUE'`, `' false '`) has no test. mutmut never generates "drop `.lower()`/`.strip()`" mutants, so its 100% score for `_parse_bool` doesn't cover this. +- `ApplyOverridesTests` asserts three keys one by one, and doesn't check that the input dict is left unmutated. +- `save_settings` survivors #79/#80 (`indent`, `sort_keys`): decide whether the file layout is a contract. +- `if os.path.exists(...): os.remove(...)` inside the subTest loop of `test_invalid_peer_limits_are_dropped_silently` (about line 318). + +## Acceptance criteria +- `main(settings_path=None)` falls back to `TRANSMISSION_SETTINGS_FILE`, then a module constant `DEFAULT_SETTINGS_FILE = "/config/settings.json"`. The default-path test asserts the constant and runs hermetically. +- `TRANSMISSION_PEER_LIMIT_GLOBAL=1` is kept as 1. +- `TRANSMISSION_PORT_FORWARDING='TRUE'` and `' false '` parse. +- `ApplyOverridesTests` uses one `assertEqual(merged, {...})` and asserts the input is unchanged. +- Either pin the settings file text (`indent=4`, sorted keys) in one golden test, or document #79/#80 as accepted equivalents in the README targets section. +- Each subTest case gets its own temp dir, with no `if` in the test body. diff --git a/tasks/32-atomic-torrent-writes.md b/tasks/32-atomic-torrent-writes.md new file mode 100644 index 0000000..db13e2b --- /dev/null +++ b/tasks/32-atomic-torrent-writes.md @@ -0,0 +1,15 @@ +# 32 — Atomic `.torrent` writes (partial file treated as "existing" forever) + +**Priority**: P2 (bug) · **Class**: NO_REFACTOR (characterize, then behaviour fix) · **Depends on**: — + +## Evidence +- `fetch_torrents.py:534` (`download_torrent`) and `:1115` (old-release add) write straight to the final `/watch/.torrent` path. +- If the process dies or the disk fills mid-write, a truncated file stays behind. On the next run `os.path.exists(dest)` returns `"existing"`, so it is never re-fetched, and Transmission rejects or ignores the broken file. +- `configure_transmission.save_settings` and the `save_*` state writers already use tmp + `os.replace`. Only the watch-dir writes don't. + +## Acceptance criteria +1. Characterization first: a test where the write fails part-way leaves a partial `.torrent` behind today, and the next `download_torrent` call returns `"existing"`. Patch `open` to raise after the first write, or use a response whose `.content` access raises. Commit this as documenting current behaviour, or write it straight as the failing test for step 2. +2. Fix test-first: write to `.tmp` (or a `tempfile` in `watch_dir`, so transmission-daemon's watcher never sees a half file) and `os.replace` into place. On failure no `.torrent` exists, and the next call downloads again. +3. Make sure transmission-daemon ignores the temp name. It watches `*.torrent`, so a `.torrent.part` suffix or a dot-prefixed name is safer than `.torrent.tmp`. +4. The same for the old-release write at `:1115`. +5. The README/CLAUDE.md notes are updated if the watch-dir contract changes. diff --git a/tasks/33-scrape-torrent-branches-and-ssrf.md b/tasks/33-scrape-torrent-branches-and-ssrf.md new file mode 100644 index 0000000..ade5e62 --- /dev/null +++ b/tasks/33-scrape-torrent-branches-and-ssrf.md @@ -0,0 +1,23 @@ +# 33 — `scrape_torrent` fall-through branches and SSRF guard edges + +**Priority**: P2 · **Class**: NO_REFACTOR · **Depends on**: — + +## Evidence +- `torrent_scrape.py:351` `scrape_torrent`. In `ScrapeTorrentTests`: + - only the default timeout is asserted; the HTTP call's `timeout` kwarg is never checked + - UDP call args are `assert_called_once()` only + - request URLs are matched with `assertIn` substrings + - results are compared field by field, so `completed` is never asserted +- No test covers falling through to the next tracker on: + - an HTTP status error (`raise_for_status`) + - a `failure reason` response + - a response that lacks the info hash (stats `None`) + - an announce URL with no `announce` segment (`derive_scrape_url → None`) +- `_is_safe_scrape_host` (`:311`) returns `bool(addresses) and not any(...)`, but no test resolves to `[]`, so treating an empty resolution as safe would pass. `IsDisallowedIpTests` has no multicast case (`224.0.0.1`, `ff02::1`) and no IPv4-mapped loopback (`::ffff:127.0.0.1`). +- Five tests hand-build `Mock()` HTTP responses instead of `helpers.fake_response()`, and `fake_resolve` is duplicated. + +## Acceptance criteria +- Parametrised fall-through tests: tracker A fails in each of the four ways above, tracker B is good. Each asserts the whole result dict, and `mock_get.call_args == call(, timeout=3)` with `scrape_torrent(..., timeout=3)`. +- The UDP path asserts `(host, port, info_hash)` exactly. +- SSRF: `_resolve_addresses → []` is unsafe. Decide and pin the multicast, `0.0.0.0` and `::ffff:127.0.0.1` cases; if any is wrongly allowed today, fixing it is a behaviour change, so test first. +- Mock responses replaced by a `_tracker_response(info_hash, seeders, leechers)` helper over `helpers.fake_response`, and one shared `resolve_private()` fake. diff --git a/tasks/34-mechanical-test-tidy-up.md b/tasks/34-mechanical-test-tidy-up.md new file mode 100644 index 0000000..2859497 --- /dev/null +++ b/tasks/34-mechanical-test-tidy-up.md @@ -0,0 +1,40 @@ +# 34 — Mechanical test tidy-up + +**Priority**: P3 · **Class**: NO_REFACTOR · **Depends on**: — + +From `.dev-team-reports/test-design-2026-10-19.md`. None of these change production code. + +## Checklist +- [ ] **Circular oracles → literals.** These expected values come from production code; use literals instead (the Kali URL list is already pinned in `test_scrapers.py`): + - `test_fetchers.py` `FetchKaliOldVersionsTests.test_returns_only_the_non_newest_versions` builds its expected value from `ft._kali_torrent_urls` + - `test_fetchers.py` default-distros test uses `ft.DEFAULT_DISTROS` + - `test_daily_run.py` uses `ft.ALL_DISTROS` +- [ ] **CPython error wording.** Three `test_daily_run.py` config tests embed CPython's `ValueError` text (`could not convert string to float: ...`, `invalid literal for int() ...`). Assert only the project-owned `ENV_VAR: ` prefix. +- [ ] **Subprocess warnings.** `test_logging.ImportOpensNoFilesTests`: run the subprocess with `-W ignore` so an import-time warning from a dependency can't fail `stderr == ''`. +- [ ] **Dead code.** + - Delete the duplicate `if __name__ == "__main__"` block at the end of `test_cleanup.py`. + - Delete the unused `helpers.fixed_date`, or use it for a `check_old_releases_for_demand(today=None)` default test. + - Delete `helpers.use_isolated_env` unless task 35 adopts it. + - Hoist the duplicated `_rpc_port_open` stub (`test_daily_run.py`) into helpers. +- [ ] **Duplicates of stricter recorded-page tests.** Delete these `test_fetchers.py` tests, each of which a `test_scrapers` test covers more strictly: + - `FetchDevuanLatestNamingTests.test_returns_false_when_no_torrent_link_found` + - `FetchArchLatestNamingTests.test_no_available_releases_returns_false` + - the Kali single-version test +- [ ] **Duplicate timeout test.** Delete `test_logging.LogSeedRatiosViaHttpTests.test_session_id_request_has_a_timeout`, which duplicates `test_requests_follow_the_transmission_rpc_protocol`. +- [ ] **`test_scrapers.FailurePathTests`.** + - Merge `test_http_404` into `test_http_500` (same branch). + - Assert "an error was logged naming the cause" instead of exact per-fetcher prefixes. + - Rename `LATEST_FETCHERS`, since it also holds the `*_old_versions` fetchers. +- [ ] **`ScraperTestCase.fetch()`.** Return a small result object instead of stashing `self.pages`, and drop the dummy `logger.debug('fetch start')` hack. +- [ ] **`test_cleanup.test_selects_older_version_per_distro_and_type`.** Turn it into a subTest table per distro and delete the redundant `assertNotIn` loop. +- [ ] **Count-only asserts.** Assert names and the absence of `remove_torrent` calls instead (`test_daily_run` `len(transmission.torrents) == 2`, `test_cleanup` `test_missing_added_date_does_not_crash`, `test_cleanup_old_versions_keeps_versions_below_min_ratio`). +- [ ] **Weak boolean asserts.** + - Replace `assertTrue(x.startswith(...))` / `assertTrue(any(...))` with `assertRegex` or an exact value where one is known (e.g. `test_torrent_scrape` `test_adds_query_param_with_no_existing_query`). + - `test_fetchers.py` remove-watch-file test: assert the file's contents. +- [ ] **`parse_supported_distros`.** Wrap the unknown-entry and fallback cases in `assertLogs` and assert the warning text. Assert the default list as a literal. +- [ ] **Misfiled tests.** Move each beside the function it tests: + - flavour/lubuntu tests out of `ParseVersionTypeTests` + - `distro_of` into its own class + - `NetworkGuardTests` plus `test_transmission_double.py` into a `test_helpers.py` + - `LoadConfig`/`FormatRunSummary`/`MakeClientFactory` into `test_config.py`; update the module list in CLAUDE.md +- [ ] **Old-release clock.** `CheckOldReleasesForDemandTests` call `check_old_releases_for_demand` without `today`; pass it explicitly. diff --git a/tasks/35-fixture-consolidation.md b/tasks/35-fixture-consolidation.md new file mode 100644 index 0000000..c44518b --- /dev/null +++ b/tasks/35-fixture-consolidation.md @@ -0,0 +1,31 @@ +# 35 — Consolidate fixtures and the module-global path seam + +**Priority**: P3 · **Class**: NO_REFACTOR (helpers), then REFACTOR_REQUIRED (watch dir env var) · **Depends on**: 34 + +## Evidence +- Eight `setUp`s copy the same `patch.object(ft, "watch_dir" / "_file", ...)` plus `addCleanup` dance: + - `test_fetchers.py` (3) + - `test_cleanup.py` (2) + - `test_old_release_demand.py` (2) + - `test_daily_run.py` (1) + + Renaming one of these module globals touches 4 files. That is the suite's one rung-1 maintainability pocket. +- `watch_dir = "/watch"` (`fetch_torrents.py:169`) is hard-coded, unlike `log_dir`. `MainWiringTests` (the runpy test) can't redirect it; it stays safe only because no route is registered that would make it write to `/watch` (C:\watch on Windows). +- There are two RPC doubles with different rules. `test_logging`'s `LogSeedRatiosViaHttpTests` hand-builds the 409 handshake, while `test_daily_run.FakeWorld.post` (now strict) has its own. They should be one helper. +- Repeated fixture boilerplate: + - `HasUnmetDemandTests` spells out all three thresholds in every test + - about 14 `removed_history` literals carry fields the gate ignores + - `RunTests` repeats the same two-torrent setup 5 times +- `ShouldFetchTorrentRatioKeyTests` has 10 near-identical tests. Kali and Arch have no high-ratio case. +- `get_rpc_credentials()` (`fetch_torrents.py:172`) duplicates `Config.rpc_auth`. + +## Acceptance criteria +- `helpers.use_watch_dir(testcase)` and `helpers.use_state_files(testcase)` return the temp paths; all 8 setUps use them. +- `FETCH_TORRENTS_WATCH_DIR` (default `/watch`) is read once like `log_dir`, written test-first. `MainWiringTests` points it at a temp dir. Document it in the README. +- One strict Transmission-RPC HTTP fake lives in helpers (409 plus session id, records auth), used by both `test_logging` and `FakeWorld`. +- Creation Methods: + - `demand(scrape, **overrides)` in `test_old_release_demand` + - `removed(name, ratio=None)` for removed-history entries + - a `world.hold_current_and_superseded_ubuntu()` helper +- The ratio-key tests become one parameterised table, including the missing Kali and Arch high-ratio rows. +- `get_rpc_credentials` folds into `Config` (one credential rule), with `log_seed_ratios_via_http`'s fallback kept working. diff --git a/tasks/36-remaining-mutation-survivors.md b/tasks/36-remaining-mutation-survivors.md new file mode 100644 index 0000000..8e331e2 --- /dev/null +++ b/tasks/36-remaining-mutation-survivors.md @@ -0,0 +1,52 @@ +# 36 — Remaining `fetch_torrents.py` mutation survivors + +**Priority**: P3 · **Class**: NO_REFACTOR · **Depends on**: — + +Full mutmut 2.5.1 run at aa81ef3, with every test module except `test_live_check` as the runner: 1027/1076 (95.4%) honest, critical set 258/258. Of the 49 survivors, the 6 in `wait_for_transmission_rpc` are task 28. The rest are listed below. Mutant ids and line numbers are from that run; rerun `mutmut run` and `mutmut results` to refresh them. + +## To kill +- **Fetcher request timeouts never asserted** (`timeout=30` → `31`): + - `download_torrent` #324 + - `fetch_ubuntu_lts` #336 + - `fetch_debian_stable` #368 + - `fetch_kali_latest` #406 + - `fetch_kali_old_versions` #429 + - `fetch_arch_latest` #458 + - `fetch_devuan_latest` #494 + - `fetch_linuxmint_cinnamon` #511 + - `fetch_fedora_workstation` #530 + - `fetch_fedora_workstation_old_versions` #566 + + One table test over the fetchers, with a recording `requests.get` that asserts every call passes `timeout=30`, kills all ten. It also pins that no provider can hang the run. +- **Ordering and selection in scrapers:** + - `fetch_arch_latest` #476 (`continue` → `break` on a non-available row): needs an unavailable row *before* an available one. + - `fetch_arch_latest` #484 (`max` by href instead of version): needs a page where href order differs from version order. + - `fetch_fedora_workstation` #547 (`>` → `>=`): needs the same version listed twice for one arch, with different URLs. + - `fetch_fedora_workstation_old_versions` #584 (sort by url instead of version): same idea. + - `fetch_kali_latest` #424 (`return False` → `True` on no versions): assert `assertIs(result, False)`. + - `fetch_kali_old_versions` #445 (`versions[:-1]` → `[:+1]`): needs 3+ versions on the page. +- **`cleanup_old_versions`:** + - #814: the default `min_ratio=1.0` is never relied on. Call it without `min_ratio`, with a 1.5-ratio superseded torrent, which is removed under 1.0 and kept under 2.0. + - #824: `or 0.0` → `or 1.0` on the recorded ratio. Remove a zero-ratio torrent and assert history `ratio == 0.0`. +- **`log_seed_ratios_via_http` #739** (`and` → `or` in the credential fallback): a username without a password must send `auth=None`. +- **`parse_bool` #20** (default `False` → `True`): call it once without `default`. +- **`parse_supported_distros` #163/#164:** assert the "contains unknown distributions: …" warning (`assertLogs`) lists exactly the unknown entries. This also covers task 34's warning item. +- **`record_removal` #658** (`round(…, 3)` → `4`): record a ratio like `0.12345` and assert `0.123`. +- **`RunSummary` #1046/#1047** (default `elapsed`): assert `ft.RunSummary()` equals `RunSummary(elapsed=0.0, added=0, existing=0, not_found=0, failed=0, old_added=None, old_skipped=None)`. +- **`Config` #943** (`frozen=True` → `False`): assert assigning to a `Config` field raises `dataclasses.FrozenInstanceError`, if immutability is meant to be a contract. Otherwise document it as accepted. +- **Module path globals #10/#12/#84** (`ratio_history_file`, `old_release_check_state_file`, `watch_dir` → `None`): every test patches them, so their real values are never checked. Assert the defaults: `os.path.join(log_dir, …)` names and `"/watch"`, or the env-var default once task 35 adds `FETCH_TORRENTS_WATCH_DIR`. + +## Decide, then kill or accept +- **`save_old_release_check_state` / `save_ratio_history` / `save_removed_history`** (#635/#636, #881/#882, #652/#653: `indent`, `sort_keys`): the state files are only ever read back by this code. Either pin one golden file text in the `JsonStoreContract` mixin (`tests/test_persistence.py`), or list these as accepted equivalents in the README. +- **`enforce_disk_usage_limit` #858** (in-loop `total == 0` fallback): unreachable unless the disk's total drops to 0 mid-loop. Accept. +- **`enforce_disk_usage_limit` #861** (`break` → `continue` once under the threshold): only adds extra disk checks. Accept, or assert `len(disk.paths)`. + +## Accepted equivalents (record in the README) +- **#715, #729:** `.get("seeders", 0)` default. A scrape result always carries `seeders`. +- **#82:** `logger.propagate = None`, which logging treats as `False`. +- **#823, #863, #929:** the `getattr(torrent, "ratio", 0.0)` default. A real `transmission_rpc.Torrent` always has `.ratio`. + +## Acceptance criteria +- Every "To kill" item has a test that fails against the mutant (check with `mutmut apply ` or a hand edit) and passes on the real code. +- The "decide" items are either killed or added to the accepted list in `tasks/README.md`. +- A full mutmut rerun leaves only the accepted list, plus whatever task 28 hasn't handled yet. diff --git a/tasks/37-toolchain-and-q4.md b/tasks/37-toolchain-and-q4.md new file mode 100644 index 0000000..102c2f7 --- /dev/null +++ b/tasks/37-toolchain-and-q4.md @@ -0,0 +1,17 @@ +# 37 — Toolchain hardening: smoke test, shell/Dockerfile linting, ruff S, pinned versions + +**Priority**: P2 · **Class**: CI · **Depends on**: — + +From `.dev-team-reports/cd-test-architecture-linux-iso-seeder.md` (migration steps 4–6). + +## Evidence +- `scripts/smoke_test.sh` asserts the container starts, but not that the daily `fetch_torrents.py` run began. A script that crashes on import would pass. +- No `shellcheck` (`entrypoint.sh`, `scripts/*.sh`) or `hadolint` (`Dockerfile`) in CI. `su-exec` and quoting bugs show up only at runtime. +- ruff enables only 3 `S` (bandit) rules. +- `.github/workflows/docker-image.yml:40` uses `python-version: "3.x"`, and `requirements-test.txt` leaves `requests`/`beautifulsoup4` unpinned. The pip job can change behaviour with no code change; the in-image run on apk versions only partly mitigates this. + +## Acceptance criteria +- `smoke_test.sh` waits for `/logs/fetch_torrents.log` to contain `Starting torrent fetch run.` in both the root and `RUN_AS_NON_ROOT` variants. Fetch errors from `--network none` are expected and fine. +- The `test` job runs `shellcheck entrypoint.sh scripts/*.sh` and `hadolint Dockerfile` (pinned versions), with any ignores justified inline. +- ruff's `select` gains the `S` family, with per-rule ignores justified in `pyproject.toml` (e.g. `S101` in tests, the `S310`/`S603` cases that are intended). +- CI Python is pinned to the Alpine minor (e.g. `"3.14"`). `requests` and `beautifulsoup4` are pinned in `requirements-test.txt` to the apk versions, with a comment to bump them alongside `transmission-rpc`. diff --git a/tasks/38-remeasure-and-ratchet.md b/tasks/38-remeasure-and-ratchet.md new file mode 100644 index 0000000..f2d3eb5 --- /dev/null +++ b/tasks/38-remeasure-and-ratchet.md @@ -0,0 +1,12 @@ +# 38 — Re-measure, raise the coverage ratchet, record the baseline + +**Priority**: P3 · **Class**: — · **Depends on**: 28–37 (or whichever of them are done) + +## Steps +1. Full mutmut runs on all three modules, with the whole suite as the runner. `test_live_check` needs `scripts/` in the mutation working copy; copy it in, or leave the module out. +2. Coverage. It is 99% total at aa81ef3 against `fail_under = 94` in `pyproject.toml`. Raise `fail_under` to a value CI can hold (e.g. 97). CI's pip-installed versions can cover slightly different lines than local runs, so check the CI number before ratcheting to the local one. +3. Update the "Targets" table in `tasks/README.md`: tests, coverage, mutation scores, Farley, and the accepted-equivalent survivor list from task 36. +4. Optionally re-run `/test-health` to produce a fresh report and compare it with `.dev-team-reports/test-health-2026-10-19.md`. + +## Optional: business-readable seeding policy (Q2) +The suite has no business-facing (Q2) layer: no `.feature` files, and the seeding policy exists only as code-level examples. A short README or `docs/` section would let a non-developer review the policy without adopting a BDD runner. It would be a plain table of when an ISO is fetched, kept, removed or re-seeded, with each row naming the test that pins it. diff --git a/tasks/README.md b/tasks/README.md index fda95c7..fc213fb 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -71,3 +71,40 @@ The list is in a scratch run, not the repo: rerun `mutmut run` and All tasks are done. `tests/test_fetch_torrents.py`, which the task files refer to, was split by concern in task 26 (see CLAUDE.md for the new module list). + +## Round 2 (from the 2026-10-19 test-health review) + +Source: `.dev-team-reports/test-health-2026-10-19.md` and +`.dev-team-reports/test-design-2026-10-19.md` (local, gitignored). Farley Score was 8.41 +across 383 tests at 7080bbd. + +The review's top five items were done straight away, without task files: + +| Item | Commit | +| --- | --- | +| `FakeDisk` honours `delete_data`; disk-pressure and stagnation boundaries pinned | 90c7f8f | +| Old-release check tested sociably (real evaluation, faked network only) | db38abe | +| Daily-run counters, timing, disk report, RPC credentials | b4d010a | +| Logging routing, `parse_log_level`, always-log prefix contract | aa81ef3 | + +Re-measured at aa81ef3: 423 tests green, 99% branch coverage. `fetch_torrents.py` +mutation score is 95.4% (1027/1076), up from 88.6%, with the critical set still +258/258. `torrent_scrape.py` is at 100% and `configure_transmission.py` at 96.7% +(87/90), both unchanged. + +| # | Task | Priority | Class | Depends on | Status | +| --- | --- | --- | --- | --- | --- | +| 28 | [Make `wait_for_transmission_rpc` testable and test it](28-rpc-wait-seam.md) | P2 | REFACTOR_REQUIRED | — | open | +| 29 | [Inject the UDP transaction id; assert wire payloads](29-udp-transaction-id-seam.md) | P3 | REFACTOR_REQUIRED | — | open | +| 30 | [Test `scripts/live_check.py` beyond `problems()`](30-live-check-tests.md) | P2 | REFACTOR_REQUIRED | — | open | +| 31 | [`configure_transmission`: hermetic default path, boundaries](31-configure-transmission-gaps.md) | P3 | NO_REFACTOR | — | open | +| 32 | [Atomic `.torrent` writes](32-atomic-torrent-writes.md) | P2 (bug) | NO_REFACTOR | — | open | +| 33 | [`scrape_torrent` fall-through branches and SSRF edges](33-scrape-torrent-branches-and-ssrf.md) | P2 | NO_REFACTOR | — | open | +| 34 | [Mechanical test tidy-up](34-mechanical-test-tidy-up.md) | P3 | NO_REFACTOR | — | open | +| 35 | [Consolidate fixtures and the path-global seam](35-fixture-consolidation.md) | P3 | NO_REFACTOR → REFACTOR_REQUIRED | 34 | open | +| 36 | [Remaining `fetch_torrents.py` mutation survivors](36-remaining-mutation-survivors.md) | P3 | NO_REFACTOR | — | open | +| 37 | [Toolchain: smoke test, shellcheck/hadolint, ruff S, pins](37-toolchain-and-q4.md) | P2 | CI | — | open | +| 38 | [Re-measure, raise the coverage ratchet, record the baseline](38-remeasure-and-ratchet.md) | P3 | — | 28–37 | open | + +Suggested order: 32 (a real bug), then 28, 30 and 33 (resilience and drift-job trust), +then 37, then 36, then the clean-up tasks 29, 31, 34 and 35, and finally 38. From 8e5f4bac6be3e0efafa11b71c9b5c77401f19485 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 15:47:48 +0200 Subject: [PATCH 31/46] Write watch-dir .torrent files atomically (task 32) download_torrent() and the old-release add wrote straight to /watch/.torrent, so a write that died part-way left a partial file that counted as "existing" forever. Both now write a dot-prefixed ..torrent.part (outside transmission's *.torrent glob) and os.replace() it into place. A failed old-release write is logged, skipped and left unrecorded so the next run retries it. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- fetch_torrents.py | 32 ++++++++++++++--- tasks/README.md | 2 +- tests/helpers.py | 29 +++++++++++++++ tests/test_fetchers.py | 54 ++++++++++++++++++++++++++++ tests/test_old_release_demand.py | 60 ++++++++++++++++++++++++++++++++ 6 files changed, 173 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ea4634d..95ac2c2 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ by default: |---|---| | `/config` | Transmission configuration files | | `/downloads` | Downloaded ISO files (seeding storage) | -| `/watch` | Torrent watch folder | +| `/watch` | Torrent watch folder. `.torrent` files are written as a hidden `..torrent.part` first and renamed into place, so Transmission never picks up a half-written file. A leftover `.part` file is safe to delete. | | `/logs` | Persistent logs for fetch script runs, plus `fetch_torrents_ratio_history.json` - a small, bounded per-torrent ratio history used to detect stagnant (no-traction) torrents. Deleting it just resets stagnation tracking; it's not required for anything else. `fetch_torrents.log` itself is size-bounded (rotated, see `FETCH_TORRENTS_LOG_MAX_BYTES`/`FETCH_TORRENTS_LOG_BACKUP_COUNT`), so this volume won't grow without limit even if left running indefinitely. | --- diff --git a/fetch_torrents.py b/fetch_torrents.py index bbc585f..8bfa403 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -511,6 +511,25 @@ def _safe_watch_path(name): return os.path.join(watch_dir, f"{name}.torrent") +def _write_watch_file(dest, data): + """Write data to dest atomically: into a dot-prefixed .part file beside + it, then os.replace() into place. transmission-daemon watches *.torrent + and download_torrent() treats an existing dest as already fetched, so a + write that dies part-way (disk full, process killed) must never leave a + partial file under dest's name.""" + tmp_path = os.path.join(os.path.dirname(dest), f".{os.path.basename(dest)}.part") + try: + with open(tmp_path, "wb") as f: + f.write(data) + os.replace(tmp_path, dest) + except BaseException: + try: + os.remove(tmp_path) + except OSError: + pass + raise + + def download_torrent(name, url): dest = _safe_watch_path(name) if dest is None: @@ -531,8 +550,7 @@ def download_torrent(name, url): logger.info("%s not found upstream (404) – likely no longer hosted.", url) return "not_found" r.raise_for_status() - with open(dest, "wb") as f: - f.write(r.content) + _write_watch_file(dest, r.content) logger.info(f"Saved {dest}") return "added" except Exception as e: @@ -1112,8 +1130,14 @@ def check_old_releases_for_demand( } if keep: - with open(dest, "wb") as f: - f.write(torrent_bytes) + try: + _write_watch_file(dest, torrent_bytes) + except OSError as e: + logger.error("Failed to save old release %s: %s", name, e) + # Not recorded as checked, so the next run retries it. + del check_state[name] + skipped += 1 + continue logger.info( "Found unmet demand on old release %s (%d leechers via %s) - added.", name, diff --git a/tasks/README.md b/tasks/README.md index fc213fb..b02669c 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -98,7 +98,7 @@ mutation score is 95.4% (1027/1076), up from 88.6%, with the critical set still | 29 | [Inject the UDP transaction id; assert wire payloads](29-udp-transaction-id-seam.md) | P3 | REFACTOR_REQUIRED | — | open | | 30 | [Test `scripts/live_check.py` beyond `problems()`](30-live-check-tests.md) | P2 | REFACTOR_REQUIRED | — | open | | 31 | [`configure_transmission`: hermetic default path, boundaries](31-configure-transmission-gaps.md) | P3 | NO_REFACTOR | — | open | -| 32 | [Atomic `.torrent` writes](32-atomic-torrent-writes.md) | P2 (bug) | NO_REFACTOR | — | open | +| 32 | [Atomic `.torrent` writes](32-atomic-torrent-writes.md) | P2 (bug) | NO_REFACTOR | — | done | | 33 | [`scrape_torrent` fall-through branches and SSRF edges](33-scrape-torrent-branches-and-ssrf.md) | P2 | NO_REFACTOR | — | open | | 34 | [Mechanical test tidy-up](34-mechanical-test-tidy-up.md) | P3 | NO_REFACTOR | — | open | | 35 | [Consolidate fixtures and the path-global seam](35-fixture-consolidation.md) | P3 | NO_REFACTOR → REFACTOR_REQUIRED | 34 | open | diff --git a/tests/helpers.py b/tests/helpers.py index 8b95ca7..d1c2e9b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -16,6 +16,7 @@ def setUpModule(): helpers.isolate_module() """ +import errno import glob import inspect import json @@ -210,6 +211,34 @@ def __call__(self, path): return (self.total, used, self.total - used) +def disk_full_open(module): + """Patch `module`'s own `open` so a binary write keeps half its bytes and + then fails with ENOSPC, as when the disk fills mid-write. Scoped to the + module (not builtins) so logging's own file writes are unaffected.""" + real_open = open + + class HalfWritingFile: + def __init__(self, f): + self.f = f + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.f.close() + return False + + def write(self, data): + self.f.write(data[: len(data) // 2]) + raise OSError(errno.ENOSPC, "No space left on device") + + def fake_open(path, mode="r", *args, **kwargs): + f = real_open(path, mode, *args, **kwargs) + return HalfWritingFile(f) if "wb" in mode else f + + return unittest.mock.patch.object(module, "open", fake_open, create=True) + + PAGES_DIR = os.path.join( os.path.dirname(os.path.abspath(__file__)), "fixtures", "pages" ) diff --git a/tests/test_fetchers.py b/tests/test_fetchers.py index 646d0e6..5b12c32 100644 --- a/tests/test_fetchers.py +++ b/tests/test_fetchers.py @@ -458,6 +458,60 @@ def test_successful_download_is_still_added(self): self.assertEqual(f.read(), b"fake torrent bytes") +class DownloadTorrentAtomicWriteTests(unittest.TestCase): + """transmission-daemon watches *.torrent, and download_torrent() treats + an existing .torrent as already fetched - so a write that dies + part-way must never leave a partial file under that name.""" + + NAME = "ubuntu-24.04-desktop-amd64.iso" + + def setUp(self): + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) + patcher.start() + self.addCleanup(patcher.stop) + + def _download(self): + response = helpers.fake_response(b"d8:announce3:urle" * 64) + with unittest.mock.patch.object(ft.requests, "get", return_value=response): + return ft.download_torrent(self.NAME, "https://example.invalid/x.torrent") + + def test_write_failing_part_way_leaves_no_file_behind(self): + with helpers.disk_full_open(ft): + status = self._download() + + self.assertEqual(status, "failed") + self.assertEqual(os.listdir(self.tmp_watch_dir), []) + + def test_next_run_downloads_again_after_a_failed_write(self): + with helpers.disk_full_open(ft): + self._download() + + status = self._download() + + self.assertEqual(status, "added") + dest = os.path.join(self.tmp_watch_dir, f"{self.NAME}.torrent") + with open(dest, "rb") as f: + self.assertEqual(f.read(), b"d8:announce3:urle" * 64) + self.assertEqual(os.listdir(self.tmp_watch_dir), [f"{self.NAME}.torrent"]) + + def test_partial_file_is_never_named_like_a_torrent(self): + # Whatever exists mid-write must not match transmission's *.torrent + # watch glob. + seen = [] + real_replace = os.replace + + def spy_replace(src, dst): + seen.extend(os.listdir(self.tmp_watch_dir)) + real_replace(src, dst) + + with unittest.mock.patch.object(ft.os, "replace", spy_replace): + self.assertEqual(self._download(), "added") + + self.assertEqual(len(seen), 1) + self.assertFalse(seen[0].endswith(".torrent"), seen[0]) + + class SafeWatchPathTests(unittest.TestCase): """_safe_watch_path() is the shared guard download_torrent(), _remove_watch_file(), and check_old_releases_for_demand() all rely on to diff --git a/tests/test_old_release_demand.py b/tests/test_old_release_demand.py index 0a6cefd..e7ecf2c 100644 --- a/tests/test_old_release_demand.py +++ b/tests/test_old_release_demand.py @@ -631,6 +631,66 @@ def test_no_demand_result_is_still_recorded_for_future_recheck_gating(self): self.assertEqual(new_state[name], {"last_checked": "2026-01-08", "leechers": 0}) +class CheckOldReleasesAtomicWriteTests(unittest.TestCase): + """An old release whose .torrent can't be written in full must leave no + .torrent behind (it would count as present forever) and must not + be recorded as checked, so the next run tries it again.""" + + NAME = "kali-linux-2026.1-installer-amd64.iso" + + def setUp(self): + self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) + patcher.start() + self.addCleanup(patcher.stop) + + def _check(self): + candidates = {self.NAME: "https://example.invalid/x.torrent"} + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, + "evaluate_old_release_demand", + return_value=( + True, + b"torrent bytes" * 32, + {"tracker": "t", "seeders": 1, "leechers": 30, "completed": 0}, + ), + ), + ): + return ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state={}, + removed_history={}, + today=date(2026, 1, 8), + ) + + def test_failed_write_leaves_no_file_and_is_not_recorded(self): + with helpers.disk_full_open(ft): + added, skipped, state = self._check() + + self.assertEqual((added, skipped), (0, 1)) + self.assertEqual(os.listdir(self.tmp_watch_dir), []) + self.assertNotIn(self.NAME, state) + + def test_next_run_adds_it_after_a_failed_write(self): + with helpers.disk_full_open(ft): + self._check() + + added, _, _ = self._check() + + self.assertEqual(added, 1) + with open(os.path.join(self.tmp_watch_dir, f"{self.NAME}.torrent"), "rb") as f: + self.assertEqual(f.read(), b"torrent bytes" * 32) + self.assertEqual(os.listdir(self.tmp_watch_dir), [f"{self.NAME}.torrent"]) + + class OldReleaseTracker: """Plays both network boundaries of the old-release check: requests.get for the candidate's .torrent (`routes`: url -> response, or an exception From c6d42ca712cdf7d650bdb8abb8a25cbdc645544f Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 15:49:06 +0200 Subject: [PATCH 32/46] Inject the RPC wait's connect, clock and sleep; test it (task 28) wait_for_transmission_rpc() now takes connect/clock/sleep seams and uses time.monotonic() rather than wall-clock time. run() takes the readiness check as wait_for_rpc, so the daily-run tests drive the "RPC unavailable" path through it instead of patching both the function and socket.create_connection. Co-Authored-By: Claude Opus 5.5 --- fetch_torrents.py | 23 ++++++--- tasks/README.md | 2 +- tests/test_daily_run.py | 105 +++++++++++++++++++++++++++++++--------- 3 files changed, 101 insertions(+), 29 deletions(-) diff --git a/fetch_torrents.py b/fetch_torrents.py index 8bfa403..aa8dc30 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -182,14 +182,24 @@ def get_rpc_credentials(): return username or None, password or None -def wait_for_transmission_rpc(host="localhost", port=9091, timeout=30): - start = time.time() - while time.time() - start < timeout: +def wait_for_transmission_rpc( + host="localhost", + port=9091, + timeout=30, + *, + connect=socket.create_connection, + clock=time.monotonic, + sleep=time.sleep, +): + """True once host:port accepts a TCP connection, retrying once a second; + False if it still doesn't after `timeout` seconds.""" + start = clock() + while clock() - start < timeout: try: - with socket.create_connection((host, port), timeout=3): + with connect((host, port), timeout=3): return True except OSError: - time.sleep(1) + sleep(1) return False @@ -1797,6 +1807,7 @@ def run( fetchers=DEFAULT_FETCHERS, disk_usage=shutil.disk_usage, download_path="/downloads", + wait_for_rpc=wait_for_transmission_rpc, ): """The daily run: fetch new releases (ratio-gated on the previous run's ratio log), optionally check old releases for demand, log seed ratios @@ -1818,7 +1829,7 @@ def run( if config.check_old_releases: _check_old_releases(config, today, summary) - if wait_for_transmission_rpc(): + if wait_for_rpc(): try: log_seed_ratios_via_http(auth=config.rpc_auth) except Exception as exc: diff --git a/tasks/README.md b/tasks/README.md index b02669c..93b89f6 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -94,7 +94,7 @@ mutation score is 95.4% (1027/1076), up from 88.6%, with the critical set still | # | Task | Priority | Class | Depends on | Status | | --- | --- | --- | --- | --- | --- | -| 28 | [Make `wait_for_transmission_rpc` testable and test it](28-rpc-wait-seam.md) | P2 | REFACTOR_REQUIRED | — | open | +| 28 | [Make `wait_for_transmission_rpc` testable and test it](28-rpc-wait-seam.md) | P2 | REFACTOR_REQUIRED | — | done | | 29 | [Inject the UDP transaction id; assert wire payloads](29-udp-transaction-id-seam.md) | P3 | REFACTOR_REQUIRED | — | open | | 30 | [Test `scripts/live_check.py` beyond `problems()`](30-live-check-tests.md) | P2 | REFACTOR_REQUIRED | — | open | | 31 | [`configure_transmission`: hermetic default path, boundaries](31-configure-transmission-gaps.md) | P3 | NO_REFACTOR | — | open | diff --git a/tests/test_daily_run.py b/tests/test_daily_run.py index d1ee1a8..057060d 100644 --- a/tests/test_daily_run.py +++ b/tests/test_daily_run.py @@ -3,8 +3,8 @@ RunTests call run() directly with everything external faked at its boundary: HTTP (ft.requests.get/post), the Transmission client (a -client_factory returning helpers.FakeTransmission), the RPC port check -(socket.create_connection), disk usage and the clock. State files live in a +client_factory returning helpers.FakeTransmission), the RPC readiness +check (wait_for_rpc), disk usage and the clock. State files live in a per-test temp dir. MainWiringTests runs the script via runpy exactly as the container does, to @@ -70,6 +70,7 @@ def __init__(self, testcase): self.events = [] self.transmission = helpers.FakeTransmission() self.rpc_auths = [] + self.rpc_up = True self.disk_used_percent = 10 self.disk_paths = [] @@ -97,6 +98,9 @@ def post(self, url, **kwargs): ] return helpers.fake_response(json.dumps({"arguments": {"torrents": torrents}})) + def wait_for_rpc(self): + return self.rpc_up + def connect(self): self.events.append("connect") return self.transmission @@ -162,6 +166,77 @@ def route_arch(self): ) +class FakeClock: + """A monotonic clock that only moves when `sleep` is called, recording + each sleep.""" + + def __init__(self): + self.now = 0.0 + self.sleeps = [] + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.sleeps.append(seconds) + self.now += seconds + + +class WaitForTransmissionRpcTests(unittest.TestCase): + def setUp(self): + self.clock = FakeClock() + self.attempts = [] + + def _connect_after(self, refusals): + @contextlib.contextmanager + def connect(address, timeout): + self.attempts.append((address, timeout)) + if len(self.attempts) <= refusals: + raise ConnectionRefusedError + yield MagicMock() + + return connect + + def _wait(self, refusals, **kwargs): + return ft.wait_for_transmission_rpc( + connect=self._connect_after(refusals), + clock=self.clock, + sleep=self.clock.sleep, + **kwargs, + ) + + def test_connects_on_the_first_try_without_sleeping(self): + self.assertTrue(self._wait(refusals=0)) + + self.assertEqual(self.attempts, [(("localhost", 9091), 3)]) + self.assertEqual(self.clock.sleeps, []) + + def test_retries_once_a_second_until_the_port_opens(self): + self.assertTrue(self._wait(refusals=4)) + + self.assertEqual(self.clock.sleeps, [1] * 4) + self.assertEqual(len(self.attempts), 5) + + def test_gives_up_once_the_timeout_has_elapsed(self): + self.assertFalse(self._wait(refusals=1000)) + + # Attempts at t=0..29; at t=30 elapsed is no longer < 30. + self.assertEqual(len(self.attempts), 30) + self.assertEqual(self.clock.now, 30) + + def test_host_port_and_timeout_are_honoured(self): + self.assertFalse( + self._wait(refusals=1000, host="transmission", port=9999, timeout=2) + ) + + self.assertEqual(self.attempts, [(("transmission", 9999), 3)] * 2) + + def test_succeeds_on_the_last_attempt_before_the_timeout(self): + self.assertTrue(self._wait(refusals=29)) + + self.assertEqual(len(self.attempts), 30) + + class RunTestCase(unittest.TestCase): """Base for tests of ft.run() inside a FakeWorld; defines no tests.""" @@ -170,9 +245,6 @@ def setUp(self): patches = [ unittest.mock.patch.object(ft.requests, "get", self.world.get), unittest.mock.patch.object(ft.requests, "post", self.world.post), - unittest.mock.patch.object( - ft.socket, "create_connection", self._rpc_port_open - ), unittest.mock.patch.object(ft, "watch_dir", self.world.watch_dir), ] for attr, filename in ( @@ -191,11 +263,6 @@ def setUp(self): patcher.start() self.addCleanup(patcher.stop) - @staticmethod - @contextlib.contextmanager - def _rpc_port_open(*args, **kwargs): - yield MagicMock() - def _run(self, fetchers=ft.DEFAULT_FETCHERS, now=lambda: 1000.0, **env): """Runs ft.run() with load_config(env); returns (summary, messages).""" with self.assertLogs(ft.logger, level="DEBUG") as logs: @@ -206,6 +273,7 @@ def _run(self, fetchers=ft.DEFAULT_FETCHERS, now=lambda: 1000.0, **env): now=now, fetchers=fetchers, disk_usage=self.world.disk_usage, + wait_for_rpc=self.world.wait_for_rpc, ) return summary, [record.getMessage() for record in logs.records] @@ -295,6 +363,7 @@ def test_custom_fetchers_replace_the_default_set(self): today=date(2026, 9, 2), now=lambda: 1000.0, disk_usage=self.world.disk_usage, + wait_for_rpc=self.world.wait_for_rpc, fetchers=(("arch", lambda: {"archlinux-2026.10.01": ARCH_TORRENT}),), ) @@ -405,22 +474,14 @@ def test_disk_usage_limit_error_is_logged_not_raised(self): self.assertTrue(messages[-1].startswith("Run complete in")) def test_rpc_unavailable_skips_ratio_logging_but_still_cleans_up(self): - def refuse(*args, **kwargs): - raise ConnectionRefusedError - + self.world.rpc_up = False self.world.transmission.torrents = [ torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), ] - with ( - unittest.mock.patch.object( - ft, "wait_for_transmission_rpc", return_value=False - ), - unittest.mock.patch.object(ft.socket, "create_connection", refuse), - ): - _summary, messages = self._run( - FETCH_TORRENTS_DISTROS="arch", CLEANUP_KEEP_ONLY_LATEST_VERSION="true" - ) + _summary, messages = self._run( + FETCH_TORRENTS_DISTROS="arch", CLEANUP_KEEP_ONLY_LATEST_VERSION="true" + ) self.assertIn( "Transmission RPC not available on localhost:9091; skipping ratio logs.", From f76defeae7aa002ce3641c8a31c316494ef1cc4e Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 15:51:38 +0200 Subject: [PATCH 33/46] Inject live_check's network and test every check and exit code (task 30) is_reachable(), check_scrapers(), check_tracker() and main() take their collaborators (HEAD/GET, fetchers, reachability, scrape, the checks map) as defaulted arguments, so the drift job's HEAD-405 fallback, MAY_BE_EMPTY, "one reachable is enough", the tracker for/else and integer validation, and exit codes 0/1/2 are all tested offline. The test module now loads live_check from its path instead of putting scripts/ on sys.path, and fake_response() marks its body consumed so a response can be used as a context manager like a real one. Co-Authored-By: Claude Opus 5.5 --- scripts/live_check.py | 32 ++-- tasks/README.md | 2 +- tests/helpers.py | 5 +- tests/test_live_check.py | 308 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 325 insertions(+), 22 deletions(-) diff --git a/scripts/live_check.py b/scripts/live_check.py index 5f31d2e..1f1f9bb 100644 --- a/scripts/live_check.py +++ b/scripts/live_check.py @@ -62,20 +62,20 @@ def problems(distro, name, url): return found -def is_reachable(url): +def is_reachable(url, *, head=requests.head, get=requests.get): try: - r = requests.head(url, allow_redirects=True, timeout=30) + r = head(url, allow_redirects=True, timeout=30) if r.status_code == 405: # HEAD not allowed: ask for the body instead - with requests.get(url, stream=True, timeout=30) as r: + with get(url, stream=True, timeout=30) as r: return r.status_code == 200 return r.status_code == 200 except requests.RequestException: return False -def check_scrapers(): +def check_scrapers(fetchers=FETCHERS, reachable=is_reachable): failures = [] - for fetcher_name, fetcher in FETCHERS: + for fetcher_name, fetcher in fetchers: distro = distro_of(fetcher_name) candidates = fetcher() or {} print(f"{fetcher_name}: {len(candidates)} candidates") @@ -93,19 +93,21 @@ def check_scrapers(): # ESM-only Ubuntu releases etc. are expected to 404 (download_torrent() # counts those as "not found upstream"), so one reachable is enough. - reachable = next((url for url in well_formed if is_reachable(url)), None) - if reachable: - print(f" reachable: {reachable}") + first_reachable = next((url for url in well_formed if reachable(url)), None) + if first_reachable: + print(f" reachable: {first_reachable}") elif well_formed: failures.append(f"{fetcher_name}: none of its torrent URLs returned 200") return failures -def check_tracker(): - candidates = ft.fetch_ubuntu_lts() or {} +def check_tracker( + fetch=ft.fetch_ubuntu_lts, get=requests.get, scrape=torrent_scrape.scrape_torrent +): + candidates = fetch() or {} desktop = [url for name, url in candidates.items() if "-desktop-" in name] for url in desktop: # newest first; older ones may be gone (404) - r = requests.get(url, timeout=30) + r = get(url, timeout=30) if r.status_code == 200: break else: @@ -115,7 +117,7 @@ def check_tracker(): path = os.path.join(tmp, "ubuntu.torrent") with open(path, "wb") as f: f.write(r.content) - result = torrent_scrape.scrape_torrent(path) + result = scrape(path) print(f"tracker: {url} -> {result}") if result is None: return ["tracker: no tracker answered the scrape"] @@ -124,8 +126,10 @@ def check_tracker(): return [] -def main(argv): - checks = {"scrapers": check_scrapers, "tracker": check_tracker} +CHECKS = {"scrapers": check_scrapers, "tracker": check_tracker} + + +def main(argv, checks=CHECKS): if len(argv) != 1 or argv[0] not in checks: print(__doc__, file=sys.stderr) return 2 diff --git a/tasks/README.md b/tasks/README.md index 93b89f6..07ad367 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -96,7 +96,7 @@ mutation score is 95.4% (1027/1076), up from 88.6%, with the critical set still | --- | --- | --- | --- | --- | --- | | 28 | [Make `wait_for_transmission_rpc` testable and test it](28-rpc-wait-seam.md) | P2 | REFACTOR_REQUIRED | — | done | | 29 | [Inject the UDP transaction id; assert wire payloads](29-udp-transaction-id-seam.md) | P3 | REFACTOR_REQUIRED | — | open | -| 30 | [Test `scripts/live_check.py` beyond `problems()`](30-live-check-tests.md) | P2 | REFACTOR_REQUIRED | — | open | +| 30 | [Test `scripts/live_check.py` beyond `problems()`](30-live-check-tests.md) | P2 | REFACTOR_REQUIRED | — | done | | 31 | [`configure_transmission`: hermetic default path, boundaries](31-configure-transmission-gaps.md) | P3 | NO_REFACTOR | — | open | | 32 | [Atomic `.torrent` writes](32-atomic-torrent-writes.md) | P2 (bug) | NO_REFACTOR | — | done | | 33 | [`scrape_torrent` fall-through branches and SSRF edges](33-scrape-torrent-branches-and-ssrf.md) | P2 | NO_REFACTOR | — | open | diff --git a/tests/helpers.py b/tests/helpers.py index d1c2e9b..61c391b 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -246,13 +246,16 @@ def fake_open(path, mode="r", *args, **kwargs): def fake_response(body=b"", status=200, url="", headers=None): """A real requests.Response with the given body (str or bytes), so - .text, .content, .json() and raise_for_status() behave as for real.""" + .text, .content, .json(), raise_for_status() and `with` behave as for + real.""" response = requests.Response() response.status_code = status response.reason = {200: "OK", 404: "Not Found", 500: "Internal Server Error"}.get( status, "" ) response._content = body.encode("utf-8") if isinstance(body, str) else body + # Already read, so close() (and so `with response:`) needn't touch .raw. + response._content_consumed = True response.encoding = "utf-8" response.url = url response.headers.update(headers or {}) diff --git a/tests/test_live_check.py b/tests/test_live_check.py index 1cc3036..207d297 100644 --- a/tests/test_live_check.py +++ b/tests/test_live_check.py @@ -1,18 +1,26 @@ -"""Tests for the pure part of scripts/live_check.py (the weekly drift job): -deciding whether a live fetcher's candidate looks like a real release. +"""Tests for scripts/live_check.py (the weekly drift job): deciding whether +a live fetcher's candidate looks like a real release, and the scraper and +tracker checks and exit codes built on that, with the network injected. Run with: python -m unittest discover -s tests """ +import contextlib +import importlib.util +import io import os -import sys import unittest import helpers +import requests -sys.path.insert(0, os.path.join(helpers.REPO_ROOT, "scripts")) - -import live_check +# Loaded from its path so scripts/ never lands on sys.path for other tests. +_spec = importlib.util.spec_from_file_location( + "live_check", os.path.join(helpers.REPO_ROOT, "scripts", "live_check.py") +) +assert _spec and _spec.loader +live_check = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(live_check) def setUpModule(): @@ -120,5 +128,293 @@ def test_old_release_fetchers_are_checked_as_their_distro(self): self.assertEqual(live_check.distro_of("mint"), "mint") +def quietly(fn, *args, **kwargs): + """Calls fn, swallowing what it prints; returns its result.""" + with ( + contextlib.redirect_stdout(io.StringIO()), + contextlib.redirect_stderr(io.StringIO()), + ): + return fn(*args, **kwargs) + + +UBUNTU_URL = "https://releases.ubuntu.com/noble/ubuntu-24.04-desktop-amd64.iso.torrent" +ARCH_URL = "https://archlinux.org/releng/releases/2026.10.01/torrent/" + + +class IsReachableTests(unittest.TestCase): + def _reachable(self, head, get=None): + def unexpected_get(url, **kwargs): + raise AssertionError("GET should not be needed") + + return live_check.is_reachable(UBUNTU_URL, head=head, get=get or unexpected_get) + + def test_head_200_is_reachable(self): + self.assertTrue(self._reachable(lambda url, **kw: helpers.fake_response())) + + def test_head_404_is_not_reachable(self): + self.assertFalse( + self._reachable(lambda url, **kw: helpers.fake_response(status=404)) + ) + + def test_head_follows_redirects(self): + heads = [] + + def head(url, **kwargs): + heads.append((url, kwargs)) + return helpers.fake_response() + + self._reachable(head) + + self.assertEqual( + heads, [(UBUNTU_URL, {"allow_redirects": True, "timeout": 30})] + ) + + def test_head_not_allowed_falls_back_to_a_streamed_get(self): + gets = [] + + def get(url, **kwargs): + gets.append((url, kwargs)) + return helpers.fake_response(b"d8:announce") + + self.assertTrue( + self._reachable( + lambda url, **kw: helpers.fake_response(status=405), get=get + ) + ) + self.assertEqual(gets, [(UBUNTU_URL, {"stream": True, "timeout": 30})]) + + def test_get_fallback_that_fails_is_not_reachable(self): + self.assertFalse( + self._reachable( + lambda url, **kw: helpers.fake_response(status=405), + get=lambda url, **kw: helpers.fake_response(status=404), + ) + ) + + def test_request_error_is_not_reachable(self): + def head(url, **kwargs): + raise requests.ConnectionError("down") + + self.assertFalse(self._reachable(head)) + + +class CheckScrapersTests(unittest.TestCase): + def _check(self, fetchers, reachable=lambda url: True): + return quietly( + live_check.check_scrapers, fetchers=fetchers, reachable=reachable + ) + + def test_well_formed_reachable_candidates_pass(self): + fetchers = (("arch", lambda: {"archlinux-2026.10.01": ARCH_URL}),) + + self.assertEqual(self._check(fetchers), []) + + def test_fetcher_returning_nothing_fails(self): + for empty in ({}, None): + with self.subTest(empty=empty): + fetchers = (("arch", lambda empty=empty: empty),) + + self.assertEqual(self._check(fetchers), ["arch: no candidates"]) + + def test_old_release_fetchers_may_be_empty(self): + fetchers = (("kali-old", dict), ("fedora-old", dict)) + + self.assertEqual(self._check(fetchers), []) + + def test_malformed_candidates_are_reported_and_not_probed(self): + probed = [] + fetchers = (("arch", lambda: {"archlinux-2026.10.01": "http://x/a.iso"}),) + + failures = self._check(fetchers, reachable=probed.append) + + self.assertEqual( + failures, + [ + "arch: URL 'http://x/a.iso' is not https", + "arch: URL 'http://x/a.iso' is not a .torrent", + ], + ) + self.assertEqual(probed, []) + + def test_every_candidate_unreachable_fails(self): + fetchers = (("ubuntu", lambda: {"ubuntu-24.04-desktop-amd64.iso": UBUNTU_URL}),) + + self.assertEqual( + self._check(fetchers, reachable=lambda url: False), + ["ubuntu: none of its torrent URLs returned 200"], + ) + + def test_one_reachable_candidate_is_enough_and_stops_the_probing(self): + older = UBUNTU_URL.replace("24.04", "22.04") + newest = UBUNTU_URL.replace("24.04", "24.10") + probed = [] + + def reachable(url): + probed.append(url) + return url == UBUNTU_URL + + fetchers = ( + ( + "ubuntu", + lambda: { + "ubuntu-22.04-desktop-amd64.iso": older, + "ubuntu-24.04-desktop-amd64.iso": UBUNTU_URL, + "ubuntu-24.10-desktop-amd64.iso": newest, + }, + ), + ) + + self.assertEqual(self._check(fetchers, reachable=reachable), []) + self.assertEqual(probed, [older, UBUNTU_URL]) + + def test_failures_from_every_fetcher_are_collected(self): + fetchers = ( + ("arch", dict), + ("ubuntu", lambda: {"ubuntu-24.04-desktop-amd64.iso": UBUNTU_URL}), + ) + + self.assertEqual( + self._check(fetchers, reachable=lambda url: False), + ["arch: no candidates", "ubuntu: none of its torrent URLs returned 200"], + ) + + def test_default_fetchers_include_the_old_release_ones(self): + names = [name for name, _fetcher in live_check.FETCHERS] + + self.assertEqual( + names, + [name for name, _fetcher in live_check.ft.DEFAULT_FETCHERS] + + ["kali-old", "fedora-old"], + ) + + +SERVER_URL = UBUNTU_URL.replace("desktop", "live-server") +OLDER_URL = UBUNTU_URL.replace("24.04", "22.04") +UBUNTU_CANDIDATES = { + "ubuntu-24.04-live-server-amd64.iso": SERVER_URL, + "ubuntu-24.04-desktop-amd64.iso": UBUNTU_URL, + "ubuntu-22.04-desktop-amd64.iso": OLDER_URL, +} + + +class CheckTrackerTests(unittest.TestCase): + def setUp(self): + self.requested = [] + self.scraped = [] + self.routes = {} + self.result = None + + def get(self, url, timeout=None): + self.requested.append(url) + return self.routes.get(url, helpers.fake_response(status=404)) + + def scrape(self, path): + with open(path, "rb") as f: + self.scraped.append(f.read()) + return self.result + + def _check(self, candidates=UBUNTU_CANDIDATES): + return quietly( + live_check.check_tracker, + fetch=lambda: candidates, + get=self.get, + scrape=self.scrape, + ) + + def test_integer_counts_from_the_newest_desktop_torrent_pass(self): + self.routes[UBUNTU_URL] = helpers.fake_response(b"d8:announce") + self.result = {"seeders": 10, "leechers": 2} + + self.assertEqual(self._check(), []) + self.assertEqual(self.requested, [UBUNTU_URL]) + self.assertEqual(self.scraped, [b"d8:announce"]) + + def test_falls_back_to_an_older_desktop_torrent(self): + self.routes[OLDER_URL] = helpers.fake_response(b"old") + self.result = {"seeders": 1, "leechers": 0} + + self.assertEqual(self._check(), []) + self.assertEqual(self.requested, [UBUNTU_URL, OLDER_URL]) + self.assertEqual(self.scraped, [b"old"]) + + def test_no_downloadable_desktop_torrent_fails_without_scraping(self): + self.routes[SERVER_URL] = helpers.fake_response(b"server") + + self.assertEqual( + self._check(), + ["tracker: no current Ubuntu desktop torrent could be downloaded"], + ) + self.assertNotIn(SERVER_URL, self.requested) + self.assertEqual(self.scraped, []) + + def test_no_candidates_at_all_fails(self): + for candidates in ({}, None): + with self.subTest(candidates=candidates): + self.assertEqual( + self._check(candidates), + ["tracker: no current Ubuntu desktop torrent could be downloaded"], + ) + + def test_no_tracker_answer_fails(self): + self.routes[UBUNTU_URL] = helpers.fake_response(b"d8:announce") + + self.assertEqual(self._check(), ["tracker: no tracker answered the scrape"]) + + def test_non_integer_counts_fail(self): + self.routes[UBUNTU_URL] = helpers.fake_response(b"d8:announce") + for result in ( + {"seeders": "10", "leechers": 2}, + {"seeders": 10, "leechers": None}, + {"seeders": 10}, + ): + with self.subTest(result=result): + self.result = result + + self.assertEqual( + self._check(), + [f"tracker: non-integer seeders/leechers in {result}"], + ) + + +class MainTests(unittest.TestCase): + def _main(self, argv, failures=()): + calls = [] + + def check(): + calls.append("check") + return list(failures) + + err = io.StringIO() + with contextlib.redirect_stderr(err): + code = live_check.main(argv, checks={"scrapers": check}) + return code, calls, err.getvalue() + + def test_no_problems_exits_0(self): + self.assertEqual(self._main(["scrapers"]), (0, ["check"], "")) + + def test_a_problem_exits_1_and_is_reported(self): + code, _calls, err = self._main(["scrapers"], failures=["arch: no candidates"]) + + self.assertEqual(code, 1) + self.assertEqual(err, "DRIFT: arch: no candidates\n") + + def test_bad_usage_exits_2_with_the_usage_text(self): + for argv in ([], ["nope"], ["scrapers", "tracker"]): + with self.subTest(argv=argv): + code, calls, err = self._main(argv) + + self.assertEqual((code, calls), (2, [])) + self.assertIn("Usage:", err) + + def test_default_checks_are_scrapers_and_tracker(self): + self.assertEqual( + live_check.CHECKS, + { + "scrapers": live_check.check_scrapers, + "tracker": live_check.check_tracker, + }, + ) + + if __name__ == "__main__": unittest.main() From d2c438ec64797e247acc2fc67adfaca4c52ef2f7 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 15:54:07 +0200 Subject: [PATCH 34/46] Pin scrape_torrent's fall-through paths and SSRF edges (task 33) A parametrised test has tracker A fail by connection error, HTTP status, failure reason, missing stats or a non-scrapable announce URL, and checks B's exact %-encoded request URL, timeout and whole result. UDP calls are asserted with exact (host, port, info_hash, timeout). SSRF: an empty resolution is unsafe; multicast, unspecified and IPv4-mapped internal addresses are pinned as disallowed (all already were, so no production change). Hand-built Mock responses are replaced by tracker_response() over helpers.fake_response, and the duplicated resolver fakes by resolve_private(). Co-Authored-By: Claude Opus 5.5 --- tasks/README.md | 2 +- tests/test_torrent_scrape.py | 316 ++++++++++++++++++++++++----------- 2 files changed, 216 insertions(+), 102 deletions(-) diff --git a/tasks/README.md b/tasks/README.md index 07ad367..cc81670 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -99,7 +99,7 @@ mutation score is 95.4% (1027/1076), up from 88.6%, with the critical set still | 30 | [Test `scripts/live_check.py` beyond `problems()`](30-live-check-tests.md) | P2 | REFACTOR_REQUIRED | — | done | | 31 | [`configure_transmission`: hermetic default path, boundaries](31-configure-transmission-gaps.md) | P3 | NO_REFACTOR | — | open | | 32 | [Atomic `.torrent` writes](32-atomic-torrent-writes.md) | P2 (bug) | NO_REFACTOR | — | done | -| 33 | [`scrape_torrent` fall-through branches and SSRF edges](33-scrape-torrent-branches-and-ssrf.md) | P2 | NO_REFACTOR | — | open | +| 33 | [`scrape_torrent` fall-through branches and SSRF edges](33-scrape-torrent-branches-and-ssrf.md) | P2 | NO_REFACTOR | — | done | | 34 | [Mechanical test tidy-up](34-mechanical-test-tidy-up.md) | P3 | NO_REFACTOR | — | open | | 35 | [Consolidate fixtures and the path-global seam](35-fixture-consolidation.md) | P3 | NO_REFACTOR → REFACTOR_REQUIRED | 34 | open | | 36 | [Remaining `fetch_torrents.py` mutation survivors](36-remaining-mutation-survivors.md) | P3 | NO_REFACTOR | — | open | diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index ba89021..7d544f3 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -608,6 +608,27 @@ def test_link_local_is_disallowed(self): def test_public_ipv6_is_allowed(self): self.assertFalse(ts._is_disallowed_ip("2001:4860:4860::8888")) + def test_multicast_is_disallowed(self): + for ip in ("224.0.0.1", "239.255.255.250", "ff02::1"): + with self.subTest(ip=ip): + self.assertTrue(ts._is_disallowed_ip(ip)) + + def test_unspecified_address_is_disallowed(self): + # Connecting to 0.0.0.0 / :: reaches this host on most systems. + for ip in ("0.0.0.0", "::"): + with self.subTest(ip=ip): + self.assertTrue(ts._is_disallowed_ip(ip)) + + def test_ipv4_mapped_internal_addresses_are_disallowed(self): + # The Docker build runs this suite under Alpine's Python too, so + # this also holds there, not only on a developer's interpreter. + for ip in ("::ffff:127.0.0.1", "::ffff:10.0.0.1", "::ffff:169.254.169.254"): + with self.subTest(ip=ip): + self.assertTrue(ts._is_disallowed_ip(ip)) + + def test_ipv4_mapped_public_address_is_allowed(self): + self.assertFalse(ts._is_disallowed_ip("::ffff:93.184.216.34")) + class ResolveAddressesTests(unittest.TestCase): def test_returns_the_ip_of_every_getaddrinfo_entry(self): @@ -643,6 +664,10 @@ def test_any_disallowed_address_among_several_makes_host_unsafe(self): ): self.assertFalse(ts._is_safe_scrape_host("multi-homed.example.com")) + def test_resolving_to_no_addresses_is_unsafe(self): + with unittest.mock.patch.object(ts, "_resolve_addresses", return_value=[]): + self.assertFalse(ts._is_safe_scrape_host("tracker.example.com")) + def test_resolution_failure_is_unsafe(self): with unittest.mock.patch.object( ts, "_resolve_addresses", side_effect=OSError("not known") @@ -654,20 +679,59 @@ def test_empty_hostname_is_unsafe(self): self.assertFalse(ts._is_safe_scrape_host("")) +PUBLIC_IP = "93.184.216.34" + + +def resolve_private(*private_hosts): + """A _resolve_addresses stand-in: loopback for private_hosts, a public + address for every other host.""" + + def resolve(hostname): + return ["127.0.0.1"] if hostname in private_hosts else [PUBLIC_IP] + + return resolve + + +def percent_encoded(raw): + """RFC 3986 percent-encoding of raw bytes, written independently of + urllib so it serves as an oracle for build_scrape_request_url().""" + unreserved = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" + return "".join(chr(b) if b in unreserved else f"%{b:02X}" for b in raw) + + +def tracker_response(info_hash, seeders, leechers, completed=0): + """A real HTTP response carrying a bencoded scrape for info_hash.""" + return helpers.fake_response( + b"d5:filesd" + + bstr(info_hash) + + b"d8:complete" + + bint(seeders) + + b"10:incomplete" + + bint(leechers) + + b"10:downloaded" + + bint(completed) + + b"eee" + ) + + +OTHER_HASH = b"\x01" * 20 + + class ScrapeTorrentTests(unittest.TestCase): def setUp(self): # scrape_torrent() resolves each tracker's host before ever # requesting it (see IsSafeScrapeHostTests) - default every test's - # hosts to a public address so existing tracker-selection behavior - # doesn't depend on real DNS. Tests of the SSRF guard itself override - # this per-case. + # hosts to a public address so tracker-selection behavior doesn't + # depend on real DNS. Tests of the SSRF guard itself override this + # per-case. patcher = unittest.mock.patch.object( - ts, "_resolve_addresses", return_value=["93.184.216.34"] + ts, "_resolve_addresses", side_effect=resolve_private() ) patcher.start() self.addCleanup(patcher.stop) def _write_torrent(self, announce_urls): + """Writes a .torrent with these trackers; returns (path, info_hash).""" tiers = b"".join(b"l" + bstr(u) + b"e" for u in announce_urls) info_bytes = b"d6:lengthi1e4:name4:teste" torrent_bytes = ( @@ -686,19 +750,10 @@ def _write_torrent(self, announce_urls): self.addCleanup(os.remove, path) with os.fdopen(fd, "wb") as f: f.write(torrent_bytes) - return path - - def _scrape_response_bytes(self, info_hash, seeders, leechers): - return ( - b"d5:filesd" - + bstr(info_hash) - + f"d8:completei{seeders}e10:incompletei{leechers}e10:downloadedi0ee".encode() - + b"ee" - + b"e" - ) + return path, hashlib.sha1(info_bytes).digest() def test_uses_udp_tracker_when_it_succeeds(self): - path = self._write_torrent( + path, info_hash = self._write_torrent( [ "udp://tracker.example.com:80/announce", "http://tracker.example.com:6969/announce", @@ -708,103 +763,165 @@ def test_uses_udp_tracker_when_it_succeeds(self): unittest.mock.patch.object( ts, "scrape_udp_tracker", - return_value={ - "seeders": 5, - "leechers": 7, - "completed": 1, - }, + return_value={"seeders": 5, "leechers": 7, "completed": 1}, ) as mock_udp_scrape, unittest.mock.patch.object(ts.requests, "get") as mock_get, ): result = ts.scrape_torrent(path) - mock_udp_scrape.assert_called_once() - self.assertEqual(mock_udp_scrape.call_args.kwargs["timeout"], 15) + mock_udp_scrape.assert_called_once_with( + "tracker.example.com", 80, info_hash, timeout=15 + ) mock_get.assert_not_called() - self.assertEqual(result["seeders"], 5) - self.assertEqual(result["leechers"], 7) - self.assertEqual(result["tracker"], "udp://tracker.example.com:80") + self.assertEqual( + result, + { + "tracker": "udp://tracker.example.com:80", + "seeders": 5, + "leechers": 7, + "completed": 1, + }, + ) + + def test_timeout_is_passed_to_the_udp_scrape(self): + path, info_hash = self._write_torrent(["udp://tracker.example.com:80/announce"]) + with unittest.mock.patch.object( + ts, + "scrape_udp_tracker", + return_value={"seeders": 0, "leechers": 0, "completed": 0}, + ) as mock_udp_scrape: + ts.scrape_torrent(path, timeout=3) + + mock_udp_scrape.assert_called_once_with( + "tracker.example.com", 80, info_hash, timeout=3 + ) def test_falls_through_from_failing_udp_tracker_to_http_tracker(self): - path = self._write_torrent( + path, info_hash = self._write_torrent( [ "udp://tracker.example.com:80/announce", "http://tracker.example.com:6969/announce", ] ) - with open(path, "rb") as f: - info_hash = ts.compute_info_hash(f.read()) - fake_response = unittest.mock.Mock() - fake_response.content = self._scrape_response_bytes(info_hash, 5, 7) - fake_response.raise_for_status = lambda: None - with ( unittest.mock.patch.object( ts, "scrape_udp_tracker", side_effect=OSError("unreachable") ), unittest.mock.patch.object( - ts.requests, "get", return_value=fake_response + ts.requests, + "get", + return_value=tracker_response(info_hash, 5, 7, completed=2), ) as mock_get, ): - result = ts.scrape_torrent(path) + result = ts.scrape_torrent(path, timeout=3) - mock_get.assert_called_once() - self.assertIn("tracker.example.com:6969/scrape", mock_get.call_args[0][0]) - self.assertEqual(result["seeders"], 5) - self.assertEqual(result["leechers"], 7) - self.assertEqual(result["tracker"], "http://tracker.example.com:6969/scrape") + self.assertEqual( + mock_get.call_args_list, + [ + unittest.mock.call( + "http://tracker.example.com:6969/scrape?info_hash=" + + percent_encoded(info_hash), + timeout=3, + ) + ], + ) + self.assertEqual( + result, + { + "tracker": "http://tracker.example.com:6969/scrape", + "seeders": 5, + "leechers": 7, + "completed": 2, + }, + ) def test_skips_udp_tracker_with_no_port(self): """BEP 15 requires an explicit port; an announce URL without one can't be dialed, so it should be skipped like any other unusable tracker rather than raising.""" - path = self._write_torrent( + path, info_hash = self._write_torrent( [ "udp://tracker.example.com/announce", "http://tracker.example.com:6969/announce", ] ) - with open(path, "rb") as f: - info_hash = ts.compute_info_hash(f.read()) - fake_response = unittest.mock.Mock() - fake_response.content = self._scrape_response_bytes(info_hash, 5, 7) - fake_response.raise_for_status = lambda: None - with ( unittest.mock.patch.object(ts, "scrape_udp_tracker") as mock_udp_scrape, - unittest.mock.patch.object(ts.requests, "get", return_value=fake_response), + unittest.mock.patch.object( + ts.requests, "get", return_value=tracker_response(info_hash, 5, 7) + ), ): result = ts.scrape_torrent(path) mock_udp_scrape.assert_not_called() self.assertEqual(result["tracker"], "http://tracker.example.com:6969/scrape") - def test_falls_through_to_next_tracker_on_failure(self): - path = self._write_torrent( - [ - "http://dead.example.com/announce", - "http://alive.example.com/announce", - ] - ) - with open(path, "rb") as f: - info_hash = ts.compute_info_hash(f.read()) - fake_response = unittest.mock.Mock() - fake_response.content = self._scrape_response_bytes(info_hash, 1, 2) - fake_response.raise_for_status = lambda: None - - def fake_get(url, timeout=None): - if "dead" in url: - raise ConnectionError("nope") - return fake_response - - with unittest.mock.patch.object(ts.requests, "get", side_effect=fake_get): - result = ts.scrape_torrent(path) + def test_falls_through_to_the_next_http_tracker(self): + """Tracker A is unusable in each way scrape_torrent() tolerates; + tracker B answers. A's failure must not end the search.""" + good = "http://good.example.com/announce" + dead = "http://dead.example.com/announce" + # case -> (A's announce URL, what A answers: a response, an + # exception to raise, or None if A must never be requested) + cases = { + "connection error": (dead, ConnectionError("nope")), + "HTTP status error": (dead, helpers.fake_response(status=500)), + "failure reason": ( + dead, + helpers.fake_response(b"d14:failure reason11:not allowede"), + ), + "no stats for this hash": (dead, tracker_response(OTHER_HASH, 9, 9)), + "no announce segment": ("http://dead.example.com/tracker", None), + } + for case, (announce, dead_answer) in cases.items(): + with self.subTest(case): + path, info_hash = self._write_torrent([announce, good]) + query = "/scrape?info_hash=" + percent_encoded(info_hash) + dead_url = "http://dead.example.com" + query + good_url = "http://good.example.com" + query + + answers = {good_url: tracker_response(info_hash, 1, 2, completed=3)} + + def fake_get( + url, timeout=None, answers=answers, dead_answer=dead_answer + ): + if url in answers: + return answers[url] + if isinstance(dead_answer, Exception): + raise dead_answer + return dead_answer + + with unittest.mock.patch.object( + ts.requests, "get", side_effect=fake_get + ) as mock_get: + result = ts.scrape_torrent(path, timeout=3) + + requested = [good_url] if dead_answer is None else [dead_url, good_url] + self.assertEqual( + mock_get.call_args_list, + [unittest.mock.call(url, timeout=3) for url in requested], + ) + self.assertEqual( + result, + { + "tracker": "http://good.example.com/scrape", + "seeders": 1, + "leechers": 2, + "completed": 3, + }, + ) + + def test_default_http_timeout_is_15_seconds(self): + path, info_hash = self._write_torrent(["http://good.example.com/announce"]) + with unittest.mock.patch.object( + ts.requests, "get", return_value=tracker_response(info_hash, 1, 2) + ) as mock_get: + ts.scrape_torrent(path) - self.assertEqual(result["seeders"], 1) - self.assertEqual(result["tracker"], "http://alive.example.com/scrape") + self.assertEqual(mock_get.call_args.kwargs, {"timeout": 15}) def test_returns_none_when_no_tracker_usable(self): - path = self._write_torrent(["udp://only.example.com:1337/announce"]) + path, _info_hash = self._write_torrent(["udp://only.example.com:1337/announce"]) with ( unittest.mock.patch.object( ts, "scrape_udp_tracker", side_effect=OSError("unreachable") @@ -815,31 +932,29 @@ def test_returns_none_when_no_tracker_usable(self): mock_get.assert_not_called() self.assertIsNone(result) + def test_returns_none_when_no_tracker_has_stats_for_the_hash(self): + path, _info_hash = self._write_torrent(["http://good.example.com/announce"]) + with unittest.mock.patch.object( + ts.requests, "get", return_value=tracker_response(OTHER_HASH, 9, 9) + ): + self.assertIsNone(ts.scrape_torrent(path)) + def test_skips_udp_tracker_resolving_to_a_private_address(self): - path = self._write_torrent( + path, info_hash = self._write_torrent( [ "udp://internal.example.com:1337/announce", "http://real-tracker.example.com/announce", ] ) - with open(path, "rb") as f: - info_hash = ts.compute_info_hash(f.read()) - fake_response = unittest.mock.Mock() - fake_response.content = self._scrape_response_bytes(info_hash, 3, 4) - fake_response.raise_for_status = lambda: None - - def fake_resolve(hostname): - if hostname == "internal.example.com": - return ["127.0.0.1"] - return ["93.184.216.34"] - with ( unittest.mock.patch.object( - ts, "_resolve_addresses", side_effect=fake_resolve + ts, + "_resolve_addresses", + side_effect=resolve_private("internal.example.com"), ), unittest.mock.patch.object(ts, "scrape_udp_tracker") as mock_udp_scrape, unittest.mock.patch.object( - ts.requests, "get", return_value=fake_response + ts.requests, "get", return_value=tracker_response(info_hash, 3, 4) ) as mock_get, ): result = ts.scrape_torrent(path) @@ -853,39 +968,38 @@ def test_skips_tracker_resolving_to_a_private_address(self): (e.g. a compromised upstream mirror pointing at cloud metadata or this container's own services) must never be scraped - no request should go out to it at all.""" - path = self._write_torrent( + path, info_hash = self._write_torrent( [ "http://internal.example.com/announce", "http://real-tracker.example.com/announce", ] ) - with open(path, "rb") as f: - info_hash = ts.compute_info_hash(f.read()) - fake_response = unittest.mock.Mock() - fake_response.content = self._scrape_response_bytes(info_hash, 3, 4) - fake_response.raise_for_status = lambda: None - - def fake_resolve(hostname): - if hostname == "internal.example.com": - return ["127.0.0.1"] - return ["93.184.216.34"] - with ( unittest.mock.patch.object( - ts, "_resolve_addresses", side_effect=fake_resolve + ts, + "_resolve_addresses", + side_effect=resolve_private("internal.example.com"), ), unittest.mock.patch.object( - ts.requests, "get", return_value=fake_response + ts.requests, "get", return_value=tracker_response(info_hash, 3, 4) ) as mock_get, ): result = ts.scrape_torrent(path) - mock_get.assert_called_once() - self.assertIn("real-tracker.example.com", mock_get.call_args[0][0]) + self.assertEqual( + mock_get.call_args_list, + [ + unittest.mock.call( + "http://real-tracker.example.com/scrape?info_hash=" + + percent_encoded(info_hash), + timeout=15, + ) + ], + ) self.assertEqual(result["tracker"], "http://real-tracker.example.com/scrape") def test_returns_none_when_every_tracker_resolves_privately(self): - path = self._write_torrent(["http://internal.example.com/announce"]) + path, _info_hash = self._write_torrent(["http://internal.example.com/announce"]) with ( unittest.mock.patch.object( ts, "_resolve_addresses", return_value=["169.254.169.254"] From 5bdff61c72f43fe6f480df82536ca1224a4728ad Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 15:57:10 +0200 Subject: [PATCH 35/46] Harden the toolchain: fetch-run smoke check, shellcheck, hadolint, ruff S, pins (task 37) - smoke_test.sh now waits for "Starting torrent fetch run." in /logs/fetch_torrents.log in both the root and RUN_AS_NON_ROOT runs, so a fetch script that crashes on import fails the build. - The test job runs shellcheck v0.11.0 and hadolint v2.15.1 (pinned images). The two Dockerfile ignores (rolling alpine:latest, unpinned apk packages) are justified inline. - ruff adds the bandit S family. The info hash uses sha1(usedforsecurity=False); test-only ignores are justified in pyproject.toml. - CI Python is pinned to alpine:latest's 3.14, and requests and beautifulsoup4 to the apk versions (2.33.1, 4.15.0). Co-Authored-By: Claude Opus 5.5 --- .claude/CLAUDE.md | 2 +- .github/workflows/docker-image.yml | 9 ++++++++- CONTRIBUTING.md | 17 ++++++++++++++--- Dockerfile | 8 +++++++- pyproject.toml | 18 ++++++++++++++++-- requirements-test.txt | 12 +++++++----- scripts/smoke_test.sh | 22 ++++++++++++++++++++-- scripts/transmission_roundtrip.py | 7 ++++--- tasks/README.md | 2 +- tests/test_torrent_scrape.py | 4 ++-- torrent_scrape.py | 3 ++- 11 files changed, 82 insertions(+), 22 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 2a8e783..a9ac844 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -11,7 +11,7 @@ Testing rules and the import-stubbing pattern live in the root [CLAUDE.md](../CL - Test: `python -m unittest discover -s tests -v` (Windows: `py -m unittest discover -s tests -v`) - Lint/format: `ruff check .` / `ruff format .` - Typecheck: `mypy` (files and settings in `pyproject.toml`) -- CI's `test` job runs ruff check, ruff format --check and mypy before the tests; versions are pinned in `requirements-dev.txt` +- CI's `test` job runs ruff check (defaults + bandit `S`), ruff format --check, mypy, shellcheck and hadolint before the tests; Python versions are pinned in `requirements-dev.txt`, the shellcheck/hadolint images in `.github/workflows/docker-image.yml` (commands in CONTRIBUTING.md) - Coverage: `coverage run -m unittest discover -s tests` then `coverage report` (branch coverage; config in `pyproject.toml`, which also sets the `fail_under` ratchet CI enforces) - Mutation: `mutmut run` (mutmut<3; config in `pyproject.toml`, see CONTRIBUTING.md) - Dev deps: `pip install -r requirements-dev.txt` (includes `requirements-test.txt`) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 33df55c..620a0ad 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -37,7 +37,8 @@ jobs: - name: Set up Python uses: actions/setup-python@v7 with: - python-version: "3.x" + # alpine:latest's python3 minor; bump with requirements-test.txt. + python-version: "3.14" - name: Install dev dependencies run: pip install -r requirements-dev.txt @@ -51,6 +52,12 @@ jobs: - name: Type-check run: mypy + - name: Lint shell scripts + run: docker run --rm -v "$PWD:/mnt" -w /mnt koalaman/shellcheck:v0.11.0 entrypoint.sh scripts/*.sh + + - name: Lint Dockerfile + run: docker run --rm -i hadolint/hadolint:v2.15.1 < Dockerfile + - name: Run tests run: coverage run -m unittest discover -s tests -v diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f70c781..1390fdc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,9 +35,20 @@ sh scripts/smoke_test.sh linux-iso-seeder:smoke The smoke test starts the image with `--network none`, both as root and with `RUN_AS_NON_ROOT=true`, and checks that `configure_transmission.py` -applied its overrides, that `transmission-daemon` is running, and that every -module imports. When `alpine:latest` moves to a new `py3-transmission-rpc`, -bump the `transmission-rpc` pin in `requirements-test.txt` to match. +applied its overrides, that `transmission-daemon` is running, that the +daily `fetch_torrents.py` run started, and that every module imports. + +When `alpine:latest` moves to new `python3` or `py3-*` versions, bump the +pins in `requirements-test.txt` and CI's `python-version` in +`.github/workflows/docker-image.yml` to match. + +The `test` job also lints the shell scripts and the Dockerfile with pinned +images. To run them locally: + +```sh +docker run --rm -v "$PWD:/mnt" -w /mnt koalaman/shellcheck:v0.11.0 entrypoint.sh scripts/*.sh +docker run --rm -i hadolint/hadolint:v2.15.1 < Dockerfile +``` ## Upstream drift diff --git a/Dockerfile b/Dockerfile index c633d20..057a1c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,14 @@ +# Rolling on purpose: entrypoint.sh apk-upgrades on start anyway, and +# requirements-test.txt tracks what alpine:latest ships. +# hadolint ignore=DL3007 FROM alpine:latest AS runtime ARG VERSION=latest LABEL org.opencontainers.image.source=https://github.com/squazz/linux-iso-seeder LABEL org.opencontainers.image.version=$VERSION -# Install prerequisites +# Install prerequisites. Unpinned: Alpine drops superseded package +# versions from its mirrors, so pins would break the build on the next +# security update. +# hadolint ignore=DL3018 RUN apk update && \ apk add --no-cache transmission-daemon curl wget python3 py3-pip py3-requests py3-beautifulsoup4 py3-transmission-rpc su-exec diff --git a/pyproject.toml b/pyproject.toml index c177620..2c58304 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,11 +2,13 @@ # the scripts directly with Alpine's python3 (see Dockerfile). [tool.ruff] -# Alpine's python3 at the time of writing; also what CI's "3.x" must stay -# compatible with. +# The oldest Python the scripts still support. The container runs Alpine's +# python3 (3.14 on alpine:latest as of 2026-10), which CI pins too. target-version = "py312" [tool.ruff.lint] +# The defaults plus flake8-bandit (S), so new security smells fail CI. +extend-select = ["S"] ignore = [ # Broad `except Exception` is the error-handling design here: each # distro fetcher, cleanup step and tracker scrape must log and carry on @@ -14,6 +16,18 @@ ignore = [ "BLE001", ] +[tool.ruff.lint.per-file-ignores] +"tests/**" = [ + # unittest asserts via self.assert*, but a bare assert to narrow a type + # for mypy (e.g. a module spec) is fine in a test. + "S101", + # Fake RPC credentials and the 0.0.0.0 SSRF case are test inputs. + "S104", + "S106", + # Runs sys.executable on this repo's own scripts, with no shell. + "S603", +] + [tool.mypy] python_version = "3.12" files = ["fetch_torrents.py", "torrent_scrape.py", "configure_transmission.py"] diff --git a/requirements-test.txt b/requirements-test.txt index 21f42cb..029ae57 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,6 +1,8 @@ -requests -beautifulsoup4 -# The container gets this from Alpine's py3-transmission-rpc; keep the pin at -# the version alpine:latest ships (`apk info py3-transmission-rpc`) so the -# test doubles in tests/helpers.py are checked against the same API. +# The container gets these from Alpine's py3-* packages. Keep every pin at +# the version alpine:latest ships (`apk list py3-requests py3-beautifulsoup4 +# py3-transmission-rpc`) and bump them together, so the pip test job runs +# what the image runs and the test doubles in tests/helpers.py are checked +# against the same transmission-rpc API. +requests==2.33.1 +beautifulsoup4==4.15.0 transmission-rpc==7.0.11 diff --git a/scripts/smoke_test.sh b/scripts/smoke_test.sh index 09f393d..8658d54 100755 --- a/scripts/smoke_test.sh +++ b/scripts/smoke_test.sh @@ -1,8 +1,9 @@ #!/bin/sh # Starts a built image the way a deployment would and checks it comes up: # entrypoint.sh runs, configure_transmission.py applies its overrides, -# transmission-daemon is running, RUN_AS_NON_ROOT hands /config to PUID:PGID, -# and every Python module imports against the image's apk packages. +# transmission-daemon is running, the daily fetch_torrents.py run starts, +# RUN_AS_NON_ROOT hands /config to PUID:PGID, and every Python module +# imports against the image's apk packages. # # Usage: scripts/smoke_test.sh # @@ -63,6 +64,21 @@ wait_until_started() { fail "transmission-daemon not running after ${TIMEOUT_SECONDS}s" "$cid" } +# A fetch_torrents.py that crashes on import or before its first log line +# never writes this. Network errors after it (--network none) are expected. +wait_for_fetch_run() { + cid=$1 + elapsed=0 + while [ "$elapsed" -lt "$TIMEOUT_SECONDS" ]; do + if docker exec "$cid" grep -q "Starting torrent fetch run." /logs/fetch_torrents.log 2>/dev/null; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + fail "fetch_torrents.py did not start its run within ${TIMEOUT_SECONDS}s" "$cid" +} + # configure_transmission.py only writes settings.json once it has parsed the # env and applied every override, so the override being there means it ran # to completion. @@ -80,12 +96,14 @@ cid=$(start_container) CONTAINERS="$CONTAINERS $cid" wait_until_started "$cid" check_configure_ran "$cid" +wait_for_fetch_run "$cid" echo "== RUN_AS_NON_ROOT=true PUID=1234 PGID=1234 startup" cid=$(start_container -e RUN_AS_NON_ROOT=true -e PUID=1234 -e PGID=1234) CONTAINERS="$CONTAINERS $cid" wait_until_started "$cid" check_configure_ran "$cid" +wait_for_fetch_run "$cid" owner=$(docker exec "$cid" stat -c %u:%g /config) [ "$owner" = "1234:1234" ] || fail "/config owned by $owner, expected 1234:1234" "$cid" daemon_uid=$(docker exec "$cid" sh -c 'stat -c %u /proc/$(pgrep -x transmission-daemon)') diff --git a/scripts/transmission_roundtrip.py b/scripts/transmission_roundtrip.py index eba0204..fe4393d 100644 --- a/scripts/transmission_roundtrip.py +++ b/scripts/transmission_roundtrip.py @@ -33,7 +33,7 @@ def make_torrent(name): "name": name, "length": len(data), "piece length": 16384, - "pieces": hashlib.sha1(data).digest(), + "pieces": hashlib.sha1(data, usedforsecurity=False).digest(), } return torrent_scrape.bencode({"info": info}) @@ -49,7 +49,7 @@ def wait_for(description, predicate): def main(): - ft.configure_logging("/tmp") + ft.configure_logging("/tmp") # noqa: S108 - a throwaway CI container if not ft.wait_for_transmission_rpc(timeout=TIMEOUT_SECONDS): raise SystemExit("FAILED: Transmission RPC never came up") @@ -71,7 +71,8 @@ def added(): ) ft.log_seed_ratios_via_http(auth=config.rpc_auth) - with open(os.path.join("/tmp", ft.RATIO_LOG_FILE_NAME), encoding="utf-8") as f: + ratio_log = os.path.join("/tmp", ft.RATIO_LOG_FILE_NAME) # noqa: S108 - as above + with open(ratio_log, encoding="utf-8") as f: if name not in f.read(): raise SystemExit("FAILED: torrent missing from the RPC ratio log") diff --git a/tasks/README.md b/tasks/README.md index cc81670..6a4d89d 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -103,7 +103,7 @@ mutation score is 95.4% (1027/1076), up from 88.6%, with the critical set still | 34 | [Mechanical test tidy-up](34-mechanical-test-tidy-up.md) | P3 | NO_REFACTOR | — | open | | 35 | [Consolidate fixtures and the path-global seam](35-fixture-consolidation.md) | P3 | NO_REFACTOR → REFACTOR_REQUIRED | 34 | open | | 36 | [Remaining `fetch_torrents.py` mutation survivors](36-remaining-mutation-survivors.md) | P3 | NO_REFACTOR | — | open | -| 37 | [Toolchain: smoke test, shellcheck/hadolint, ruff S, pins](37-toolchain-and-q4.md) | P2 | CI | — | open | +| 37 | [Toolchain: smoke test, shellcheck/hadolint, ruff S, pins](37-toolchain-and-q4.md) | P2 | CI | — | done | | 38 | [Re-measure, raise the coverage ratchet, record the baseline](38-remeasure-and-ratchet.md) | P3 | — | 28–37 | open | Suggested order: 32 (a real bug), then 28, 30 and 33 (resilience and drift-job trust), diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index 7d544f3..3b1dd60 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -161,7 +161,7 @@ def test_matches_sha1_of_raw_info_bytes(self): + b"e" ) - expected = hashlib.sha1(info_bytes).digest() + expected = hashlib.sha1(info_bytes, usedforsecurity=False).digest() self.assertEqual(ts.compute_info_hash(torrent_bytes), expected) def test_raises_when_info_missing(self): @@ -750,7 +750,7 @@ def _write_torrent(self, announce_urls): self.addCleanup(os.remove, path) with os.fdopen(fd, "wb") as f: f.write(torrent_bytes) - return path, hashlib.sha1(info_bytes).digest() + return path, hashlib.sha1(info_bytes, usedforsecurity=False).digest() def test_uses_udp_tracker_when_it_succeeds(self): path, info_hash = self._write_torrent( diff --git a/torrent_scrape.py b/torrent_scrape.py index c5b4eb5..3541a9c 100644 --- a/torrent_scrape.py +++ b/torrent_scrape.py @@ -132,7 +132,8 @@ def compute_info_hash(torrent_bytes): metainfo, _ = bdecode(torrent_bytes) if not isinstance(metainfo, dict) or b"info" not in metainfo: raise BencodeError("Not a valid .torrent file: missing 'info' dictionary") - return hashlib.sha1(bencode(metainfo[b"info"])).digest() + # BitTorrent v1 defines the info hash as SHA-1; it identifies, not secures. + return hashlib.sha1(bencode(metainfo[b"info"]), usedforsecurity=False).digest() def get_announce_urls(torrent_bytes): From 9398143f1392d7467bb487715805d0ada6a0fb74 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 17:12:43 +0200 Subject: [PATCH 36/46] Kill the remaining fetch_torrents mutation survivors (task 36) New tests pin the 30 s timeout on every fetcher and download, Arch/Kali/ Fedora selection and ordering, the unknown-distro warning text, cleanup_old_versions' default floor and zero-ratio record, ratio rounding, the credential fallback needing both halves, parse_bool's default, RunSummary/Config defaults and immutability, the state-file paths, and the state files' indented, sorted JSON. Bug fix: a Fedora listing that links the newest torrent twice no longer reports the duplicate as an old release. Both Fedora fetchers now share one de-duplicating parser. Arch's max() and the disk-usage percentage were simplified so their equivalent mutants are no longer generated; the remaining accepted equivalents are listed in tasks/README.md. Also pins types-requests: requests 2.33 (the apk pin from task 37) ships no inline types, so mypy failed without it. Co-Authored-By: Claude Opus 5.5 --- .claude/CLAUDE.md | 2 +- fetch_torrents.py | 85 ++++++++++---------- requirements-dev.txt | 2 + tasks/README.md | 21 ++++- tests/test_cleanup.py | 62 +++++++++++++++ tests/test_daily_run.py | 43 ++++++++++ tests/test_fetchers.py | 162 ++++++++++++++++++++++++++++++++++++++ tests/test_logging.py | 25 ++++++ tests/test_persistence.py | 50 ++++++++++++ 9 files changed, 407 insertions(+), 45 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index a9ac844..c020ed8 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -11,7 +11,7 @@ Testing rules and the import-stubbing pattern live in the root [CLAUDE.md](../CL - Test: `python -m unittest discover -s tests -v` (Windows: `py -m unittest discover -s tests -v`) - Lint/format: `ruff check .` / `ruff format .` - Typecheck: `mypy` (files and settings in `pyproject.toml`) -- CI's `test` job runs ruff check (defaults + bandit `S`), ruff format --check, mypy, shellcheck and hadolint before the tests; Python versions are pinned in `requirements-dev.txt`, the shellcheck/hadolint images in `.github/workflows/docker-image.yml` (commands in CONTRIBUTING.md) +- CI's `test` job runs ruff check (defaults + bandit `S`), ruff format --check, mypy, shellcheck and hadolint before the tests; tool versions are pinned in `requirements-dev.txt`, the shellcheck/hadolint images in `.github/workflows/docker-image.yml` (commands in CONTRIBUTING.md) - Coverage: `coverage run -m unittest discover -s tests` then `coverage report` (branch coverage; config in `pyproject.toml`, which also sets the `fail_under` ratchet CI enforces) - Mutation: `mutmut run` (mutmut<3; config in `pyproject.toml`, see CONTRIBUTING.md) - Dev deps: `pip install -r requirements-dev.txt` (includes `requirements-test.txt`) diff --git a/fetch_torrents.py b/fetch_torrents.py index aa8dc30..27e3add 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -756,7 +756,7 @@ def fetch_arch_latest(): return False # Version strings are YYYY.MM.DD, so lexicographic order == chronological order. - version, href = max(candidates, key=lambda entry: entry[0]) + version, href = max(candidates) logger.debug(f"Arch Linux {version}: {base_url}{href}") return {f"archlinux-{version}": base_url + href} except Exception as exc: @@ -815,31 +815,44 @@ def fetch_linuxmint_cinnamon(): return False -def fetch_fedora_workstation(): - url = "https://torrent.fedoraproject.org/torrents/" - try: - r = requests.get(url, timeout=30) - r.raise_for_status() - soup = BeautifulSoup(r.text, "html.parser") +FEDORA_TORRENTS_URL = "https://torrent.fedoraproject.org/torrents/" +FEDORA_WORKSTATION_PATTERN = re.compile( + r"^Fedora-Workstation-Live-([A-Za-z0-9_]+)-(\d+)\.torrent$" +) - pattern = re.compile( - r"^Fedora-Workstation-Live-([A-Za-z0-9_]+)-(\d+)\.torrent$" - ) - latest = {} - for link in soup.find_all("a", href=True): - match = pattern.match(link["href"]) - if not match: - continue + +def _fedora_workstation_torrents(): + """{arch: [href, ...]} for every Fedora Workstation Live torrent the + listing links, each href once, newest version first.""" + r = requests.get(FEDORA_TORRENTS_URL, timeout=30) + r.raise_for_status() + soup = BeautifulSoup(r.text, "html.parser") + + by_arch: dict[str, set[tuple[int, str]]] = {} + for link in soup.find_all("a", href=True): + match = FEDORA_WORKSTATION_PATTERN.match(link["href"]) + if match: arch, version = match.group(1), int(match.group(2)) - if arch not in latest or version > latest[arch][0]: - latest[arch] = (version, link["href"]) + by_arch.setdefault(arch, set()).add((version, link["href"])) + return { + arch: [href for _version, href in sorted(entries, reverse=True)] + for arch, entries in by_arch.items() + } + + +def fetch_fedora_workstation(): + url = FEDORA_TORRENTS_URL + try: + latest = { + arch: hrefs[0] for arch, hrefs in _fedora_workstation_torrents().items() + } if not latest: logger.warning("No Fedora Workstation torrents found on %s", url) return False results = {} - for arch, (version, href) in latest.items(): + for href in latest.values(): torrent_url = url + href # The torrent's own "name" field (and Transmission's later # torrent.name) has no .iso extension for Fedora, unlike Ubuntu/ @@ -857,29 +870,11 @@ def fetch_fedora_workstation_old_versions(): """Non-latest Fedora Workstation Live torrents per arch, from the same listing fetch_fedora_workstation() already scrapes - for the old-release demand check (FETCH_TORRENTS_CHECK_OLD_RELEASES).""" - url = "https://torrent.fedoraproject.org/torrents/" + url = FEDORA_TORRENTS_URL try: - r = requests.get(url, timeout=30) - r.raise_for_status() - soup = BeautifulSoup(r.text, "html.parser") - - pattern = re.compile( - r"^Fedora-Workstation-Live-([A-Za-z0-9_]+)-(\d+)\.torrent$" - ) - by_arch = {} - for link in soup.find_all("a", href=True): - match = pattern.match(link["href"]) - if not match: - continue - arch, version = match.group(1), int(match.group(2)) - by_arch.setdefault(arch, []).append((version, link["href"])) - results = {} - for versions in by_arch.values(): - versions.sort(key=lambda entry: entry[0], reverse=True) - for _version, href in versions[ - 1: - ]: # newest is fetch_fedora_workstation()'s job + for hrefs in _fedora_workstation_torrents().values(): + for href in hrefs[1:]: # newest is fetch_fedora_workstation()'s job name = href.replace(".torrent", "") results[name] = url + href return results @@ -1358,6 +1353,12 @@ def plan_disk_pressure_cleanup(torrents, used_percent, threshold_percent): return candidates +def _used_percent(disk_usage, path): + """How full path's filesystem is, 0-100; 0.0 for a zero-size one.""" + total, used, _free = disk_usage(path) + return (used / total * 100) if total else 0.0 + + def enforce_disk_usage_limit( client_factory, threshold_percent, @@ -1373,8 +1374,7 @@ def enforce_disk_usage_limit( if threshold_percent <= 0: return - total, used, _free = disk_usage(download_path) - used_percent = (used / total * 100) if total else 0.0 + used_percent = _used_percent(disk_usage, download_path) if used_percent < threshold_percent: return @@ -1393,8 +1393,7 @@ def enforce_disk_usage_limit( removed_history = load_removed_history(removed_history_file) for torrent in candidates: - total, used, _free = disk_usage(download_path) - used_percent = (used / total * 100) if total else 0.0 + used_percent = _used_percent(disk_usage, download_path) if used_percent < threshold_percent: break ratio = float(getattr(torrent, "ratio", 0.0) or 0.0) diff --git a/requirements-dev.txt b/requirements-dev.txt index 9b55b0e..d229469 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,5 +2,7 @@ # Pinned so new upstream lint/type rules can't break CI unannounced. ruff==0.17.1 mypy==2.4.0 +# requests 2.33 (pinned to the apk version) ships no inline types. +types-requests==2.33.0.20261006 coverage==7.16.2 mutmut==2.5.1 diff --git a/tasks/README.md b/tasks/README.md index 6a4d89d..3d83f04 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -102,9 +102,28 @@ mutation score is 95.4% (1027/1076), up from 88.6%, with the critical set still | 33 | [`scrape_torrent` fall-through branches and SSRF edges](33-scrape-torrent-branches-and-ssrf.md) | P2 | NO_REFACTOR | — | done | | 34 | [Mechanical test tidy-up](34-mechanical-test-tidy-up.md) | P3 | NO_REFACTOR | — | open | | 35 | [Consolidate fixtures and the path-global seam](35-fixture-consolidation.md) | P3 | NO_REFACTOR → REFACTOR_REQUIRED | 34 | open | -| 36 | [Remaining `fetch_torrents.py` mutation survivors](36-remaining-mutation-survivors.md) | P3 | NO_REFACTOR | — | open | +| 36 | [Remaining `fetch_torrents.py` mutation survivors](36-remaining-mutation-survivors.md) | P3 | NO_REFACTOR | — | done | | 37 | [Toolchain: smoke test, shellcheck/hadolint, ruff S, pins](37-toolchain-and-q4.md) | P2 | CI | — | done | | 38 | [Re-measure, raise the coverage ratchet, record the baseline](38-remeasure-and-ratchet.md) | P3 | — | 28–37 | open | +### Accepted equivalent mutants (task 36) + +These `fetch_torrents.py` survivors can't change behaviour, so they're accepted +rather than tested. A mutmut run that leaves only these (plus string-literal +mutants) counts as clean: +- `.get("seeders", 0)` defaults in the old-release logging: a scrape result + always carries `seeders`. +- `logger.propagate = False` → `None`: logging treats `None` as false. +- `getattr(torrent, "ratio", 0.0)` defaults in cleanup and disk pressure: a + real `transmission_rpc.Torrent` always has `.ratio`. +- `break` → `continue` once `enforce_disk_usage_limit` is under the + threshold: every later candidate fails the same check, so the only + difference is extra disk-usage reads. + +Task 36 removed three other equivalents by simplifying the code, so mutmut +no longer generates them: Arch's `max(..., key=...)` became `max()`, Fedora's +two parsers share one de-duplicating `_fedora_workstation_torrents()`, and the +duplicated zero-size-disk guard moved into `_used_percent()`. + Suggested order: 32 (a real bug), then 28, 30 and 33 (resilience and drift-job trust), then 37, then 36, then the clean-up tasks 29, 31, 34 and 35, and finally 38. diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index e385fab..ab8145a 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -1350,6 +1350,68 @@ def test_records_removal_reason_as_disk_pressure(self): ) +class CleanupOldVersionsDefaultsTests(unittest.TestCase): + """cleanup_old_versions() called the way the daily run calls it.""" + + def setUp(self): + self.history_path = os.path.join( + helpers.make_temp_dir(self, "fetch_torrents_cleanup_"), "removed.json" + ) + for attr, value in ( + ("removed_history_file", self.history_path), + ("watch_dir", helpers.make_temp_dir(self, "fetch_torrents_test_watch_")), + ): + patcher = unittest.mock.patch.object(ft, attr, value) + patcher.start() + self.addCleanup(patcher.stop) + + def _transmission(self, old_ratio): + return helpers.FakeTransmission( + [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), + helpers.FakeTorrent( + "ubuntu-23.10-desktop-amd64.iso", 2, ratio=old_ratio + ), + ] + ) + + def test_default_min_ratio_is_one(self): + for old_ratio, kept in ((1.5, False), (0.99, True)): + with self.subTest(old_ratio=old_ratio): + transmission = self._transmission(old_ratio) + + ft.cleanup_old_versions(transmission, today=date(2026, 8, 1)) + + self.assertEqual( + "ubuntu-23.10-desktop-amd64.iso" in transmission.names, kept + ) + + def test_a_zero_ratio_removal_is_recorded_as_zero(self): + transmission = self._transmission(0.0) + + ft.cleanup_old_versions( + transmission, today=date(2026, 8, 1), skip_ratio_check=True + ) + + self.assertEqual( + ft.load_removed_history(self.history_path), + { + "ubuntu-23.10-desktop-amd64.iso": { + "removed_date": "2026-08-01", + "reason": "keep_only_latest", + "ratio": 0.0, + } + }, + ) + + +class RecordRemovalRatioTests(unittest.TestCase): + def test_ratio_is_rounded_to_three_decimals(self): + history = ft.record_removal({}, "x.iso", date(2026, 8, 1), "stagnant", 0.12345) + + self.assertEqual(history["x.iso"]["ratio"], 0.123) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_daily_run.py b/tests/test_daily_run.py index 057060d..2d7b3b9 100644 --- a/tests/test_daily_run.py +++ b/tests/test_daily_run.py @@ -14,6 +14,7 @@ """ import contextlib +import dataclasses import io import json import logging @@ -1113,5 +1114,47 @@ def test_factory_connects_only_when_called(self): self.assertTrue(callable(factory)) +class ParseBoolTests(unittest.TestCase): + def test_unset_is_false_by_default(self): + self.assertIs(ft.parse_bool("X", env={}), False) + + def test_unset_or_blank_uses_the_given_default(self): + for value in ({}, {"X": " "}): + with self.subTest(env=value): + self.assertIs(ft.parse_bool("X", default=True, env=value), True) + + def test_truthy_and_falsy_spellings(self): + for value in ("1", "true", "YES", " On "): + with self.subTest(value=value): + self.assertIs(ft.parse_bool("X", env={"X": value}), True) + for value in ("0", "false", "no", "off", "maybe"): + with self.subTest(value=value): + self.assertIs(ft.parse_bool("X", default=True, env={"X": value}), False) + + +class RunValueTypesTests(unittest.TestCase): + def test_run_summary_starts_at_zero_with_no_old_release_counts(self): + self.assertEqual( + ft.RunSummary(), + ft.RunSummary( + elapsed=0.0, + added=0, + existing=0, + not_found=0, + failed=0, + old_added=None, + old_skipped=None, + ), + ) + + def test_config_is_immutable(self): + # run() and its steps share one Config; none may change it for the + # steps after it. + config = ft.Config() + + with self.assertRaises(dataclasses.FrozenInstanceError): + config.skip_ratio_check = True # type: ignore[misc] + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_fetchers.py b/tests/test_fetchers.py index 5b12c32..aa1f210 100644 --- a/tests/test_fetchers.py +++ b/tests/test_fetchers.py @@ -574,5 +574,167 @@ def test_remove_watch_file_refuses_unsafe_name_without_touching_disk(self): self.assertTrue(os.path.exists(outside)) +class FetcherRequestTimeoutTests(unittest.TestCase): + """Every upstream request has a timeout, so no slow or stalled distro + site can hang the daily run.""" + + def test_every_fetcher_and_the_download_pass_a_30_second_timeout(self): + watch = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + calls = { + **{name: fetcher for name, fetcher in ft.DEFAULT_FETCHERS}, + "kali-old": ft.fetch_kali_old_versions, + "fedora-old": ft.fetch_fedora_workstation_old_versions, + "download": lambda: ft.download_torrent( + "x.iso", "https://example.invalid/x" + ), + } + for name, call in calls.items(): + with self.subTest(name): + timeouts = [] + + def get(url, timeouts=timeouts, **kwargs): + timeouts.append(kwargs) + return helpers.fake_response("") + + with ( + unittest.mock.patch.object(ft.requests, "get", get), + unittest.mock.patch.object(ft, "watch_dir", watch), + self.assertNoLogs(ft.logger, level="CRITICAL"), + ): + call() + + self.assertTrue(timeouts) + self.assertEqual(timeouts, [{"timeout": 30}] * len(timeouts)) + + +def _get_page(html): + return unittest.mock.patch.object( + ft.requests, "get", return_value=helpers.fake_response(html) + ) + + +class FetchArchLatestSelectionTests(unittest.TestCase): + def test_unavailable_row_before_an_available_one_is_skipped(self): + html = ( + "" + "" + "" + "" + "" + "
t
t
" + ) + + with _get_page(html): + results = ft.fetch_arch_latest() + + self.assertEqual( + results, + { + "archlinux-2026.09.01": "https://archlinux.org/releng/releases/2026.09.01/torrent/" + }, + ) + + def test_row_without_a_torrent_link_is_skipped(self): + html = ( + "" + "" + "" + "" + "
no link
t
" + ) + + with _get_page(html): + results = ft.fetch_arch_latest() + + self.assertEqual(list(results), ["archlinux-2026.09.01"]) + + +class FetchKaliSelectionTests(unittest.TestCase): + def test_page_without_a_release_number_returns_false(self): + with _get_page("no installers here"): + self.assertIs(ft.fetch_kali_latest(), False) + + def test_old_versions_are_every_version_but_the_newest(self): + html = "".join( + f'kali-{v}' + for v in ("2025.4", "2026.2", "2025.10", "2026.1") + ) + + with _get_page(f"{html}"): + results = ft.fetch_kali_old_versions() + + expected = {} + for version in ("2025.4", "2025.10", "2026.1"): + for url in ft._kali_torrent_urls(version): + expected[os.path.basename(url).removesuffix(".torrent")] = url + self.assertEqual(results, expected) + + +class FetchFedoraSelectionTests(unittest.TestCase): + BASE = "https://torrent.fedoraproject.org/torrents/" + + def _page(self, *versions, arch="x86_64"): + links = "".join( + f't' + for v in versions + ) + return _get_page(f"{links}") + + def _entry(self, version, arch="x86_64"): + name = f"Fedora-Workstation-Live-{arch}-{version}" + return name, f"{self.BASE}{name}.torrent" + + def test_latest_compares_versions_numerically(self): + with self._page(100, 99, 98): + results = ft.fetch_fedora_workstation() + + self.assertEqual(results, dict([self._entry(100)])) + + def test_old_versions_compare_numerically_and_skip_only_the_newest(self): + with self._page(99, 100, 98): + results = ft.fetch_fedora_workstation_old_versions() + + self.assertEqual(results, dict([self._entry(99), self._entry(98)])) + + def test_same_version_listed_twice_is_one_candidate(self): + with self._page(44, 44, 43): + latest = ft.fetch_fedora_workstation() + with self._page(44, 44, 43): + old = ft.fetch_fedora_workstation_old_versions() + + self.assertEqual(latest, dict([self._entry(44)])) + self.assertEqual(old, dict([self._entry(43)])) + + +class ParseSupportedDistrosWarningTests(unittest.TestCase): + def test_unknown_entries_are_named_in_the_warning(self): + with self.assertLogs(ft.logger, level="WARNING") as logs: + result = ft.parse_supported_distros( + env={"FETCH_TORRENTS_DISTROS": "Ubuntu, gentoo,arch,slackware"} + ) + + self.assertEqual(result, ["ubuntu", "arch"]) + self.assertEqual( + [r.getMessage() for r in logs.records], + [ + "FETCH_TORRENTS_DISTROS contains unknown distributions: gentoo, " + "slackware. Valid values: " + ", ".join(ft.ALL_DISTROS) + ], + ) + + def test_all_unknown_warns_twice_and_falls_back(self): + with self.assertLogs(ft.logger, level="WARNING") as logs: + result = ft.parse_supported_distros( + env={"FETCH_TORRENTS_DISTROS": "gentoo"} + ) + + self.assertEqual(result, list(ft.DEFAULT_DISTROS)) + self.assertEqual( + [r.getMessage() for r in logs.records][1], + "FETCH_TORRENTS_DISTROS did not specify any valid distros. " + "Falling back to defaults: " + ", ".join(ft.DEFAULT_DISTROS), + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_logging.py b/tests/test_logging.py index 7bc5ade..9fca972 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -422,5 +422,30 @@ def test_explicit_auth_argument_still_wins(self): self.assertEqual(second_call_kwargs.get("auth"), ("bob", "other")) +class RpcCredentialFallbackTests(unittest.TestCase): + """Without an explicit auth, log_seed_ratios_via_http() authenticates + only when both TRANSMISSION_RPC_USERNAME and _PASSWORD are set.""" + + def _auth_sent(self, **env): + session = helpers.fake_response( + status=409, headers={"X-Transmission-Session-Id": "sid"} + ) + torrents = helpers.fake_response('{"arguments": {"torrents": []}}') + with ( + helpers.isolated_env(**env), + unittest.mock.patch.object( + ft.requests, "post", side_effect=[session, torrents] + ) as mock_post, + ): + ft.log_seed_ratios_via_http() + return mock_post.call_args_list[1].kwargs["auth"] + + def test_username_without_password_sends_no_auth(self): + self.assertIsNone(self._auth_sent(TRANSMISSION_RPC_USERNAME="alice")) + + def test_password_without_username_sends_no_auth(self): + self.assertIsNone(self._auth_sent(TRANSMISSION_RPC_PASSWORD="hunter2")) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_persistence.py b/tests/test_persistence.py index e306c25..1afc604 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -132,5 +132,55 @@ def load_reporting(self, path): return data, stderr.getvalue().strip() +class StateFileLocationTests(unittest.TestCase): + """Every test patches these paths, so their real values are pinned + here: state lives under the log dir, torrents go to /watch.""" + + def test_state_files_live_in_the_log_dir(self): + self.assertEqual( + ( + ft.ratio_log_file, + ft.ratio_history_file, + ft.removed_history_file, + ft.old_release_check_state_file, + ), + tuple( + os.path.join(ft.log_dir, name) + for name in ( + "fetch_torrents_ratios.log", + "fetch_torrents_ratio_history.json", + "fetch_torrents_removed_history.json", + "fetch_torrents_old_release_check_state.json", + ) + ), + ) + + def test_torrents_are_written_to_the_watch_volume(self): + self.assertEqual(ft.watch_dir, "/watch") + + +class StateFileFormatTests(unittest.TestCase): + """The state files sit in the /logs volume for people to read and diff, + so they are written indented, with keys sorted.""" + + def test_state_files_are_indented_with_sorted_keys(self): + directory = helpers.make_temp_dir(self, "persistence_") + for save in ( + ft.save_ratio_history, + ft.save_removed_history, + ft.save_old_release_check_state, + ): + with self.subTest(save.__name__): + path = os.path.join(directory, f"{save.__name__}.json") + + save(path, {"b.iso": {"y": 1, "x": 2}, "a.iso": []}) + + with open(path, encoding="utf-8") as f: + self.assertEqual( + f.read(), + '{\n "a.iso": [],\n "b.iso": {\n "x": 2,\n "y": 1\n }\n}', + ) + + if __name__ == "__main__": unittest.main() From e7d88e86c78883da30e02663bae62ad7f3fb39d7 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 17:15:29 +0200 Subject: [PATCH 37/46] Inject the UDP transaction id; assert both payloads on the wire (task 29) scrape_udp_tracker() takes new_transaction_id (default _random_transaction_id), so tests pass a fixed id sequence instead of patching os.urandom process-wide. The UDP tests use an autospec'd socket and assert the connect and scrape datagrams byte for byte. The test that guards the fresh scrape id now says so. Every bare assertRaises in test_torrent_scrape.py now checks a message fragment, and the duplicate short-response tests are folded together. Co-Authored-By: Claude Opus 5.5 --- tasks/README.md | 2 +- tests/test_torrent_scrape.py | 262 +++++++++++++++++++++-------------- torrent_scrape.py | 8 +- 3 files changed, 161 insertions(+), 111 deletions(-) diff --git a/tasks/README.md b/tasks/README.md index 3d83f04..97ec171 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -95,7 +95,7 @@ mutation score is 95.4% (1027/1076), up from 88.6%, with the critical set still | # | Task | Priority | Class | Depends on | Status | | --- | --- | --- | --- | --- | --- | | 28 | [Make `wait_for_transmission_rpc` testable and test it](28-rpc-wait-seam.md) | P2 | REFACTOR_REQUIRED | — | done | -| 29 | [Inject the UDP transaction id; assert wire payloads](29-udp-transaction-id-seam.md) | P3 | REFACTOR_REQUIRED | — | open | +| 29 | [Inject the UDP transaction id; assert wire payloads](29-udp-transaction-id-seam.md) | P3 | REFACTOR_REQUIRED | — | done | | 30 | [Test `scripts/live_check.py` beyond `problems()`](30-live-check-tests.md) | P2 | REFACTOR_REQUIRED | — | done | | 31 | [`configure_transmission`: hermetic default path, boundaries](31-configure-transmission-gaps.md) | P3 | NO_REFACTOR | — | open | | 32 | [Atomic `.torrent` writes](32-atomic-torrent-writes.md) | P2 (bug) | NO_REFACTOR | — | done | diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index 3b1dd60..9d9871c 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -7,6 +7,7 @@ """ import hashlib +import inspect import os import socket import struct @@ -69,7 +70,9 @@ def test_bdecode_nested(self): self.assertEqual(value, {b"info": {b"name": b"test", b"length": 10}}) def test_bdecode_invalid_prefix_raises(self): - with self.assertRaises(ts.BencodeError): + with self.assertRaisesRegex( + ts.BencodeError, r"Invalid bencode prefix b'x' at index 0" + ): ts.bdecode(b"x123") def test_bdecode_malformed_input_raises_bencode_error(self): @@ -87,10 +90,11 @@ def test_bdecode_malformed_input_raises_bencode_error(self): b"d3:cow3:moo": "unterminated dict", } for data, problem in cases.items(): - with self.subTest(problem, data=data): - with self.assertRaises(ts.BencodeError) as ctx: - ts.bdecode(data) - self.assertRegex(str(ctx.exception), r"index \d+") + with ( + self.subTest(problem, data=data), + self.assertRaisesRegex(ts.BencodeError, r"index \d+"), + ): + ts.bdecode(data) def test_bdecode_truncated_string_reports_declared_and_remaining_bytes(self): with self.assertRaisesRegex( @@ -105,7 +109,7 @@ def test_bdecode_unterminated_list_reports_end_of_data(self): ts.bdecode(b"l4:spam") def test_bdecode_truncated_string_inside_container_raises(self): - with self.assertRaises(ts.BencodeError): + with self.assertRaisesRegex(ts.BencodeError, "declares 10 bytes"): ts.bdecode(b"d4:info10:shorte") def test_bdecode_string_ending_exactly_at_end_of_data_is_accepted(self): @@ -142,7 +146,12 @@ def test_mixed_keys_are_sorted_by_byte_value(self): def test_unsupported_types_raise(self): for value in (True, 1.5, None, ("tuple",)): - with self.subTest(value=value), self.assertRaises(ts.BencodeError): + with ( + self.subTest(value=value), + self.assertRaisesRegex( + ts.BencodeError, "is not a bencode type|Cannot bencode type" + ), + ): ts.bencode(value) @@ -168,7 +177,7 @@ def test_raises_when_info_missing(self): torrent_bytes = ( b"d" + bstr("announce") + bstr("http://tracker.example.com/announce") + b"e" ) - with self.assertRaises(ts.BencodeError): + with self.assertRaisesRegex(ts.BencodeError, "missing 'info' dictionary"): ts.compute_info_hash(torrent_bytes) @@ -311,7 +320,7 @@ def test_returns_none_when_hash_absent(self): def test_raises_on_failure_reason(self): response = b"d14:failure reason7:blockede" - with self.assertRaises(ts.BencodeError): + with self.assertRaisesRegex(ts.BencodeError, "^blocked$"): ts.parse_scrape_response(response, b"A" * 20) @@ -371,21 +380,27 @@ def test_parses_valid_response(self): def test_transaction_id_mismatch_raises(self): response = _connect_response(transaction_id=42, connection_id=1) - with self.assertRaises(ts.UdpTrackerError): + with self.assertRaisesRegex( + ts.UdpTrackerError, "Transaction ID mismatch in connect response" + ): ts.parse_udp_connect_response(response, transaction_id=99) def test_wrong_action_raises(self): response = _connect_response(transaction_id=42, connection_id=1, action=2) - with self.assertRaises(ts.UdpTrackerError): + with self.assertRaisesRegex( + ts.UdpTrackerError, "Unexpected action 2 in connect response" + ): ts.parse_udp_connect_response(response, transaction_id=42) - def test_short_response_raises(self): - with self.assertRaises(ts.UdpTrackerError): - ts.parse_udp_connect_response(b"\x00" * 8, transaction_id=42) - - def test_short_response_reports_its_length(self): - with self.assertRaisesRegex(ts.UdpTrackerError, r"too short \(15 bytes\)"): - ts.parse_udp_connect_response(b"\x00" * 15, transaction_id=42) + def test_short_response_raises_with_its_length(self): + for length in (8, 15): + with ( + self.subTest(length=length), + self.assertRaisesRegex( + ts.UdpTrackerError, rf"too short \({length} bytes\)" + ), + ): + ts.parse_udp_connect_response(b"\x00" * length, transaction_id=42) def test_trailing_bytes_after_connect_response_are_ignored(self): response = _connect_response(transaction_id=42, connection_id=7) + b"xx" @@ -400,15 +415,15 @@ def test_error_action_with_empty_message_is_still_a_tracker_error(self): def test_error_action_for_another_transaction_is_not_reported_as_ours(self): response = _error_response(transaction_id=99, message="not for you") - with self.assertRaises(ts.UdpTrackerError) as ctx: + with self.assertRaisesRegex( + ts.UdpTrackerError, "^Unexpected action 3 in connect response$" + ): ts.parse_udp_connect_response(response, transaction_id=42) - self.assertNotIn("not for you", str(ctx.exception)) def test_tracker_error_action_raises_with_message(self): response = _error_response(transaction_id=42, message="bad request") - with self.assertRaises(ts.UdpTrackerError) as ctx: + with self.assertRaisesRegex(ts.UdpTrackerError, "^Tracker error: bad request$"): ts.parse_udp_connect_response(response, transaction_id=42) - self.assertIn("bad request", str(ctx.exception)) class BuildParseUdpScrapeRequestTests(unittest.TestCase): @@ -426,7 +441,9 @@ def test_request_matches_bep15_layout(self): self.assertEqual(request[16:], info_hash) def test_wrong_length_info_hash_raises(self): - with self.assertRaises(ts.UdpTrackerError): + with self.assertRaisesRegex( + ts.UdpTrackerError, "info_hash must be 20 bytes, got 5" + ): ts.build_udp_scrape_request( connection_id=1, transaction_id=1, info_hash=b"short" ) @@ -442,24 +459,31 @@ def test_transaction_id_mismatch_raises(self): response = _scrape_response( transaction_id=7, seeders=1, completed=1, leechers=1 ) - with self.assertRaises(ts.UdpTrackerError): + with self.assertRaisesRegex( + ts.UdpTrackerError, "Transaction ID mismatch in scrape response" + ): ts.parse_udp_scrape_response(response, transaction_id=8) def test_wrong_action_raises(self): response = _scrape_response( transaction_id=7, seeders=1, completed=1, leechers=1, action=0 ) - with self.assertRaises(ts.UdpTrackerError): + with self.assertRaisesRegex( + ts.UdpTrackerError, "Unexpected action 0 in scrape response" + ): ts.parse_udp_scrape_response(response, transaction_id=7) - def test_short_response_raises(self): - with self.assertRaises(ts.UdpTrackerError): - ts.parse_udp_scrape_response(b"\x00" * 4, transaction_id=7) - - def test_response_one_byte_short_of_one_hash_raises(self): - response = _scrape_response(7, seeders=1, completed=1, leechers=1)[:19] - with self.assertRaisesRegex(ts.UdpTrackerError, r"too short \(19 bytes\)"): - ts.parse_udp_scrape_response(response, transaction_id=7) + def test_short_response_raises_with_its_length(self): + # 19 bytes is one short of the stats for a single hash. + full = _scrape_response(7, seeders=1, completed=1, leechers=1) + for length in (4, 19): + with ( + self.subTest(length=length), + self.assertRaisesRegex( + ts.UdpTrackerError, rf"too short \({length} bytes\)" + ), + ): + ts.parse_udp_scrape_response(full[:length], transaction_id=7) def test_trailing_bytes_after_first_hash_are_ignored(self): response = ( @@ -472,121 +496,145 @@ def test_trailing_bytes_after_first_hash_are_ignored(self): def test_tracker_error_action_raises_with_message(self): response = _error_response(transaction_id=7, message="scrape not supported") - with self.assertRaises(ts.UdpTrackerError) as ctx: + with self.assertRaisesRegex( + ts.UdpTrackerError, "^Tracker error: scrape not supported$" + ): ts.parse_udp_scrape_response(response, transaction_id=7) - self.assertIn("scrape not supported", str(ctx.exception)) class ScrapeUdpTrackerTests(unittest.TestCase): - """Exercises scrape_udp_tracker()'s socket I/O via a mocked socket, so - no real network access is required - same spirit as ScrapeTorrentTests - mocking ts.requests.get for the HTTP path.""" - - def _make_fake_socket(self, recvfrom_side_effect): - fake_sock = unittest.mock.Mock() - fake_sock.recvfrom.side_effect = recvfrom_side_effect - return fake_sock + """Exercises scrape_udp_tracker()'s socket I/O against an autospec'd + socket (so a mistyped socket method fails) and a fixed transaction id + sequence, so no real network access or randomness is involved.""" + + INFO_HASH = bytes(range(20)) + ADDR = ("tracker.example.com", 1337) + + def _scrape(self, replies, ids=(1, 2), **kwargs): + """Runs scrape_udp_tracker with `replies` as what recvfrom returns + (or raises); returns (result or raised exception, fake socket).""" + fake_sock = unittest.mock.create_autospec(socket.socket, instance=True) + fake_sock.recvfrom.side_effect = replies + with unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock): + result = ts.scrape_udp_tracker( + *self.ADDR, + self.INFO_HASH, + new_transaction_id=iter(ids).__next__, + **kwargs, + ) + return result, fake_sock - def test_full_round_trip_returns_stats(self): - info_hash = b"A" * 20 - connect_resp = _connect_response( - transaction_id=0x00000001, connection_id=0xCAFEBABE - ) - scrape_resp = _scrape_response( - transaction_id=0x00000002, seeders=5, completed=2, leechers=3 - ) - fake_sock = self._make_fake_socket( + def test_full_round_trip_sends_both_bep15_requests_and_returns_stats(self): + stats, fake_sock = self._scrape( [ - (connect_resp, ("1.2.3.4", 1337)), - (scrape_resp, ("1.2.3.4", 1337)), - ] + (_connect_response(transaction_id=1, connection_id=0xCAFEBABE), None), + (_scrape_response(2, seeders=5, completed=2, leechers=3), None), + ], + timeout=5, ) - with ( - unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), - unittest.mock.patch.object( - ts.os, "urandom", side_effect=[b"\x00\x00\x00\x01", b"\x00\x00\x00\x02"] - ), - ): - stats = ts.scrape_udp_tracker( - "tracker.example.com", 1337, info_hash, timeout=5 - ) - self.assertEqual(stats, {"seeders": 5, "leechers": 3, "completed": 2}) fake_sock.settimeout.assert_called_once_with(5) self.assertEqual( - [call.args[1] for call in fake_sock.sendto.call_args_list], - [("tracker.example.com", 1337)] * 2, - ) - # Second (scrape) request must carry the connection_id from the connect response. - scrape_request_bytes = fake_sock.sendto.call_args_list[1][0][0] - connection_id_sent, action_sent = struct.unpack( - ">QI", scrape_request_bytes[:12] - ) - self.assertEqual(connection_id_sent, 0xCAFEBABE) - self.assertEqual(action_sent, 2) - fake_sock.close.assert_called_once() - - def test_scrape_response_echoing_the_connect_id_raises(self): - info_hash = b"A" * 20 - fake_sock = self._make_fake_socket( + fake_sock.sendto.call_args_list, [ - (_connect_response(transaction_id=1, connection_id=9), None), - (_scrape_response(1, seeders=5, completed=2, leechers=3), None), - ] + unittest.mock.call(struct.pack(">QII", 0x41727101980, 0, 1), self.ADDR), + unittest.mock.call( + struct.pack(">QII", 0xCAFEBABE, 2, 2) + self.INFO_HASH, self.ADDR + ), + ], ) + fake_sock.close.assert_called_once() - with ( - unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), - unittest.mock.patch.object( - ts.os, "urandom", side_effect=[b"\x00\x00\x00\x01", b"\x00\x00\x00\x02"] - ), - self.assertRaisesRegex(ts.UdpTrackerError, "mismatch in scrape response"), + def test_scrape_uses_a_fresh_transaction_id_not_the_connect_one(self): + # A reply to the scrape that echoes the connect's id is not ours. + with self.assertRaisesRegex( + ts.UdpTrackerError, "Transaction ID mismatch in scrape response" ): - ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) - - fake_sock.close.assert_called_once() + self._scrape( + [ + (_connect_response(transaction_id=1, connection_id=9), None), + (_scrape_response(1, seeders=5, completed=2, leechers=3), None), + ] + ) def test_default_timeout_is_15_seconds(self): - fake_sock = self._make_fake_socket(TimeoutError("timed out")) + fake_sock = unittest.mock.create_autospec(socket.socket, instance=True) + fake_sock.recvfrom.side_effect = TimeoutError("timed out") with ( unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), - self.assertRaises(socket.timeout), + self.assertRaisesRegex(socket.timeout, "timed out"), ): - ts.scrape_udp_tracker("tracker.example.com", 1337, b"A" * 20) + ts.scrape_udp_tracker( + *self.ADDR, self.INFO_HASH, new_transaction_id=lambda: 1 + ) fake_sock.settimeout.assert_called_once_with(15) def test_connect_transaction_mismatch_raises_and_closes_socket(self): - info_hash = b"A" * 20 - bad_connect_resp = _connect_response(transaction_id=0x00000002, connection_id=1) - fake_sock = self._make_fake_socket([(bad_connect_resp, ("1.2.3.4", 1337))]) + fake_sock = unittest.mock.create_autospec(socket.socket, instance=True) + fake_sock.recvfrom.side_effect = [ + (_connect_response(transaction_id=2, connection_id=1), None) + ] with ( unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), - unittest.mock.patch.object( - ts.os, "urandom", return_value=b"\x00\x00\x00\x01" + self.assertRaisesRegex( + ts.UdpTrackerError, "Transaction ID mismatch in connect response" ), - self.assertRaises(ts.UdpTrackerError), ): - ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) + ts.scrape_udp_tracker( + *self.ADDR, self.INFO_HASH, new_transaction_id=lambda: 1 + ) fake_sock.close.assert_called_once() - def test_socket_timeout_propagates(self): - info_hash = b"A" * 20 - fake_sock = unittest.mock.Mock() + def test_socket_timeout_propagates_and_closes_socket(self): + fake_sock = unittest.mock.create_autospec(socket.socket, instance=True) fake_sock.recvfrom.side_effect = TimeoutError("timed out") with ( unittest.mock.patch.object(ts.socket, "socket", return_value=fake_sock), - self.assertRaises(socket.timeout), + self.assertRaisesRegex(socket.timeout, "timed out"), ): - ts.scrape_udp_tracker("tracker.example.com", 1337, info_hash, timeout=5) + ts.scrape_udp_tracker(*self.ADDR, self.INFO_HASH, timeout=5) fake_sock.close.assert_called_once() + def test_opens_a_udp_socket(self): + fake_sock = unittest.mock.create_autospec(socket.socket, instance=True) + fake_sock.recvfrom.side_effect = TimeoutError("timed out") + + with ( + unittest.mock.patch.object( + ts.socket, "socket", return_value=fake_sock + ) as make_socket, + self.assertRaises(socket.timeout), + ): + ts.scrape_udp_tracker(*self.ADDR, self.INFO_HASH) + + make_socket.assert_called_once_with(socket.AF_INET, socket.SOCK_DGRAM) + + +class RandomTransactionIdTests(unittest.TestCase): + def test_is_four_random_bytes_read_big_endian(self): + with unittest.mock.patch.object( + ts.os, "urandom", return_value=b"\x01\x02\x03\x04" + ) as urandom: + self.assertEqual(ts._random_transaction_id(), 0x01020304) + + urandom.assert_called_once_with(4) + + def test_is_the_default_id_source(self): + default = ( + inspect.signature(ts.scrape_udp_tracker) + .parameters["new_transaction_id"] + .default + ) + + self.assertIs(default, ts._random_transaction_id) + class IsDisallowedIpTests(unittest.TestCase): def test_public_ipv4_is_allowed(self): diff --git a/torrent_scrape.py b/torrent_scrape.py index 3541a9c..89a2cf7 100644 --- a/torrent_scrape.py +++ b/torrent_scrape.py @@ -258,7 +258,9 @@ def _random_transaction_id(): return int.from_bytes(os.urandom(4), "big") -def scrape_udp_tracker(host, port, info_hash, timeout=15): +def scrape_udp_tracker( + host, port, info_hash, timeout=15, *, new_transaction_id=_random_transaction_id +): """Scrapes a BEP 15 UDP tracker for info_hash's stats via a connect round trip followed by a scrape round trip. Returns {'seeders', 'leechers', 'completed'}. Raises (UdpTrackerError, OSError/socket.timeout, @@ -269,12 +271,12 @@ def scrape_udp_tracker(host, port, info_hash, timeout=15): sock.settimeout(timeout) addr = (host, port) - connect_transaction_id = _random_transaction_id() + connect_transaction_id = new_transaction_id() sock.sendto(build_udp_connect_request(connect_transaction_id), addr) data, _ = sock.recvfrom(UDP_RECV_BUFFER_BYTES) connection_id = parse_udp_connect_response(data, connect_transaction_id) - scrape_transaction_id = _random_transaction_id() + scrape_transaction_id = new_transaction_id() sock.sendto( build_udp_scrape_request(connection_id, scrape_transaction_id, info_hash), addr, From d508fb35dc98d0df32744c3d70e79873928b149b Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 17:16:49 +0200 Subject: [PATCH 38/46] configure_transmission: hermetic default path, boundaries, whole results (task 31) main(settings_path=None) falls back to TRANSMISSION_SETTINGS_FILE, then the new DEFAULT_SETTINGS_FILE constant. The default-path test patches that constant, so it no longer depends on whether this machine has a real /config/settings.json. New tests pin a peer limit of exactly 1, case/whitespace-insensitive TRANSMISSION_PORT_FORWARDING, and apply_overrides' whole result with its input unchanged. Each subTest case gets its own temp dir. The settings.json layout is recorded as an accepted equivalent. Co-Authored-By: Claude Opus 5.5 --- configure_transmission.py | 10 +++- tasks/README.md | 8 ++- tests/test_configure_transmission.py | 73 +++++++++++++++++++++++----- 3 files changed, 76 insertions(+), 15 deletions(-) diff --git a/configure_transmission.py b/configure_transmission.py index 9eba1f6..93e4fe0 100644 --- a/configure_transmission.py +++ b/configure_transmission.py @@ -14,6 +14,8 @@ import os import sys +DEFAULT_SETTINGS_FILE = "/config/settings.json" + def build_rpc_overrides(env): overrides = {} @@ -136,8 +138,12 @@ def save_settings(path, settings): os.replace(tmp_path, path) -def main(): - settings_path = os.getenv("TRANSMISSION_SETTINGS_FILE", "/config/settings.json") +def main(settings_path=None): + """settings_path defaults to TRANSMISSION_SETTINGS_FILE, then + DEFAULT_SETTINGS_FILE.""" + settings_path = settings_path or os.getenv( + "TRANSMISSION_SETTINGS_FILE", DEFAULT_SETTINGS_FILE + ) settings_file_exists = os.path.exists(settings_path) overrides = { **build_rpc_overrides(os.environ), diff --git a/tasks/README.md b/tasks/README.md index 97ec171..9de6e3e 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -97,7 +97,7 @@ mutation score is 95.4% (1027/1076), up from 88.6%, with the critical set still | 28 | [Make `wait_for_transmission_rpc` testable and test it](28-rpc-wait-seam.md) | P2 | REFACTOR_REQUIRED | — | done | | 29 | [Inject the UDP transaction id; assert wire payloads](29-udp-transaction-id-seam.md) | P3 | REFACTOR_REQUIRED | — | done | | 30 | [Test `scripts/live_check.py` beyond `problems()`](30-live-check-tests.md) | P2 | REFACTOR_REQUIRED | — | done | -| 31 | [`configure_transmission`: hermetic default path, boundaries](31-configure-transmission-gaps.md) | P3 | NO_REFACTOR | — | open | +| 31 | [`configure_transmission`: hermetic default path, boundaries](31-configure-transmission-gaps.md) | P3 | NO_REFACTOR | — | done | | 32 | [Atomic `.torrent` writes](32-atomic-torrent-writes.md) | P2 (bug) | NO_REFACTOR | — | done | | 33 | [`scrape_torrent` fall-through branches and SSRF edges](33-scrape-torrent-branches-and-ssrf.md) | P2 | NO_REFACTOR | — | done | | 34 | [Mechanical test tidy-up](34-mechanical-test-tidy-up.md) | P3 | NO_REFACTOR | — | open | @@ -120,6 +120,12 @@ mutants) counts as clean: threshold: every later candidate fails the same check, so the only difference is extra disk-usage reads. +In `configure_transmission.py` (task 31), `save_settings`'s `indent=4` and +`sort_keys=True` are accepted too. transmission-daemon owns settings.json and +rewrites it in its own layout when it saves, so ours is never relied on. The +fetch_torrents state files are different: only this code writes them, so +their layout is pinned in `tests/test_persistence.py`. + Task 36 removed three other equivalents by simplifying the code, so mutmut no longer generates them: Arch's `max(..., key=...)` became `max()`, Fedora's two parsers share one de-duplicating `_fedora_workstation_torrents()`, and the diff --git a/tests/test_configure_transmission.py b/tests/test_configure_transmission.py index 97744e8..ea590fa 100644 --- a/tests/test_configure_transmission.py +++ b/tests/test_configure_transmission.py @@ -135,6 +135,18 @@ def test_non_numeric_env_vars_are_ignored(self): self.assertEqual(overrides, {}) + def test_a_limit_of_one_is_kept(self): + overrides = ct.build_peer_limit_overrides( + { + "TRANSMISSION_PEER_LIMIT_GLOBAL": "1", + "TRANSMISSION_PEER_LIMIT_PER_TORRENT": " 1 ", + } + ) + + self.assertEqual( + overrides, {"peer-limit-global": 1, "peer-limit-per-torrent": 1} + ) + def test_zero_and_negative_values_are_ignored(self): overrides = ct.build_peer_limit_overrides( { @@ -173,6 +185,16 @@ def test_explicit_false_disables_it_on_a_later_run(self): self.assertEqual(overrides, {"port-forwarding-enabled": False}) + def test_explicit_value_ignores_case_and_surrounding_whitespace(self): + for value, expected in (("TRUE", True), (" false ", False), ("True\n", True)): + with self.subTest(value=value): + overrides = ct.build_port_forwarding_overrides( + {"TRANSMISSION_PORT_FORWARDING": value}, + settings_file_exists=True, + ) + + self.assertEqual(overrides, {"port-forwarding-enabled": expected}) + def test_blank_env_var_on_first_run_falls_back_to_default(self): overrides = ct.build_port_forwarding_overrides( {"TRANSMISSION_PORT_FORWARDING": " "}, @@ -233,9 +255,22 @@ def test_overrides_take_precedence_but_other_keys_survive(self): merged = ct.apply_overrides(settings, overrides) - self.assertEqual(merged["peer-port"], 51413) - self.assertEqual(merged["rpc-whitelist"], "10.0.0.*") - self.assertTrue(merged["rpc-whitelist-enabled"]) + self.assertEqual( + merged, + { + "peer-port": 51413, + "rpc-whitelist-enabled": True, + "rpc-whitelist": "10.0.0.*", + }, + ) + self.assertEqual( + settings, + { + "peer-port": 51413, + "rpc-whitelist-enabled": True, + "rpc-whitelist": "127.0.0.1", + }, + ) class MainTests(unittest.TestCase): @@ -243,7 +278,10 @@ class MainTests(unittest.TestCase): settings.json out.""" def setUp(self): - self.path = os.path.join( + self.path = self._new_settings_path() + + def _new_settings_path(self): + return os.path.join( helpers.make_temp_dir(self, "configure_transmission_main_"), "settings.json", ) @@ -319,8 +357,7 @@ def test_invalid_peer_limits_are_dropped_silently(self): # Pins current behavior; whether these should warn is a product call. for value in ("abc", "0"): with self.subTest(value=value): - if os.path.exists(self.path): - os.remove(self.path) + self.path = self._new_settings_path() stderr = self._run(TRANSMISSION_PEER_LIMIT_GLOBAL=value) @@ -352,17 +389,29 @@ def test_running_the_script_calls_main(self): self.assertEqual(self._read_settings(), {"port-forwarding-enabled": False}) - def test_defaults_to_config_settings_path(self): + def test_default_settings_file_is_in_the_config_volume(self): + self.assertEqual(ct.DEFAULT_SETTINGS_FILE, "/config/settings.json") + + def test_without_an_env_var_the_default_settings_file_is_used(self): + # The default is patched to a temp path, so this never reads (or + # depends on) a real /config/settings.json on the machine. with ( + unittest.mock.patch.object(ct, "DEFAULT_SETTINGS_FILE", self.path), unittest.mock.patch.dict(os.environ, {}, clear=True), - unittest.mock.patch.object(ct, "save_settings") as save, - unittest.mock.patch.object(ct, "load_settings", return_value={}), ): ct.main() - save.assert_called_once_with( - "/config/settings.json", {"port-forwarding-enabled": False} - ) + self.assertEqual(self._read_settings(), {"port-forwarding-enabled": False}) + + def test_an_explicit_settings_path_wins_over_the_env_var(self): + env_path = self._new_settings_path() + with unittest.mock.patch.dict( + os.environ, {"TRANSMISSION_SETTINGS_FILE": env_path}, clear=True + ): + ct.main(settings_path=self.path) + + self.assertEqual(self._read_settings(), {"port-forwarding-enabled": False}) + self.assertFalse(os.path.exists(env_path)) if __name__ == "__main__": From c9b167311f7660d98bd4da32f3e8fc69d55cc552 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 17:28:30 +0200 Subject: [PATCH 39/46] Tidy the test suite: literal oracles, no CPython wording, misfiled tests (task 34) Expected values that came from production (ft._kali_torrent_urls, ft.DEFAULT_DISTROS, ft.ALL_DISTROS) are now written out: the Kali image list moved to helpers.kali_candidates(), shared with test_scrapers.py. Config tests assert only the project's "ENV_VAR: " prefix, not CPython's float()/int() error text. The import subprocess runs with -W ignore. Count-only and startswith/any() asserts now check names, exact log lines and that nothing was removed (new FakeTransmission.removals). parse_supported_distros' unknown-entry and fallback cases assert their warnings. The plan_cleanup per-distro test is a subTest table plus a combined run. ScraperTestCase.fetch() returns a Fetched result instead of stashing self.pages and logging a dummy line; FailurePathTests checks that one error naming the cause was logged, merges 404 into one HTTP status test and renames LATEST_FETCHERS to SINGLE_PAGE_FETCHERS. Removed duplicates of stricter tests: the Devuan no-link, Arch no-releases and Kali single-version fetcher tests, the session-id timeout test, and the second fallback warning test. Also removed the duplicate __main__ block in test_cleanup.py and the unused helpers.use_isolated_env. helpers.fixed_date now backs a test that check_old_releases_for_demand() defaults today to date.today(); the other old-release tests pass today explicitly. Moved: config, parse_bool, client factory, run summary and value-type tests to test_config.py; test_transmission_double.py plus NetworkGuardTests to test_helpers.py; get_distro and the lubuntu ratio gate out of ParseVersionTypeTests; distro_of into its own class. CLAUDE.md lists the new modules. Only one _rpc_port_open stub was left, so there was nothing to hoist. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 8 +- tasks/34-mechanical-test-tidy-up.md | 28 +- tasks/README.md | 2 +- tests/helpers.py | 45 ++- tests/test_cleanup.py | 110 +++--- tests/test_config.py | 307 +++++++++++++++ tests/test_configure_transmission.py | 6 +- tests/test_daily_run.py | 349 ++---------------- tests/test_fetchers.py | 117 +++--- ...transmission_double.py => test_helpers.py} | 37 +- tests/test_live_check.py | 2 + tests/test_logging.py | 17 +- tests/test_old_release_demand.py | 36 ++ tests/test_ratio_gate.py | 46 +-- tests/test_scrapers.py | 279 +++++++------- tests/test_torrent_scrape.py | 6 +- 16 files changed, 749 insertions(+), 646 deletions(-) create mode 100644 tests/test_config.py rename tests/{test_transmission_double.py => test_helpers.py} (68%) diff --git a/CLAUDE.md b/CLAUDE.md index 6bb0f78..c0d253a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -40,7 +40,7 @@ bump the pin when `alpine:latest` moves), so they run without a container or a live Transmission daemon. Never talk to a real `Client` in a test: use `tests/helpers.py` `FakeTransmission` and `FakeTorrent`, which are checked against the pinned library's `Client` signatures and `Torrent` fields (see -`tests/test_transmission_double.py`), so they can't drift from it. +`tests/test_helpers.py`), so they can't drift from it. Importing the module opens no files: log handlers are only attached by `configure_logging()`, which the `__main__` block calls. Tests that need it @@ -59,8 +59,10 @@ faked with `helpers.fake_response()` (a real `requests.Response`) or `helpers.RecordedPages` (pages captured by `scripts/record_fixtures.py`). `fetch_torrents` tests are split by concern: `test_ratio_gate.py`, `test_fetchers.py`, `test_scrapers.py`, `test_cleanup.py`, -`test_persistence.py`, `test_logging.py`, `test_old_release_demand.py` and -`test_daily_run.py` (config, `run()`, `__main__`). +`test_persistence.py`, `test_logging.py`, `test_old_release_demand.py`, +`test_config.py` (`load_config()`, `parse_bool()`, `make_client_factory()`, +the run summary) and `test_daily_run.py` (`run()`, `__main__`). +`test_helpers.py` tests the doubles in `tests/helpers.py` themselves. Business logic that needs to be unit-testable should be kept as pure functions (taking plain data in, returning plain data out) separate from diff --git a/tasks/34-mechanical-test-tidy-up.md b/tasks/34-mechanical-test-tidy-up.md index 2859497..7abae0f 100644 --- a/tasks/34-mechanical-test-tidy-up.md +++ b/tasks/34-mechanical-test-tidy-up.md @@ -5,36 +5,36 @@ From `.dev-team-reports/test-design-2026-10-19.md`. None of these change production code. ## Checklist -- [ ] **Circular oracles → literals.** These expected values come from production code; use literals instead (the Kali URL list is already pinned in `test_scrapers.py`): +- [x] **Circular oracles → literals.** These expected values come from production code; use literals instead (the Kali URL list is already pinned in `test_scrapers.py`): - `test_fetchers.py` `FetchKaliOldVersionsTests.test_returns_only_the_non_newest_versions` builds its expected value from `ft._kali_torrent_urls` - `test_fetchers.py` default-distros test uses `ft.DEFAULT_DISTROS` - `test_daily_run.py` uses `ft.ALL_DISTROS` -- [ ] **CPython error wording.** Three `test_daily_run.py` config tests embed CPython's `ValueError` text (`could not convert string to float: ...`, `invalid literal for int() ...`). Assert only the project-owned `ENV_VAR: ` prefix. -- [ ] **Subprocess warnings.** `test_logging.ImportOpensNoFilesTests`: run the subprocess with `-W ignore` so an import-time warning from a dependency can't fail `stderr == ''`. -- [ ] **Dead code.** +- [x] **CPython error wording.** Three `test_daily_run.py` config tests embed CPython's `ValueError` text (`could not convert string to float: ...`, `invalid literal for int() ...`). Assert only the project-owned `ENV_VAR: ` prefix. +- [x] **Subprocess warnings.** `test_logging.ImportOpensNoFilesTests`: run the subprocess with `-W ignore` so an import-time warning from a dependency can't fail `stderr == ''`. +- [x] **Dead code.** - Delete the duplicate `if __name__ == "__main__"` block at the end of `test_cleanup.py`. - Delete the unused `helpers.fixed_date`, or use it for a `check_old_releases_for_demand(today=None)` default test. - Delete `helpers.use_isolated_env` unless task 35 adopts it. - Hoist the duplicated `_rpc_port_open` stub (`test_daily_run.py`) into helpers. -- [ ] **Duplicates of stricter recorded-page tests.** Delete these `test_fetchers.py` tests, each of which a `test_scrapers` test covers more strictly: +- [x] **Duplicates of stricter recorded-page tests.** Delete these `test_fetchers.py` tests, each of which a `test_scrapers` test covers more strictly: - `FetchDevuanLatestNamingTests.test_returns_false_when_no_torrent_link_found` - `FetchArchLatestNamingTests.test_no_available_releases_returns_false` - the Kali single-version test -- [ ] **Duplicate timeout test.** Delete `test_logging.LogSeedRatiosViaHttpTests.test_session_id_request_has_a_timeout`, which duplicates `test_requests_follow_the_transmission_rpc_protocol`. -- [ ] **`test_scrapers.FailurePathTests`.** +- [x] **Duplicate timeout test.** Delete `test_logging.LogSeedRatiosViaHttpTests.test_session_id_request_has_a_timeout`, which duplicates `test_requests_follow_the_transmission_rpc_protocol`. +- [x] **`test_scrapers.FailurePathTests`.** - Merge `test_http_404` into `test_http_500` (same branch). - Assert "an error was logged naming the cause" instead of exact per-fetcher prefixes. - Rename `LATEST_FETCHERS`, since it also holds the `*_old_versions` fetchers. -- [ ] **`ScraperTestCase.fetch()`.** Return a small result object instead of stashing `self.pages`, and drop the dummy `logger.debug('fetch start')` hack. -- [ ] **`test_cleanup.test_selects_older_version_per_distro_and_type`.** Turn it into a subTest table per distro and delete the redundant `assertNotIn` loop. -- [ ] **Count-only asserts.** Assert names and the absence of `remove_torrent` calls instead (`test_daily_run` `len(transmission.torrents) == 2`, `test_cleanup` `test_missing_added_date_does_not_crash`, `test_cleanup_old_versions_keeps_versions_below_min_ratio`). -- [ ] **Weak boolean asserts.** +- [x] **`ScraperTestCase.fetch()`.** Return a small result object instead of stashing `self.pages`, and drop the dummy `logger.debug('fetch start')` hack. +- [x] **`test_cleanup.test_selects_older_version_per_distro_and_type`.** Turn it into a subTest table per distro and delete the redundant `assertNotIn` loop. +- [x] **Count-only asserts.** Assert names and the absence of `remove_torrent` calls instead (`test_daily_run` `len(transmission.torrents) == 2`, `test_cleanup` `test_missing_added_date_does_not_crash`, `test_cleanup_old_versions_keeps_versions_below_min_ratio`). +- [x] **Weak boolean asserts.** - Replace `assertTrue(x.startswith(...))` / `assertTrue(any(...))` with `assertRegex` or an exact value where one is known (e.g. `test_torrent_scrape` `test_adds_query_param_with_no_existing_query`). - `test_fetchers.py` remove-watch-file test: assert the file's contents. -- [ ] **`parse_supported_distros`.** Wrap the unknown-entry and fallback cases in `assertLogs` and assert the warning text. Assert the default list as a literal. -- [ ] **Misfiled tests.** Move each beside the function it tests: +- [x] **`parse_supported_distros`.** Wrap the unknown-entry and fallback cases in `assertLogs` and assert the warning text. Assert the default list as a literal. +- [x] **Misfiled tests.** Move each beside the function it tests: - flavour/lubuntu tests out of `ParseVersionTypeTests` - `distro_of` into its own class - `NetworkGuardTests` plus `test_transmission_double.py` into a `test_helpers.py` - `LoadConfig`/`FormatRunSummary`/`MakeClientFactory` into `test_config.py`; update the module list in CLAUDE.md -- [ ] **Old-release clock.** `CheckOldReleasesForDemandTests` call `check_old_releases_for_demand` without `today`; pass it explicitly. +- [x] **Old-release clock.** `CheckOldReleasesForDemandTests` call `check_old_releases_for_demand` without `today`; pass it explicitly. diff --git a/tasks/README.md b/tasks/README.md index 9de6e3e..4f37061 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -100,7 +100,7 @@ mutation score is 95.4% (1027/1076), up from 88.6%, with the critical set still | 31 | [`configure_transmission`: hermetic default path, boundaries](31-configure-transmission-gaps.md) | P3 | NO_REFACTOR | — | done | | 32 | [Atomic `.torrent` writes](32-atomic-torrent-writes.md) | P2 (bug) | NO_REFACTOR | — | done | | 33 | [`scrape_torrent` fall-through branches and SSRF edges](33-scrape-torrent-branches-and-ssrf.md) | P2 | NO_REFACTOR | — | done | -| 34 | [Mechanical test tidy-up](34-mechanical-test-tidy-up.md) | P3 | NO_REFACTOR | — | open | +| 34 | [Mechanical test tidy-up](34-mechanical-test-tidy-up.md) | P3 | NO_REFACTOR | — | done | | 35 | [Consolidate fixtures and the path-global seam](35-fixture-consolidation.md) | P3 | NO_REFACTOR → REFACTOR_REQUIRED | 34 | open | | 36 | [Remaining `fetch_torrents.py` mutation survivors](36-remaining-mutation-survivors.md) | P3 | NO_REFACTOR | — | done | | 37 | [Toolchain: smoke test, shellcheck/hadolint, ruff S, pins](37-toolchain-and-q4.md) | P2 | CI | — | done | diff --git a/tests/helpers.py b/tests/helpers.py index 61c391b..37d1dbb 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -67,20 +67,13 @@ def isolated_env(**overrides): """patch.dict replacing os.environ with scrubbed_environ(**overrides). Usable as a context manager, or started/stopped manually (see - use_isolated_env()). The scrubbed copy is taken when this is called, not + isolate_module()). The scrubbed copy is taken when this is called, not when it is entered, so call it at the point of use.""" return unittest.mock.patch.dict( os.environ, scrubbed_environ(**overrides), clear=True ) -def use_isolated_env(testcase, **overrides): - """Apply isolated_env() for the rest of a test (call from setUp).""" - patcher = isolated_env(**overrides) - patcher.start() - testcase.addCleanup(patcher.stop) - - def isolate_module(): """The baseline for a test module (call from setUpModule): the developer's exported SKIP_RATIO_CHECK/CLEANUP_*/TRANSMISSION_*/ @@ -167,7 +160,7 @@ class FakeTransmission: Every call is bound against the real Client's signature first, so an argument the pinned library wouldn't accept raises TypeError here too - (see tests/test_transmission_double.py).""" + (see tests/test_helpers.py).""" def __init__(self, torrents=()): self.torrents = list(torrents) @@ -190,6 +183,11 @@ def remove_torrent(self, *args, **kwargs): def names(self): return sorted(t.name for t in self.torrents) + @property + def removals(self): + """The remove_torrent calls made so far, as (ids, delete_data).""" + return [call[1:] for call in self.calls if call != "get_torrents"] + class FakeDisk: """shutil.disk_usage() stand-in whose used space is `base_used` plus the @@ -300,6 +298,35 @@ def get(self, url, timeout=None, **kwargs): return fake_response(b"not recorded", status=404, url=url) +# Every image fetch_kali_latest()/fetch_kali_old_versions() offer for a +# release, written out by hand (not taken from fetch_torrents) so tests can +# check the scrapers against it. +KALI_SUFFIXES = ( + "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-amd64.iso", + "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-netinst-amd64.iso", + "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-everything-amd64.iso", + "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-arm64.iso", + "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-netinst-arm64.iso", + "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-purple-amd64.iso", + "kali.download/arm-images/kali-{v}/kali-linux-{v}-raspberry-pi-armhf.img.xz", + "kali.download/arm-images/kali-{v}/kali-linux-{v}-raspberry-pi-zero-2-w-armhf.img.xz", + "kali.download/arm-images/kali-{v}/kali-linux-{v}-raspberry-pi-zero-w-armel.img.xz", + "kali.download/cloud-images/kali-{v}/kali-linux-{v}-cloud-genericcloud-amd64.tar.xz", + "kali.download/cloud-images/kali-{v}/kali-linux-{v}-cloud-genericcloud-arm64.tar.xz", +) + + +def kali_candidates(*versions): + """{name: torrent URL} for every Kali image of each version, in order.""" + return { + os.path.basename(suffix.format(v=version)): ( + "https://" + suffix.format(v=version) + ".torrent" + ) + for version in versions + for suffix in KALI_SUFFIXES + } + + def _refuse_network(*args, **kwargs): raise RuntimeError("tests must not open network connections") diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index ab8145a..b3e617b 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -23,54 +23,64 @@ def setUpModule(): class PlanCleanupTests(unittest.TestCase): + # (group, newer, older): one (distro, type) group per row. + VERSION_PAIRS = ( + ( + "ubuntu desktop", + "ubuntu-24.04-desktop-amd64.iso", + "ubuntu-23.10-desktop-amd64.iso", + ), + ( + "ubuntu live-server", + "ubuntu-24.04.1-live-server-amd64.iso", + "ubuntu-22.04.1-live-server-amd64.iso", + ), + ("debian", "debian-12.5.0-amd64-DVD-1.iso", "debian-12.4.0-amd64-DVD-1.iso"), + ( + "kali", + "kali-linux-2024.1-installer-amd64.iso", + "kali-linux-2023.4-installer-amd64.iso", + ), + ("arch", "archlinux-2024.05.01-x86_64.iso", "archlinux-2024.04.01-x86_64.iso"), + ( + "mint", + "linuxmint-22.3-cinnamon-64bit.iso", + "linuxmint-22.2-cinnamon-64bit.iso", + ), + ( + "fedora", + "Fedora-Workstation-Live-x86_64-44", + "Fedora-Workstation-Live-x86_64-43", + ), + ) + def test_selects_older_version_per_distro_and_type(self): - torrents = [ - helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=2.0), - helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2, ratio=2.0), - helpers.FakeTorrent("ubuntu-24.04.1-live-server-amd64.iso", 3, ratio=2.0), - helpers.FakeTorrent("ubuntu-22.04.1-live-server-amd64.iso", 4, ratio=2.0), - helpers.FakeTorrent("debian-12.5.0-amd64-DVD-1.iso", 5, ratio=2.0), - helpers.FakeTorrent("debian-12.4.0-amd64-DVD-1.iso", 6, ratio=2.0), - helpers.FakeTorrent("kali-linux-2024.1-installer-amd64.iso", 7, ratio=2.0), - helpers.FakeTorrent("kali-linux-2023.4-installer-amd64.iso", 8, ratio=2.0), - helpers.FakeTorrent("archlinux-2024.05.01-x86_64.iso", 9, ratio=2.0), - helpers.FakeTorrent("archlinux-2024.04.01-x86_64.iso", 10, ratio=2.0), - helpers.FakeTorrent("linuxmint-22.3-cinnamon-64bit.iso", 11, ratio=2.0), - helpers.FakeTorrent("linuxmint-22.2-cinnamon-64bit.iso", 12, ratio=2.0), - helpers.FakeTorrent("Fedora-Workstation-Live-x86_64-44", 13, ratio=2.0), - helpers.FakeTorrent("Fedora-Workstation-Live-x86_64-43", 14, ratio=2.0), - ] + for group, newer, older in self.VERSION_PAIRS: + with self.subTest(group): + to_remove, to_keep_low_ratio = ft.plan_cleanup( + [ + helpers.FakeTorrent(newer, 1, ratio=2.0), + helpers.FakeTorrent(older, 2, ratio=2.0), + ] + ) + + self.assertEqual(helpers.names_of(to_remove), [older]) + self.assertEqual(to_keep_low_ratio, []) + + def test_groups_planned_together_do_not_affect_each_other(self): + torrents = [] + for _group, newer, older in self.VERSION_PAIRS: + torrents.append(helpers.FakeTorrent(newer, len(torrents), ratio=2.0)) + torrents.append(helpers.FakeTorrent(older, len(torrents), ratio=2.0)) to_remove, to_keep_low_ratio = ft.plan_cleanup(torrents) self.assertEqual( helpers.names_of(to_remove), - [ - "Fedora-Workstation-Live-x86_64-43", - "archlinux-2024.04.01-x86_64.iso", - "debian-12.4.0-amd64-DVD-1.iso", - "kali-linux-2023.4-installer-amd64.iso", - "linuxmint-22.2-cinnamon-64bit.iso", - "ubuntu-22.04.1-live-server-amd64.iso", - "ubuntu-23.10-desktop-amd64.iso", - ], + sorted(older for _group, _newer, older in self.VERSION_PAIRS), ) self.assertEqual(to_keep_low_ratio, []) - # The newest torrent in each (distro, type) group must never be - # scheduled for removal. - removed_names = set(helpers.names_of(to_remove)) - for newest in ( - "ubuntu-24.04-desktop-amd64.iso", - "ubuntu-24.04.1-live-server-amd64.iso", - "debian-12.5.0-amd64-DVD-1.iso", - "kali-linux-2024.1-installer-amd64.iso", - "archlinux-2024.05.01-x86_64.iso", - "linuxmint-22.3-cinnamon-64bit.iso", - "Fedora-Workstation-Live-x86_64-44", - ): - self.assertNotIn(newest, removed_names) - def test_single_torrent_per_group_is_left_alone(self): torrents = [helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=0.0)] @@ -322,7 +332,9 @@ def test_missing_added_date_does_not_crash(self): to_remove, _ = ft.plan_cleanup(torrents) - self.assertEqual(len(to_remove), 1) + # Tied (missing) dates: the stable sort keeps the first one listed as + # the current release, so only the other is a candidate. + self.assertEqual(helpers.names_of(to_remove), ["devuan_excalibur"]) def test_release_without_added_date_sorts_as_oldest(self): undated = helpers.DuckTorrent("devuan_daedalus", 1, ratio=2.0, added_date=None) @@ -592,7 +604,7 @@ def test_old_version_with_no_history_at_all_is_kept(self): ) self.assertEqual(to_remove, []) - self.assertEqual(len(to_keep), 1) + self.assertEqual(to_keep, [(torrents[1], None)]) def test_unrelated_single_torrent_groups_are_ignored(self): today = date(2026, 8, 29) @@ -841,7 +853,11 @@ def test_cleanup_old_versions_keeps_versions_below_min_ratio(self): ft.cleanup_old_versions(transmission, today=date(2026, 8, 1), min_ratio=2.0) - self.assertEqual(len(transmission.torrents), 2) + self.assertEqual( + transmission.names, + ["ubuntu-23.10-desktop-amd64.iso", "ubuntu-24.04-desktop-amd64.iso"], + ) + self.assertEqual(transmission.removals, []) def test_cleanup_stagnant_torrents_removes_the_watch_file(self): old_path = self._touch_watch_file("ubuntu-23.10-desktop-amd64.iso") @@ -925,7 +941,11 @@ def test_cleanup_stagnant_torrents_keeps_growth_above_the_default_delta(self): with self.assertLogs(ft.logger, level="INFO") as logs: ft.cleanup_stagnant_torrents(transmission, today=today) - self.assertEqual(len(transmission.torrents), 2) + self.assertEqual( + transmission.names, + ["ubuntu-23.10-desktop-amd64.iso", "ubuntu-24.04-desktop-amd64.iso"], + ) + self.assertEqual(transmission.removals, []) self.assertIn( "Keeping old version ubuntu-23.10-desktop-amd64.iso – ratio grew by " "0.500 over the last 30 days.", @@ -1414,7 +1434,3 @@ def test_ratio_is_rounded_to_three_decimals(self): if __name__ == "__main__": unittest.main() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..912e100 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,307 @@ +"""Tests for the daily run's configuration and value types: load_config() +and Config, parse_bool(), make_client_factory(), RunSummary and +format_run_summary(). + +Run with: python -m unittest discover -s tests +""" + +import dataclasses +import logging +import unittest +import unittest.mock + +import helpers + +import fetch_torrents as ft + + +def setUpModule(): + helpers.isolate_module() + + +class FormatRunSummaryTests(unittest.TestCase): + """format_run_summary() builds the single end-of-run log line - it must + surface how many old-release torrents FETCH_TORRENTS_CHECK_OLD_RELEASES + introduced, not just the counts from the normal latest-release fetch.""" + + def test_without_old_release_check(self): + summary = ft.format_run_summary( + elapsed=12.5, + success_count=3, + existing_count=1, + not_found_count=0, + failure_count=0, + ) + self.assertEqual( + summary, + "Run complete in 12.50 seconds. 3 added, 1 existing, 0 not found upstream, 0 failed.", + ) + + def test_with_old_release_check_appends_counts(self): + summary = ft.format_run_summary( + elapsed=12.5, + success_count=3, + existing_count=1, + not_found_count=0, + failure_count=0, + old_added=2, + old_skipped=6, + ) + self.assertEqual( + summary, + "Run complete in 12.50 seconds. 3 added, 1 existing, 0 not found upstream, 0 failed. " + "Old releases (FETCH_TORRENTS_CHECK_OLD_RELEASES): 2 added, 6 skipped.", + ) + + def test_is_let_through_by_the_always_log_filter(self): + summary = ft.format_run_summary( + elapsed=1.0, + success_count=0, + existing_count=0, + not_found_count=0, + failure_count=0, + old_added=0, + old_skipped=0, + ) + record = logging.LogRecord( + "fetch_torrents", logging.INFO, __file__, 0, summary, None, None + ) + always_log = ft.ImportantMessageFilter( + logging.ERROR, ft.get_always_log_prefixes() + ) + + self.assertIs(always_log.filter(record), True) + + +class LoadConfigTests(unittest.TestCase): + """load_config() is the one place the daily run reads its environment: + each env var must land on the right field, and a bad number must be + deferred (recorded, then raised by require()) rather than crash the + whole run up front.""" + + def test_empty_env_gives_the_documented_defaults(self): + self.assertEqual(ft.load_config({}), ft.Config()) + self.assertEqual( + ft.Config(), + ft.Config( + distros=("ubuntu", "debian", "kali", "mint", "fedora"), + include_low_demand=False, + skip_ratio_check=False, + check_old_releases=False, + old_release_min_leechers=10, + old_release_min_leecher_ratio=0.1, + old_release_allow_zero_seeders=False, + old_release_recheck_days=7, + old_release_skip_removed=True, + cleanup_keep_only_latest=False, + cleanup_skip_ratio_check=False, + cleanup_min_ratio=1.0, + stagnation_window_days=30, + stagnation_min_ratio_delta=0.02, + disk_usage_threshold_percent=95.0, + rpc_username=None, + rpc_password=None, + invalid={}, + ), + ) + + def test_each_env_var_sets_its_field(self): + config = ft.load_config( + { + "FETCH_TORRENTS_DISTROS": "devuan, arch", + "FETCH_TORRENTS_INCLUDE_LOW_DEMAND": "true", + "SKIP_RATIO_CHECK": "yes", + "FETCH_TORRENTS_CHECK_OLD_RELEASES": "1", + "FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS": "3", + "FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO": "0.5", + "FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS": "true", + "FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS": "2", + "FETCH_TORRENTS_OLD_RELEASE_SKIP_REMOVED": "false", + "CLEANUP_KEEP_ONLY_LATEST_VERSION": "true", + "CLEANUP_SKIP_RATIO_CHECK": "true", + "CLEANUP_MIN_RATIO": "1.5", + "CLEANUP_STAGNATION_WINDOW_DAYS": "14", + "CLEANUP_STAGNATION_MIN_RATIO_DELTA": "0.1", + "CLEANUP_DISK_USAGE_THRESHOLD_PERCENT": "80", + "TRANSMISSION_RPC_USERNAME": " alice ", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + } + ) + + self.assertEqual( + config, + ft.Config( + # In the order the distros are known, not as listed. + distros=("arch", "devuan"), + include_low_demand=True, + skip_ratio_check=True, + check_old_releases=True, + old_release_min_leechers=3, + old_release_min_leecher_ratio=0.5, + old_release_allow_zero_seeders=True, + old_release_recheck_days=2, + old_release_skip_removed=False, + cleanup_keep_only_latest=True, + cleanup_skip_ratio_check=True, + cleanup_min_ratio=1.5, + stagnation_window_days=14, + stagnation_min_ratio_delta=0.1, + disk_usage_threshold_percent=80.0, + rpc_username="alice", + rpc_password="hunter2", + ), + ) + + def test_reads_only_the_env_it_is_given(self): + with helpers.isolated_env(SKIP_RATIO_CHECK="true", CLEANUP_MIN_RATIO="9"): + config = ft.load_config({}) + + self.assertFalse(config.skip_ratio_check) + self.assertEqual(config.cleanup_min_ratio, 1.0) + + def test_invalid_number_is_recorded_not_raised(self): + config = ft.load_config( + {"CLEANUP_MIN_RATIO": "high", "CLEANUP_STAGNATION_WINDOW_DAYS": "1.5"} + ) + + self.assertIsNone(config.cleanup_min_ratio) + self.assertIsNone(config.stagnation_window_days) + # Only the env var prefix is this project's; the rest is Python's + # own float()/int() error text. + self.assertEqual( + sorted(config.invalid), ["cleanup_min_ratio", "stagnation_window_days"] + ) + self.assertRegex(config.invalid["cleanup_min_ratio"], r"^CLEANUP_MIN_RATIO: ") + self.assertRegex( + config.invalid["stagnation_window_days"], + r"^CLEANUP_STAGNATION_WINDOW_DAYS: ", + ) + + def test_require_raises_the_first_invalid_field_named(self): + config = ft.load_config({"CLEANUP_MIN_RATIO": "high"}) + + config.require("stagnation_window_days") # valid: no error + with self.assertRaisesRegex(ValueError, "^CLEANUP_MIN_RATIO: "): + config.require("stagnation_window_days", "cleanup_min_ratio") + + def test_rpc_auth_needs_both_username_and_password(self): + cases = ( + ({}, None), + ({"TRANSMISSION_RPC_USERNAME": "alice"}, None), + ({"TRANSMISSION_RPC_PASSWORD": "hunter2"}, None), + ( + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": " ", + }, + None, + ), + ( + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, + ("alice", "hunter2"), + ), + ) + for env, expected in cases: + with self.subTest(env=env): + self.assertEqual(ft.load_config(env).rpc_auth, expected) + + +class MakeClientFactoryTests(unittest.TestCase): + """The daily run's own Transmission connections (cleanup, disk-usage + limit) must authenticate the same way configure_transmission.py + configured the daemon. Without this, a user who sets + TRANSMISSION_RPC_USERNAME/PASSWORD (as the README requires once + TRANSMISSION_RPC_WHITELIST is widened beyond localhost) finds cleanup + silently stops working forever: every RPC call fails with 401 and the + failure is only logged, never surfaced.""" + + def _connect(self, env): + config = ft.load_config(env) + with unittest.mock.patch.object(ft, "Client", autospec=True) as client_cls: + client = ft.make_client_factory(config)() + return client_cls, client + + def test_passes_configured_credentials(self): + client_cls, client = self._connect( + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + } + ) + + client_cls.assert_called_once_with( + host="localhost", port=9091, username="alice", password="hunter2" + ) + self.assertIs(client, client_cls.return_value) + + def test_passes_no_credentials_when_unset(self): + client_cls, _client = self._connect({}) + + client_cls.assert_called_once_with( + host="localhost", port=9091, username=None, password=None + ) + + def test_blank_credentials_count_as_unset(self): + client_cls, _client = self._connect( + {"TRANSMISSION_RPC_USERNAME": " ", "TRANSMISSION_RPC_PASSWORD": ""} + ) + + client_cls.assert_called_once_with( + host="localhost", port=9091, username=None, password=None + ) + + def test_factory_connects_only_when_called(self): + with unittest.mock.patch.object(ft, "Client", autospec=True) as client_cls: + factory = ft.make_client_factory(ft.load_config({})) + + client_cls.assert_not_called() + self.assertTrue(callable(factory)) + + +class ParseBoolTests(unittest.TestCase): + def test_unset_is_false_by_default(self): + self.assertIs(ft.parse_bool("X", env={}), False) + + def test_unset_or_blank_uses_the_given_default(self): + for value in ({}, {"X": " "}): + with self.subTest(env=value): + self.assertIs(ft.parse_bool("X", default=True, env=value), True) + + def test_truthy_and_falsy_spellings(self): + for value in ("1", "true", "YES", " On "): + with self.subTest(value=value): + self.assertIs(ft.parse_bool("X", env={"X": value}), True) + for value in ("0", "false", "no", "off", "maybe"): + with self.subTest(value=value): + self.assertIs(ft.parse_bool("X", default=True, env={"X": value}), False) + + +class RunValueTypesTests(unittest.TestCase): + def test_run_summary_starts_at_zero_with_no_old_release_counts(self): + self.assertEqual( + ft.RunSummary(), + ft.RunSummary( + elapsed=0.0, + added=0, + existing=0, + not_found=0, + failed=0, + old_added=None, + old_skipped=None, + ), + ) + + def test_config_is_immutable(self): + # run() and its steps share one Config; none may change it for the + # steps after it. + config = ft.Config() + + with self.assertRaises(dataclasses.FrozenInstanceError): + config.skip_ratio_check = True # type: ignore[misc] + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_configure_transmission.py b/tests/test_configure_transmission.py index ea590fa..24eedf8 100644 --- a/tests/test_configure_transmission.py +++ b/tests/test_configure_transmission.py @@ -350,7 +350,11 @@ def test_existing_file_and_no_overrides_is_left_untouched(self): def test_whitelist_without_credentials_warns_on_stderr(self): stderr = self._run(TRANSMISSION_RPC_WHITELIST="10.0.0.*") - self.assertTrue(stderr.startswith("WARNING: TRANSMISSION_RPC_WHITELIST is set")) + # The warning text itself is pinned by the whitelist warning tests above. + self.assertRegex( + stderr, + r"\AWARNING: TRANSMISSION_RPC_WHITELIST is set .+ authentication\.\n\Z", + ) self.assertEqual(self._read_settings()["rpc-whitelist"], "10.0.0.*") def test_invalid_peer_limits_are_dropped_silently(self): diff --git a/tests/test_daily_run.py b/tests/test_daily_run.py index 2d7b3b9..8778eef 100644 --- a/tests/test_daily_run.py +++ b/tests/test_daily_run.py @@ -14,11 +14,11 @@ """ import contextlib -import dataclasses import io import json import logging import os +import re import runpy import shutil import socket @@ -54,6 +54,11 @@ def setUpModule(): "" ) ARCH_TORRENT = "https://archlinux.org/releng/releases/2026.09.01/torrent/" +BOTH_UBUNTUS = ["ubuntu-23.10-desktop-amd64.iso", "ubuntu-24.04-desktop-amd64.iso"] +# The last line of a run in which the arch fetcher found nothing (unrouted). +ARCH_FAILED_SUMMARY = ( + "Run complete in 0.00 seconds. 0 added, 0 existing, 0 not found upstream, 1 failed." +) def torrent(name, id_, ratio): @@ -278,6 +283,10 @@ def _run(self, fetchers=ft.DEFAULT_FETCHERS, now=lambda: 1000.0, **env): ) return summary, [record.getMessage() for record in logs.records] + def assertNothingRemoved(self): + self.assertEqual(self.world.transmission.names, BOTH_UBUNTUS) + self.assertEqual(self.world.transmission.removals, []) + class RunTests(RunTestCase): def test_counts_each_download_outcome(self): @@ -397,7 +406,7 @@ def test_keep_only_latest_honors_min_ratio(self): CLEANUP_MIN_RATIO="3", ) - self.assertEqual(len(self.world.transmission.torrents), 2) + self.assertNothingRemoved() def test_default_cleanup_is_stagnation_based(self): # No ratio history yet, so the stagnation cleanup can't judge the old @@ -409,7 +418,7 @@ def test_default_cleanup_is_stagnation_based(self): self._run(FETCH_TORRENTS_DISTROS="arch") - self.assertEqual(len(self.world.transmission.torrents), 2) + self.assertNothingRemoved() self.assertEqual( ft.load_ratio_history(ft.ratio_history_file), { @@ -434,13 +443,12 @@ def test_invalid_cleanup_setting_skips_cleanup_and_continues(self): CLEANUP_MIN_RATIO="high", ) - self.assertEqual(len(self.world.transmission.torrents), 2) - self.assertIn( - "Could not clean up old versions: CLEANUP_MIN_RATIO: could not " - "convert string to float: 'high'", - messages, + self.assertNothingRemoved() + self.assertRegex( + "\n".join(messages), + r"(?m)^Could not clean up old versions: CLEANUP_MIN_RATIO: ", ) - self.assertTrue(messages[-1].startswith("Run complete in")) + self.assertEqual(messages[-1], ARCH_FAILED_SUMMARY) def test_disk_usage_limit_runs_after_cleanup(self): self.world.transmission.torrents = [ @@ -466,13 +474,12 @@ def test_disk_usage_limit_error_is_logged_not_raised(self): FETCH_TORRENTS_DISTROS="arch", CLEANUP_DISK_USAGE_THRESHOLD_PERCENT="lots" ) - self.assertIn( - "Could not enforce disk usage limit: " - "CLEANUP_DISK_USAGE_THRESHOLD_PERCENT: could not convert string to " - "float: 'lots'", - messages, + self.assertRegex( + "\n".join(messages), + r"(?m)^Could not enforce disk usage limit: " + r"CLEANUP_DISK_USAGE_THRESHOLD_PERCENT: ", ) - self.assertTrue(messages[-1].startswith("Run complete in")) + self.assertEqual(messages[-1], ARCH_FAILED_SUMMARY) def test_rpc_unavailable_skips_ratio_logging_but_still_cleans_up(self): self.world.rpc_up = False @@ -488,7 +495,7 @@ def test_rpc_unavailable_skips_ratio_logging_but_still_cleans_up(self): "Transmission RPC not available on localhost:9091; skipping ratio logs.", messages, ) - self.assertFalse(any(m.startswith("[ratio]") for m in messages)) + self.assertEqual([m for m in messages if m.startswith("[ratio]")], []) self.assertEqual( self.world.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"] ) @@ -621,8 +628,10 @@ def test_elapsed_time_is_measured_from_start_to_finish(self): ) self.assertEqual(summary.elapsed, 12.5) - self.assertTrue( - messages[-1].startswith("Run complete in 12.50 seconds."), messages[-1] + self.assertEqual( + messages[-1], + "Run complete in 12.50 seconds. 0 added, 0 existing, " + "0 not found upstream, 0 failed.", ) def test_reports_downloads_folder_usage_in_whole_gigabytes(self): @@ -713,10 +722,7 @@ def test_every_always_log_prefix_matches_a_message_the_run_emits(self): for prefix in ft.get_always_log_prefixes(): with self.subTest(prefix=prefix): - self.assertTrue( - any(m.startswith(prefix) for m in messages), - f"no message starts with {prefix!r}", - ) + self.assertRegex("\n".join(messages), "(?m)^" + re.escape(prefix)) def test_config_and_saved_state_drive_the_check(self): recent = self._candidate(1, "installer", 50, 1) @@ -764,11 +770,11 @@ def test_config_and_saved_state_drive_the_check(self): "recheck interval: 10 days)...", messages, ) - self.assertTrue( - messages[-1].endswith( - "Old releases (FETCH_TORRENTS_CHECK_OLD_RELEASES): 2 added, 4 skipped." - ), + self.assertEqual( messages[-1], + "Run complete in 0.00 seconds. 0 added, 0 existing, 0 not found " + "upstream, 0 failed. Old releases (FETCH_TORRENTS_CHECK_OLD_RELEASES): " + "2 added, 4 skipped.", ) def test_previously_removed_release_is_rechecked_when_skip_removed_is_off(self): @@ -856,7 +862,11 @@ def rpc_port_open(*args, **kwargs): messages = [record.getMessage() for record in logs.records] self.assertIn("Selected distros for this run: arch", messages) - self.assertTrue(messages[-1].startswith("Run complete in")) + self.assertRegex( + messages[-1], + r"\ARun complete in \d+\.\d\d seconds\. 0 added, 0 existing, " + r"0 not found upstream, 1 failed\.\Z", + ) client_cls.assert_called_with( host="localhost", port=9091, username="alice", password="hunter2" ) @@ -871,290 +881,5 @@ def rpc_port_open(*args, **kwargs): self.assertTrue(os.path.exists(os.path.join(log_dir, "fetch_torrents.log"))) -class FormatRunSummaryTests(unittest.TestCase): - """format_run_summary() builds the single end-of-run log line - it must - surface how many old-release torrents FETCH_TORRENTS_CHECK_OLD_RELEASES - introduced, not just the counts from the normal latest-release fetch.""" - - def test_without_old_release_check(self): - summary = ft.format_run_summary( - elapsed=12.5, - success_count=3, - existing_count=1, - not_found_count=0, - failure_count=0, - ) - self.assertEqual( - summary, - "Run complete in 12.50 seconds. 3 added, 1 existing, 0 not found upstream, 0 failed.", - ) - - def test_with_old_release_check_appends_counts(self): - summary = ft.format_run_summary( - elapsed=12.5, - success_count=3, - existing_count=1, - not_found_count=0, - failure_count=0, - old_added=2, - old_skipped=6, - ) - self.assertEqual( - summary, - "Run complete in 12.50 seconds. 3 added, 1 existing, 0 not found upstream, 0 failed. " - "Old releases (FETCH_TORRENTS_CHECK_OLD_RELEASES): 2 added, 6 skipped.", - ) - - def test_is_let_through_by_the_always_log_filter(self): - summary = ft.format_run_summary( - elapsed=1.0, - success_count=0, - existing_count=0, - not_found_count=0, - failure_count=0, - old_added=0, - old_skipped=0, - ) - record = logging.LogRecord( - "fetch_torrents", logging.INFO, __file__, 0, summary, None, None - ) - always_log = ft.ImportantMessageFilter( - logging.ERROR, ft.get_always_log_prefixes() - ) - - self.assertIs(always_log.filter(record), True) - - -class LoadConfigTests(unittest.TestCase): - """load_config() is the one place the daily run reads its environment: - each env var must land on the right field, and a bad number must be - deferred (recorded, then raised by require()) rather than crash the - whole run up front.""" - - def test_empty_env_gives_the_documented_defaults(self): - self.assertEqual(ft.load_config({}), ft.Config()) - self.assertEqual( - ft.Config(), - ft.Config( - distros=ft.DEFAULT_DISTROS, - include_low_demand=False, - skip_ratio_check=False, - check_old_releases=False, - old_release_min_leechers=10, - old_release_min_leecher_ratio=0.1, - old_release_allow_zero_seeders=False, - old_release_recheck_days=7, - old_release_skip_removed=True, - cleanup_keep_only_latest=False, - cleanup_skip_ratio_check=False, - cleanup_min_ratio=1.0, - stagnation_window_days=30, - stagnation_min_ratio_delta=0.02, - disk_usage_threshold_percent=95.0, - rpc_username=None, - rpc_password=None, - invalid={}, - ), - ) - - def test_each_env_var_sets_its_field(self): - config = ft.load_config( - { - "FETCH_TORRENTS_DISTROS": "devuan, arch", - "FETCH_TORRENTS_INCLUDE_LOW_DEMAND": "true", - "SKIP_RATIO_CHECK": "yes", - "FETCH_TORRENTS_CHECK_OLD_RELEASES": "1", - "FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS": "3", - "FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHER_RATIO": "0.5", - "FETCH_TORRENTS_OLD_RELEASE_ALLOW_ZERO_SEEDERS": "true", - "FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS": "2", - "FETCH_TORRENTS_OLD_RELEASE_SKIP_REMOVED": "false", - "CLEANUP_KEEP_ONLY_LATEST_VERSION": "true", - "CLEANUP_SKIP_RATIO_CHECK": "true", - "CLEANUP_MIN_RATIO": "1.5", - "CLEANUP_STAGNATION_WINDOW_DAYS": "14", - "CLEANUP_STAGNATION_MIN_RATIO_DELTA": "0.1", - "CLEANUP_DISK_USAGE_THRESHOLD_PERCENT": "80", - "TRANSMISSION_RPC_USERNAME": " alice ", - "TRANSMISSION_RPC_PASSWORD": "hunter2", - } - ) - - self.assertEqual( - config, - ft.Config( - distros=tuple(d for d in ft.ALL_DISTROS if d in {"arch", "devuan"}), - include_low_demand=True, - skip_ratio_check=True, - check_old_releases=True, - old_release_min_leechers=3, - old_release_min_leecher_ratio=0.5, - old_release_allow_zero_seeders=True, - old_release_recheck_days=2, - old_release_skip_removed=False, - cleanup_keep_only_latest=True, - cleanup_skip_ratio_check=True, - cleanup_min_ratio=1.5, - stagnation_window_days=14, - stagnation_min_ratio_delta=0.1, - disk_usage_threshold_percent=80.0, - rpc_username="alice", - rpc_password="hunter2", - ), - ) - - def test_reads_only_the_env_it_is_given(self): - with helpers.isolated_env(SKIP_RATIO_CHECK="true", CLEANUP_MIN_RATIO="9"): - config = ft.load_config({}) - - self.assertFalse(config.skip_ratio_check) - self.assertEqual(config.cleanup_min_ratio, 1.0) - - def test_invalid_number_is_recorded_not_raised(self): - config = ft.load_config( - {"CLEANUP_MIN_RATIO": "high", "CLEANUP_STAGNATION_WINDOW_DAYS": "1.5"} - ) - - self.assertIsNone(config.cleanup_min_ratio) - self.assertIsNone(config.stagnation_window_days) - self.assertEqual( - config.invalid, - { - "cleanup_min_ratio": ( - "CLEANUP_MIN_RATIO: could not convert string to float: 'high'" - ), - "stagnation_window_days": ( - "CLEANUP_STAGNATION_WINDOW_DAYS: invalid literal for int() " - "with base 10: '1.5'" - ), - }, - ) - - def test_require_raises_the_first_invalid_field_named(self): - config = ft.load_config({"CLEANUP_MIN_RATIO": "high"}) - - config.require("stagnation_window_days") # valid: no error - with self.assertRaisesRegex(ValueError, "^CLEANUP_MIN_RATIO: "): - config.require("stagnation_window_days", "cleanup_min_ratio") - - def test_rpc_auth_needs_both_username_and_password(self): - cases = ( - ({}, None), - ({"TRANSMISSION_RPC_USERNAME": "alice"}, None), - ({"TRANSMISSION_RPC_PASSWORD": "hunter2"}, None), - ( - { - "TRANSMISSION_RPC_USERNAME": "alice", - "TRANSMISSION_RPC_PASSWORD": " ", - }, - None, - ), - ( - { - "TRANSMISSION_RPC_USERNAME": "alice", - "TRANSMISSION_RPC_PASSWORD": "hunter2", - }, - ("alice", "hunter2"), - ), - ) - for env, expected in cases: - with self.subTest(env=env): - self.assertEqual(ft.load_config(env).rpc_auth, expected) - - -class MakeClientFactoryTests(unittest.TestCase): - """The daily run's own Transmission connections (cleanup, disk-usage - limit) must authenticate the same way configure_transmission.py - configured the daemon. Without this, a user who sets - TRANSMISSION_RPC_USERNAME/PASSWORD (as the README requires once - TRANSMISSION_RPC_WHITELIST is widened beyond localhost) finds cleanup - silently stops working forever: every RPC call fails with 401 and the - failure is only logged, never surfaced.""" - - def _connect(self, env): - config = ft.load_config(env) - with unittest.mock.patch.object(ft, "Client", autospec=True) as client_cls: - client = ft.make_client_factory(config)() - return client_cls, client - - def test_passes_configured_credentials(self): - client_cls, client = self._connect( - { - "TRANSMISSION_RPC_USERNAME": "alice", - "TRANSMISSION_RPC_PASSWORD": "hunter2", - } - ) - - client_cls.assert_called_once_with( - host="localhost", port=9091, username="alice", password="hunter2" - ) - self.assertIs(client, client_cls.return_value) - - def test_passes_no_credentials_when_unset(self): - client_cls, _client = self._connect({}) - - client_cls.assert_called_once_with( - host="localhost", port=9091, username=None, password=None - ) - - def test_blank_credentials_count_as_unset(self): - client_cls, _client = self._connect( - {"TRANSMISSION_RPC_USERNAME": " ", "TRANSMISSION_RPC_PASSWORD": ""} - ) - - client_cls.assert_called_once_with( - host="localhost", port=9091, username=None, password=None - ) - - def test_factory_connects_only_when_called(self): - with unittest.mock.patch.object(ft, "Client", autospec=True) as client_cls: - factory = ft.make_client_factory(ft.load_config({})) - - client_cls.assert_not_called() - self.assertTrue(callable(factory)) - - -class ParseBoolTests(unittest.TestCase): - def test_unset_is_false_by_default(self): - self.assertIs(ft.parse_bool("X", env={}), False) - - def test_unset_or_blank_uses_the_given_default(self): - for value in ({}, {"X": " "}): - with self.subTest(env=value): - self.assertIs(ft.parse_bool("X", default=True, env=value), True) - - def test_truthy_and_falsy_spellings(self): - for value in ("1", "true", "YES", " On "): - with self.subTest(value=value): - self.assertIs(ft.parse_bool("X", env={"X": value}), True) - for value in ("0", "false", "no", "off", "maybe"): - with self.subTest(value=value): - self.assertIs(ft.parse_bool("X", default=True, env={"X": value}), False) - - -class RunValueTypesTests(unittest.TestCase): - def test_run_summary_starts_at_zero_with_no_old_release_counts(self): - self.assertEqual( - ft.RunSummary(), - ft.RunSummary( - elapsed=0.0, - added=0, - existing=0, - not_found=0, - failed=0, - old_added=None, - old_skipped=None, - ), - ) - - def test_config_is_immutable(self): - # run() and its steps share one Config; none may change it for the - # steps after it. - config = ft.Config() - - with self.assertRaises(dataclasses.FrozenInstanceError): - config.skip_ratio_check = True # type: ignore[misc] - - if __name__ == "__main__": unittest.main() diff --git a/tests/test_fetchers.py b/tests/test_fetchers.py index aa1f210..a77de14 100644 --- a/tests/test_fetchers.py +++ b/tests/test_fetchers.py @@ -19,6 +19,12 @@ def setUpModule(): helpers.isolate_module() +# Written out rather than read from ft.DEFAULT_DISTROS/ft.ALL_DISTROS, so a +# change to either has to be made here on purpose. +DEFAULT_DISTROS = ["ubuntu", "debian", "kali", "mint", "fedora"] +ALL_DISTROS_TEXT = "ubuntu, debian, kali, arch, mint, fedora, devuan" + + class FetchUbuntuLtsNamingTests(unittest.TestCase): """The names fetch_ubuntu_lts() uses as dict keys are also what gets passed to should_fetch_torrent() before download, so they must match the @@ -155,16 +161,6 @@ def test_picks_only_the_single_newest_release(self): }, ) - def test_no_available_releases_returns_false(self): - html = "
" - - response = helpers.fake_response(html) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - result = ft.fetch_arch_latest() - - self.assertFalse(result) - class FetchDevuanLatestNamingTests(unittest.TestCase): """Devuan's get-devuan page links a single release torrent whose @@ -190,14 +186,6 @@ def test_extracts_current_release_torrent(self): {"devuan_excalibur": "https://files.devuan.org/devuan_excalibur.torrent"}, ) - def test_returns_false_when_no_torrent_link_found(self): - response = helpers.fake_response("nothing here") - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - result = ft.fetch_devuan_latest() - - self.assertFalse(result) - class FetchKaliOldVersionsTests(unittest.TestCase): """fetch_kali_old_versions() discovers the prior release(s) still listed @@ -219,25 +207,7 @@ def test_returns_only_the_non_newest_versions(self): with unittest.mock.patch.object(ft.requests, "get", return_value=response): results = ft.fetch_kali_old_versions() - # The URL templates themselves are pinned in tests/test_scrapers.py; - # what matters here is which version they're built for. - self.assertEqual( - results, - { - os.path.basename(url).removesuffix(".torrent"): url - for url in ft._kali_torrent_urls("2026.1") - }, - ) - - def test_single_version_folder_returns_nothing(self): - html = 'currentkali-2026.2' - - response = helpers.fake_response(html) - - with unittest.mock.patch.object(ft.requests, "get", return_value=response): - results = ft.fetch_kali_old_versions() - - self.assertEqual(results, {}) + self.assertEqual(results, helpers.kali_candidates("2026.1")) class FetchFedoraWorkstationOldVersionsTests(unittest.TestCase): @@ -289,40 +259,61 @@ class ParseSupportedDistrosTests(unittest.TestCase): def test_default_excludes_opt_in_distros(self): with helpers.isolated_env(): result = ft.parse_supported_distros() - self.assertNotIn("arch", result) - self.assertNotIn("devuan", result) - for expected in ("ubuntu", "debian", "kali", "mint", "fedora"): - self.assertIn(expected, result) + self.assertEqual(result, DEFAULT_DISTROS) def test_explicit_opt_in_distros_are_honored(self): with unittest.mock.patch.dict( os.environ, {"FETCH_TORRENTS_DISTROS": "arch,devuan"} ): result = ft.parse_supported_distros() - self.assertEqual(sorted(result), ["arch", "devuan"]) + self.assertEqual(result, ["arch", "devuan"]) def test_can_mix_default_and_opt_in_distros(self): with unittest.mock.patch.dict( os.environ, {"FETCH_TORRENTS_DISTROS": "ubuntu,arch"} ): result = ft.parse_supported_distros() - self.assertEqual(sorted(result), ["arch", "ubuntu"]) + self.assertEqual(result, ["ubuntu", "arch"]) def test_unknown_entries_are_dropped_but_valid_ones_kept(self): - with unittest.mock.patch.dict( - os.environ, {"FETCH_TORRENTS_DISTROS": "ubuntu,nonsense"} + with ( + unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "ubuntu,nonsense"} + ), + self.assertLogs(ft.logger, level="WARNING") as logs, ): result = ft.parse_supported_distros() self.assertEqual(result, ["ubuntu"]) + self.assertEqual( + [r.getMessage() for r in logs.records], + [ + "FETCH_TORRENTS_DISTROS contains unknown distributions: nonsense. " + "Valid values: " + ALL_DISTROS_TEXT + ], + ) def test_all_invalid_falls_back_to_default_not_all_distros(self): - with unittest.mock.patch.dict( - os.environ, {"FETCH_TORRENTS_DISTROS": "nonsense"} + with ( + unittest.mock.patch.dict( + os.environ, {"FETCH_TORRENTS_DISTROS": "nonsense"} + ), + self.assertLogs(ft.logger, level="WARNING") as logs, ): result = ft.parse_supported_distros() - self.assertEqual(sorted(result), sorted(ft.DEFAULT_DISTROS)) - self.assertNotIn("arch", result) - self.assertNotIn("devuan", result) + self.assertEqual(result, DEFAULT_DISTROS) + self.assertEqual( + [r.getMessage() for r in logs.records], + [ + ( + "FETCH_TORRENTS_DISTROS contains unknown distributions: " + "nonsense. Valid values: " + ALL_DISTROS_TEXT + ), + ( + "FETCH_TORRENTS_DISTROS did not specify any valid distros. " + "Falling back to defaults: ubuntu, debian, kali, mint, fedora" + ), + ], + ) class LowDemandVariantTests(unittest.TestCase): @@ -571,7 +562,8 @@ def test_remove_watch_file_refuses_unsafe_name_without_touching_disk(self): with unittest.mock.patch.object(ft, "watch_dir", watch_dir): ft._remove_watch_file("../evil") - self.assertTrue(os.path.exists(outside)) + with open(outside, encoding="utf-8") as f: + self.assertEqual(f.read(), "must survive") class FetcherRequestTimeoutTests(unittest.TestCase): @@ -663,11 +655,9 @@ def test_old_versions_are_every_version_but_the_newest(self): with _get_page(f"{html}"): results = ft.fetch_kali_old_versions() - expected = {} - for version in ("2025.4", "2025.10", "2026.1"): - for url in ft._kali_torrent_urls(version): - expected[os.path.basename(url).removesuffix(".torrent")] = url - self.assertEqual(results, expected) + self.assertEqual( + results, helpers.kali_candidates("2025.4", "2025.10", "2026.1") + ) class FetchFedoraSelectionTests(unittest.TestCase): @@ -718,23 +708,10 @@ def test_unknown_entries_are_named_in_the_warning(self): [r.getMessage() for r in logs.records], [ "FETCH_TORRENTS_DISTROS contains unknown distributions: gentoo, " - "slackware. Valid values: " + ", ".join(ft.ALL_DISTROS) + "slackware. Valid values: " + ALL_DISTROS_TEXT ], ) - def test_all_unknown_warns_twice_and_falls_back(self): - with self.assertLogs(ft.logger, level="WARNING") as logs: - result = ft.parse_supported_distros( - env={"FETCH_TORRENTS_DISTROS": "gentoo"} - ) - - self.assertEqual(result, list(ft.DEFAULT_DISTROS)) - self.assertEqual( - [r.getMessage() for r in logs.records][1], - "FETCH_TORRENTS_DISTROS did not specify any valid distros. " - "Falling back to defaults: " + ", ".join(ft.DEFAULT_DISTROS), - ) - if __name__ == "__main__": unittest.main() diff --git a/tests/test_transmission_double.py b/tests/test_helpers.py similarity index 68% rename from tests/test_transmission_double.py rename to tests/test_helpers.py index cc5083a..facd1ee 100644 --- a/tests/test_transmission_double.py +++ b/tests/test_helpers.py @@ -1,7 +1,10 @@ -"""The Transmission doubles in tests/helpers.py must not drift from the real -transmission_rpc library (pinned in requirements-test.txt to the version -Alpine's py3-transmission-rpc ships): a call production code makes that the -real Client wouldn't accept has to fail here, not in the container. +"""Tests for the test doubles and guards in tests/helpers.py. + +The Transmission doubles must not drift from the real transmission_rpc +library (pinned in requirements-test.txt to the version Alpine's +py3-transmission-rpc ships): a call production code makes that the real +Client wouldn't accept has to fail here, not in the container. And +isolate_module() must really keep tests off the network. Run with: python -m unittest discover -s tests """ @@ -9,9 +12,14 @@ import unittest import helpers +import requests import transmission_rpc +def setUpModule(): + helpers.isolate_module() + + class FakeTransmissionMatchesClientTests(unittest.TestCase): def test_accepts_every_call_shape_the_real_client_accepts(self): transmission = helpers.FakeTransmission( @@ -40,6 +48,21 @@ def test_every_public_method_exists_on_the_real_client(self): self.assertEqual(own - set(dir(transmission_rpc.Client)), set()) + def test_removals_lists_only_the_remove_torrent_calls(self): + transmission = helpers.FakeTransmission( + [ + helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1), + helpers.FakeTorrent("ubuntu-23.10-desktop-amd64.iso", 2), + ] + ) + + transmission.get_torrents() + transmission.remove_torrent(2, delete_data=True) + transmission.get_torrents() + transmission.remove_torrent(1) + + self.assertEqual(transmission.removals, [(2, True), (1, False)]) + class FakeDiskTests(unittest.TestCase): """The real Client's remove_torrent() defaults to delete_data=False, @@ -78,5 +101,11 @@ def test_is_a_real_torrent_reading_real_rpc_field_names(self): ) +class NetworkGuardTests(unittest.TestCase): + def test_real_connections_are_refused(self): + with self.assertRaisesRegex(RuntimeError, "must not open network"): + requests.get("http://192.0.2.1/", timeout=1) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_live_check.py b/tests/test_live_check.py index 207d297..b6fba30 100644 --- a/tests/test_live_check.py +++ b/tests/test_live_check.py @@ -122,6 +122,8 @@ def test_name_without_a_parseable_version(self): ["name 'linuxmint-latest-cinnamon' has no parseable version"], ) + +class DistroOfTests(unittest.TestCase): def test_old_release_fetchers_are_checked_as_their_distro(self): self.assertEqual(live_check.distro_of("kali-old"), "kali") self.assertEqual(live_check.distro_of("fedora-old"), "fedora") diff --git a/tests/test_logging.py b/tests/test_logging.py index 9fca972..899ec59 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -96,7 +96,9 @@ def test_import_leaves_the_log_dir_untouched(self): "ft.logger.warning('logged before configure_logging()')\n" ) result = subprocess.run( - [sys.executable, "-c", script], + # -W ignore: a dependency warning at import must not fail the + # stderr check below. + [sys.executable, "-W", "ignore", "-c", script], check=False, env=dict(os.environ, FETCH_TORRENTS_LOG_DIR=log_dir), capture_output=True, @@ -151,7 +153,7 @@ def test_calling_again_replaces_its_own_handlers_only(self): [h for h in ft.logger.handlers if h is not other], list(second) ) self.assertIn(other, ft.logger.handlers) - self.assertTrue(all(h not in ft.logger.handlers for h in first)) + self.assertEqual([h for h in first if h in ft.logger.handlers], []) class ParseLogLevelTests(unittest.TestCase): @@ -347,17 +349,6 @@ def test_logs_ratios_highest_first_with_none_as_zero(self): ], ) - def test_session_id_request_has_a_timeout(self): - with unittest.mock.patch.object( - ft.requests, - "post", - side_effect=[self.session_id_response(), self.torrent_get_response()], - ) as mock_post: - ft.log_seed_ratios_via_http() - - first_call_kwargs = mock_post.call_args_list[0].kwargs - self.assertEqual(first_call_kwargs, {"timeout": 15}) - def test_uses_configured_credentials_when_auth_not_explicitly_passed(self): with ( unittest.mock.patch.dict( diff --git a/tests/test_old_release_demand.py b/tests/test_old_release_demand.py index e7ecf2c..8d6c6ed 100644 --- a/tests/test_old_release_demand.py +++ b/tests/test_old_release_demand.py @@ -353,6 +353,7 @@ def fake_evaluate( min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, + today=date(2026, 1, 8), ) self.assertEqual(added, 1) @@ -375,6 +376,7 @@ def test_skips_distros_without_old_release_discovery(self): min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, + today=date(2026, 1, 8), ) self.assertEqual((added, skipped), (0, 0)) @@ -401,6 +403,7 @@ def test_already_present_torrent_is_not_re_evaluated(self): min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, + today=date(2026, 1, 8), ) mock_evaluate.assert_not_called() @@ -425,6 +428,7 @@ def test_unsafe_candidate_name_is_skipped_without_evaluation(self): min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, + today=date(2026, 1, 8), ) mock_evaluate.assert_not_called() @@ -449,6 +453,7 @@ def test_low_demand_variant_is_filtered_before_evaluation(self): min_leechers=1, min_leecher_ratio=0.0, allow_zero_seeders=False, + today=date(2026, 1, 8), ) mock_evaluate.assert_not_called() @@ -597,6 +602,37 @@ def test_due_candidate_is_evaluated_and_recorded(self): new_state[name], {"last_checked": "2026-01-08", "leechers": 30} ) + def test_today_defaults_to_the_current_date(self): + name = "kali-linux-2026.1-installer-amd64.iso" + candidates = {name: "https://example.invalid/x.torrent"} + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, + "evaluate_old_release_demand", + return_value=( + False, + b"bytes", + {"tracker": "t", "seeders": 1, "leechers": 0, "completed": 0}, + ), + ), + unittest.mock.patch.object( + ft, "date", helpers.fixed_date(date(2026, 3, 14)) + ), + ): + _added, _skipped, new_state = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + ) + + self.assertEqual(new_state[name], {"last_checked": "2026-03-14", "leechers": 0}) + def test_no_demand_result_is_still_recorded_for_future_recheck_gating(self): name = "kali-linux-2026.1-installer-arm64.iso" candidates = {name: "https://example.invalid/x.torrent"} diff --git a/tests/test_ratio_gate.py b/tests/test_ratio_gate.py index 12ea07c..4c59e3d 100644 --- a/tests/test_ratio_gate.py +++ b/tests/test_ratio_gate.py @@ -1,4 +1,4 @@ -"""Tests for the ratio gate: parse_version_type(), should_fetch_torrent() +"""Tests for the ratio gate: get_distro(), parse_version_type(), should_fetch_torrent() (and its removed-history / Devuan variants) and reading the previous run's ratio log with get_previous_ratios(). @@ -57,27 +57,6 @@ class ParseVersionTypeTests(unittest.TestCase): ("anything-1.0", "not-a-distro", ("", "")), ) - def test_every_ubuntu_flavour_fetched_is_recognised_as_ubuntu(self): - # fetch_ubuntu_lts() fetches ubuntu, lubuntu and xubuntu images; a - # flavour get_distro() doesn't recognise skips the ratio gate and - # cleanup grouping entirely. - for name in ( - "ubuntu-24.04-desktop-amd64.iso", - "lubuntu-24.04-desktop-amd64.iso", - "xubuntu-24.04-desktop-amd64.iso", - ): - with self.subTest(name): - self.assertEqual(ft.get_distro(name), "ubuntu") - - def test_lubuntu_is_ratio_gated_on_its_own_previous_release(self): - ratios = { - "lubuntu-23.10-desktop-amd64.iso": 0.2, - "ubuntu-23.10-desktop-amd64.iso": 5.0, - } - self.assertFalse( - ft.should_fetch_torrent("lubuntu-24.04-desktop-amd64.iso", ratios) - ) - def test_version_and_type_per_distro(self): for name, distro, expected in self.CASES: with self.subTest(name): @@ -114,11 +93,34 @@ def test_architectures_of_the_same_release_get_different_types(self): ) +class GetDistroTests(unittest.TestCase): + def test_every_ubuntu_flavour_fetched_is_recognised_as_ubuntu(self): + # fetch_ubuntu_lts() fetches ubuntu, lubuntu and xubuntu images; a + # flavour get_distro() doesn't recognise skips the ratio gate and + # cleanup grouping entirely. + for name in ( + "ubuntu-24.04-desktop-amd64.iso", + "lubuntu-24.04-desktop-amd64.iso", + "xubuntu-24.04-desktop-amd64.iso", + ): + with self.subTest(name): + self.assertEqual(ft.get_distro(name), "ubuntu") + + class ShouldFetchTorrentRatioKeyTests(unittest.TestCase): """Candidate names must key into the same ratio-lookup bucket as the names Transmission actually reports (which include the arch suffix and .iso extension), for all supported distros.""" + def test_lubuntu_is_ratio_gated_on_its_own_previous_release(self): + ratios = { + "lubuntu-23.10-desktop-amd64.iso": 0.2, + "ubuntu-23.10-desktop-amd64.iso": 5.0, + } + self.assertFalse( + ft.should_fetch_torrent("lubuntu-24.04-desktop-amd64.iso", ratios) + ) + def test_finds_previous_version_and_honors_low_ratio_ubuntu(self): ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.5} self.assertFalse( diff --git a/tests/test_scrapers.py b/tests/test_scrapers.py index 4fecb71..baf0f5c 100644 --- a/tests/test_scrapers.py +++ b/tests/test_scrapers.py @@ -10,6 +10,8 @@ Run with: python -m unittest discover -s tests """ +import dataclasses +import logging import os import unittest import unittest.mock @@ -40,45 +42,56 @@ def setUpModule(): "https://torrent.ubuntu.com/xubuntu/releases/{c}/release/desktop/xubuntu-{v}-desktop-amd64.iso.torrent", "https://torrent.ubuntu.com/xubuntu/releases/{c}/release/minimal/xubuntu-{v}-minimal-amd64.iso.torrent", ) -KALI_SUFFIXES = ( - "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-amd64.iso", - "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-netinst-amd64.iso", - "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-everything-amd64.iso", - "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-arm64.iso", - "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-netinst-arm64.iso", - "cdimage.kali.org/kali-{v}/kali-linux-{v}-installer-purple-amd64.iso", - "kali.download/arm-images/kali-{v}/kali-linux-{v}-raspberry-pi-armhf.img.xz", - "kali.download/arm-images/kali-{v}/kali-linux-{v}-raspberry-pi-zero-2-w-armhf.img.xz", - "kali.download/arm-images/kali-{v}/kali-linux-{v}-raspberry-pi-zero-w-armel.img.xz", - "kali.download/cloud-images/kali-{v}/kali-linux-{v}-cloud-genericcloud-amd64.tar.xz", - "kali.download/cloud-images/kali-{v}/kali-linux-{v}-cloud-genericcloud-arm64.tar.xz", -) FEDORA = "https://torrent.fedoraproject.org/torrents/" DEBIAN = "https://cdimage.debian.org/debian-cd/current/" -def kali_candidates(version): - return { - os.path.basename(suffix.format(v=version)): ( - "https://" + suffix.format(v=version) + ".torrent" - ) - for suffix in KALI_SUFFIXES - } +class _RecordingHandler(logging.Handler): + def __init__(self): + super().__init__(logging.DEBUG) + self.records = [] + + def emit(self, record): + self.records.append(record) + + +@dataclasses.dataclass +class Fetched: + """One fetcher run: what it returned, every message it logged (any + level, in order), the ERROR ones on their own, and the URLs it asked + for.""" + + result: object + logs: list + errors: list + requested: list class ScraperTestCase(unittest.TestCase): def fetch(self, fetcher, overrides=None): - """Runs fetcher against the recorded pages; returns (result, logs).""" + """Runs fetcher against the recorded pages, plus overrides.""" pages = helpers.RecordedPages(overrides) - with ( - unittest.mock.patch.object(ft.requests, "get", pages.get), - self.assertLogs(ft.logger, level="DEBUG") as logs, - ): - ft.logger.debug("fetch start") # assertLogs needs at least one - result = fetcher() + handler = _RecordingHandler() + level, propagate = ft.logger.level, ft.logger.propagate + ft.logger.addHandler(handler) + ft.logger.setLevel(logging.DEBUG) + ft.logger.propagate = False + try: + with unittest.mock.patch.object(ft.requests, "get", pages.get): + result = fetcher() + finally: + ft.logger.removeHandler(handler) + ft.logger.setLevel(level) + ft.logger.propagate = propagate self.assertEqual(pages.unexpected, []) - self.pages = pages - return result, [r.getMessage() for r in logs.records[1:]] + return Fetched( + result=result, + logs=[r.getMessage() for r in handler.records], + errors=[ + r.getMessage() for r in handler.records if r.levelno >= logging.ERROR + ], + requested=pages.requested, + ) class RecordedSnapshotTests(ScraperTestCase): @@ -86,21 +99,21 @@ class RecordedSnapshotTests(ScraperTestCase): exactly the candidates the snapshot lists.""" def test_ubuntu_lists_every_supported_lts_newest_first(self): - result, _logs = self.fetch(ft.fetch_ubuntu_lts) + fetched = self.fetch(ft.fetch_ubuntu_lts) expected = {} for codename, version in UBUNTU_SUPPORTED: for template in UBUNTU_TEMPLATES: url = template.format(c=codename, v=version) expected[os.path.basename(url)[: -len(".torrent")]] = url - self.assertEqual(result, expected) - self.assertEqual(list(result), list(expected)) + self.assertEqual(fetched.result, expected) + self.assertEqual(list(fetched.result), list(expected)) def test_debian_takes_the_first_torrent_of_each_listing(self): - result, logs = self.fetch(ft.fetch_debian_stable) + fetched = self.fetch(ft.fetch_debian_stable) self.assertEqual( - result, + fetched.result, { "debian-13.7.0-amd64-DVD-1.iso": DEBIAN + "amd64/bt-dvd/debian-13.7.0-amd64-DVD-1.iso.torrent", @@ -112,24 +125,24 @@ def test_debian_takes_the_first_torrent_of_each_listing(self): + "arm64/bt-cd/debian-13.7.0-arm64-netinst.iso.torrent", }, ) - self.assertEqual(logs, []) + self.assertEqual(fetched.logs, []) def test_kali_latest_uses_the_newest_installer_version(self): - result, _logs = self.fetch(ft.fetch_kali_latest) + fetched = self.fetch(ft.fetch_kali_latest) - self.assertEqual(result, kali_candidates("2026.2")) - self.assertEqual(self.pages.requested, ["https://www.kali.org/get-kali/"]) + self.assertEqual(fetched.result, helpers.kali_candidates("2026.2")) + self.assertEqual(fetched.requested, ["https://www.kali.org/get-kali/"]) def test_kali_old_versions_lists_every_folder_but_the_newest(self): - result, _logs = self.fetch(ft.fetch_kali_old_versions) + fetched = self.fetch(ft.fetch_kali_old_versions) - self.assertEqual(result, kali_candidates("2026.1")) + self.assertEqual(fetched.result, helpers.kali_candidates("2026.1")) def test_arch_takes_the_newest_available_release(self): - result, _logs = self.fetch(ft.fetch_arch_latest) + fetched = self.fetch(ft.fetch_arch_latest) self.assertEqual( - result, + fetched.result, { "archlinux-2026.10.01": ( "https://archlinux.org/releng/releases/2026.10.01/torrent/" @@ -138,18 +151,18 @@ def test_arch_takes_the_newest_available_release(self): ) def test_devuan_takes_the_linked_release_torrent(self): - result, _logs = self.fetch(ft.fetch_devuan_latest) + fetched = self.fetch(ft.fetch_devuan_latest) self.assertEqual( - result, + fetched.result, {"devuan_excalibur": "https://files.devuan.org/devuan_excalibur.torrent"}, ) def test_mint_reads_the_version_from_the_title(self): - result, _logs = self.fetch(ft.fetch_linuxmint_cinnamon) + fetched = self.fetch(ft.fetch_linuxmint_cinnamon) self.assertEqual( - result, + fetched.result, { "linuxmint-22.3-cinnamon-64bit.iso": ( "https://www.linuxmint.com/torrents/" @@ -160,10 +173,10 @@ def test_mint_reads_the_version_from_the_title(self): def test_fedora_takes_the_newest_final_release_per_arch(self): # 45_Beta is listed too, but isn't a final release. - result, _logs = self.fetch(ft.fetch_fedora_workstation) + fetched = self.fetch(ft.fetch_fedora_workstation) self.assertEqual( - result, + fetched.result, { "Fedora-Workstation-Live-aarch64-44": ( FEDORA + "Fedora-Workstation-Live-aarch64-44.torrent" @@ -175,10 +188,10 @@ def test_fedora_takes_the_newest_final_release_per_arch(self): ) def test_fedora_old_versions_lists_the_older_final_releases(self): - result, _logs = self.fetch(ft.fetch_fedora_workstation_old_versions) + fetched = self.fetch(ft.fetch_fedora_workstation_old_versions) self.assertEqual( - result, + fetched.result, { "Fedora-Workstation-Live-aarch64-43": ( FEDORA + "Fedora-Workstation-Live-aarch64-43.torrent" @@ -190,46 +203,18 @@ def test_fedora_old_versions_lists_the_older_final_releases(self): ) -# (fetcher, the URL it reads, the error it logs, what it returns on failure) -LATEST_FETCHERS = ( - ( - ft.fetch_ubuntu_lts, - "https://changelogs.ubuntu.com/meta-release-lts", - "Ubuntu fetch error", - False, - ), - (ft.fetch_kali_latest, "https://www.kali.org/get-kali/", "Kali fetch error", False), - ( - ft.fetch_kali_old_versions, - "https://cdimage.kali.org/", - "Kali old-release fetch error", - {}, - ), - ( - ft.fetch_arch_latest, - "https://archlinux.org/releng/releases/", - "Arch Linux fetch error", - False, - ), - ( - ft.fetch_devuan_latest, - "https://www.devuan.org/get-devuan", - "Devuan fetch error", - False, - ), - ( - ft.fetch_linuxmint_cinnamon, - "https://www.linuxmint.com/download.php", - "Linux Mint fetch error", - False, - ), - (ft.fetch_fedora_workstation, FEDORA, "Fedora fetch error", False), - ( - ft.fetch_fedora_workstation_old_versions, - FEDORA, - "Fedora old-release fetch error", - {}, - ), +# Every fetcher that reads a single page (Debian reads four; see +# test_debian_logs_each_failing_listing_and_keeps_the_rest), with that +# page's URL and what the fetcher returns when it fails. +SINGLE_PAGE_FETCHERS = ( + (ft.fetch_ubuntu_lts, "https://changelogs.ubuntu.com/meta-release-lts", False), + (ft.fetch_kali_latest, "https://www.kali.org/get-kali/", False), + (ft.fetch_kali_old_versions, "https://cdimage.kali.org/", {}), + (ft.fetch_arch_latest, "https://archlinux.org/releng/releases/", False), + (ft.fetch_devuan_latest, "https://www.devuan.org/get-devuan", False), + (ft.fetch_linuxmint_cinnamon, "https://www.linuxmint.com/download.php", False), + (ft.fetch_fedora_workstation, FEDORA, False), + (ft.fetch_fedora_workstation_old_versions, FEDORA, {}), ) @@ -238,17 +223,15 @@ class FailurePathTests(ScraperTestCase): wrong and return its empty result - never raise into the daily run, and never pass an error page off as "no releases".""" - def _assert_fails_with(self, overrides, expected_in_log): - for fetcher, url, error, empty in LATEST_FETCHERS: + def _assert_fails_with(self, overrides, cause): + for fetcher, url, empty in SINGLE_PAGE_FETCHERS: with self.subTest(fetcher=fetcher.__name__): - result, logs = self.fetch(fetcher, {url: overrides(url)}) + fetched = self.fetch(fetcher, {url: overrides(url)}) - self.assertIs(type(result), type(empty)) - self.assertEqual(result, empty) - self.assertTrue( - any(m.startswith(error) and expected_in_log in m for m in logs), - logs, - ) + self.assertIs(type(fetched.result), type(empty)) + self.assertEqual(fetched.result, empty) + self.assertEqual(len(fetched.errors), 1, fetched.logs) + self.assertIn(cause, fetched.errors[0]) def test_connection_error(self): self._assert_fails_with( @@ -256,23 +239,21 @@ def test_connection_error(self): "connection refused", ) - def test_http_500(self): - self._assert_fails_with( - lambda url: helpers.fake_response("oops", status=500, url=url), - "500 Server Error", - ) - - def test_http_404(self): - self._assert_fails_with( - lambda url: helpers.fake_response("gone", status=404, url=url), - "404 Client Error", - ) + def test_http_error_status(self): + for status, cause in ((500, "500 Server Error"), (404, "404 Client Error")): + with self.subTest(status=status): + self._assert_fails_with( + lambda url, status=status: helpers.fake_response( + "oops", status=status, url=url + ), + cause, + ) def test_debian_logs_each_failing_listing_and_keeps_the_rest(self): amd64_dvd = DEBIAN + "amd64/bt-dvd/" arm64_cd = DEBIAN + "arm64/bt-cd/" - result, logs = self.fetch( + fetched = self.fetch( ft.fetch_debian_stable, { amd64_dvd: requests.ConnectionError("refused"), @@ -281,12 +262,12 @@ def test_debian_logs_each_failing_listing_and_keeps_the_rest(self): ) self.assertEqual( - sorted(result), + sorted(fetched.result), ["debian-13.7.0-amd64-netinst.iso", "debian-13.7.0-arm64-DVD-1.iso"], ) - self.assertEqual([m.split(":")[0] for m in logs], ["Debian fetch error"] * 2) - self.assertIn("refused", logs[0]) - self.assertIn("500 Server Error", logs[1]) + self.assertEqual(len(fetched.errors), 2, fetched.logs) + self.assertIn("refused", fetched.errors[0]) + self.assertIn("500 Server Error", fetched.errors[1]) class LayoutChangeTests(ScraperTestCase): @@ -299,26 +280,26 @@ def _page(self, url, body): def test_kali_page_without_a_version(self): url = "https://www.kali.org/get-kali/" - result, logs = self.fetch( + fetched = self.fetch( ft.fetch_kali_latest, self._page(url, "Get Kali") ) - self.assertIs(result, False) + self.assertIs(fetched.result, False) self.assertEqual( - logs, + fetched.logs, [f"Could not detect a Kali release number on {url}#kali-installer-images"], ) def test_kali_cdimage_with_a_single_version_has_no_old_ones(self): - result, logs = self.fetch( + fetched = self.fetch( ft.fetch_kali_old_versions, self._page("https://cdimage.kali.org/", 'x'), ) - self.assertEqual((result, logs), ({}, [])) + self.assertEqual((fetched.result, fetched.logs), ({}, [])) def test_ubuntu_block_without_a_version_line(self): - result, logs = self.fetch( + fetched = self.fetch( ft.fetch_ubuntu_lts, self._page( "https://changelogs.ubuntu.com/meta-release-lts", @@ -326,9 +307,9 @@ def test_ubuntu_block_without_a_version_line(self): ), ) - self.assertIs(result, False) - self.assertEqual(len(logs), 1) - self.assertTrue(logs[0].startswith("Ubuntu fetch error")) + self.assertIs(fetched.result, False) + self.assertEqual(len(fetched.logs), 1) + self.assertRegex(fetched.logs[0], r"^Ubuntu fetch error: ") def test_debian_listing_without_torrents(self): overrides = {} @@ -340,24 +321,24 @@ def test_debian_listing_without_torrents(self): ): overrides.update(self._page(DEBIAN + listing, "x")) - result, logs = self.fetch(ft.fetch_debian_stable, overrides) + fetched = self.fetch(ft.fetch_debian_stable, overrides) - self.assertEqual(result, {}) - self.assertEqual(logs, ["No Debian DVD-1 torrent found."] * 4) + self.assertEqual(fetched.result, {}) + self.assertEqual(fetched.logs, ["No Debian DVD-1 torrent found."] * 4) def test_arch_page_without_the_release_table(self): - result, logs = self.fetch( + fetched = self.fetch( ft.fetch_arch_latest, self._page("https://archlinux.org/releng/releases/", ""), ) - self.assertIs(result, False) - self.assertEqual(len(logs), 1) - self.assertTrue(logs[0].startswith("Arch Linux fetch error")) + self.assertIs(fetched.result, False) + self.assertEqual(len(fetched.logs), 1) + self.assertRegex(fetched.logs[0], r"^Arch Linux fetch error: ") def test_arch_table_without_available_releases(self): url = "https://archlinux.org/releng/releases/" - result, logs = self.fetch( + fetched = self.fetch( ft.fetch_arch_latest, self._page( url, @@ -367,43 +348,43 @@ def test_arch_table_without_available_releases(self): ), ) - self.assertIs(result, False) - self.assertEqual(logs, [f"No available Arch Linux releases found on {url}"]) + self.assertIs(fetched.result, False) + self.assertEqual( + fetched.logs, [f"No available Arch Linux releases found on {url}"] + ) def test_devuan_page_without_a_torrent_link(self): url = "https://www.devuan.org/get-devuan" - result, logs = self.fetch(ft.fetch_devuan_latest, self._page(url, "")) + fetched = self.fetch(ft.fetch_devuan_latest, self._page(url, "")) - self.assertIs(result, False) + self.assertIs(fetched.result, False) self.assertEqual( - logs, [f"Could not find a Devuan release torrent link on {url}"] + fetched.logs, [f"Could not find a Devuan release torrent link on {url}"] ) def test_mint_page_without_a_versioned_title(self): url = "https://www.linuxmint.com/download.php" - result, logs = self.fetch( + fetched = self.fetch( ft.fetch_linuxmint_cinnamon, self._page(url, "Linux Mint") ) - self.assertIs(result, False) - self.assertEqual(logs, [f"Could not detect a Linux Mint version on {url}"]) + self.assertIs(fetched.result, False) + self.assertEqual( + fetched.logs, [f"Could not detect a Linux Mint version on {url}"] + ) def test_fedora_listing_without_workstation_torrents(self): - result, logs = self.fetch( + fetched = self.fetch( ft.fetch_fedora_workstation, self._page(FEDORA, "x"), ) - self.assertIs(result, False) - self.assertEqual(logs, [f"No Fedora Workstation torrents found on {FEDORA}"]) - - -class NetworkGuardTests(unittest.TestCase): - def test_real_connections_are_refused(self): - with self.assertRaisesRegex(RuntimeError, "must not open network"): - requests.get("http://192.0.2.1/", timeout=1) + self.assertIs(fetched.result, False) + self.assertEqual( + fetched.logs, [f"No Fedora Workstation torrents found on {FEDORA}"] + ) if __name__ == "__main__": diff --git a/tests/test_torrent_scrape.py b/tests/test_torrent_scrape.py index 9d9871c..c074918 100644 --- a/tests/test_torrent_scrape.py +++ b/tests/test_torrent_scrape.py @@ -276,7 +276,11 @@ class BuildScrapeRequestUrlTests(unittest.TestCase): def test_adds_query_param_with_no_existing_query(self): info_hash = bytes(range(20)) url = ts.build_scrape_request_url("http://example.com/scrape", info_hash) - self.assertTrue(url.startswith("http://example.com/scrape?info_hash=")) + self.assertEqual( + url, + "http://example.com/scrape?info_hash=" + "%00%01%02%03%04%05%06%07%08%09%0A%0B%0C%0D%0E%0F%10%11%12%13", + ) def test_appends_to_existing_query(self): info_hash = bytes(range(20)) From 374d4ba2cd72e0f8488a51ae5e178b8c1e9f6da1 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 17:38:32 +0200 Subject: [PATCH 40/46] Consolidate fixtures; FETCH_TORRENTS_WATCH_DIR; one credential rule (task 35) watch_dir is now FETCH_TORRENTS_WATCH_DIR (default /watch), read once at import like log_dir, and documented in the README. Tests check the default and the override in a fresh interpreter. MainWiringTests points it, and FETCH_TORRENTS_LOG_DIR, at FakeWorld's temp dirs, and now routes Arch so the runpy run really writes a torrent there. get_rpc_credentials() is gone. load_config() and log_seed_ratios_via_http()'s env fallback both use _rpc_credentials(env), and Config.rpc_auth is the only rule for when both halves count. A test pins the fallback to Config.rpc_auth. New helpers: use_watch_dir(), use_state_files() (both return the temp paths) and removed(name, ratio=None). Also FakeTransmissionRpc, the one strict HTTP RPC fake: 409 plus session id, auth recorded, method and fields checked against the pinned transmission_rpc. test_logging and FakeWorld both use it in place of their own handshakes. All of these are tested in test_helpers.py, and removed() is checked against record_removal(). Every setUp that patched watch_dir or a state-file global now calls the helpers. test_old_release_demand gets demand(scrape, **overrides) with the daily run's default thresholds. FakeWorld gets hold_current_and_superseded_ubuntu(). The ratio-key tests are one table, which adds the missing Kali and Arch high-ratio rows. CLAUDE.md documents the new helpers. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 9 +- README.md | 3 +- fetch_torrents.py | 29 ++-- tasks/README.md | 2 +- tests/helpers.py | 106 +++++++++++++++ tests/test_cleanup.py | 46 +------ tests/test_daily_run.py | 132 +++++++----------- tests/test_fetchers.py | 24 +--- tests/test_helpers.py | 160 ++++++++++++++++++++++ tests/test_logging.py | 184 ++++++++++--------------- tests/test_old_release_demand.py | 140 +++++--------------- tests/test_persistence.py | 32 ++++- tests/test_ratio_gate.py | 221 ++++++++++++------------------- 13 files changed, 566 insertions(+), 522 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index c0d253a..030d7ab 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,7 +49,14 @@ The daily run itself is `run(config, client_factory=..., today=...)` with its I/O (Transmission client, clock, disk usage) injected, so it is tested directly with fakes from `tests/helpers.py` (`FakeTransmission`, `FakeDisk`); `tests/test_daily_run.py` keeps one runpy test for the `__main__` wiring. -Env vars are read once by `load_config(env)`. +Env vars are read once by `load_config(env)`. Transmission's HTTP RPC (the +ratio log) is faked with `helpers.FakeTransmissionRpc` as `requests.post`: +409 plus session id first, like the daemon, recording each call's auth. +The file locations are module globals read at import +(`FETCH_TORRENTS_LOG_DIR`, `FETCH_TORRENTS_WATCH_DIR`); in-process tests +point them at temp dirs with `helpers.use_watch_dir(self)` and +`helpers.use_state_files(self)`, which return the paths, instead of +patching them by hand. Every test module starts with `import helpers` (which also puts the repo root on `sys.path`) and a `setUpModule()` calling `helpers.isolate_module()`: diff --git a/README.md b/README.md index 95ac2c2..25418f2 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ by default: |---|---| | `/config` | Transmission configuration files | | `/downloads` | Downloaded ISO files (seeding storage) | -| `/watch` | Torrent watch folder. `.torrent` files are written as a hidden `..torrent.part` first and renamed into place, so Transmission never picks up a half-written file. A leftover `.part` file is safe to delete. | +| `/watch` | Torrent watch folder. `.torrent` files are written as a hidden `..torrent.part` first and renamed into place, so Transmission never picks up a half-written file. A leftover `.part` file is safe to delete. The fetch script's side of this path is `FETCH_TORRENTS_WATCH_DIR`; leave it at its default here. | | `/logs` | Persistent logs for fetch script runs, plus `fetch_torrents_ratio_history.json` - a small, bounded per-torrent ratio history used to detect stagnant (no-traction) torrents. Deleting it just resets stagnation tracking; it's not required for anything else. `fetch_torrents.log` itself is size-bounded (rotated, see `FETCH_TORRENTS_LOG_MAX_BYTES`/`FETCH_TORRENTS_LOG_BACKUP_COUNT`), so this volume won't grow without limit even if left running indefinitely. | --- @@ -156,6 +156,7 @@ below), remember to remove that forwarding rule too. | `TRANSMISSION_PORT_FORWARDING` | *(unset, defaults to `false` on first run only)* | Controls Transmission's own `port-forwarding-enabled` (UPnP/NAT-PMP) - see [Peer port forwarding](#-peer-port-forwarding-optional). Only applied the first time the container starts (before `settings.json` exists); afterwards it's left alone - so it won't fight a value you or Transmission's web UI later change. Set explicitly to force it either way on every start. | | `FETCH_TORRENTS_LOG_MAX_BYTES` | `5242880` (5 MB) | Maximum size of `fetch_torrents.log` before it's rotated. | | `FETCH_TORRENTS_LOG_BACKUP_COUNT` | `3` | Number of rotated `fetch_torrents.log.N` backups kept before the oldest is deleted. | +| `FETCH_TORRENTS_WATCH_DIR` | `/watch` | Folder the fetch script writes `.torrent` files to. Leave it unset in the container: transmission-daemon always watches `/watch` (see `entrypoint.sh`), so this is only for running `fetch_torrents.py` outside the container, against a daemon watching that folder. | --- diff --git a/fetch_torrents.py b/fetch_torrents.py index 27e3add..540c5bd 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -166,20 +166,25 @@ def configure_logging(log_dir, env=None): return stream_handler, file_handler, ratio_handler -watch_dir = "/watch" +# Where transmission-daemon picks up new .torrent files; read once, like +# log_dir. +watch_dir = os.getenv("FETCH_TORRENTS_WATCH_DIR", "/watch") -def get_rpc_credentials(): - """TRANSMISSION_RPC_USERNAME/PASSWORD, read here so this script's own - RPC calls (cleanup, ratio logging) authenticate the same way +def _rpc_credentials(env): + """Config's rpc_username/rpc_password from TRANSMISSION_RPC_USERNAME/ + PASSWORD (blank counts as unset), read so this script's own RPC calls + (cleanup, ratio logging) authenticate the same way configure_transmission.py configured the daemon. Without this, setting rpc-authentication-required (as the README instructs whenever TRANSMISSION_RPC_WHITELIST is widened beyond localhost) makes every one of this script's own localhost RPC calls fail with 401 - silently, since - callers only log the failure - permanently disabling cleanup.""" - username = os.getenv("TRANSMISSION_RPC_USERNAME", "").strip() - password = os.getenv("TRANSMISSION_RPC_PASSWORD", "").strip() - return username or None, password or None + callers only log the failure - permanently disabling cleanup. + Config.rpc_auth decides whether they are complete enough to send.""" + return { + "rpc_username": env.get("TRANSMISSION_RPC_USERNAME", "").strip() or None, + "rpc_password": env.get("TRANSMISSION_RPC_PASSWORD", "").strip() or None, + } def wait_for_transmission_rpc( @@ -1188,8 +1193,7 @@ def log_seed_ratios_via_http( rpc_url="http://localhost:9091/transmission/rpc", auth: tuple | None = None ): if auth is None: - username, password = get_rpc_credentials() - auth = (username, password) if username and password else None + auth = Config(**_rpc_credentials(os.environ)).rpc_auth logger.info("Querying Transmission RPC for seed ratios...") r = requests.post(rpc_url, timeout=15) @@ -1661,15 +1665,14 @@ def number(field_name, key, default, convert): 95.0, float, ), - rpc_username=env.get("TRANSMISSION_RPC_USERNAME", "").strip() or None, - rpc_password=env.get("TRANSMISSION_RPC_PASSWORD", "").strip() or None, + **_rpc_credentials(env), invalid=invalid, ) def make_client_factory(config): """Connects to the local daemon with the credentials configure_transmission.py - configured it with (see get_rpc_credentials() for why that matters).""" + configured it with (see _rpc_credentials() for why that matters).""" def connect(): return Client( diff --git a/tasks/README.md b/tasks/README.md index 4f37061..3154171 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -101,7 +101,7 @@ mutation score is 95.4% (1027/1076), up from 88.6%, with the critical set still | 32 | [Atomic `.torrent` writes](32-atomic-torrent-writes.md) | P2 (bug) | NO_REFACTOR | — | done | | 33 | [`scrape_torrent` fall-through branches and SSRF edges](33-scrape-torrent-branches-and-ssrf.md) | P2 | NO_REFACTOR | — | done | | 34 | [Mechanical test tidy-up](34-mechanical-test-tidy-up.md) | P3 | NO_REFACTOR | — | done | -| 35 | [Consolidate fixtures and the path-global seam](35-fixture-consolidation.md) | P3 | NO_REFACTOR → REFACTOR_REQUIRED | 34 | open | +| 35 | [Consolidate fixtures and the path-global seam](35-fixture-consolidation.md) | P3 | NO_REFACTOR → REFACTOR_REQUIRED | 34 | done | | 36 | [Remaining `fetch_torrents.py` mutation survivors](36-remaining-mutation-survivors.md) | P3 | NO_REFACTOR | — | done | | 37 | [Toolchain: smoke test, shellcheck/hadolint, ruff S, pins](37-toolchain-and-q4.md) | P2 | CI | — | done | | 38 | [Re-measure, raise the coverage ratchet, record the baseline](38-remeasure-and-ratchet.md) | P3 | — | 28–37 | open | diff --git a/tests/helpers.py b/tests/helpers.py index 37d1dbb..5796214 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -25,6 +25,7 @@ def setUpModule(): import socket import sys import tempfile +import types import unittest import unittest.mock from datetime import date @@ -32,6 +33,7 @@ def setUpModule(): import requests import transmission_rpc +import transmission_rpc.constants REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if REPO_ROOT not in sys.path: @@ -100,6 +102,17 @@ def names_of(torrents): return sorted(t.name for t in torrents) +def removed(name, ratio=None, *, on="2026-01-01"): + """A removed-history entry for `name`, as record_removal() writes it + ({name: {...}}; merge several with {**a, **b}). Only the ratio matters + to the ratio gate, and only the date to the old-release log line; no + ratio makes a legacy entry from before ratios were recorded.""" + entry = {"removed_date": on, "reason": "stagnant"} + if ratio is not None: + entry["ratio"] = ratio + return {name: entry} + + def make_temp_dir(testcase, prefix): """mkdtemp() that is removed again when the test finishes.""" path = tempfile.mkdtemp(prefix=prefix) @@ -209,6 +222,99 @@ def __call__(self, path): return (self.total, used, self.total - used) +class FakeTransmissionRpc: + """requests.post stand-in for transmission-daemon's HTTP RPC, as strict + as the daemon: a request without the session id it handed out is + refused with a 409 carrying that id. A torrent-get with it records its + auth in `auths` and lists `transmission`'s torrents, with just the + requested fields. A method or field the pinned transmission_rpc doesn't + know raises ValueError. `calls` records every post, as mock.call()s.""" + + URL = "http://localhost:9091/transmission/rpc" + SESSION_ID = "sid" + + def __init__(self, transmission): + self.transmission = transmission + self.calls = [] + self.auths = [] + + def post(self, url, **kwargs): + self.calls.append(unittest.mock.call(url, **kwargs)) + if url != self.URL: + return fake_response(b"not found", status=404, url=url) + headers = kwargs.get("headers") or {} + if headers.get("X-Transmission-Session-Id") != self.SESSION_ID: + return fake_response( + status=409, + url=url, + headers={"X-Transmission-Session-Id": self.SESSION_ID}, + ) + payload = kwargs.get("json") or {} + method = payload.get("method") + RpcMethod = transmission_rpc.constants.RpcMethod + if method not in {m.value for m in RpcMethod}: + raise ValueError(f"the pinned transmission_rpc has no method {method!r}") + if method != RpcMethod.TorrentGet.value: + raise ValueError(f"method {method!r}: only torrent-get is faked") + fields = payload["arguments"]["fields"] + unknown = set(fields) - set(transmission_rpc.constants.TORRENT_GET_ARGS) + if unknown: + raise ValueError( + f"torrent-get fields the pinned transmission_rpc lacks: {unknown}" + ) + self.auths.append(kwargs.get("auth")) + torrents = [ + {field: t.fields[field] for field in fields} + for t in self.transmission.torrents + ] + return fake_response( + json.dumps({"arguments": {"torrents": torrents}, "result": "success"}), + url=url, + ) + + +def use_watch_dir(testcase): + """Points fetch_torrents.watch_dir at a fresh temp dir for the rest of + `testcase`; returns that dir.""" + import fetch_torrents + + path = make_temp_dir(testcase, "fetch_torrents_test_watch_") + _patch_for_test(testcase, fetch_torrents, "watch_dir", path) + return path + + +# fetch_torrents' state file globals and the file each is named after. +STATE_FILES = { + "ratio_log_file": "fetch_torrents_ratios.log", + "ratio_history_file": "fetch_torrents_ratio_history.json", + "removed_history_file": "fetch_torrents_removed_history.json", + "old_release_check_state_file": "fetch_torrents_old_release_check_state.json", +} + + +def use_state_files(testcase): + """Points every fetch_torrents state file global (STATE_FILES) at a + not-yet-existing file in a fresh temp dir for the rest of `testcase`. + Returns a namespace holding `dir` and each patched path, by the + global's name (e.g. `.removed_history_file`).""" + import fetch_torrents + + state = types.SimpleNamespace( + dir=make_temp_dir(testcase, "fetch_torrents_test_state_") + ) + for attr, filename in STATE_FILES.items(): + path = os.path.join(state.dir, filename) + setattr(state, attr, path) + _patch_for_test(testcase, fetch_torrents, attr, path) + return state + + +def _patch_for_test(testcase, target, attr, value): + patcher = unittest.mock.patch.object(target, attr, value) + patcher.start() + testcase.addCleanup(patcher.stop) + + def disk_full_open(module): """Patch `module`'s own `open` so a binary write keeps half its bytes and then fails with ENOSPC, as when the disk fills mid-write. Scoped to the diff --git a/tests/test_cleanup.py b/tests/test_cleanup.py index b3e617b..190113b 100644 --- a/tests/test_cleanup.py +++ b/tests/test_cleanup.py @@ -778,21 +778,8 @@ class CleanupRemovesWatchFileTests(unittest.TestCase): by default) ever cleans them up.""" def setUp(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_") - for attr, filename in ( - ("removed_history_file", "removed_history.json"), - ("ratio_history_file", "ratio_history.json"), - ): - patcher = unittest.mock.patch.object( - ft, attr, os.path.join(tmp_dir, filename) - ) - patcher.start() - self.addCleanup(patcher.stop) - - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) + helpers.use_state_files(self) + self.tmp_watch_dir = helpers.use_watch_dir(self) def _touch_watch_file(self, name): path = os.path.join(self.tmp_watch_dir, f"{name}.torrent") @@ -1156,21 +1143,8 @@ class EnforceDiskUsageLimitTests(unittest.TestCase): holds.""" def setUp(self): - tmp_dir = helpers.make_temp_dir(self, "fetch_torrents_cleanup_") - for attr, filename in ( - ("removed_history_file", "removed_history.json"), - ("ratio_history_file", "ratio_history.json"), - ): - patcher = unittest.mock.patch.object( - ft, attr, os.path.join(tmp_dir, filename) - ) - patcher.start() - self.addCleanup(patcher.stop) - - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) + helpers.use_state_files(self) + self.tmp_watch_dir = helpers.use_watch_dir(self) self.transmission = helpers.FakeTransmission( [ @@ -1374,16 +1348,8 @@ class CleanupOldVersionsDefaultsTests(unittest.TestCase): """cleanup_old_versions() called the way the daily run calls it.""" def setUp(self): - self.history_path = os.path.join( - helpers.make_temp_dir(self, "fetch_torrents_cleanup_"), "removed.json" - ) - for attr, value in ( - ("removed_history_file", self.history_path), - ("watch_dir", helpers.make_temp_dir(self, "fetch_torrents_test_watch_")), - ): - patcher = unittest.mock.patch.object(ft, attr, value) - patcher.start() - self.addCleanup(patcher.stop) + self.history_path = helpers.use_state_files(self).removed_history_file + helpers.use_watch_dir(self) def _transmission(self, old_ratio): return helpers.FakeTransmission( diff --git a/tests/test_daily_run.py b/tests/test_daily_run.py index 8778eef..57e63fa 100644 --- a/tests/test_daily_run.py +++ b/tests/test_daily_run.py @@ -66,16 +66,22 @@ def torrent(name, id_, ratio): class FakeWorld: - """HTTP routes, a Transmission daemon and a disk, all in memory.""" + """HTTP routes, a Transmission daemon (its HTTP RPC is `rpc`) and a disk, + all in memory. State files and torrents go to `state_dir` and + `watch_dir`: fresh temp dirs unless given.""" - def __init__(self, testcase): - self.state_dir = helpers.make_temp_dir(testcase, "daily_run_state_") - self.watch_dir = helpers.make_temp_dir(testcase, "daily_run_watch_") + def __init__(self, testcase, state_dir=None, watch_dir=None): + self.state_dir = state_dir or helpers.make_temp_dir( + testcase, "daily_run_state_" + ) + self.watch_dir = watch_dir or helpers.make_temp_dir( + testcase, "daily_run_watch_" + ) self.routes = {} self.requested = [] self.events = [] self.transmission = helpers.FakeTransmission() - self.rpc_auths = [] + self.rpc = helpers.FakeTransmissionRpc(self.transmission) self.rpc_up = True self.disk_used_percent = 10 self.disk_paths = [] @@ -89,20 +95,13 @@ def get(self, url, timeout=None): raise route return route - def post(self, url, **kwargs): - """Transmission's RPC: a request without the session id is refused - with a 409 carrying it; a torrent-get with it records its auth and - lists what `transmission` holds.""" - headers = kwargs.get("headers") or {} - if headers.get("X-Transmission-Session-Id") != "sid": - return helpers.fake_response( - status=409, headers={"X-Transmission-Session-Id": "sid"} - ) - self.rpc_auths.append(kwargs.get("auth")) - torrents = [ - {"name": t.name, "uploadRatio": t.ratio} for t in self.transmission.torrents + def hold_current_and_superseded_ubuntu(self, superseded_ratio=2.0): + """Transmission holds Ubuntu 24.04 (ratio 2.0) and the 23.10 it + superseded.""" + self.transmission.torrents = [ + torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), + torrent("ubuntu-23.10-desktop-amd64.iso", 2, superseded_ratio), ] - return helpers.fake_response(json.dumps({"arguments": {"torrents": torrents}})) def wait_for_rpc(self): return self.rpc_up @@ -247,25 +246,13 @@ class RunTestCase(unittest.TestCase): """Base for tests of ft.run() inside a FakeWorld; defines no tests.""" def setUp(self): - self.world = FakeWorld(self) - patches = [ - unittest.mock.patch.object(ft.requests, "get", self.world.get), - unittest.mock.patch.object(ft.requests, "post", self.world.post), - unittest.mock.patch.object(ft, "watch_dir", self.world.watch_dir), - ] - for attr, filename in ( - ("ratio_log_file", "fetch_torrents_ratios.log"), - ("ratio_history_file", "fetch_torrents_ratio_history.json"), - ("removed_history_file", "fetch_torrents_removed_history.json"), - ( - "old_release_check_state_file", - "fetch_torrents_old_release_check_state.json", - ), - ): - patches.append( - unittest.mock.patch.object(ft, attr, self.world.state_path(filename)) - ) - for patcher in patches: + self.world = FakeWorld( + self, + state_dir=helpers.use_state_files(self).dir, + watch_dir=helpers.use_watch_dir(self), + ) + for name, fake in (("get", self.world.get), ("post", self.world.rpc.post)): + patcher = unittest.mock.patch.object(ft.requests, name, fake) patcher.start() self.addCleanup(patcher.stop) @@ -381,10 +368,7 @@ def test_custom_fetchers_replace_the_default_set(self): self.assertEqual(summary.not_found, 1) def test_keep_only_latest_selects_the_old_version_cleanup(self): - self.world.transmission.torrents = [ - torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), - torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), - ] + self.world.hold_current_and_superseded_ubuntu() self._run( FETCH_TORRENTS_DISTROS="arch", CLEANUP_KEEP_ONLY_LATEST_VERSION="true" @@ -395,10 +379,7 @@ def test_keep_only_latest_selects_the_old_version_cleanup(self): ) def test_keep_only_latest_honors_min_ratio(self): - self.world.transmission.torrents = [ - torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), - torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), - ] + self.world.hold_current_and_superseded_ubuntu() self._run( FETCH_TORRENTS_DISTROS="arch", @@ -411,10 +392,7 @@ def test_keep_only_latest_honors_min_ratio(self): def test_default_cleanup_is_stagnation_based(self): # No ratio history yet, so the stagnation cleanup can't judge the old # version and keeps it - unlike keep-only-latest, which removes it. - self.world.transmission.torrents = [ - torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), - torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), - ] + self.world.hold_current_and_superseded_ubuntu() self._run(FETCH_TORRENTS_DISTROS="arch") @@ -432,10 +410,7 @@ def test_default_cleanup_is_stagnation_based(self): ) def test_invalid_cleanup_setting_skips_cleanup_and_continues(self): - self.world.transmission.torrents = [ - torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), - torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), - ] + self.world.hold_current_and_superseded_ubuntu() _summary, messages = self._run( FETCH_TORRENTS_DISTROS="arch", @@ -451,10 +426,7 @@ def test_invalid_cleanup_setting_skips_cleanup_and_continues(self): self.assertEqual(messages[-1], ARCH_FAILED_SUMMARY) def test_disk_usage_limit_runs_after_cleanup(self): - self.world.transmission.torrents = [ - torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), - torrent("ubuntu-23.10-desktop-amd64.iso", 2, 0.1), - ] + self.world.hold_current_and_superseded_ubuntu(superseded_ratio=0.1) self.world.disk_used_percent = 99 self._run( @@ -483,10 +455,7 @@ def test_disk_usage_limit_error_is_logged_not_raised(self): def test_rpc_unavailable_skips_ratio_logging_but_still_cleans_up(self): self.world.rpc_up = False - self.world.transmission.torrents = [ - torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), - torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), - ] + self.world.hold_current_and_superseded_ubuntu() _summary, messages = self._run( FETCH_TORRENTS_DISTROS="arch", CLEANUP_KEEP_ONLY_LATEST_VERSION="true" ) @@ -522,12 +491,12 @@ def test_rpc_credentials_from_config_are_used_for_ratio_logging(self): TRANSMISSION_RPC_PASSWORD="hunter2", ) - self.assertEqual(self.world.rpc_auths, [("alice", "hunter2")]) + self.assertEqual(self.world.rpc.auths, [("alice", "hunter2")]) def test_ratio_logging_sends_no_credentials_when_none_are_configured(self): self._run(FETCH_TORRENTS_DISTROS="arch") - self.assertEqual(self.world.rpc_auths, [None]) + self.assertEqual(self.world.rpc.auths, [None]) def test_every_download_outcome_is_counted_each_time(self): # Two of each outcome (four failures: two downloads, two fetchers @@ -549,16 +518,7 @@ def url(name): "w", encoding="utf-8", ) as f: - json.dump( - { - "archlinux-2026.08.01": { - "removed_date": "2026-08-20", - "reason": "stagnant", - "ratio": 0.4, - } - }, - f, - ) + json.dump(helpers.removed("archlinux-2026.08.01", 0.4), f) def found(*names): return lambda: {name: url(name) for name in names} @@ -739,7 +699,7 @@ def test_config_and_saved_state_drive_the_check(self): ) self._write_state( "fetch_torrents_removed_history.json", - {removed[0]: {"removed_date": "2026-08-01", "reason": "stagnant"}}, + helpers.removed(removed[0]), ) summary, messages = self._check( @@ -781,7 +741,7 @@ def test_previously_removed_release_is_rechecked_when_skip_removed_is_off(self): removed = self._candidate(1, "live", 50, 1) self._write_state( "fetch_torrents_removed_history.json", - {removed[0]: {"removed_date": "2026-08-01", "reason": "stagnant"}}, + helpers.removed(removed[0]), ) summary, _ = self._check( @@ -811,11 +771,8 @@ class MainWiringTests(unittest.TestCase): def test_running_the_script_performs_a_daily_run(self): world = FakeWorld(self) - world.transmission.torrents = [ - torrent("ubuntu-24.04-desktop-amd64.iso", 1, 2.0), - torrent("ubuntu-23.10-desktop-amd64.iso", 2, 2.0), - ] - log_dir = helpers.make_temp_dir(self, "daily_run_main_logs_") + world.hold_current_and_superseded_ubuntu() + world.route_arch() logger = logging.getLogger("fetch_torrents") handlers_before = list(logger.handlers) @@ -826,7 +783,8 @@ def rpc_port_open(*args, **kwargs): with contextlib.ExitStack() as stack: stack.enter_context( helpers.isolated_env( - FETCH_TORRENTS_LOG_DIR=log_dir, + FETCH_TORRENTS_LOG_DIR=world.state_dir, + FETCH_TORRENTS_WATCH_DIR=world.watch_dir, FETCH_TORRENTS_DISTROS="arch", CLEANUP_KEEP_ONLY_LATEST_VERSION="true", TRANSMISSION_RPC_USERNAME="alice", @@ -834,7 +792,7 @@ def rpc_port_open(*args, **kwargs): ) ) stack.enter_context(unittest.mock.patch("requests.get", world.get)) - stack.enter_context(unittest.mock.patch("requests.post", world.post)) + stack.enter_context(unittest.mock.patch("requests.post", world.rpc.post)) client_cls = stack.enter_context( unittest.mock.patch.object( sys.modules["transmission_rpc"], @@ -864,21 +822,23 @@ def rpc_port_open(*args, **kwargs): self.assertIn("Selected distros for this run: arch", messages) self.assertRegex( messages[-1], - r"\ARun complete in \d+\.\d\d seconds\. 0 added, 0 existing, " - r"0 not found upstream, 1 failed\.\Z", + r"\ARun complete in \d+\.\d\d seconds\. 1 added, 0 existing, " + r"0 not found upstream, 0 failed\.\Z", ) + self.assertEqual(os.listdir(world.watch_dir), ["archlinux-2026.09.01.torrent"]) client_cls.assert_called_with( host="localhost", port=9091, username="alice", password="hunter2" ) + self.assertEqual(world.rpc.auths, [("alice", "hunter2")]) self.assertEqual(world.transmission.names, ["ubuntu-24.04-desktop-amd64.iso"]) self.assertEqual( - ft.get_previous_ratios(os.path.join(log_dir, "fetch_torrents_ratios.log")), + ft.get_previous_ratios(world.state_path("fetch_torrents_ratios.log")), { "ubuntu-24.04-desktop-amd64.iso": 2.0, "ubuntu-23.10-desktop-amd64.iso": 2.0, }, ) - self.assertTrue(os.path.exists(os.path.join(log_dir, "fetch_torrents.log"))) + self.assertTrue(os.path.exists(world.state_path("fetch_torrents.log"))) if __name__ == "__main__": diff --git a/tests/test_fetchers.py b/tests/test_fetchers.py index a77de14..3cc1847 100644 --- a/tests/test_fetchers.py +++ b/tests/test_fetchers.py @@ -403,10 +403,7 @@ class DownloadTorrent404Tests(unittest.TestCase): same way as a genuine download failure.""" def setUp(self): - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) + self.tmp_watch_dir = helpers.use_watch_dir(self) def test_404_response_is_not_found_not_failed(self): response = helpers.fake_response(status=404) @@ -457,10 +454,7 @@ class DownloadTorrentAtomicWriteTests(unittest.TestCase): NAME = "ubuntu-24.04-desktop-amd64.iso" def setUp(self): - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) + self.tmp_watch_dir = helpers.use_watch_dir(self) def _download(self): response = helpers.fake_response(b"d8:announce3:urle" * 64) @@ -511,14 +505,12 @@ class SafeWatchPathTests(unittest.TestCase): upstream content into a path escape.""" def setUp(self): - patcher = unittest.mock.patch.object(ft, "watch_dir", "/watch") - patcher.start() - self.addCleanup(patcher.stop) + self.watch_dir = helpers.use_watch_dir(self) def test_normal_name_resolves_under_watch_dir(self): self.assertEqual( ft._safe_watch_path("ubuntu-24.04-desktop-amd64.iso"), - os.path.join("/watch", "ubuntu-24.04-desktop-amd64.iso.torrent"), + os.path.join(self.watch_dir, "ubuntu-24.04-desktop-amd64.iso.torrent"), ) def test_rejects_path_separators(self): @@ -535,10 +527,7 @@ def test_rejects_empty_or_dot_names(self): class DownloadTorrentUnsafeNameTests(unittest.TestCase): def setUp(self): - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) + self.tmp_watch_dir = helpers.use_watch_dir(self) def test_download_torrent_refuses_unsafe_name_without_any_request(self): with unittest.mock.patch.object(ft.requests, "get") as mock_get: @@ -571,7 +560,7 @@ class FetcherRequestTimeoutTests(unittest.TestCase): site can hang the daily run.""" def test_every_fetcher_and_the_download_pass_a_30_second_timeout(self): - watch = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") + helpers.use_watch_dir(self) calls = { **{name: fetcher for name, fetcher in ft.DEFAULT_FETCHERS}, "kali-old": ft.fetch_kali_old_versions, @@ -590,7 +579,6 @@ def get(url, timeouts=timeouts, **kwargs): with ( unittest.mock.patch.object(ft.requests, "get", get), - unittest.mock.patch.object(ft, "watch_dir", watch), self.assertNoLogs(ft.logger, level="CRITICAL"), ): call() diff --git a/tests/test_helpers.py b/tests/test_helpers.py index facd1ee..8065044 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -9,12 +9,17 @@ Run with: python -m unittest discover -s tests """ +import os import unittest +import unittest.mock +from datetime import date import helpers import requests import transmission_rpc +import fetch_torrents as ft + def setUpModule(): helpers.isolate_module() @@ -101,6 +106,161 @@ def test_is_a_real_torrent_reading_real_rpc_field_names(self): ) +RPC_URL = "http://localhost:9091/transmission/rpc" +TORRENT_GET = {"method": "torrent-get", "arguments": {"fields": ["name"]}} + + +class FakeTransmissionRpcTests(unittest.TestCase): + """The HTTP RPC double behaves like the daemon's CSRF handshake: no + answer without the session id it handed out in a 409.""" + + def setUp(self): + self.rpc = helpers.FakeTransmissionRpc( + helpers.FakeTransmission( + [helpers.FakeTorrent("ubuntu-24.04-desktop-amd64.iso", 1, ratio=1.5)] + ) + ) + + def _session_headers(self): + refused = self.rpc.post(RPC_URL, timeout=15) + return { + "X-Transmission-Session-Id": refused.headers["X-Transmission-Session-Id"] + } + + def test_a_request_without_the_session_id_gets_a_409_carrying_it(self): + for headers in (None, {"X-Transmission-Session-Id": "stale"}): + with self.subTest(headers=headers): + response = self.rpc.post( + RPC_URL, json=TORRENT_GET, headers=headers, timeout=15 + ) + + self.assertEqual(response.status_code, 409) + self.assertEqual( + response.headers["X-Transmission-Session-Id"], + helpers.FakeTransmissionRpc.SESSION_ID, + ) + self.assertEqual(self.rpc.auths, []) + + def test_torrent_get_lists_the_requested_fields_and_records_auth(self): + response = self.rpc.post( + RPC_URL, + json={ + "method": "torrent-get", + "arguments": {"fields": ["name", "uploadRatio"]}, + }, + headers=self._session_headers(), + auth=("alice", "hunter2"), + timeout=15, + ) + + response.raise_for_status() + self.assertEqual( + response.json(), + { + "arguments": { + "torrents": [ + {"name": "ubuntu-24.04-desktop-amd64.iso", "uploadRatio": 1.5} + ] + }, + "result": "success", + }, + ) + self.assertEqual(self.rpc.auths, [("alice", "hunter2")]) + + def test_every_post_is_recorded_in_order(self): + headers = self._session_headers() + self.rpc.post(RPC_URL, json=TORRENT_GET, headers=headers, timeout=15) + + self.assertEqual( + self.rpc.calls, + [ + unittest.mock.call(RPC_URL, timeout=15), + unittest.mock.call( + RPC_URL, json=TORRENT_GET, headers=headers, timeout=15 + ), + ], + ) + + def test_rejects_what_the_pinned_library_does_not_know(self): + headers = self._session_headers() + for payload in ( + {"method": "torrent-fetch", "arguments": {"fields": ["name"]}}, + {"method": "torrent-get", "arguments": {"fields": ["ratio"]}}, + ): + with ( + self.subTest(payload=payload), + self.assertRaisesRegex(ValueError, "transmission_rpc"), + ): + self.rpc.post(RPC_URL, json=payload, headers=headers, timeout=15) + + def test_other_urls_are_not_found(self): + response = self.rpc.post("http://localhost:9091/rpc", timeout=15) + + self.assertEqual(response.status_code, 404) + + +class PathFixtureTests(unittest.TestCase): + """use_watch_dir()/use_state_files() point fetch_torrents' module path + globals at fresh temp files for one test, and put them back after.""" + + def test_use_watch_dir_patches_watch_dir_until_the_test_ends(self): + original = ft.watch_dir + test = unittest.FunctionTestCase(lambda: None) + + path = helpers.use_watch_dir(test) + + self.assertEqual(ft.watch_dir, path) + self.assertEqual(os.listdir(path), []) + test.doCleanups() + self.assertEqual(ft.watch_dir, original) + self.assertFalse(os.path.exists(path)) + + def test_use_state_files_patches_every_state_file_until_the_test_ends(self): + names = ( + "ratio_log_file", + "ratio_history_file", + "removed_history_file", + "old_release_check_state_file", + ) + originals = [getattr(ft, name) for name in names] + test = unittest.FunctionTestCase(lambda: None) + + state = helpers.use_state_files(test) + + self.assertEqual( + [getattr(ft, name) for name in names], + [ + os.path.join(state.dir, "fetch_torrents_ratios.log"), + os.path.join(state.dir, "fetch_torrents_ratio_history.json"), + os.path.join(state.dir, "fetch_torrents_removed_history.json"), + os.path.join(state.dir, "fetch_torrents_old_release_check_state.json"), + ], + ) + self.assertEqual( + [getattr(state, name) for name in names], [getattr(ft, n) for n in names] + ) + self.assertEqual(os.listdir(state.dir), []) + test.doCleanups() + self.assertEqual([getattr(ft, name) for name in names], originals) + self.assertFalse(os.path.exists(state.dir)) + + +class RemovedTests(unittest.TestCase): + def test_matches_what_record_removal_writes(self): + self.assertEqual( + helpers.removed("ubuntu-23.10-desktop-amd64.iso", 0.4), + ft.record_removal( + {}, "ubuntu-23.10-desktop-amd64.iso", date(2026, 1, 1), "stagnant", 0.4 + ), + ) + + def test_without_a_ratio_is_a_legacy_entry(self): + self.assertEqual( + helpers.removed("devuan_daedalus", on="2025-12-01"), + {"devuan_daedalus": {"removed_date": "2025-12-01", "reason": "stagnant"}}, + ) + + class NetworkGuardTests(unittest.TestCase): def test_real_connections_are_refused(self): with self.assertRaisesRegex(RuntimeError, "must not open network"): diff --git a/tests/test_logging.py b/tests/test_logging.py index 899ec59..7f97765 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -6,7 +6,6 @@ """ import io -import json import logging import logging.handlers import os @@ -16,7 +15,6 @@ import unittest.mock import helpers -import transmission_rpc.constants import fetch_torrents as ft @@ -268,71 +266,45 @@ class LogSeedRatiosViaHttpTests(unittest.TestCase): credentials configure_transmission.py configured the daemon with when the caller doesn't explicitly override auth.""" - @staticmethod - def session_id_response(): - return helpers.fake_response( - status=409, headers={"X-Transmission-Session-Id": "abc"} - ) - - @staticmethod - def torrent_get_response(torrents=()): - return helpers.fake_response( - json.dumps({"arguments": {"torrents": list(torrents)}}) - ) + def setUp(self): + self.transmission = helpers.FakeTransmission() + self.rpc = helpers.FakeTransmissionRpc(self.transmission) + patcher = unittest.mock.patch.object(ft.requests, "post", self.rpc.post) + patcher.start() + self.addCleanup(patcher.stop) def test_requests_follow_the_transmission_rpc_protocol(self): - # The pinned transmission_rpc's own tables of methods and torrent-get - # fields stand in for the RPC spec, so a renamed field fails here. - with unittest.mock.patch.object( - ft.requests, - "post", - side_effect=[self.session_id_response(), self.torrent_get_response()], - ) as mock_post: - ft.log_seed_ratios_via_http(auth=("alice", "hunter2")) + # FakeTransmissionRpc checks the method and torrent-get fields + # against the pinned transmission_rpc's own tables, which stand in + # for the RPC spec, so a renamed field fails here. + ft.log_seed_ratios_via_http(auth=("alice", "hunter2")) - session_call, get_call = mock_post.call_args_list - self.assertEqual( - session_call, - unittest.mock.call("http://localhost:9091/transmission/rpc", timeout=15), - ) self.assertEqual( - get_call, - unittest.mock.call( - "http://localhost:9091/transmission/rpc", - json={ - "method": "torrent-get", - "arguments": {"fields": ["name", "uploadRatio"]}, - }, - headers={"X-Transmission-Session-Id": "abc"}, - auth=("alice", "hunter2"), - timeout=15, - ), - ) - payload = get_call.kwargs["json"] - self.assertEqual( - payload["method"], transmission_rpc.constants.RpcMethod.TorrentGet.value - ) - self.assertLessEqual( - set(payload["arguments"]["fields"]), - set(transmission_rpc.constants.TORRENT_GET_ARGS), + self.rpc.calls, + [ + unittest.mock.call( + "http://localhost:9091/transmission/rpc", timeout=15 + ), + unittest.mock.call( + "http://localhost:9091/transmission/rpc", + json={ + "method": "torrent-get", + "arguments": {"fields": ["name", "uploadRatio"]}, + }, + headers={"X-Transmission-Session-Id": "sid"}, + auth=("alice", "hunter2"), + timeout=15, + ), + ], ) def test_logs_ratios_highest_first_with_none_as_zero(self): - response = self.torrent_get_response( - [ - {"name": "low.iso", "uploadRatio": 0.5}, - {"name": "unset.iso", "uploadRatio": None}, - {"name": "high.iso", "uploadRatio": 2.25}, - ] - ) - with ( - unittest.mock.patch.object( - ft.requests, - "post", - side_effect=[self.session_id_response(), response], - ), - self.assertLogs(ft.logger, level="INFO") as logs, - ): + self.transmission.torrents = [ + helpers.FakeTorrent("low.iso", 1, ratio=0.5), + helpers.FakeTorrent("unset.iso", 2, ratio=None), + helpers.FakeTorrent("high.iso", 3, ratio=2.25), + ] + with self.assertLogs(ft.logger, level="INFO") as logs: ft.log_seed_ratios_via_http(auth=None) ratio_lines = [ @@ -350,67 +322,34 @@ def test_logs_ratios_highest_first_with_none_as_zero(self): ) def test_uses_configured_credentials_when_auth_not_explicitly_passed(self): - with ( - unittest.mock.patch.dict( - os.environ, - { - "TRANSMISSION_RPC_USERNAME": "alice", - "TRANSMISSION_RPC_PASSWORD": "hunter2", - }, - ), - unittest.mock.patch.object( - ft.requests, - "post", - side_effect=[ - self.session_id_response(), - self.torrent_get_response(), - ], - ) as mock_post, + with unittest.mock.patch.dict( + os.environ, + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, ): ft.log_seed_ratios_via_http() - second_call_kwargs = mock_post.call_args_list[1].kwargs - self.assertEqual(second_call_kwargs.get("auth"), ("alice", "hunter2")) + self.assertEqual(self.rpc.auths, [("alice", "hunter2")]) def test_no_auth_when_credentials_unset(self): - with ( - helpers.isolated_env(), - unittest.mock.patch.object( - ft.requests, - "post", - side_effect=[ - self.session_id_response(), - self.torrent_get_response(), - ], - ) as mock_post, - ): + with helpers.isolated_env(): ft.log_seed_ratios_via_http() - second_call_kwargs = mock_post.call_args_list[1].kwargs - self.assertIsNone(second_call_kwargs.get("auth")) + self.assertEqual(self.rpc.auths, [None]) def test_explicit_auth_argument_still_wins(self): - with ( - unittest.mock.patch.dict( - os.environ, - { - "TRANSMISSION_RPC_USERNAME": "alice", - "TRANSMISSION_RPC_PASSWORD": "hunter2", - }, - ), - unittest.mock.patch.object( - ft.requests, - "post", - side_effect=[ - self.session_id_response(), - self.torrent_get_response(), - ], - ) as mock_post, + with unittest.mock.patch.dict( + os.environ, + { + "TRANSMISSION_RPC_USERNAME": "alice", + "TRANSMISSION_RPC_PASSWORD": "hunter2", + }, ): ft.log_seed_ratios_via_http(auth=("bob", "other")) - second_call_kwargs = mock_post.call_args_list[1].kwargs - self.assertEqual(second_call_kwargs.get("auth"), ("bob", "other")) + self.assertEqual(self.rpc.auths, [("bob", "other")]) class RpcCredentialFallbackTests(unittest.TestCase): @@ -418,18 +357,14 @@ class RpcCredentialFallbackTests(unittest.TestCase): only when both TRANSMISSION_RPC_USERNAME and _PASSWORD are set.""" def _auth_sent(self, **env): - session = helpers.fake_response( - status=409, headers={"X-Transmission-Session-Id": "sid"} - ) - torrents = helpers.fake_response('{"arguments": {"torrents": []}}') + rpc = helpers.FakeTransmissionRpc(helpers.FakeTransmission()) with ( helpers.isolated_env(**env), - unittest.mock.patch.object( - ft.requests, "post", side_effect=[session, torrents] - ) as mock_post, + unittest.mock.patch.object(ft.requests, "post", rpc.post), ): ft.log_seed_ratios_via_http() - return mock_post.call_args_list[1].kwargs["auth"] + (auth,) = rpc.auths + return auth def test_username_without_password_sends_no_auth(self): self.assertIsNone(self._auth_sent(TRANSMISSION_RPC_USERNAME="alice")) @@ -437,6 +372,21 @@ def test_username_without_password_sends_no_auth(self): def test_password_without_username_sends_no_auth(self): self.assertIsNone(self._auth_sent(TRANSMISSION_RPC_PASSWORD="hunter2")) + def test_fallback_applies_the_config_credential_rule(self): + # One rule for which credentials count: Config.rpc_auth, the same + # one run() passes in explicitly. + with unittest.mock.patch.object( + ft.Config, + "rpc_auth", + new_callable=unittest.mock.PropertyMock, + return_value=("from", "config"), + ): + auth = self._auth_sent( + TRANSMISSION_RPC_USERNAME="alice", TRANSMISSION_RPC_PASSWORD="hunter2" + ) + + self.assertEqual(auth, ("from", "config")) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_old_release_demand.py b/tests/test_old_release_demand.py index 8d6c6ed..bb3fef9 100644 --- a/tests/test_old_release_demand.py +++ b/tests/test_old_release_demand.py @@ -19,6 +19,19 @@ def setUpModule(): helpers.isolate_module() +def demand(scrape, **overrides): + """has_unmet_demand(scrape) with the daily run's default thresholds + (10 leechers, 0.1 leechers per seeder, zero-seeder swarms excluded), + each overridable by keyword.""" + thresholds = { + "min_leechers": 10, + "min_leecher_ratio": 0.1, + "allow_zero_seeders": False, + **overrides, + } + return ft.has_unmet_demand(scrape, **thresholds) + + class HasUnmetDemandTests(unittest.TestCase): """Demand requires an absolute floor (enough leechers to be worth the disk/bandwidth), a relative one (leechers reach some fraction of the @@ -29,17 +42,13 @@ class HasUnmetDemandTests(unittest.TestCase): served) shouldn't either.""" def test_none_scrape_result_is_no_demand(self): - self.assertFalse( - ft.has_unmet_demand( - None, min_leechers=1, min_leecher_ratio=1.2, allow_zero_seeders=False - ) - ) + self.assertFalse(demand(None, min_leechers=1, min_leecher_ratio=1.2)) def test_missing_counts_mean_zero(self): # A tracker's scrape can omit a count; missing must read as 0, not # as some peer. self.assertIs( - ft.has_unmet_demand( + demand( {"seeders": 0}, min_leechers=1, min_leecher_ratio=0.0, @@ -47,68 +56,31 @@ def test_missing_counts_mean_zero(self): ), False, ) - self.assertIs( - ft.has_unmet_demand( - {"leechers": 5}, - min_leechers=1, - min_leecher_ratio=0.1, - allow_zero_seeders=False, - ), - False, - ) + self.assertIs(demand({"leechers": 5}, min_leechers=1), False) def test_below_absolute_threshold_is_no_demand(self): # Seeded swarm and no ratio requirement, so only the leecher floor # can turn this down. self.assertFalse( - ft.has_unmet_demand( - {"leechers": 0, "seeders": 5}, - min_leechers=1, - min_leecher_ratio=0.0, - allow_zero_seeders=False, - ) + demand({"leechers": 0, "seeders": 5}, min_leechers=1, min_leecher_ratio=0.0) ) def test_single_leecher_on_huge_swarm_is_no_demand(self): self.assertFalse( - ft.has_unmet_demand( - {"leechers": 1, "seeders": 1000}, - min_leechers=1, - min_leecher_ratio=1.2, - allow_zero_seeders=False, + demand( + {"leechers": 1, "seeders": 1000}, min_leechers=1, min_leecher_ratio=1.2 ) ) def test_meets_absolute_but_not_ratio_is_no_demand(self): # 11 leechers clears a min_leechers=10 floor, but 11 < 1.2x the 10 seeders. - self.assertFalse( - ft.has_unmet_demand( - {"leechers": 11, "seeders": 10}, - min_leechers=10, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - ) + self.assertFalse(demand({"leechers": 11, "seeders": 10}, min_leecher_ratio=1.2)) def test_meets_both_absolute_and_ratio_thresholds_is_demand(self): - self.assertTrue( - ft.has_unmet_demand( - {"leechers": 25, "seeders": 10}, - min_leechers=10, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - ) + self.assertTrue(demand({"leechers": 25, "seeders": 10}, min_leecher_ratio=1.2)) def test_exactly_at_ratio_threshold_is_demand(self): - self.assertTrue( - ft.has_unmet_demand( - {"leechers": 12, "seeders": 10}, - min_leechers=10, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - ) + self.assertTrue(demand({"leechers": 12, "seeders": 10}, min_leecher_ratio=1.2)) def test_zero_seeders_is_excluded_by_default(self): # A scrape can't tell us whether the leechers collectively hold every @@ -116,20 +88,12 @@ def test_zero_seeders_is_excluded_by_default(self): # swarm at all, so we could end up leeching something we can never # finish and therefore never seed back. Excluded unless the operator # opts into the gamble via allow_zero_seeders. - self.assertFalse( - ft.has_unmet_demand( - {"leechers": 10, "seeders": 0}, - min_leechers=10, - min_leecher_ratio=1.2, - allow_zero_seeders=False, - ) - ) + self.assertFalse(demand({"leechers": 10, "seeders": 0}, min_leecher_ratio=1.2)) def test_zero_seeders_counts_as_demand_when_explicitly_allowed(self): self.assertTrue( - ft.has_unmet_demand( + demand( {"leechers": 10, "seeders": 0}, - min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=True, ) @@ -137,9 +101,8 @@ def test_zero_seeders_counts_as_demand_when_explicitly_allowed(self): def test_zero_seeders_still_needs_the_absolute_floor_when_allowed(self): self.assertFalse( - ft.has_unmet_demand( + demand( {"leechers": 5, "seeders": 0}, - min_leechers=10, min_leecher_ratio=1.2, allow_zero_seeders=True, ) @@ -153,35 +116,14 @@ def test_lopsided_but_substantial_swarms_still_count_as_demand_at_default_ratio( # capacity or are bandwidth-limited, so leechers don't need to # approach or outnumber seeders to represent real demand - 10:100 # and 100:1000 both clearly indicate room for another seeder. - self.assertTrue( - ft.has_unmet_demand( - {"leechers": 10, "seeders": 100}, - min_leechers=1, - min_leecher_ratio=0.1, - allow_zero_seeders=False, - ) - ) - self.assertTrue( - ft.has_unmet_demand( - {"leechers": 100, "seeders": 1000}, - min_leechers=1, - min_leecher_ratio=0.1, - allow_zero_seeders=False, - ) - ) + self.assertTrue(demand({"leechers": 10, "seeders": 100}, min_leechers=1)) + self.assertTrue(demand({"leechers": 100, "seeders": 1000}, min_leechers=1)) def test_extreme_oversupply_still_fails_at_default_ratio(self): # 3 leechers on 673 seeders (an actual old Kali release scraped # during development) is still not demand, even at the more # permissive 0.1 default - it needs at least 67.3 leechers. - self.assertFalse( - ft.has_unmet_demand( - {"leechers": 3, "seeders": 673}, - min_leechers=1, - min_leecher_ratio=0.1, - allow_zero_seeders=False, - ) - ) + self.assertFalse(demand({"leechers": 3, "seeders": 673}, min_leechers=1)) class EvaluateOldReleaseDemandTests(unittest.TestCase): @@ -311,10 +253,7 @@ def test_meets_absolute_floor_but_already_well_seeded_is_not_kept(self): class CheckOldReleasesForDemandTests(unittest.TestCase): def setUp(self): - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) + self.tmp_watch_dir = helpers.use_watch_dir(self) def test_adds_only_torrents_with_unmet_demand(self): # Neither name matches LOW_DEMAND_PATTERN - that filtering is @@ -505,10 +444,7 @@ class CheckOldReleasesForDemandStateTests(unittest.TestCase): network, and must record a fresh check for anything it does evaluate.""" def setUp(self): - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) + self.tmp_watch_dir = helpers.use_watch_dir(self) def test_recently_checked_candidate_is_skipped_without_network_call(self): name = "kali-linux-2026.1-installer-amd64.iso" @@ -542,7 +478,7 @@ def test_recently_checked_candidate_is_skipped_without_network_call(self): def test_previously_removed_candidate_is_skipped_without_network_call(self): name = "kali-linux-2026.1-installer-amd64.iso" candidates = {name: "https://example.invalid/x.torrent"} - removed_history = {name: {"removed_date": "2026-01-01", "reason": "stagnant"}} + removed_history = helpers.removed(name) with ( unittest.mock.patch.dict( @@ -675,10 +611,7 @@ class CheckOldReleasesAtomicWriteTests(unittest.TestCase): NAME = "kali-linux-2026.1-installer-amd64.iso" def setUp(self): - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) + self.tmp_watch_dir = helpers.use_watch_dir(self) def _check(self): candidates = {self.NAME: "https://example.invalid/x.torrent"} @@ -897,10 +830,7 @@ class CheckOldReleasesSociableTests(unittest.TestCase): TRACKER = "udp://tracker.example:6969" def setUp(self): - self.tmp_watch_dir = helpers.make_temp_dir(self, "fetch_torrents_test_watch_") - patcher = unittest.mock.patch.object(ft, "watch_dir", self.tmp_watch_dir) - patcher.start() - self.addCleanup(patcher.stop) + self.tmp_watch_dir = helpers.use_watch_dir(self) self.tracker = OldReleaseTracker() self.tracker.patch(self) @@ -947,8 +877,8 @@ def test_every_outcome_is_counted_and_later_candidates_still_checked(self): f.write(b"already seeding") check_state = {recent: {"last_checked": "2026-01-05", "leechers": 0}} removed_history = { - name: {"removed_date": "2025-12-01", "reason": "stagnant"} - for name in (removed_a, removed_b) + **helpers.removed(removed_a, on="2025-12-01"), + **helpers.removed(removed_b, on="2025-12-01"), } with self.assertLogs(ft.logger, level="DEBUG") as logs: diff --git a/tests/test_persistence.py b/tests/test_persistence.py index 1afc604..76a8076 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -9,6 +9,8 @@ import contextlib import io import os +import subprocess +import sys import unittest from typing import ClassVar @@ -155,8 +157,34 @@ def test_state_files_live_in_the_log_dir(self): ), ) - def test_torrents_are_written_to_the_watch_volume(self): - self.assertEqual(ft.watch_dir, "/watch") + def _watch_dir_on_import(self, **env): + """ft.watch_dir as a fresh interpreter importing the module under + `env` sees it (this process imported it long ago).""" + script = ( + "import sys\n" + f"sys.path.insert(0, {helpers.REPO_ROOT!r})\n" + "import fetch_torrents as ft\n" + "print(ft.watch_dir)\n" + ) + result = subprocess.run( + [sys.executable, "-W", "ignore", "-c", script], + check=False, + env=helpers.scrubbed_environ(**env), + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout.rstrip("\n") + + def test_torrents_are_written_to_the_watch_volume_by_default(self): + self.assertEqual(self._watch_dir_on_import(), "/watch") + + def test_fetch_torrents_watch_dir_moves_the_watch_dir(self): + self.assertEqual( + self._watch_dir_on_import(FETCH_TORRENTS_WATCH_DIR="/srv/torrents"), + "/srv/torrents", + ) class StateFileFormatTests(unittest.TestCase): diff --git a/tests/test_ratio_gate.py b/tests/test_ratio_gate.py index 4c59e3d..e5c3d29 100644 --- a/tests/test_ratio_gate.py +++ b/tests/test_ratio_gate.py @@ -112,6 +112,77 @@ class ShouldFetchTorrentRatioKeyTests(unittest.TestCase): actually reports (which include the arch suffix and .iso extension), for all supported distros.""" + # (previous version as Transmission reports it, its ratio, candidate, + # whether the candidate is fetched). A low ratio blocks only if the + # previous version is found; a high one shows the floor is applied, not + # just any ratio. + CASES = ( + ( + "ubuntu-23.10-desktop-amd64.iso", + 0.5, + "ubuntu-24.04-desktop-amd64.iso", + False, + ), + ("ubuntu-23.10-desktop-amd64.iso", 1.5, "ubuntu-24.04-desktop-amd64.iso", True), + ("debian-12.4.0-amd64-DVD-1.iso", 0.5, "debian-12.5.0-amd64-DVD-1.iso", False), + ("debian-12.4.0-amd64-DVD-1.iso", 1.5, "debian-12.5.0-amd64-DVD-1.iso", True), + ( + "kali-linux-2023.4-installer-amd64.iso", + 0.3, + "kali-linux-2024.1-installer-amd64.iso", + False, + ), + ( + "kali-linux-2023.4-installer-amd64.iso", + 1.5, + "kali-linux-2024.1-installer-amd64.iso", + True, + ), + ( + "archlinux-2024.04.01-x86_64.iso", + 0.2, + "archlinux-2024.05.01-x86_64.iso", + False, + ), + ( + "archlinux-2024.04.01-x86_64.iso", + 1.5, + "archlinux-2024.05.01-x86_64.iso", + True, + ), + ( + "linuxmint-22.2-cinnamon-64bit.iso", + 0.4, + "linuxmint-22.3-cinnamon-64bit.iso", + False, + ), + ( + "linuxmint-22.2-cinnamon-64bit.iso", + 1.5, + "linuxmint-22.3-cinnamon-64bit.iso", + True, + ), + ( + "Fedora-Workstation-Live-x86_64-43", + 0.4, + "Fedora-Workstation-Live-x86_64-44", + False, + ), + ( + "Fedora-Workstation-Live-x86_64-43", + 1.5, + "Fedora-Workstation-Live-x86_64-44", + True, + ), + ) + + def test_previous_version_is_found_and_its_ratio_gates_the_candidate(self): + for previous, ratio, candidate, fetched in self.CASES: + with self.subTest(candidate=candidate, previous_ratio=ratio): + self.assertIs( + ft.should_fetch_torrent(candidate, {previous: ratio}), fetched + ) + def test_lubuntu_is_ratio_gated_on_its_own_previous_release(self): ratios = { "lubuntu-23.10-desktop-amd64.iso": 0.2, @@ -121,66 +192,6 @@ def test_lubuntu_is_ratio_gated_on_its_own_previous_release(self): ft.should_fetch_torrent("lubuntu-24.04-desktop-amd64.iso", ratios) ) - def test_finds_previous_version_and_honors_low_ratio_ubuntu(self): - ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.5} - self.assertFalse( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) - ) - - def test_finds_previous_version_and_honors_low_ratio_debian(self): - ratios = {"debian-12.4.0-amd64-DVD-1.iso": 0.5} - self.assertFalse( - ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios) - ) - - def test_finds_previous_version_and_honors_low_ratio_kali(self): - ratios = {"kali-linux-2023.4-installer-amd64.iso": 0.3} - self.assertFalse( - ft.should_fetch_torrent("kali-linux-2024.1-installer-amd64.iso", ratios) - ) - - def test_finds_previous_version_and_honors_low_ratio_arch(self): - ratios = {"archlinux-2024.04.01-x86_64.iso": 0.2} - self.assertFalse( - ft.should_fetch_torrent("archlinux-2024.05.01-x86_64.iso", ratios) - ) - - def test_finds_previous_version_and_allows_high_ratio_ubuntu(self): - ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.5} - self.assertTrue( - ft.should_fetch_torrent("ubuntu-24.04-desktop-amd64.iso", ratios) - ) - - def test_finds_previous_version_and_allows_high_ratio_debian(self): - ratios = {"debian-12.4.0-amd64-DVD-1.iso": 1.5} - self.assertTrue( - ft.should_fetch_torrent("debian-12.5.0-amd64-DVD-1.iso", ratios) - ) - - def test_finds_previous_version_and_honors_low_ratio_mint(self): - ratios = {"linuxmint-22.2-cinnamon-64bit.iso": 0.4} - self.assertFalse( - ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios) - ) - - def test_finds_previous_version_and_allows_high_ratio_mint(self): - ratios = {"linuxmint-22.2-cinnamon-64bit.iso": 1.5} - self.assertTrue( - ft.should_fetch_torrent("linuxmint-22.3-cinnamon-64bit.iso", ratios) - ) - - def test_finds_previous_version_and_honors_low_ratio_fedora(self): - ratios = {"Fedora-Workstation-Live-x86_64-43": 0.4} - self.assertFalse( - ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios) - ) - - def test_finds_previous_version_and_allows_high_ratio_fedora(self): - ratios = {"Fedora-Workstation-Live-x86_64-43": 1.5} - self.assertTrue( - ft.should_fetch_torrent("Fedora-Workstation-Live-x86_64-44", ratios) - ) - class ShouldFetchTorrentRemovedHistoryFallbackTests(unittest.TestCase): """Once a previous version is removed by cleanup, Transmission stops @@ -191,13 +202,7 @@ class ShouldFetchTorrentRemovedHistoryFallbackTests(unittest.TestCase): removed_history (record_removal()'s output) is the fallback source.""" def test_falls_back_to_removed_history_ratio_when_not_live(self): - removed_history = { - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.4, - }, - } + removed_history = helpers.removed("ubuntu-23.10-desktop-amd64.iso", 0.4) self.assertFalse( ft.should_fetch_torrent( "ubuntu-24.04-desktop-amd64.iso", @@ -207,13 +212,7 @@ def test_falls_back_to_removed_history_ratio_when_not_live(self): ) def test_removed_history_ratio_above_floor_allows_fetch(self): - removed_history = { - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-01-01", - "reason": "keep_only_latest", - "ratio": 1.5, - }, - } + removed_history = helpers.removed("ubuntu-23.10-desktop-amd64.iso", 1.5) self.assertTrue( ft.should_fetch_torrent( "ubuntu-24.04-desktop-amd64.iso", @@ -226,13 +225,7 @@ def test_live_ratio_takes_priority_over_removed_history(self): # Same name in both is not expected in practice (a torrent Transmission # still reports hasn't been removed), but live data winning if it ever # happens is the safer default. - removed_history = { - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.1, - }, - } + removed_history = helpers.removed("ubuntu-23.10-desktop-amd64.iso", 0.1) ratios = {"ubuntu-23.10-desktop-amd64.iso": 1.5} self.assertTrue( ft.should_fetch_torrent( @@ -243,13 +236,7 @@ def test_live_ratio_takes_priority_over_removed_history(self): ) def test_low_live_ratio_is_not_overridden_by_high_removed_history(self): - removed_history = { - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 1.5, - } - } + removed_history = helpers.removed("ubuntu-23.10-desktop-amd64.iso", 1.5) ratios = {"ubuntu-23.10-desktop-amd64.iso": 0.4} self.assertFalse( ft.should_fetch_torrent( @@ -262,12 +249,7 @@ def test_low_live_ratio_is_not_overridden_by_high_removed_history(self): def test_removed_history_entry_without_ratio_is_ignored(self): # Legacy entries recorded before this field existed have no 'ratio' # key - must not crash, and must not count as a usable prior version. - removed_history = { - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - }, - } + removed_history = helpers.removed("ubuntu-23.10-desktop-amd64.iso") # Skipped up front, not by tripping over the missing key and logging # a parse-error warning. with self.assertNoLogs(ft.logger, level="WARNING"): @@ -280,15 +262,8 @@ def test_removed_history_entry_without_ratio_is_ignored(self): def test_skipped_removed_history_entry_does_not_hide_later_ones(self): removed_history = { - "ubuntu-23.04-desktop-amd64.iso": { - "removed_date": "2025-06-01", - "reason": "stagnant", - }, - "ubuntu-23.10-desktop-amd64.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.4, - }, + **helpers.removed("ubuntu-23.04-desktop-amd64.iso"), + **helpers.removed("ubuntu-23.10-desktop-amd64.iso", 0.4), } self.assertFalse( ft.should_fetch_torrent( @@ -299,13 +274,7 @@ def test_skipped_removed_history_entry_does_not_hide_later_ones(self): ) def test_removed_history_from_a_different_distro_is_not_consulted(self): - removed_history = { - "debian-12.4.0-amd64-DVD-1.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.1, - }, - } + removed_history = helpers.removed("debian-12.4.0-amd64-DVD-1.iso", 0.1) self.assertTrue( ft.should_fetch_torrent( "ubuntu-24.04-desktop-amd64.iso", @@ -339,13 +308,7 @@ def test_already_holding_this_exact_release_is_always_fetched(self): self.assertTrue(ft.should_fetch_torrent("devuan_excalibur", ratios)) def test_falls_back_to_removed_history_when_no_live_ratio(self): - removed_history = { - "devuan_daedalus": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.4, - }, - } + removed_history = helpers.removed("devuan_daedalus", 0.4) self.assertFalse( ft.should_fetch_torrent( "devuan_excalibur", ratios={}, removed_history=removed_history @@ -353,13 +316,7 @@ def test_falls_back_to_removed_history_when_no_live_ratio(self): ) def test_other_distros_removed_history_not_consulted_for_devuan(self): - removed_history = { - "debian-12.4.0-amd64-DVD-1.iso": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.1, - }, - } + removed_history = helpers.removed("debian-12.4.0-amd64-DVD-1.iso", 0.1) self.assertTrue( ft.should_fetch_torrent( "devuan_excalibur", ratios={}, removed_history=removed_history @@ -427,13 +384,7 @@ def test_devuan_ignores_non_devuan_ratios(self): self.assertFalse(ft.should_fetch_torrent("devuan_excalibur", ratios)) def test_devuan_removed_history_for_the_same_release_is_not_a_predecessor(self): - removed_history = { - "devuan_excalibur": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.1, - } - } + removed_history = helpers.removed("devuan_excalibur", 0.1) self.assertTrue( ft.should_fetch_torrent( "devuan_excalibur", ratios={}, removed_history=removed_history @@ -441,13 +392,7 @@ def test_devuan_removed_history_for_the_same_release_is_not_a_predecessor(self): ) def test_devuan_live_ratio_beats_removed_history(self): - removed_history = { - "devuan_chimaera": { - "removed_date": "2026-01-01", - "reason": "stagnant", - "ratio": 0.1, - } - } + removed_history = helpers.removed("devuan_chimaera", 0.1) self.assertTrue( ft.should_fetch_torrent( "devuan_excalibur", From 209292c66c8cdec48f99b64c2b8726ec1744d277 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 18:13:21 +0200 Subject: [PATCH 41/46] Kill the survivors the re-measure found; raise the coverage ratchet to 97 The full mutmut run for task 38 found three new behavioural survivors: - fetch_kali_latest's "if not results" branch was unreachable (_kali_torrent_urls always returns its fixed list), so it is removed. - The old-release write-failure branch from task 32 now has a test where a skip comes before the failed write and a later candidate still gets added, which kills "skipped = 1" and "continue -> break". Coverage is 99.13% (branch), so fail_under goes from 94 to 97. Co-Authored-By: Claude Opus 5.5 --- fetch_torrents.py | 4 --- pyproject.toml | 8 +++--- tests/test_old_release_demand.py | 46 ++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/fetch_torrents.py b/fetch_torrents.py index 540c5bd..aade4a5 100644 --- a/fetch_torrents.py +++ b/fetch_torrents.py @@ -677,10 +677,6 @@ def fetch_kali_latest(): name = os.path.basename(turl).replace(".torrent", "") results[name] = turl - if not results: - logger.warning("No Kali torrents found.") - return False - return results except Exception as exc: diff --git a/pyproject.toml b/pyproject.toml index 2c58304..be7cd28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,10 +39,10 @@ omit = ["tests/*"] [tool.coverage.report] show_missing = true -# Ratchet: about 2 points under the measured total (96.78% on 2026-10-19), -# so coverage can only go up. Raise it as coverage improves; never lower it -# to get a change through. -fail_under = 94 +# Ratchet: about 2 points under the measured total (99.13% on 2026-10-19, +# task 38), so coverage can only go up. Raise it as coverage improves; never +# lower it to get a change through. +fail_under = 97 [tool.mutmut] # mutmut<3 (see requirements-dev.txt). It edits these files in place while diff --git a/tests/test_old_release_demand.py b/tests/test_old_release_demand.py index bb3fef9..1957b52 100644 --- a/tests/test_old_release_demand.py +++ b/tests/test_old_release_demand.py @@ -659,6 +659,52 @@ def test_next_run_adds_it_after_a_failed_write(self): self.assertEqual(f.read(), b"torrent bytes" * 32) self.assertEqual(os.listdir(self.tmp_watch_dir), [f"{self.NAME}.torrent"]) + def test_a_failed_write_is_counted_and_later_candidates_still_added(self): + names = [ + f"kali-linux-2026.1-installer-{arch}.iso" for arch in ("amd64", "arm64") + ] + # An unsafe name is skipped first, so the failed write is a second skip. + candidates = {"kali-linux/../evil": "https://example.invalid/evil.torrent"} + candidates |= { + name: f"https://example.invalid/{name}.torrent" for name in names + } + real_write = ft._write_watch_file + + def write(dest, data): + if os.path.basename(dest) == f"{names[0]}.torrent": + raise OSError(28, "No space left on device") + real_write(dest, data) + + with ( + unittest.mock.patch.dict( + ft.OLD_RELEASE_DISCOVERY, {"kali": lambda: candidates}, clear=True + ), + unittest.mock.patch.object( + ft, + "evaluate_old_release_demand", + return_value=( + True, + b"torrent bytes", + {"tracker": "t", "seeders": 1, "leechers": 30, "completed": 0}, + ), + ), + unittest.mock.patch.object(ft, "_write_watch_file", write), + ): + added, skipped, state = ft.check_old_releases_for_demand( + ["kali"], + include_low_demand=False, + min_leechers=1, + min_leecher_ratio=0.0, + allow_zero_seeders=False, + check_state={}, + removed_history={}, + today=date(2026, 1, 8), + ) + + self.assertEqual((added, skipped), (1, 2)) + self.assertEqual(os.listdir(self.tmp_watch_dir), [f"{names[1]}.torrent"]) + self.assertEqual(list(state), [names[1]]) + class OldReleaseTracker: """Plays both network boundaries of the old-release check: requests.get From 01b77ce6e0b907e440d6f58dbbade091db6a4bc9 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 18:45:39 +0200 Subject: [PATCH 42/46] Record the round-2 baseline (task 38) Full mutmut 2.5.1 runs at 209292c: fetch_torrents.py 99.3% (1040/1047), torrent_scrape.py 99.6% (248/249), configure_transmission.py 96.8% (90/93), critical set still 100%. Every behavioural survivor left is an accepted equivalent, now including the FIPS-only usedforsecurity flag. Branch coverage is 99.1% against the new fail_under = 97. The mutmut docs note the stale-.pyc and Windows-encoding pitfalls hit here. Co-Authored-By: Claude Opus 5.5 --- .claude/CLAUDE.md | 2 +- CONTRIBUTING.md | 5 +++- tasks/README.md | 65 +++++++++++++++++++++++++++++------------------ 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index c020ed8..f6f4591 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -13,7 +13,7 @@ Testing rules and the import-stubbing pattern live in the root [CLAUDE.md](../CL - Typecheck: `mypy` (files and settings in `pyproject.toml`) - CI's `test` job runs ruff check (defaults + bandit `S`), ruff format --check, mypy, shellcheck and hadolint before the tests; tool versions are pinned in `requirements-dev.txt`, the shellcheck/hadolint images in `.github/workflows/docker-image.yml` (commands in CONTRIBUTING.md) - Coverage: `coverage run -m unittest discover -s tests` then `coverage report` (branch coverage; config in `pyproject.toml`, which also sets the `fail_under` ratchet CI enforces) -- Mutation: `mutmut run` (mutmut<3; config in `pyproject.toml`, see CONTRIBUTING.md) +- Mutation: `mutmut run` (mutmut<3; config in `pyproject.toml`, see CONTRIBUTING.md; run with `PYTHONDONTWRITEBYTECODE=1`) - Dev deps: `pip install -r requirements-dev.txt` (includes `requirements-test.txt`) ## Active agent templates diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1390fdc..eca736a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,7 +79,10 @@ mutmut show # show one mutant's diff ``` mutmut edits the source files in place while it runs, so run it on a clean -working tree (or a copy of the repo). A full run takes a while (~40 minutes on +working tree (or a copy of the repo, e.g. a `git worktree`), with +`PYTHONDONTWRITEBYTECODE=1`: a mutant that keeps the file's size can +otherwise be run from a stale `.pyc`. On Windows, also set `PYTHONUTF8=1` +when redirecting its output to a file. A full run takes a while (~40 minutes on Windows). ### Handling surviving mutants diff --git a/tasks/README.md b/tasks/README.md index 3154171..1a0a2b7 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -13,19 +13,23 @@ so search for the function name if they have drifted. ## Targets ("a better place") -Re-measured on 2026-10-19 after tasks 10–26 (task 27): 383 tests green. -Mutation scores are honest (string-literal-only survivors excluded). They -come from full mutmut 2.5.1 runs with the whole suite as the runner. - -| Metric | Baseline (84761d3) | Target | Now | -| --- | --- | --- | --- | -| Mutation score, critical `fetch_torrents.py` functions¹ | 70.9% | ≥ 90% | **100%** (258/258) | -| Mutation score, critical `torrent_scrape.py` functions | 82.6% | ≥ 90% | **100%** (247/247, every function) | -| Mutation score, `configure_transmission.py` | 89.8% | — | 96.7% (87/90) | -| Mutation score, all of `fetch_torrents.py` | — | — | 88.6% (915/1033) | -| Branch coverage, `fetch_torrents.py` | 75% | ≥ 90% | **96%** (total 96.8%; CI `fail_under = 94`) | -| CI gates | unittest only | unittest + ruff + mypy + container smoke | **met**: also the in-image unit suite, hostile env, coverage ratchet | -| Scraper drift detection | none | recorded pages in the merge check + a scheduled live job | **met**: `tests/test_scrapers.py` + `.github/workflows/drift.yml` | +Re-measured on 2026-10-19 at 209292c after round 2 (task 38): 493 tests +green. Mutation scores are honest: string-literal-only survivors (log and +error text) are excluded, and a timeout counts as a kill. They come from +full mutmut 2.5.1 runs over all three modules, with the whole suite as the +runner. The task 27 numbers are kept for comparison. + +| Metric | Baseline (84761d3) | Target | Task 27 | Now (209292c) | +| --- | --- | --- | --- | --- | +| Mutation score, critical `fetch_torrents.py` functions¹ | 70.9% | ≥ 90% | 100% (258/258) | **100%** | +| Mutation score, critical `torrent_scrape.py` functions | 82.6% | ≥ 90% | 100% (247/247) | **100%** | +| Mutation score, all of `fetch_torrents.py` | — | — | 88.6% (915/1033) | **99.3%** (1040/1047) | +| Mutation score, all of `torrent_scrape.py` | — | — | — | **99.6%** (248/249) | +| Mutation score, `configure_transmission.py` | 89.8% | — | 96.7% (87/90) | **96.8%** (90/93) | +| Branch coverage, total | — | — | 96.8% | **99.1%** (CI `fail_under = 97`) | +| Branch coverage, `fetch_torrents.py` | 75% | ≥ 90% | 96% | **99.2%** | +| CI gates | unittest only | unittest + ruff + mypy + container smoke | met | **met**, plus ruff `S`, shellcheck, hadolint, a smoke check that the daily fetch run starts, and CI Python and deps pinned to Alpine's | +| Scraper drift detection | none | recorded pages in the merge check + a scheduled live job | met | **met**; `scripts/live_check.py` itself is now tested offline | ¹ The ratio gate, the cleanup planners and the parsers they rely on: `should_fetch_torrent`, `_should_fetch_devuan`, `parse_version_type`, @@ -34,17 +38,25 @@ come from full mutmut 2.5.1 runs with the whole suite as the runner. `update_ratio_history`, `plan_disk_pressure_cleanup`, `has_unmet_demand`, `should_recheck_old_release`, `_safe_watch_path`, `filter_low_demand`. -Remaining survivors (not critical logic, left as follow-ups) are mostly in -the I/O and orchestration layer: -- `run` (9), `_fetch_latest` (5), `_check_old_releases` (2) -- `check_old_releases_for_demand` (21), `evaluate_old_release_demand` (7) -- `enforce_disk_usage_limit` (10), `cleanup_stagnant_torrents` (5), - `cleanup_old_versions` (3) -- `configure_logging` (6), `parse_log_level` (3), `wait_for_transmission_rpc` (6) -- the `save_*` temp-file handling (2 each) - -The list is in a scratch run, not the repo: rerun `mutmut run` and -`mutmut results` to see the current set. +Every remaining behavioural survivor is an accepted equivalent (listed under +"Accepted equivalent mutants" below): +- `fetch_torrents.py` (7): `logger.propagate = None`, the two + `.get("seeders", 0)` defaults, the three `getattr(torrent, "ratio", 0.0)` + defaults, and `break` → `continue` in `enforce_disk_usage_limit`. +- `torrent_scrape.py` (1): `usedforsecurity=False` → `True` on the info-hash + SHA-1, which only differs on a FIPS-restricted OpenSSL build. +- `configure_transmission.py` (3): `save_settings`' `indent` and + `sort_keys`, plus the `__main__` guard on the last line, which mutmut + leaves untested. + +Farley Score was not re-measured; it was 8.41 at 7080bbd. Rerun +`/test-health` for a fresh report. + +The survivor list is in a scratch run, not the repo: rerun `mutmut run` and +`mutmut results` to see the current set. Run it with +`PYTHONDONTWRITEBYTECODE=1` (mutmut rewrites files in place, and a mutant +that keeps the file size can otherwise run from a stale `.pyc`), and on +Windows with `PYTHONUTF8=1` if output is redirected to a file. ## Order @@ -104,7 +116,7 @@ mutation score is 95.4% (1027/1076), up from 88.6%, with the critical set still | 35 | [Consolidate fixtures and the path-global seam](35-fixture-consolidation.md) | P3 | NO_REFACTOR → REFACTOR_REQUIRED | 34 | done | | 36 | [Remaining `fetch_torrents.py` mutation survivors](36-remaining-mutation-survivors.md) | P3 | NO_REFACTOR | — | done | | 37 | [Toolchain: smoke test, shellcheck/hadolint, ruff S, pins](37-toolchain-and-q4.md) | P2 | CI | — | done | -| 38 | [Re-measure, raise the coverage ratchet, record the baseline](38-remeasure-and-ratchet.md) | P3 | — | 28–37 | open | +| 38 | [Re-measure, raise the coverage ratchet, record the baseline](38-remeasure-and-ratchet.md) | P3 | — | 28–37 | done | ### Accepted equivalent mutants (task 36) @@ -119,6 +131,9 @@ mutants) counts as clean: - `break` → `continue` once `enforce_disk_usage_limit` is under the threshold: every later candidate fails the same check, so the only difference is extra disk-usage reads. +- `usedforsecurity=False` → `True` on `compute_info_hash`'s SHA-1 (task 38): + identical except on a FIPS-restricted OpenSSL, where `True` would refuse + SHA-1. The flag just says the hash isn't a security control. In `configure_transmission.py` (task 31), `save_settings`'s `indent=4` and `sort_keys=True` are accepted too. transmission-daemon owns settings.json and From 1fc226258f430fc4d04a0812c5f90737dd9a25eb Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 19:11:18 +0200 Subject: [PATCH 43/46] Fix ruff EXE001 on CI: mark scripts executable, drop module shebang fetch_torrents.py and configure_transmission.py are run as scripts, so record them as 100755 in git. torrent_scrape.py is only imported, so its shebang goes instead. Ruff on Windows can't see the exec bit, which is why this only failed on the Linux runner. Co-Authored-By: Claude Opus 5.5 --- configure_transmission.py | 0 fetch_torrents.py | 0 torrent_scrape.py | 1 - 3 files changed, 1 deletion(-) mode change 100644 => 100755 configure_transmission.py mode change 100644 => 100755 fetch_torrents.py diff --git a/configure_transmission.py b/configure_transmission.py old mode 100644 new mode 100755 diff --git a/fetch_torrents.py b/fetch_torrents.py old mode 100644 new mode 100755 diff --git a/torrent_scrape.py b/torrent_scrape.py index 89a2cf7..3079c91 100644 --- a/torrent_scrape.py +++ b/torrent_scrape.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 """Bencode parsing and BitTorrent tracker "scrape" support. Scrape is a separate endpoint from announce: it returns aggregate From cda44f6c68c527d2a7f293e74cc2a66cd45a9e15 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 19:16:26 +0200 Subject: [PATCH 44/46] Test that shebang files are executable in git; fix entrypoint.sh Ruff's EXE001 only sees the exec bit on Linux, so a 100644 shebang file passes lint on Windows and fails on CI. test_repo_hygiene reads modes from the git index, which works on every OS. It also caught entrypoint.sh, which the Dockerfile chmods but git stored as 100644. Both CLAUDE.md files note the rule and the update-index fix. Co-Authored-By: Claude Opus 5.5 --- .claude/CLAUDE.md | 1 + CLAUDE.md | 4 +- entrypoint.sh | 0 tests/test_repo_hygiene.py | 76 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) mode change 100644 => 100755 entrypoint.sh create mode 100644 tests/test_repo_hygiene.py diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index f6f4591..f989e51 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -12,6 +12,7 @@ Testing rules and the import-stubbing pattern live in the root [CLAUDE.md](../CL - Lint/format: `ruff check .` / `ruff format .` - Typecheck: `mypy` (files and settings in `pyproject.toml`) - CI's `test` job runs ruff check (defaults + bandit `S`), ruff format --check, mypy, shellcheck and hadolint before the tests; tool versions are pinned in `requirements-dev.txt`, the shellcheck/hadolint images in `.github/workflows/docker-image.yml` (commands in CONTRIBUTING.md) +- Exec bit: a tracked file with a `#!` shebang must be `100755` in git, or ruff's EXE001 fails on CI. Ruff can't see the bit on Windows, so it passes locally; `tests/test_repo_hygiene.py` catches it instead. Fix with `git update-index --chmod=+x ` (`chmod` does nothing with `core.fileMode=false`), or drop the shebang from a module that is only imported - Coverage: `coverage run -m unittest discover -s tests` then `coverage report` (branch coverage; config in `pyproject.toml`, which also sets the `fail_under` ratchet CI enforces) - Mutation: `mutmut run` (mutmut<3; config in `pyproject.toml`, see CONTRIBUTING.md; run with `PYTHONDONTWRITEBYTECODE=1`) - Dev deps: `pip install -r requirements-dev.txt` (includes `requirements-test.txt`) diff --git a/CLAUDE.md b/CLAUDE.md index 030d7ab..5f143fb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -69,7 +69,9 @@ faked with `helpers.fake_response()` (a real `requests.Response`) or `test_persistence.py`, `test_logging.py`, `test_old_release_demand.py`, `test_config.py` (`load_config()`, `parse_bool()`, `make_client_factory()`, the run summary) and `test_daily_run.py` (`run()`, `__main__`). -`test_helpers.py` tests the doubles in `tests/helpers.py` themselves. +`test_helpers.py` tests the doubles in `tests/helpers.py` themselves; +`test_repo_hygiene.py` checks the repository itself (shebang files are +executable in git). Business logic that needs to be unit-testable should be kept as pure functions (taking plain data in, returning plain data out) separate from diff --git a/entrypoint.sh b/entrypoint.sh old mode 100644 new mode 100755 diff --git a/tests/test_repo_hygiene.py b/tests/test_repo_hygiene.py new file mode 100644 index 0000000..a15c8a4 --- /dev/null +++ b/tests/test_repo_hygiene.py @@ -0,0 +1,76 @@ +"""Checks on the repository itself rather than on a module's behavior: +every tracked file with a shebang is executable in git, so ruff's EXE001 +(which only sees the exec bit on Linux, i.e. on CI) can't fail a push that +passed lint on Windows. + +Run with: python -m unittest discover -s tests +""" + +import os +import shutil +import subprocess +import unittest + +import helpers + + +def setUpModule(): + helpers.isolate_module() + + +def shebang_files_not_executable(entries): + """The paths in `entries` ((git mode, path, first two bytes) triples) + that start with a shebang but aren't recorded as 100755.""" + return [path for mode, path, head in entries if head == b"#!" and mode != "100755"] + + +def tracked_files(git): + """(git mode, path, first two bytes) for every file in the index.""" + listing = subprocess.run( + [git, "ls-files", "-s", "-z"], + cwd=helpers.REPO_ROOT, + check=True, + capture_output=True, + timeout=30, + ).stdout.decode() + for record in filter(None, listing.split("\0")): + meta, path = record.split("\t", 1) + mode = meta.split()[0] + full = os.path.join(helpers.REPO_ROOT, path) + if mode == "120000" or not os.path.isfile(full): + continue # a symlink, or deleted in the working tree + with open(full, "rb") as f: + yield mode, path, f.read(2) + + +class ShebangFilesNotExecutableTests(unittest.TestCase): + def test_flags_a_shebang_file_without_the_exec_bit(self): + self.assertEqual( + shebang_files_not_executable([("100644", "a.py", b"#!")]), + ["a.py"], + ) + + def test_passes_an_executable_shebang_file(self): + self.assertEqual(shebang_files_not_executable([("100755", "a.py", b"#!")]), []) + + def test_ignores_files_without_a_shebang(self): + self.assertEqual(shebang_files_not_executable([("100644", "a.py", b'""')]), []) + + +class RepositoryTests(unittest.TestCase): + def test_every_tracked_shebang_file_is_executable_in_git(self): + git = shutil.which("git") + if git is None or not os.path.exists(os.path.join(helpers.REPO_ROOT, ".git")): + self.skipTest("not a git checkout") + + self.assertEqual( + shebang_files_not_executable(tracked_files(git)), + [], + "mark scripts executable with `git update-index --chmod=+x " + "` (chmod does nothing on Windows), or drop the shebang " + "from a module that is only imported", + ) + + +if __name__ == "__main__": + unittest.main() From 3b6347fe5440d6a14d1c2a8af0115b2166342e05 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 19:16:47 +0200 Subject: [PATCH 45/46] Adding more tasks --- tasks/39-ship-the-tested-image.md | 17 +++++++++++ .../40-https-scrape-and-non-utf8-failures.md | 19 +++++++++++++ tasks/41-narrow-literal-exclusion-rule.md | 13 +++++++++ tasks/42-seal-the-network-guard.md | 15 ++++++++++ tasks/43-remove-first-party-doubling.md | 21 ++++++++++++++ tasks/44-require-rpc-auth.md | 13 +++++++++ tasks/45-scrape-torrent-seams.md | 15 ++++++++++ tasks/46-remaining-error-paths.md | 18 ++++++++++++ tasks/47-test-consolidation.md | 17 +++++++++++ tasks/48-ci-leftovers.md | 14 ++++++++++ tasks/49-offline-tests-for-fixture-scripts.md | 24 ++++++++++++++++ tasks/50-remeasure-round-3.md | 12 ++++++++ tasks/README.md | 28 +++++++++++++++++++ 13 files changed, 226 insertions(+) create mode 100644 tasks/39-ship-the-tested-image.md create mode 100644 tasks/40-https-scrape-and-non-utf8-failures.md create mode 100644 tasks/41-narrow-literal-exclusion-rule.md create mode 100644 tasks/42-seal-the-network-guard.md create mode 100644 tasks/43-remove-first-party-doubling.md create mode 100644 tasks/44-require-rpc-auth.md create mode 100644 tasks/45-scrape-torrent-seams.md create mode 100644 tasks/46-remaining-error-paths.md create mode 100644 tasks/47-test-consolidation.md create mode 100644 tasks/48-ci-leftovers.md create mode 100644 tasks/49-offline-tests-for-fixture-scripts.md create mode 100644 tasks/50-remeasure-round-3.md diff --git a/tasks/39-ship-the-tested-image.md b/tasks/39-ship-the-tested-image.md new file mode 100644 index 0000000..26830fe --- /dev/null +++ b/tasks/39-ship-the-tested-image.md @@ -0,0 +1,17 @@ +# 39 — Push the image that was tested, not a rebuild + +**Priority**: P1 · **Class**: CI · **Depends on**: — + +From `.dev-team-reports/cd-test-architecture-linux-iso-seeder.md` (migration steps 1–2) and `test-health-2026-10-19-r2.md`. + +## Evidence +- `.github/workflows/docker-image.yml`, `build` job: "Build image for smoke test" builds `linux-iso-seeder:smoke` and "Smoke-test the image" tests it. Then "Build and push latest image for main branch" and "Build and push versioned image for tag" **rebuild from scratch** with `docker/build-push-action`. +- The Dockerfile uses `FROM alpine:latest` and unpinned `apk add` (`Dockerfile:4`, `:12-13`), so the rebuild can pull different packages than the image that passed. The rebuild also passes `VERSION=...` while the smoke build passes no build args, so even the layers differ. +- Nothing checks the pushed tag after the push. The weekly `published-image` job in `drift.yml` is the first check. + +## Acceptance criteria +1. The image is built once per workflow run, with the final `VERSION` build arg (`latest` on main, the tag version on tags). The smoke test runs against that image ID. +2. The push publishes that exact image: `docker tag` + `docker push`, or `build-push-action` with `load: true` and then a push of the same image. No second `docker build` after the smoke test. For tags, `latest` and `` point to the same digest. +3. After the push, a step pulls the pushed tag, checks that its digest matches the image that was smoke-tested, and fails the job if it doesn't. +4. The in-image unit-test stage (`docker build --target test .`) still runs before the build, on the same base. If the cache doesn't already share the base between the stages, pinning `alpine` by digest in the Dockerfile (bumped by Dependabot) is an acceptable way to make "tested == shipped". +5. CI-only change, so the test-first rule doesn't apply. Prove it with one green run on a branch and one dry tag push (or a `workflow_dispatch` path that skips the push). diff --git a/tasks/40-https-scrape-and-non-utf8-failures.md b/tasks/40-https-scrape-and-non-utf8-failures.md new file mode 100644 index 0000000..5eb4ea3 --- /dev/null +++ b/tasks/40-https-scrape-and-non-utf8-failures.md @@ -0,0 +1,19 @@ +# 40 — https scrape URLs and non-UTF-8 tracker failure reasons + +**Priority**: P2 · **Class**: NO_REFACTOR · **Depends on**: — + +Found by re-running mutmut on `torrent_scrape.py` at 01b77ce. These survivors only change a string literal, so the round-2 scoring rule excluded them (see task 41). + +## Evidence +- `torrent_scrape.py:165` `derive_scrape_url`: the mutant `("http", "XXhttpsXX")` survives. `DeriveScrapeUrlTests` (`tests/test_torrent_scrape.py:250`) has no `https://` case, so a change that drops https support passes. An https-only tracker would then never be scraped, and the release would be classed as "no demand". +- `torrent_scrape.py:168`: `rfind("/")` → `rfind("XX/XX")` survives. No case has "announce" in a *non-final* path segment (e.g. `http://example.com/announce/x` must give `None`), so the "last segment only" rule is unpinned. +- `torrent_scrape.py:341` (HTTP `failure reason`) and the UDP error-text decode (`decode('utf-8', 'replace')` in the UDP response parser): the mutant `'replace'` → `'XXreplaceXX'` survives. No test sends a non-UTF-8 failure message, so a change that raises `LookupError`/`UnicodeDecodeError` instead of a readable tracker error passes. + +## Acceptance criteria +1. `DeriveScrapeUrlTests` cases table gains: + - `https://example.com/announce` → `https://example.com/scrape` + - `https://example.com/x/announce.php?a=1` → `https://example.com/x/scrape.php?a=1` + - `http://example.com/announce/x` → `None` +2. An HTTP scrape whose bencoded `failure reason` is invalid UTF-8 (e.g. `b"bad \xff"`) raises `BencodeError` with the replacement character in the message. The same for a UDP action-3 error response, with `UdpTrackerError`. +3. Confirm each new test kills its mutant (`mutmut run --paths-to-mutate torrent_scrape.py` then `mutmut results`, or `mutmut apply ` and run the test). +4. If a test shows a real bug, fix the production code test-first. diff --git a/tasks/41-narrow-literal-exclusion-rule.md b/tasks/41-narrow-literal-exclusion-rule.md new file mode 100644 index 0000000..e081158 --- /dev/null +++ b/tasks/41-narrow-literal-exclusion-rule.md @@ -0,0 +1,13 @@ +# 41 — Only exclude message-text literals from the mutation score + +**Priority**: P3 · **Class**: process (docs) · **Depends on**: 40 + +## Evidence +- `tasks/README.md` ("Targets": "string-literal-only survivors (log and error text) are excluded") and `CONTRIBUTING.md` ("Log and error-message strings") treat **every** string-literal survivor as message text. +- In `torrent_scrape.py`, 5 of the 22 literal survivors at 01b77ce were not message text: a URL scheme (`"https"`), a path separator (`"/"`), `quote_from_bytes(safe="")`, and two codec error handlers (`"replace"`). Two of them hid real gaps (task 40). + +## Acceptance criteria +1. Both documents say that only literals inside log calls and exception messages are excluded. Any other literal survivor is triaged like a behavioural survivor: test it, simplify the code, or accept it as an equivalent with a reason. +2. `safe=""` → `safe="XXXX"` in `build_scrape_request_url` is added to "Accepted equivalent mutants" with its reason: trackers decode `%58` and `X` the same, so leaving `X` unescaped changes nothing. +3. The `configure_transmission.py` default-`""` survivors (`value or ""`, `env.get(..., "")`) are added as accepted equivalents: `"XXXX"` fails to parse just as `""` does. +4. Docs only, so the test-first rule doesn't apply. diff --git a/tasks/42-seal-the-network-guard.md b/tasks/42-seal-the-network-guard.md new file mode 100644 index 0000000..aa2d0ec --- /dev/null +++ b/tasks/42-seal-the-network-guard.md @@ -0,0 +1,15 @@ +# 42 — Make the network guard refuse UDP and DNS; stop the stderr leak + +**Priority**: P2 · **Class**: NO_REFACTOR · **Depends on**: — + +## Evidence +- `tests/helpers.py:440` `block_network()` patches only `socket.socket.connect` and `connect_ex`. A real `sendto` (UDP tracker scrape), `sendall` on an unconnected socket, or `socket.getaddrinfo` (DNS) still leaves the machine. CLAUDE.md says the guard "refuses any real network connection". +- `tests/test_helpers.py:265` `test_real_connections_are_refused` passes, but leaves an unclosed socket (ResourceWarning). +- `tests/test_logging.py:248` `test_lines_carry_a_timestamp_and_the_logger_does_not_propagate` leaves the configured StreamHandler on the real stderr, so every run prints `WARNING: plain warning`. + +## Acceptance criteria +1. Test first in `NetworkGuardTests`: `sendto` on a real UDP socket and `socket.getaddrinfo("example.com", 80)` both raise the guard's `RuntimeError`. Confirm they fail before the change. +2. `block_network()` also patches `socket.socket.sendto` and `socket.getaddrinfo` (and `socket.create_connection` if it doesn't go through the patched `connect`). Mocked sockets stay unaffected. +3. Tests that already stub `ts.socket.getaddrinfo` (`test_torrent_scrape.py:686`) still pass. Their patch replaces the guard for that test, which is intended. +4. The guard test closes its sockets (`with socket.socket() as s:`). `python -W error::ResourceWarning -m unittest test_helpers` is clean. +5. The logging test points the stream handler at a `StringIO` (as `_route()` does). A full suite run prints nothing outside unittest's own output. diff --git a/tasks/43-remove-first-party-doubling.md b/tasks/43-remove-first-party-doubling.md new file mode 100644 index 0000000..a9d9606 --- /dev/null +++ b/tasks/43-remove-first-party-doubling.md @@ -0,0 +1,21 @@ +# 43 — Remove first-party doubling the sociable tests already cover; drop the LOW_VALUE test + +**Priority**: P2 · **Class**: NO_REFACTOR · **Depends on**: — + +From `.dev-team-reports/test-design-2026-10-19-r2.md`: error rows 1, 2, 3 and 5, warnings 9 and 11. These rows are why test-review fails. + +## Evidence +- `tests/test_old_release_demand.py:285`: 11 tests patch our own `evaluate_old_release_demand` (with a branching `fake_evaluate` at `:266`) and `ft._write_watch_file` (`:691`). `CheckOldReleasesSociableTests` (from `:869`) already runs the real path with only the network faked. +- `ft.OLD_RELEASE_DISCOVERY` is replaced with `patch.dict` 13× in that file, plus `tests/test_daily_run.py:607` and `:658`. +- `scrape_torrent` is doubled at `test_old_release_demand.py:135`, `:173`, `:201`, `:231`, `:752` and `test_daily_run.py:630`. That is a legitimate boundary (it opens sockets) but it carries no waiver comment. +- `EvaluateOldReleaseDemandTests` (`:129-251`) repeats `HasUnmetDemandTests` and the boundary tests (`:768-862`) through a patched `scrape_torrent`. +- `tests/test_logging.py:378` `test_fallback_applies_the_config_credential_rule` patches the pure `Config.rpc_auth` with a `PropertyMock`. `tests/test_config.py:187` already pins the rule through real objects. +- **LOW_VALUE:** `tests/test_config.py:227` `test_passes_configured_credentials` has no branching in the code under test and only asserts a mock call. `tests/test_daily_run.py:829` asserts the same `Client(...)` call one layer up. + +## Acceptance criteria +1. Before deleting anything, list each patched old-release test and the sociable or boundary test that covers its behaviour. Move any case with no cover onto `OldReleaseTracker` (real `evaluate_old_release_demand`, faked `requests.get`/tracker), asserting outcomes, not calls. Fail writes at the leaf (`helpers.disk_full_open`), not by patching `_write_watch_file`. +2. Then delete the superseded patched tests and `fake_evaluate`. +3. `OLD_RELEASE_DISCOVERY`: serve the kali/fedora pages through the existing `requests.get` fake, or add a `discovery=` parameter to `check_old_releases_for_demand` (test-first, like `run(fetchers=)`). +4. Every remaining `scrape_torrent` double has a one-line reason comment (`# boundary double: opens UDP/HTTP tracker sockets`). +5. Delete `test_logging.py:378` and `test_config.py:227`. Optionally fold `test_config.py:240`/`:247` into one table test. +6. Run mutmut on `fetch_torrents.py` afterwards: the score must not drop (99.3%, critical set 100%). A survivor that appears was only killed by a deleted test; restore a sociable test for it. diff --git a/tasks/44-require-rpc-auth.md b/tasks/44-require-rpc-auth.md new file mode 100644 index 0000000..c4f43ae --- /dev/null +++ b/tasks/44-require-rpc-auth.md @@ -0,0 +1,13 @@ +# 44 — Make `log_seed_ratios_via_http`'s `auth` required + +**Priority**: P3 · **Class**: REFACTOR_REQUIRED · **Depends on**: 43 (row 5 deletes one fallback test) + +## Evidence +- `fetch_torrents.py:1188`: `auth=None` re-reads `os.environ` through `_rpc_credentials`, so `None` can't mean "send no auth". Both callers always pass `auth=config.rpc_auth`. +- Five tests (`tests/test_logging.py:324-388`) exist only for that fallback. + +## Acceptance criteria +1. Test first: `log_seed_ratios_via_http(auth=None)` sends no credentials even when `TRANSMISSION_RPC_USERNAME`/`PASSWORD` are set in the environment. It fails today. +2. Make `auth` a required keyword argument with no env fallback (or keep `None` meaning "no auth"), and remove the env read. +3. Delete the fallback-only tests. Keep the run-level credential tests (`tests/test_daily_run.py:487`, `:496`). +4. `scripts/` callers, if any, pass `auth` explicitly. mypy passes. diff --git a/tasks/45-scrape-torrent-seams.md b/tasks/45-scrape-torrent-seams.md new file mode 100644 index 0000000..ff59c9e --- /dev/null +++ b/tasks/45-scrape-torrent-seams.md @@ -0,0 +1,15 @@ +# 45 — `scrape_torrent` seams for address resolution and the UDP scrape + +**Priority**: P3 · **Class**: REFACTOR_REQUIRED · **Depends on**: 42 + +From `test-design-2026-10-19-r2.md` error row 4. + +## Evidence +- `tests/test_torrent_scrape.py:779` onwards patches our own `ts._resolve_addresses` (9 sites) and `ts.scrape_udp_tracker` (`:815`…`:1007`). The latter is verified only by `assert_called_once_with`/`assert_not_called`. +- `new_transaction_id=` (task 29) already shows the injection style. + +## Acceptance criteria +1. `scrape_torrent` takes keyword-only `resolve=` and `udp_scrape=` parameters, defaulting to the real functions. Add a test first that passes a fake for each and fails before the parameters exist. +2. The patching tests switch to the parameters, and assert the returned outcome dict instead of call counts. Where the wire matters, keep one test per path that goes through the autospec socket and asserts `sendto`. +3. No `mock.patch.object(ts, "_resolve_addresses" | "scrape_udp_tracker")` is left in `tests/`. +4. The `torrent_scrape.py` mutation score stays at 100% excluding message literals (task 41's rule). diff --git a/tasks/46-remaining-error-paths.md b/tasks/46-remaining-error-paths.md new file mode 100644 index 0000000..efb2b25 --- /dev/null +++ b/tasks/46-remaining-error-paths.md @@ -0,0 +1,18 @@ +# 46 — The last untested error paths; raise the ratchet + +**Priority**: P3 · **Class**: NO_REFACTOR · **Depends on**: — + +## Evidence +Branch coverage at 01b77ce is 99% (9 lines and 5 partial branches missed): +- `fetch_torrents.py:1833-1834`: `run()` logs `Could not query Transmission: ...` when `log_seed_ratios_via_http` raises (401/500, or no session header → `KeyError`). No test covers it. +- `fetch_torrents.py:458`, `:475-478` (`get_previous_ratios`): the parse-error warnings for stored and removed-history ratio entries, and the other-distro branch in the removed-history loop. +- `fetch_torrents.py:543-544`: cleaning up the temp file itself fails after a failed atomic write. +- `fetch_torrents.py:1290-1291`: `_remove_watch_file` hits an `OSError` other than `FileNotFoundError` (e.g. a directory with the torrent's name). +- `torrent_scrape.py:91`: a bencoded dict with a non-bytes key. `:147->145` and `:151->156`: duplicate announce-list URLs and a primary `announce` already in the list. +- The four JSON stores' `OSError` branch (`tests/test_persistence.py:60`; test-design row 29). + +## Acceptance criteria +1. A `RunTests` case where `FakeTransmissionRpc` returns 500 (or raises): the error is logged with that message, and cleanup and the run summary still happen. +2. One test per remaining path above, asserting the observable outcome (return value, log message, file state), not just that the line ran. +3. `coverage report` shows 100% on all three modules, or each remaining miss has a `# pragma: no cover` with its reason. +4. Raise `fail_under` in `pyproject.toml` to about 1 point under the CI-measured total. Check the CI number first, as in task 38. diff --git a/tasks/47-test-consolidation.md b/tasks/47-test-consolidation.md new file mode 100644 index 0000000..7d2d506 --- /dev/null +++ b/tasks/47-test-consolidation.md @@ -0,0 +1,17 @@ +# 47 — Consolidate tests: oversized scenarios, conditional logic, duplicate HTTP fakes + +**Priority**: P3 · **Class**: NO_REFACTOR · **Depends on**: 43 (it removes some of the duplicates first) + +From `test-design-2026-10-19-r2.md` rows 12–16, 19, 23, 33 and 35. Rows 16, 33 and 35 were already open in round 1. + +## Acceptance criteria +1. **One HTTP fake.** The routes-dict `requests.get` fake with a 404 default is written 4× (`FakeWorld` in `tests/test_daily_run.py:89`, `OldReleaseTracker` at `:733`, `tests/test_live_check.py:309`, `RecordedPages`). Make it a single `helpers.FakeHttp`, and move `OldReleaseTracker`/`swarm()` into `helpers.py`. Add a `test_helpers.py` test for the new double. +2. **Split the eager twin scenarios.** `test_daily_run.py:687` `test_config_and_saved_state_drive_the_check` should keep one wiring assertion per Config field. `test_old_release_demand.py:902` (11 candidates, Farley 5.74) should split counts, saved state and logs over a shared Creation Method. +3. **No conditional test logic** in `test_torrent_scrape.py:937`: put the expected requests in the cases table instead of computing them with `if` at `:951`. +4. **`MainWiringTests`** (`test_daily_run.py:772`): `addCleanup` instead of try/finally. Drop the facts `RunTests` already pins (`rpc.auths` at `:832` duplicates `:494`). +5. **Mystery Guest.** `route_ubuntu()` (`test_daily_run.py:114`) hides the 404 default and a pre-existing watch file that `test_counts_each_download_outcome` depends on. Inline that setup, or use named per-outcome routes. +6. **Logging reset.** Replace the hand save/restore of the private `ft._configured_handlers` (`test_logging.py:30`) with a public `reset_logging()` in `fetch_torrents.py` (test-first). +7. **Delete duplicates:** + - the mid-range twins that the boundary tests now cover (`test_cleanup.py:103`, `:392` and the others listed in the test-design scratch tables) + - the hand-written page tests in `test_fetchers.py` that repeat `RecordedSnapshotTests`. Keep only cases a recording can't express: unsorted rows, 99 vs 100, duplicates +8. Mutation scores don't drop. Re-run mutmut on any module whose tests were deleted. diff --git a/tasks/48-ci-leftovers.md b/tasks/48-ci-leftovers.md new file mode 100644 index 0000000..c4cb792 --- /dev/null +++ b/tasks/48-ci-leftovers.md @@ -0,0 +1,14 @@ +# 48 — CI leftovers: drift-job Python pin, actionlint + +**Priority**: P3 · **Class**: CI · **Depends on**: — + +From `.dev-team-reports/cd-test-architecture-linux-iso-seeder.md` (migration steps 3 and 5). + +## Evidence +- `drift.yml:29`, `:57`, `:109` use `python-version: "3.x"`, while the `test` job pins 3.14 to match Alpine. A drift failure could come from a Python bump, not from upstream drift. +- No workflow linter. A typo in an `if:` or a `${{ }}` expression only shows up when that path runs, which for the tag-push steps is rare. + +## Acceptance criteria +1. The drift jobs pin the same Python minor as the `test` job. Add a comment to bump them together. +2. The `test` job runs a pinned `rhysd/actionlint` image over `.github/workflows/`, with any ignores justified inline. CONTRIBUTING.md lists the command. +3. CI-only, so the test-first rule doesn't apply. diff --git a/tasks/49-offline-tests-for-fixture-scripts.md b/tasks/49-offline-tests-for-fixture-scripts.md new file mode 100644 index 0000000..685fa80 --- /dev/null +++ b/tasks/49-offline-tests-for-fixture-scripts.md @@ -0,0 +1,24 @@ +# 49 — Offline tests for `record_fixtures.py` and `transmission_roundtrip.py` + +**Priority**: P3 · **Class**: REFACTOR_REQUIRED · **Depends on**: — + +From the test-design advisor (`test-design-2026-10-19-r2.md`, "Design recommendations"). + +## Evidence +- `scripts/record_fixtures.py` and `scripts/transmission_roundtrip.py` have no tests. The drift job relies on both: `record_fixtures` re-records the pages when a live scraper fails, and `transmission_roundtrip` is the real-daemon check. +- Blockers: + - `record_fixtures.record` calls `requests.get` directly and writes to a module-global directory + - `record_fixtures.main` calls `datetime.now` directly + - `transmission_roundtrip.wait_for` uses `time.time`/`time.sleep` and a global `TIMEOUT_SECONDS` +- Today only the drift job checks that `record_fixtures.ENDPOINTS` matches the URLs in the recorded `.meta.json` fixtures, and only after a live failure. + +## Acceptance criteria +1. First, a test with no seam needed: every `ENDPOINTS` URL matches the `url` in its fixture's `.meta.json`, and every fixture has an endpoint. +2. Test-first, add `record(..., get=requests.get, out_dir=...)` and `main(..., now=...)`. Tests: + - `record` writes the body and meta under `/.` in a temp dir, using `helpers.fake_response` + - `record` passes HTTP errors through + - the `main` filter calls `record` for the requested `(distro, slug)` set (spy) +3. Test-first, add `wait_for(..., clock=time.monotonic, sleep=time.sleep, timeout=...)`, the same shape as `wait_for_transmission_rpc`. Test the success, timeout and `SystemExit` paths with a fake clock that `sleep` advances. +4. `make_torrent`: assert its structure with `torrent_scrape.bdecode`. +5. `transmission_roundtrip.main` stays untested offline: it is the integration check itself. +6. Test modules follow the CLAUDE.md conventions (`import helpers`, `setUpModule` calling `helpers.isolate_module()`). diff --git a/tasks/50-remeasure-round-3.md b/tasks/50-remeasure-round-3.md new file mode 100644 index 0000000..e87f53f --- /dev/null +++ b/tasks/50-remeasure-round-3.md @@ -0,0 +1,12 @@ +# 50 — Re-measure round 3; optional seeding-policy table + +**Priority**: P3 · **Class**: — · **Depends on**: 39–49 (or whichever are done) + +## Steps +1. Full mutmut runs on all three modules with task 41's rule (only message literals excluded). Every other survivor is killed or listed under "Accepted equivalent mutants" with its reason. +2. Coverage. Raise `fail_under` if task 46 moved it, after checking the CI number. +3. Update the round-3 rows and the Targets table in `tasks/README.md`: tests, coverage, mutation scores, and Farley (8.33 across 493 tests at 01b77ce). +4. Re-run `/test-health` and compare with `.dev-team-reports/test-health-2026-10-19-r2.md`. + +## Optional: business-readable seeding policy (Q2) +Carried over from task 38, still not done. Add a plain table in the README or a `docs/` page listing when an ISO is fetched, kept, removed or re-seeded. Each row names the test that pins it, so a non-developer can review the policy without a BDD runner. diff --git a/tasks/README.md b/tasks/README.md index 1a0a2b7..6e81481 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -148,3 +148,31 @@ duplicated zero-size-disk guard moved into `_used_percent()`. Suggested order: 32 (a real bug), then 28, 30 and 33 (resilience and drift-job trust), then 37, then 36, then the clean-up tasks 29, 31, 34 and 35, and finally 38. + +## Round 3 (from the second 2026-10-19 test-health review) + +Source: `.dev-team-reports/test-health-2026-10-19-r2.md`, plus the updated +`cd-test-architecture-linux-iso-seeder.md` and `test-design-2026-10-19-r2.md` +(local, gitignored). At 01b77ce: 493 tests green, 99.1% branch coverage, Farley 8.33, +and no flakiness across repeated, per-module and 8 shuffled-order runs. Re-running +mutmut on `torrent_scrape.py` and `configure_transmission.py` reproduced the +task 38 scores. It also showed that excluding *every* string-literal survivor +hides real gaps (tasks 40 and 41). + +| # | Task | Priority | Class | Depends on | Status | +| --- | --- | --- | --- | --- | --- | +| 39 | [Push the image that was tested, not a rebuild](39-ship-the-tested-image.md) | P1 | CI | — | open | +| 40 | [https scrape URLs and non-UTF-8 tracker failure reasons](40-https-scrape-and-non-utf8-failures.md) | P2 | NO_REFACTOR | — | open | +| 41 | [Only exclude message-text literals from the mutation score](41-narrow-literal-exclusion-rule.md) | P3 | docs | 40 | open | +| 42 | [Network guard refuses UDP and DNS; stop the stderr leak](42-seal-the-network-guard.md) | P2 | NO_REFACTOR | — | open | +| 43 | [Remove first-party doubling; drop the LOW_VALUE test](43-remove-first-party-doubling.md) | P2 | NO_REFACTOR | — | open | +| 44 | [Make `log_seed_ratios_via_http`'s `auth` required](44-require-rpc-auth.md) | P3 | REFACTOR_REQUIRED | 43 | open | +| 45 | [`scrape_torrent` seams for resolution and UDP scrape](45-scrape-torrent-seams.md) | P3 | REFACTOR_REQUIRED | 42 | open | +| 46 | [The last untested error paths; raise the ratchet](46-remaining-error-paths.md) | P3 | NO_REFACTOR | — | open | +| 47 | [Consolidate tests: oversized scenarios, conditional logic, HTTP fakes](47-test-consolidation.md) | P3 | NO_REFACTOR | 43 | open | +| 48 | [CI leftovers: drift-job Python pin, actionlint](48-ci-leftovers.md) | P3 | CI | — | open | +| 49 | [Offline tests for `record_fixtures.py` / `transmission_roundtrip.py`](49-offline-tests-for-fixture-scripts.md) | P3 | REFACTOR_REQUIRED | — | open | +| 50 | [Re-measure round 3; optional seeding-policy table](50-remeasure-round-3.md) | P3 | — | 39–49 | open | + +Suggested order: 39 (untested bits reach users), then 40 and 41 (a hidden scrape gap +and the rule that hid it), 42, 43 then 44 and 47, 45, 46, 49 and 48, and finally 50. From 6d46a78382a9f9b73a3c1e04a969007517025914 Mon Sep 17 00:00:00 2001 From: Squazz Date: Mon, 19 Oct 2026 19:43:27 +0200 Subject: [PATCH 46/46] Add task 51: guard the old-release step in the daily run run() calls _check_old_releases() unguarded, so with FETCH_TORRENTS_CHECK_OLD_RELEASES=true one invalid FETCH_TORRENTS_OLD_RELEASE_* value aborts every daily run before ratio logging, cleanup and the disk-usage limit. Found while reviewing this branch's production changes; the behavior predates the branch. Co-Authored-By: Claude Opus 5.5 --- tasks/51-guard-old-release-step.md | 15 +++++++++++++++ tasks/README.md | 7 +++++-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 tasks/51-guard-old-release-step.md diff --git a/tasks/51-guard-old-release-step.md b/tasks/51-guard-old-release-step.md new file mode 100644 index 0000000..902611b --- /dev/null +++ b/tasks/51-guard-old-release-step.md @@ -0,0 +1,15 @@ +# 51 — A failing old-release check must not abort the daily run + +**Priority**: P2 (bug) · **Class**: NO_REFACTOR · **Depends on**: — + +## Evidence +- `fetch_torrents.py:1827`: `run()` calls `_check_old_releases()` with no `try`. Every later step does have one: ratio logging (`Could not query Transmission: ...`), cleanup (`Could not clean up old versions: ...`) and the disk limit (`Could not enforce disk usage limit: ...`). +- With `FETCH_TORRENTS_CHECK_OLD_RELEASES=true`, an unparseable `FETCH_TORRENTS_OLD_RELEASE_MIN_LEECHERS`, `_MIN_LEECHER_RATIO` or `_RECHECK_DAYS` makes `config.require()` (`:1742`) raise `ValueError` out of `run()`. The same happens with any other exception from the step, for example an `OSError` from `save_old_release_check_state()`. The run then skips ratio logging, cleanup, the disk-usage limit and the run summary. +- `entrypoint.sh:59-62` reruns the script every 24 hours with the same environment, so a single bad value disables cleanup and the disk-usage limit on every run. Nothing is logged by this script's own logger. The traceback only goes to the container's stderr, and `/downloads` can fill up. +- The behavior predates the task 18 refactor, which kept it on purpose. `tests/test_daily_run.py:754` `test_invalid_old_release_setting_aborts_the_run` pins it, and its comment says the step "has never been guarded". + +## Acceptance criteria +1. Test first: replace `test_invalid_old_release_setting_aborts_the_run` with `test_invalid_old_release_setting_skips_the_check_and_continues`, modelled on `test_invalid_cleanup_setting_skips_cleanup_and_continues` (`:412`). With `FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS="weekly"`, `run()` returns normally. It logs `Could not check old releases: FETCH_TORRENTS_OLD_RELEASE_RECHECK_DAYS: ...`, still logs seed ratios, still runs cleanup (a superseded torrent is removed) and still logs the run summary. Confirm the test fails today because the `ValueError` escapes. +2. A second test where the step fails partway through (for example, the check-state file can't be written; fail it at the leaf with `helpers.disk_full_open`, not by patching our own function). The run continues the same way, and the summary still reports the counts that were set. Decide and pin what `old_added`/`old_skipped` show when the step fails before producing counts (probably `None`, which `format_run_summary` already treats as "check not run"). +3. Wrap the `_check_old_releases()` call in `run()` in the same `try`/`except Exception` + `logger.error(...)` pattern as the other steps. Update `run()`'s docstring so it says every step logs its own failure and lets the rest carry on. +4. Full suite, ruff, ruff format and mypy are green. Coverage stays at or above `fail_under`. diff --git a/tasks/README.md b/tasks/README.md index 6e81481..8480bda 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -172,7 +172,10 @@ hides real gaps (tasks 40 and 41). | 47 | [Consolidate tests: oversized scenarios, conditional logic, HTTP fakes](47-test-consolidation.md) | P3 | NO_REFACTOR | 43 | open | | 48 | [CI leftovers: drift-job Python pin, actionlint](48-ci-leftovers.md) | P3 | CI | — | open | | 49 | [Offline tests for `record_fixtures.py` / `transmission_roundtrip.py`](49-offline-tests-for-fixture-scripts.md) | P3 | REFACTOR_REQUIRED | — | open | -| 50 | [Re-measure round 3; optional seeding-policy table](50-remeasure-round-3.md) | P3 | — | 39–49 | open | +| 50 | [Re-measure round 3; optional seeding-policy table](50-remeasure-round-3.md) | P3 | — | 39–49, 51 | open | +| 51 | [A failing old-release check must not abort the daily run](51-guard-old-release-step.md) | P2 (bug) | NO_REFACTOR | — | open | -Suggested order: 39 (untested bits reach users), then 40 and 41 (a hidden scrape gap +Task 51 came from a review of this branch's production changes, not from the test-health report. + +Suggested order: 39 (untested bits reach users), then 51 (a bad setting silently disables cleanup), then 40 and 41 (a hidden scrape gap and the rule that hid it), 42, 43 then 44 and 47, 45, 46, 49 and 48, and finally 50.