From ac18ee956eaf0c029a15e780095349a49470fb20 Mon Sep 17 00:00:00 2001 From: Gustavo Almeida Barrios Date: Thu, 3 Sep 2026 15:49:38 -0300 Subject: [PATCH 001/325] =?UTF-8?q?Guardar=20e=20verificar=20os=20modelos?= =?UTF-8?q?=20Whisper=20antes=20de=20a=20transcri=C3=A7=C3=A3o=20existir?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit O catálogo dos seis modelos é um contrato com um repositório remoto, então carrega o que permite conferi-lo: a revisão (commit sha, nunca `main`), o tamanho exato de cada arquivo e o sha256 do model.bin. Sem a revisão pinada, uma publicação nova invalidaria os bytes verificados ou trocaria os pesos em silêncio; sem os tamanhos por arquivo, "modelo corrompido" seria uma promessa que nada consegue cumprir. A decisão de dispositivo é pura e recebe a sondagem como dado, porque uma sondagem feita no início do programa descreve uma máquina que já não existe quando o modelo sobe — é contra memória livre que se planeja. E o int8 nunca chega a uma placa de compute capability 12.0 ou maior: o CTranslate2 o desabilitou no sm_120, que é justamente a geração de quem pediu a função. O download escreve em .part, confere durante a transmissão e só então renomeia, de modo que nenhum instante deixa no diretório um conjunto que a checagem de "instalado" aceite sem estar íntegro. Só arquivo com digest é retomado por Range: os auxiliares são conferidos apenas por tamanho, e um prefixo velho emendado numa cauda nova passaria por todas as verificações, inclusive a mais estrita. Como o model.bin é 99,885% do modelo, a regra não custa quase nada. A raiz vive em global_dir() para que 3 GB não sejam duplicados por conta, e por isso mesmo as mutações passam por um lock entre processos. Raízes são comparadas com canonical_dir e não com abspath — no Windows o abspath não dobra maiúsculas nem resolve junction, e a versão anterior apagava o modelo das duas raízes tratando-o como duplicata de si mesmo. A camada de TLS vem junto porque é o que faz qualquer download funcionar numa máquina cujo HTTPS é interceptado: ali o certifi não conhece a raiz injetada e falha igual contra o Hugging Face, o nodejs.org e a API do GitHub — o download do Node e o auto-updater já sofriam disso. A sessão passa a validar pelo repositório de certificados do sistema quando dá, e cai no comportamento anterior em silêncio quando não dá. Co-Authored-By: Claude Opus 5 --- client/coord_locks.py | 28 + client/core/tls_trust.py | 129 ++ client/core/transcription/__init__.py | 7 + client/core/transcription/device.py | 493 ++++++ client/core/transcription/errors.py | 126 ++ client/core/transcription/model_catalog.py | 268 ++++ client/core/transcription/model_store.py | 1024 ++++++++++++ client/languages/en-US.json | 27 +- client/languages/es-ES.json | 27 +- client/languages/pl.json | 27 +- client/languages/pt-BR.json | 27 +- client/languages/pt-PT.json | 27 +- client/ui/dialogs/node_download.py | 5 +- client/updater.py | 8 +- pytest.ini | 4 + tests/test_coord_locks.py | 36 + tests/test_tls_trust.py | 241 +++ tests/test_transcription_core.py | 809 ++++++++++ tests/test_transcription_model_store.py | 1690 ++++++++++++++++++++ tests/test_updater_checksum.py | 20 +- 20 files changed, 5007 insertions(+), 16 deletions(-) create mode 100644 client/core/tls_trust.py create mode 100644 client/core/transcription/__init__.py create mode 100644 client/core/transcription/device.py create mode 100644 client/core/transcription/errors.py create mode 100644 client/core/transcription/model_catalog.py create mode 100644 client/core/transcription/model_store.py create mode 100644 tests/test_tls_trust.py create mode 100644 tests/test_transcription_core.py create mode 100644 tests/test_transcription_model_store.py diff --git a/client/coord_locks.py b/client/coord_locks.py index f977b75c5..e0a66cb9e 100644 --- a/client/coord_locks.py +++ b/client/coord_locks.py @@ -345,6 +345,34 @@ def sessions_lock(global_dir: str, timeout: float = 20.0) -> NamedLock: ) +def models_lock(models_root: str, lock_dir: str, timeout: float = 60.0) -> NamedLock: + """Guards the SHARED Whisper models directory (download / repair / remove / + move). That directory lives under global_dir() precisely so every account + process shares one copy of a model that can be 3 GB, which also means two + accounts can ask for the same model at the same moment: both would open the + same ``model.bin.part``, interleave their writes, both fail the digest, and + each one's cleanup would delete the file the other is still streaming into. + + Keyed on the models root rather than on ``lock_dir``, because part 5 lets + the user move that root anywhere — two roots are two independent critical + sections. The lock FILE, though, is written to ``lock_dir`` (the global + data dir) under a name derived from that key: the flock fallback creates it, + and creating a file inside a folder the user chose would both litter it and + defeat model_store's "leave no empty directory of ours behind" cleanup. + + One known gap, documented rather than solved: canonical_dir() cannot resolve + the true spelling of a directory that does not exist yet, so two accounts + racing the very FIRST download into a root they spell differently can end up + on two lock names. It closes as soon as the directory exists, i.e. after one + cycle, and only that one download is unguarded.""" + h = _hash_dir(models_root) + return NamedLock( + f"WinZappModels_{h}", + os.path.join(lock_dir, f"models_{h}.lock"), + timeout=timeout, + ) + + # ── fork safety (POSIX) ────────────────────────────────────────────────────── def _reset_locks_after_fork() -> None: """Reset all inherited lock state in a forked child (GPT r4 #5). diff --git a/client/core/tls_trust.py b/client/core/tls_trust.py new file mode 100644 index 000000000..0a66afa66 --- /dev/null +++ b/client/core/tls_trust.py @@ -0,0 +1,129 @@ +"""HTTPS that verifies the way the rest of Windows does. + +`requests` verifies against the CA list baked into `certifi`, which is a fixed +file and knows nothing about the machine it is running on. On a machine whose +TLS is intercepted locally — an antivirus with HTTPS scanning switched on, a +corporate proxy, an MDM profile — the certificate the app is shown is signed by +a root that only exists in *Windows'* own store, and every download fails with +CERTIFICATE_VERIFY_FAILED. Measured here against huggingface.co, nodejs.org and +api.github.com: all three fail, and all three succeed once verification goes +through the system store. That is the same reason pip carries `truststore`. + +It matters more than it looks: the three things this affects are the Whisper +models (up to 3 GB, and the transcription simply cannot happen without them), +the portable Node.js runtime WinZapp needs to talk to WhatsApp at all, and the +updater. A user in that situation currently sees three unrelated features fail +for reasons none of them explain. + +**Never raises for its own sake.** `truststore` missing, or refusing to build a +context on some future Windows, leaves the caller with an ordinary Session +verifying the ordinary way — exactly the behaviour of the code that was here +before. Downloads failing is a bad day; the updater dying on import because a +certificate helper was unavailable would be a worse one. Which path was taken +is logged once, because "why does it work on his machine" is the question this +module exists to answer. +""" + +from __future__ import annotations + +import logging +import ssl + +import requests +from requests.adapters import HTTPAdapter + +# Logged on the first session built and not again: this is asked once per +# process, and a line per HTTP request would bury the log of a 3 GB download. +_reported = False + + +class _SystemTrustAdapter(HTTPAdapter): + """An HTTPAdapter whose connections verify against the OS trust store.""" + + def __init__(self, ssl_context, **kwargs): + self._ssl_context = ssl_context + super().__init__(**kwargs) + + def init_poolmanager(self, *args, **kwargs): + kwargs["ssl_context"] = self._ssl_context + return super().init_poolmanager(*args, **kwargs) + + def proxy_manager_for(self, *args, **kwargs): + # The proxy path needs it just as much, and is not the same code path: + # a machine that intercepts TLS is very often the same machine that + # routes through a proxy, and missing this would fix the plain case and + # leave the intercepted one broken. + kwargs["ssl_context"] = self._ssl_context + # And once more for the leg *to* the proxy, when the proxy's own URL is + # https://. urllib3 verifies that hop against `proxy_ssl_context`, a + # separate setting that would otherwise fall back to the bundled CA + # list — on the corporate machines this module exists for. + kwargs["proxy_ssl_context"] = self._ssl_context + return super().proxy_manager_for(*args, **kwargs) + + +def system_ssl_context(): + """An SSLContext verifying through the OS trust store, or None. + + None means "carry on as before" and is not an error: on a machine with no + truststore, or a Python too old for it, the bundled CA list is what there + always was. + """ + global _reported + try: + import truststore + + context = truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + except Exception as exc: + if not _reported: + _reported = True + logging.info( + "[tls] verifying HTTPS against the bundled CA list — the system " + "trust store is unavailable (%s: %s)", type(exc).__name__, exc, + ) + return None + if not _reported: + _reported = True + logging.info("[tls] verifying HTTPS against the system trust store") + return context + + +def create_session() -> requests.Session: + """A `requests.Session` that verifies through the OS trust store if it can. + + Plain HTTP is left alone — there is nothing to verify — so only the https + adapter is replaced. + """ + session = requests.Session() + context = system_ssl_context() + if context is not None: + try: + session.mount("https://", _SystemTrustAdapter(context)) + except Exception as exc: + # Building the adapter runs HTTPAdapter.__init__ → init_poolmanager, + # which hands `ssl_context` to urllib3. urllib3 is a pinned + # dependency that gets bumped like any other, and a version that + # renames or validates that keyword would raise here — taking the + # model download, the Node download and the background update check + # down together, which is the single failure this module exists to + # prevent. Losing the system trust store is survivable; losing every + # download is not. + logging.warning( + "[tls] the system-trust adapter could not be installed " + "(%s: %s) — falling back to the bundled CA list", + type(exc).__name__, exc, + ) + return session + + +def get(url, **kwargs): + """`requests.get`, verifying through the OS trust store if it can. + + A session per call, closed afterwards, which is exactly what + `requests.get()` itself does — so a caller swapped over to this one keeps + the connection lifetime it had, including for `stream=True`, where the + response goes on reading from a connection the closed session no longer + pools. + """ + with create_session() as session: + return session.get(url, **kwargs) diff --git a/client/core/transcription/__init__.py b/client/core/transcription/__init__.py new file mode 100644 index 000000000..d684d5a83 --- /dev/null +++ b/client/core/transcription/__init__.py @@ -0,0 +1,7 @@ +# Local (offline) transcription of voice/audio messages — pure logic only. +# +# Nothing in this package may import faster_whisper/ctranslate2 at module +# level: the backend is an optional component an install may simply not have, +# and the menu item that offers to install it has to be reachable on a machine +# where the import would fail. Every backend import lives inside the function +# that needs it. diff --git a/client/core/transcription/device.py b/client/core/transcription/device.py new file mode 100644 index 000000000..ffc323896 --- /dev/null +++ b/client/core/transcription/device.py @@ -0,0 +1,493 @@ +"""Where a transcription runs, and in which precision. + +Probing and deciding are two different things and are kept apart on purpose. +Probing means loading the NVIDIA driver's own DLL and asking CTranslate2 how +many CUDA devices it can see — machine-specific, allowed to fail in a dozen +ways, and impossible to reproduce in a test. Deciding is a handful of +comparisons. So `probe_hardware()` is the only function here that touches the +machine, it returns its findings as a plain frozen dataclass, and every +decision function takes that dataclass as an argument. That is what lets the +"RTX 50xx picks the wrong precision" family of bugs be pinned by a test on a +machine with no GPU at all. + +Nothing in this module raises for a hardware reason. A user who asked for CUDA +on a machine that has none gets the CPU and a reason code saying so — being +told is the point, failing is not, and for a blind user an error dialog in +place of a transcription is strictly worse than a slower transcription. + +**Probe immediately before deciding, and never reuse a probe.** +available_memory_mb() plans against memory that is *free*, so when the probe +ran is part of the answer. A probe taken at startup is taken before Chromium +has loaded WhatsApp Web, before the media cache has filled and possibly before +the screen reader is even up: it reports the high number this module already +says is misleading, and auto_select_model() would then pick a model against +memory that no longer exists by the time it loads. probe_hardware() costs one +DLL load and three driver calls, which is nothing next to a transcription, so +call it at the point of decision every time. +""" + +from __future__ import annotations + +import ctypes +import logging +import os +import sys +from dataclasses import dataclass + +from core.transcription import model_catalog + +DEVICE_CPU = "cpu" +DEVICE_CUDA = "cuda" + +# What the user picked in the settings. "auto" is the default and the only one +# that is allowed to change its mind between launches. +PREFERENCE_AUTO = "auto" +PREFERENCE_CUDA = "cuda" +PREFERENCE_CPU = "cpu" + +COMPUTE_INT8 = "int8" +COMPUTE_FLOAT16 = "float16" +COMPUTE_FLOAT32 = "float32" + +# Why the device below was chosen. Symbolic, never a sentence: the UI maps them +# through DEVICE_REASON_I18N_KEYS so the announcement is in the user's language. +REASON_CUDA_SELECTED = "cuda_selected" +REASON_CPU_REQUESTED = "cpu_requested" +# The user asked for CUDA and there is none — this one has to be announced. +REASON_CUDA_UNAVAILABLE = "cuda_unavailable" +# Same situation under "auto", where the user asked for nothing in particular +# and does not need to hear it as a thwarted request. +REASON_NO_CUDA_FOUND = "no_cuda_found" +# CUDA devices were counted and then could not be questioned — a driver fault +# on a machine that really does have a card, which the user can act on. +# probe_hardware() never produces this reason on its own, and re-probing cannot +# either: a fresh probe would simply count the device again and land on +# CUDA_SELECTED. It takes a caller that demotes the card deliberately after a +# run has failed on it — dataclasses.replace(probe, cuda_available=False, +# cuda_device_count=0, cuda_probe_error=) — which is the +# only state where "your driver is broken" is the right thing to say. +REASON_CUDA_DRIVER_ERROR = "cuda_driver_error" + +DEVICE_REASON_I18N_KEYS = { + REASON_CUDA_SELECTED: "transcription_device_cuda_selected", + REASON_CPU_REQUESTED: "transcription_device_cpu_requested", + REASON_CUDA_UNAVAILABLE: "transcription_device_cuda_unavailable", + REASON_NO_CUDA_FOUND: "transcription_device_no_cuda_found", + REASON_CUDA_DRIVER_ERROR: "transcription_device_cuda_driver_error", +} + +# float16 needs Volta or newer to be worth anything; on Pascal and older the +# half-precision path is emulated and slower than float32, when it works. +_FLOAT16_MIN_CAPABILITY = (7, 0) + +# Blackwell (sm_120) and anything after it. +_INT8_UNSUPPORTED_FROM_CAPABILITY = (12, 0) + +# How much room auto_select_model() insists on beyond a model's recommended +# minimum. The minimums in model_catalog already cover weights, activations and +# the CUDA context; the extra 25% is for what else is on the machine while +# WinZapp transcribes — Chromium running WhatsApp Web (which also takes VRAM +# for compositing), the screen reader, and whatever the user was doing. Getting +# this wrong is not a graceful degradation: CTranslate2 aborts the run with an +# allocation error, after the user already waited for the model to load. +_MEMORY_HEADROOM = 1.25 + +# Where nvml.dll lives. The bare name first, then the two absolute paths pynvml +# itself falls back to: the DLL is installed by the display driver, and on a +# machine whose search order does not reach System32 (or that still has the +# legacy NVSMI layout) the bare name fails while the card is perfectly usable. +# That miss is not free — it drops the machine into the "VRAM unknown" branch, +# which is exactly where auto_select_model() has the least to work with. +_NVML_LIBRARY_PATHS = ( + "nvml.dll", + os.path.join(os.environ.get("WINDIR", r"C:\Windows"), "System32", "nvml.dll"), + os.path.join( + os.environ.get("ProgramFiles", r"C:\Program Files"), + "NVIDIA Corporation", "NVSMI", "nvml.dll", + ), +) + + +@dataclass(frozen=True) +class HardwareProbe: + """What `probe_hardware()` managed to find out. Every field may be absent. + + A field being None means "unknown", never "zero": a decision that treats an + unknown VRAM figure as 0 would silently hide every GPU whose driver DLL + failed to load. + + The two error fields are not interchangeable, and conflating them is what + made a machine with no NVIDIA hardware at all report a graphics driver + fault. `driver_error` is an **aggregate** of everything that failed during + the probe — a missing ctranslate2, NVML, the Windows memory API — and is + log-facing only, like TranscriptionError's detail. `cuda_probe_error` is the + narrow one: a GPU was counted and then could not be described. Only that one + may drive a spoken reason, because the aggregate is non-empty on every + machine that has not installed the transcription backend yet — which is all + of them until the user asks for it. + """ + + cuda_available: bool = False + cuda_device_count: int = 0 + compute_capability: tuple[int, int] | None = None + total_vram_mb: int | None = None + free_vram_mb: int | None = None + total_ram_mb: int | None = None + available_ram_mb: int | None = None + cuda_probe_error: str | None = None + driver_error: str | None = None + + +def resolve_device(preference, probe) -> tuple[str, str]: + """(device, reason) for a stated preference and a probe result. + + An unknown preference is treated as "auto" rather than rejected — it is a + settings value, and a settings file written by an older or newer WinZapp + must not be able to make transcription unavailable. + """ + if preference == PREFERENCE_CPU: + return DEVICE_CPU, REASON_CPU_REQUESTED + + if cuda_usable(probe): + return DEVICE_CUDA, REASON_CUDA_SELECTED + + if preference == PREFERENCE_CUDA: + # Asked for explicitly, so the fallback is worth saying out loud, and a + # card that is present but unreachable is worth distinguishing from "no + # GPU here": the first is usually a driver the user can reinstall. + # Keyed on cuda_probe_error, never on the driver_error aggregate — that + # one carries "ctranslate2 is not installed", i.e. every user's state + # until the backend ships, and sending those people to reinstall a + # graphics driver they may not even have is worse than saying nothing. + if probe.cuda_probe_error: + return DEVICE_CPU, REASON_CUDA_DRIVER_ERROR + return DEVICE_CPU, REASON_CUDA_UNAVAILABLE + + return DEVICE_CPU, REASON_NO_CUDA_FOUND + + +def cuda_usable(probe) -> bool: + """Whether CUDA can actually be used, not merely whether a DLL loaded.""" + return bool(probe.cuda_available) and probe.cuda_device_count > 0 + + +def gpu_supports_int8(compute_capability) -> bool: + """Whether int8 kernels exist for this GPU at all. + + This is the reason this module exists. The CTranslate2 builds that support + sm_120 (>= 4.6.3, CUDA 12.8) are compiled with INT8 *disabled* for it, so on + a Blackwell card every int8 variant — "int8", "int8_float16", + "int8_bfloat16" — fails at model load with an unsupported-compute-type + error, and the user hears nothing but "internal error". + + An unknown capability answers False: refusing int8 costs speed, offering it + to a card that cannot run it costs the transcription. + """ + if compute_capability is None: + return False + return tuple(compute_capability) < _INT8_UNSUPPORTED_FROM_CAPABILITY + + +def _int8_safe(compute, compute_capability) -> str: + """`compute`, unless it is an int8 flavour this GPU has no kernels for. + + The sm_120 rule on the execution path rather than in a comment. + select_compute_type() asks for float16 today, so this changes nothing + today — it is what keeps the rule true the day a card short of VRAM makes + int8_float16 look attractive, since that failure surfaces only at model + load, on hardware no test runner has, after the user has already waited + through a multi-gigabyte download. + """ + if str(compute).startswith("int8") and not gpu_supports_int8(compute_capability): + return COMPUTE_FLOAT16 + return compute + + +def select_compute_type(device, probe) -> str: + """The CTranslate2 compute type to load the model with.""" + if device != DEVICE_CUDA: + # int8 on the CPU is the whole reason CPU transcription is usable at + # all — roughly three times faster than float32, with no audible + # difference on speech. + return COMPUTE_INT8 + + capability = probe.compute_capability + if capability is not None and tuple(capability) < _FLOAT16_MIN_CAPABILITY: + return COMPUTE_FLOAT32 + + # float16 everywhere else, including sm_120. An unknown capability lands + # here too — a card CTranslate2 can see but NVML could not describe is far + # more likely to be new than pre-Volta. + return _int8_safe(COMPUTE_FLOAT16, capability) + + +def auto_select_model(probe, device, installed_ids, catalog=None): + """The model id to use when the user has not chosen one, or None. + + Two rules, in this order: + + 1. If a model that is already on disk fits, use the largest of those. A + silent 3 GB download nobody asked for is worse than transcribing with + what is already there — and on a metered connection it is a lot worse. + 2. Otherwise the largest model that fits, which the caller will have to + download. + + "Fits" means the device's memory covers the model's recommended minimum + with `_MEMORY_HEADROOM` to spare. When the memory could not be measured at + all, only rule 1 applies and only for the smallest installed model: guessing + upwards on an unknown machine is how a run dies half way through. + """ + models = tuple(catalog) if catalog is not None else model_catalog.list_models() + # Sorted by what each model costs *on this device* rather than trusting the + # catalogue's own order, so "the largest that fits" still means the most + # demanding one when a caller passes its own list (the tests do). + models = sorted(models, key=lambda m: (_requirement_mb(m, device), m.download_bytes)) + installed = set(installed_ids or ()) + budget_mb = available_memory_mb(probe, device) + + if budget_mb is None: + installed_models = [m for m in models if m.id in installed] + return installed_models[0].id if installed_models else None + + fitting = [m for m in models if model_fits(m, budget_mb, device)] + already_here = [m for m in fitting if m.id in installed] + if already_here: + return already_here[-1].id + return fitting[-1].id if fitting else None + + +def available_memory_mb(probe, device): + """Memory the transcription may plan around, in MB, or None if unknown. + + Free memory wins over total on both devices, and for the same reason: the + machine is not idle while WinZapp transcribes. Chromium is running WhatsApp + Web (holding RAM, and VRAM for compositing), a screen reader is running, and + the user has their own work open. A 16 GB machine with 11 GB in use is a + 5 GB machine for this purpose, and planning against the 16 buys a 3 GB + download followed by an allocation error. + """ + if device == DEVICE_CUDA: + free, total = probe.free_vram_mb, probe.total_vram_mb + else: + free, total = probe.available_ram_mb, probe.total_ram_mb + return free if free is not None else total + + +def model_fits(model, budget_mb, device) -> bool: + """Whether `model` fits in `budget_mb` on `device`, headroom included.""" + if budget_mb is None: + return False + return budget_mb >= _requirement_mb(model, device) * _MEMORY_HEADROOM + + +def _requirement_mb(model, device) -> int: + """The model's recommended minimum for the device it will run on.""" + return model.min_vram_mb if device == DEVICE_CUDA else model.min_ram_mb + + +def device_reason_i18n_key(reason) -> str: + """The i18n key announcing a device reason. + + Unknown reasons resolve to the plain CPU message rather than to a key of + their own, for the same reason as errors.error_i18n_key(): I18n.t() would + otherwise have the screen reader read the code itself. + """ + return DEVICE_REASON_I18N_KEYS.get( + reason, DEVICE_REASON_I18N_KEYS[REASON_CPU_REQUESTED] + ) + + +def probe_hardware() -> HardwareProbe: + """Ask the machine what it has. Never raises; unknown fields stay None. + + Three independent questions, each allowed to fail on its own: + + * how many CUDA devices CTranslate2 can drive — ctranslate2 is an optional + dependency, so the import lives here rather than at module level; + * VRAM and compute capability, read from the driver's own ``nvml.dll`` + through ctypes. nvidia-ml-py is deliberately not a dependency: it would + ship to every user for one call that only matters on the machines that + have the DLL anyway; + * RAM, total and available, from GlobalMemoryStatusEx. + + Only NVML's failure becomes `cuda_probe_error`, and only once a device has + actually been counted — see HardwareProbe for why the aggregate must never + be read as a driver fault. + """ + cuda_count = 0 + capability = None + total_vram = None + free_vram = None + cuda_probe_error = None + # One bag for everything that failed, joined into driver_error at the end: + # it is log-facing only, so it is more useful complete than tidy. + problems = [] + + try: + # Imported here, never at module level: ctranslate2 may not be + # installed at all, and the menu that offers to install it has to work + # on exactly those machines. + import ctranslate2 + + cuda_count = int(ctranslate2.get_cuda_device_count()) + except Exception as exc: + problems.append(f"ctranslate2: {exc}") + + # Both helpers already answer with an error string instead of raising, and + # both are still called inside a guard: "never raises" is this function's + # promise to the UI, not something it may inherit from a helper that loads + # a third-party driver DLL and calls into it through ctypes. + if cuda_count > 0: + try: + capability, total_vram, free_vram, cuda_probe_error = _probe_nvml() + except Exception as exc: + cuda_probe_error = f"nvml: {exc}" + if cuda_probe_error: + problems.append(cuda_probe_error) + + try: + total_ram, available_ram, ram_error = _probe_ram_mb() + except Exception as exc: + total_ram = available_ram = None + ram_error = f"ram: {exc}" + if ram_error: + problems.append(ram_error) + + probe = HardwareProbe( + cuda_available=cuda_count > 0, + cuda_device_count=cuda_count, + compute_capability=capability, + total_vram_mb=total_vram, + free_vram_mb=free_vram, + total_ram_mb=total_ram, + available_ram_mb=available_ram, + cuda_probe_error=cuda_probe_error, + driver_error="; ".join(problems) or None, + ) + logging.info("[transcription] hardware probe: %s", probe) + return probe + + +def _load_nvml(): + """(library, error) — the first nvml.dll that loads, or None and why not. + + Every failure is kept, not just the last one: reporting only the last leaves + the log blaming the legacy NVSMI path on every machine without an NVIDIA + card, which is the least informative of the three and says nothing about + why the normal locations failed. + + CDLL rather than WinDLL, matching pynvml, whose fallback paths these are: + NVML is a cdecl library. On x64 the two calling conventions are the same, so + this is about agreeing with the source of the convention, not about a bug. + """ + failures = [] + for path in _NVML_LIBRARY_PATHS: + try: + return ctypes.CDLL(path), None + except Exception as exc: + failures.append(f"{path}: {exc}") + return None, "; ".join(failures) + + +def _probe_nvml(): + """(capability, total_vram_mb, free_vram_mb, error) from nvml.dll. + + Device 0 only. On a single-GPU machine — which is what this is for — there + is nothing to choose. On a multi-GPU one the two device 0s need not be the + same card: NVML orders by PCI bus id while CUDA orders by FASTEST_FIRST + unless CUDA_DEVICE_ORDER=PCI_BUS_ID is set. Guessing which pairing applies + would be worse than reading the first card, since the figures only feed a + size decision that already carries 25% of headroom. + """ + if not sys.platform.startswith("win"): + return None, None, None, "nvml: not Windows" + + class _NvmlMemory(ctypes.Structure): + _fields_ = [ + ("total", ctypes.c_ulonglong), + ("free", ctypes.c_ulonglong), + ("used", ctypes.c_ulonglong), + ] + + nvml, load_error = _load_nvml() + if nvml is None: + return None, None, None, f"nvml: {load_error}" + + initialised = False + try: + if nvml.nvmlInit_v2() != 0: + return None, None, None, "nvml: nvmlInit_v2 failed" + initialised = True + + handle = ctypes.c_void_p() + if nvml.nvmlDeviceGetHandleByIndex_v2(0, ctypes.byref(handle)) != 0: + return None, None, None, "nvml: no handle for device 0" + + capability = None + major = ctypes.c_int() + minor = ctypes.c_int() + if nvml.nvmlDeviceGetCudaComputeCapability( + handle, ctypes.byref(major), ctypes.byref(minor) + ) == 0: + capability = (major.value, minor.value) + + total_mb = free_mb = None + memory = _NvmlMemory() + if nvml.nvmlDeviceGetMemoryInfo(handle, ctypes.byref(memory)) == 0: + total_mb = int(memory.total // (1024 * 1024)) + free_mb = int(memory.free // (1024 * 1024)) + + return capability, total_mb, free_mb, None + except Exception as exc: + # A driver too old for one of these entry points, or one that faults + # inside them: either way the answer is "unknown", never a crash. + return None, None, None, f"nvml: {exc}" + finally: + # Only after a successful init: nvmlShutdown() against a library that + # never initialised is not a documented no-op, and this runs on the + # path where the driver is already misbehaving. + if initialised: + try: + nvml.nvmlShutdown() + except Exception: + pass + + +def _probe_ram_mb(): + """(total_ram_mb, available_ram_mb, error) from GlobalMemoryStatusEx. + + Available as well as total, because available is what the CPU path plans + against — see available_memory_mb(). + """ + + class _MemoryStatusEx(ctypes.Structure): + _fields_ = [ + ("dwLength", ctypes.c_ulong), + ("dwMemoryLoad", ctypes.c_ulong), + ("ullTotalPhys", ctypes.c_ulonglong), + ("ullAvailPhys", ctypes.c_ulonglong), + ("ullTotalPageFile", ctypes.c_ulonglong), + ("ullAvailPageFile", ctypes.c_ulonglong), + ("ullTotalVirtual", ctypes.c_ulonglong), + ("ullAvailVirtual", ctypes.c_ulonglong), + ("ullAvailExtendedVirtual", ctypes.c_ulonglong), + ] + + try: + if sys.platform.startswith("win"): + status = _MemoryStatusEx() + status.dwLength = ctypes.sizeof(_MemoryStatusEx) + if not ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(status)): + return None, None, "ram: GlobalMemoryStatusEx failed" + return ( + int(status.ullTotalPhys // (1024 * 1024)), + int(status.ullAvailPhys // (1024 * 1024)), + None, + ) + + page_size = os.sysconf("SC_PAGE_SIZE") + total = page_size * os.sysconf("SC_PHYS_PAGES") + available = page_size * os.sysconf("SC_AVPHYS_PAGES") + return int(total // (1024 * 1024)), int(available // (1024 * 1024)), None + except Exception as exc: + return None, None, f"ram: {exc}" diff --git a/client/core/transcription/errors.py b/client/core/transcription/errors.py new file mode 100644 index 000000000..2b4290882 --- /dev/null +++ b/client/core/transcription/errors.py @@ -0,0 +1,126 @@ +"""Everything that can go wrong in a transcription, as one closed set of codes. + +Transcription fails in a lot of ways that look identical from the outside — +nothing appears — and the user is listening rather than reading a traceback. +So a failure travels as a *code*, and only the UI layer turns it into a +sentence, in the user's own language. + +The split between `code` and `detail` is the point of this module. `detail` is +technical (a CTranslate2 message, an ffmpeg exit code, a path) and belongs in +log.log; it is never spoken and never shown. Putting a backend string on screen +would say nothing useful to a blind user in Polish, and it is also where file +paths and message ids leak. +""" + +from __future__ import annotations + +# The chosen model is not on disk yet. +MODEL_NOT_INSTALLED = "model_not_installed" +# It is on disk, but a file is missing or the wrong size — an interrupted +# download that was never cleaned up. The promise is checkable because the +# catalogue pins a revision and, with it, an exact size for every file and a +# sha256 for model.bin (see model_catalog). +MODEL_CORRUPTED = "model_corrupted" +NO_DISK_SPACE = "no_disk_space" +CUDA_UNAVAILABLE = "cuda_unavailable" +INSUFFICIENT_VRAM = "insufficient_vram" +INSUFFICIENT_RAM = "insufficient_ram" +UNSUPPORTED_AUDIO_FORMAT = "unsupported_audio_format" +FFMPEG_FAILED = "ffmpeg_failed" +# The file is there but truncated: a send that died mid-upload, or a media +# file WinZapp wrote while the app was closed. +AUDIO_INCOMPLETE = "audio_incomplete" +# WhatsApp's own media download for this message has not finished, so there is +# no local file to read at all. +MEDIA_NOT_DOWNLOADED = "media_not_downloaded" +CANCELLED = "cancelled" +SAVE_FAILED = "save_failed" +# faster-whisper/CTranslate2 is not installed in this copy of WinZapp. +BACKEND_MISSING = "backend_missing" +MODEL_DOWNLOAD_FAILED = "model_download_failed" +# Moving the models folder failed part way. Deliberately not MODEL_CORRUPTED: +# nothing is corrupted — the move copies and verifies before it deletes, so the +# models are still whole in the folder they were in — and "download it again" +# would be 3 GB of advice for a problem whose answer is to pick another folder. +MODEL_MOVE_FAILED = "model_move_failed" +# Another account's process is holding the shared models directory. Its own +# code because the alternatives all say something false: the download did not +# fail, the connection is fine, and nothing is corrupted — another window is +# simply busy with the same folder, and the answer is to wait rather than to +# retry anything. +MODELS_BUSY = "models_busy" +# The catch-all: the backend raised something we have no specific answer for. +BACKEND_ERROR = "backend_error" + +# Single source of truth for "every code there is" — the i18n map below and the +# test that pins it both derive from this, so a new code cannot be added +# without a translation. +ERROR_CODES = ( + MODEL_NOT_INSTALLED, + MODEL_CORRUPTED, + NO_DISK_SPACE, + CUDA_UNAVAILABLE, + INSUFFICIENT_VRAM, + INSUFFICIENT_RAM, + UNSUPPORTED_AUDIO_FORMAT, + FFMPEG_FAILED, + AUDIO_INCOMPLETE, + MEDIA_NOT_DOWNLOADED, + CANCELLED, + SAVE_FAILED, + BACKEND_MISSING, + MODEL_DOWNLOAD_FAILED, + MODEL_MOVE_FAILED, + MODELS_BUSY, + BACKEND_ERROR, +) + +ERROR_I18N_KEYS = {code: f"transcription_error_{code}" for code in ERROR_CODES} + + +class TranscriptionError(Exception): + """A transcription failure carrying a code the UI can translate. + + `detail` is for the log only, and neither `str(exc)` nor `exc.args` carries + it. That is not tidiness: `wx.MessageBox(str(exc), ...)` is an idiom this + repository already uses (`client/ui/media_viewer.py`), so anything `__str__` + returns is one careless handler away from being read out, character by + character, to a blind user — including the file path a backend error message + usually contains. Logging wants the pair, so logging asks for `log_line`. + """ + + #: Ids a partially completed `move_models()` did manage to move before it + #: failed. Declared here, empty, rather than only where that path attaches + #: it: a caller writing `except TranscriptionError as exc: ... exc.moved` + #: around a move would otherwise hit AttributeError on every *other* way a + #: move can fail — the busy-folder path raises its own error and never + #: passes through the attach. + moved: tuple = () + + def __init__(self, code, detail=None): + super().__init__(code) + self.code = code + self.detail = detail + + def __str__(self): + return str(self.code) + + @property + def log_line(self) -> str: + """Code and technical detail on one line — for `logging.*`, only.""" + return f"{self.code}: {self.detail}" if self.detail else str(self.code) + + @property + def i18n_key(self) -> str: + return error_i18n_key(self.code) + + +def error_i18n_key(code) -> str: + """The i18n key for an error code. + + An unrecognised code resolves to the internal-error message rather than to + a key of its own: I18n.t() falls back to the raw key name, so inventing + "transcription_error_" here would have a screen reader read the + code out loud instead of a sentence. + """ + return ERROR_I18N_KEYS.get(code, ERROR_I18N_KEYS[BACKEND_ERROR]) diff --git a/client/core/transcription/model_catalog.py b/client/core/transcription/model_catalog.py new file mode 100644 index 000000000..afcb9f724 --- /dev/null +++ b/client/core/transcription/model_catalog.py @@ -0,0 +1,268 @@ +"""The Whisper models WinZapp offers, as data. + +Models are not shipped inside the installer — a single one of them is bigger +than the whole app — so they are downloaded from Hugging Face on demand. That +makes the catalogue a contract with a remote repository: the revisions, file +names, file sizes and the model.bin digest below were read from the Hugging +Face API on 2026-09-03 and are what the download and the "this model is +complete" check are measured against. + +Three details of that contract are easy to get wrong and expensive afterwards: + +* **The file list is not uniform.** The Systran tiny..medium conversions ship + ``vocabulary.txt`` and no ``preprocessor_config.json``, while large-v3 and the + turbo conversion ship ``vocabulary.json`` *and* ``preprocessor_config.json``. + Assuming one shape leaves a "complete" model that CTranslate2 refuses to load, + which is indistinguishable to the user from a corrupted download. +* **Every model is pinned to a revision.** Downloads go to + ``.../resolve//...`` rather than ``/main/``, so a new revision + published upstream cannot invalidate the byte counts here or swap the weights + under an install that already verified them. Because the revision is pinned, + every size below is a constant, not an estimate — which is what lets + MODEL_CORRUPTED mean "missing or the wrong size" for *every* file rather than + only for model.bin. +* **The totals are sums, never guesses.** They used to add an approximated + couple of megabytes for the auxiliary files, and on the turbo conversion that + guess was 181 KB *under* the real figure — an error in the wrong direction for + a number that feeds a "does this fit?" gate. + +No user-facing text lives here, only ids and i18n keys: the size classes are +what the model picker labels "fast / balanced / accurate", and translating them +is the UI layer's job. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +# Size classes, in the order the picker presents them. These drive the +# "rápido / equilibrado / preciso" labelling, so they describe what the user +# gets, not the byte count — large-v3-turbo is smaller than large-v3 and still +# belongs with it, because its output is a large model's output. +SIZE_SMALL = "small" +SIZE_MEDIUM = "medium" +SIZE_LARGE = "large" + +SIZE_CLASSES = (SIZE_SMALL, SIZE_MEDIUM, SIZE_LARGE) + +SIZE_CLASS_I18N_KEYS = { + SIZE_SMALL: "transcription_size_fast", + SIZE_MEDIUM: "transcription_size_balanced", + SIZE_LARGE: "transcription_size_accurate", +} + + +@dataclass(frozen=True) +class WhisperModel: + """One downloadable CTranslate2 conversion of a Whisper model.""" + + id: str + repo: str + # The commit the files were measured at. Everything is fetched by this sha, + # never by branch — see the module docstring. + revision: str + # model.bin's digest at that revision, so an integrity check can go past + # "the right number of bytes" for the one file where a silent corruption + # costs a multi-gigabyte re-download to discover. + model_bin_sha256: str + # (name, exact bytes) for every file the model needs, in the order the + # repository lists them. A tuple of pairs rather than a dict because the + # catalogue is shared module state a caller must not be able to edit in + # place; `dict(model.files)` is the lookup form. + files: tuple[tuple[str, int], ...] + # Pulled out of `files` for the callers that only care about the weights. + model_bin_bytes: int + # What the download costs, and what it leaves behind — both the exact sum of + # `files`. They are equal today because the files are written straight into + # the model folder (nothing is extracted, no separate blob cache is kept), + # but the picker quotes both to the user, and a future cache layout that + # duplicates blobs would move only one of them. + download_bytes: int + disk_bytes: int + size_class: str + # Recommended minimums, not hard requirements: roughly twice the weights, + # which covers activations, the KV cache at beam size 5 and the CUDA + # context. Undershooting them does not fail cleanly — CTranslate2 aborts + # mid-transcription with an allocation error — so auto_select_model() keeps + # its distance from them (see device.py). + min_vram_mb: int # float16 on the GPU + min_ram_mb: int # int8 on the CPU + + +def _model(model_id, repo, revision, model_bin_sha256, files, size_class, + min_vram_mb, min_ram_mb) -> WhisperModel: + """One catalogue entry, with everything derivable derived from `files`. + + The totals and the model.bin size are read off the file table instead of + being restated beside it: six models times three restatements is eighteen + chances for two numbers describing the same download to disagree. + + Every file name has to be a bare basename, and that is asserted rather than + assumed: the model store joins these names onto a directory the user may + point anywhere and deletes exactly them, so a nested layout + ("snapshots//model.bin", which is the shape these repositories use + internally) or a ".." would have a delete reach outside the models root. + """ + for name, _size in files: + # Both separators, not just this platform's: the catalogue is written + # once and read on Windows, where a name carrying either one escapes + # the model's own directory. + assert name and name not in (".", "..") and not set(name) & {"/", "\\"}, ( + f"{model_id}: file names must be bare basenames, got {name!r}" + ) + total = sum(size for _name, size in files) + return WhisperModel( + id=model_id, + repo=repo, + revision=revision, + model_bin_sha256=model_bin_sha256, + files=files, + model_bin_bytes=dict(files)["model.bin"], + download_bytes=total, + disk_bytes=total, + size_class=size_class, + min_vram_mb=min_vram_mb, + min_ram_mb=min_ram_mb, + ) + + +# Single source of truth. Declared in the order the repositories were verified; +# list_models() is what defines presentation order. +MODELS = ( + _model( + "tiny", + "Systran/faster-whisper-tiny", + "d90ca5fe260221311c53c58e660288d3deb8d356", + "dcb76c6586fc06cbdac6dd21f14cfd129cc4cdd9dce19bf4ffa62e59cbe6e6d1", + ( + ("config.json", 2_249), + ("model.bin", 75_538_270), + ("tokenizer.json", 2_203_239), + ("vocabulary.txt", 459_861), + ), + SIZE_SMALL, + min_vram_mb=1024, + min_ram_mb=2048, + ), + _model( + "base", + "Systran/faster-whisper-base", + "ebe41f70d5b6dfa9166e2c581c45c9c0cfc57b66", + "d01c3014881c9c6f3133c182f3d2887eb6ca1c789a7538c5c007196857a0a6a9", + ( + ("config.json", 2_309), + ("model.bin", 145_217_532), + ("tokenizer.json", 2_203_239), + ("vocabulary.txt", 459_861), + ), + SIZE_SMALL, + min_vram_mb=1024, + min_ram_mb=2048, + ), + _model( + "small", + "Systran/faster-whisper-small", + "536b0662742c02347bc0e980a01041f333bce120", + "3e305921506d8872816023e4c273e75d2419fb89b24da97b4fe7bce14170d671", + ( + ("config.json", 2_370), + ("model.bin", 483_546_902), + ("tokenizer.json", 2_203_239), + ("vocabulary.txt", 459_861), + ), + SIZE_MEDIUM, + min_vram_mb=2048, + min_ram_mb=3072, + ), + _model( + "medium", + "Systran/faster-whisper-medium", + "08e178d48790749d25932bbc082711ddcfdfbc4f", + "9b45e1009dcc4ab601eff815b61d80e60ce3fd8c74c1a14f4a282258286b51ae", + ( + ("config.json", 2_257), + ("model.bin", 1_527_906_378), + ("tokenizer.json", 2_203_239), + ("vocabulary.txt", 459_861), + ), + SIZE_MEDIUM, + min_vram_mb=4096, + min_ram_mb=6144, + ), + _model( + "large-v3", + "Systran/faster-whisper-large-v3", + "edaa852ec7e145841d8ffdb056a99866b5f0a478", + "69f74147e3334731bc3a76048724833325d2ec74642fb52620eda87352e3d4f1", + ( + ("config.json", 2_394), + ("model.bin", 3_087_284_237), + ("preprocessor_config.json", 340), + ("tokenizer.json", 2_480_617), + ("vocabulary.json", 1_068_114), + ), + SIZE_LARGE, + min_vram_mb=6144, + min_ram_mb=8192, + ), + _model( + "large-v3-turbo", + "deepdml/faster-whisper-large-v3-turbo-ct2", + "4df90f75321148c3a29a9e2351b7ddf8f5b115a8", + "e76620f83d5f5b69efd3d87e3dc180c1bd21df9fbebacfd4335e5e1efcc018da", + ( + ("config.json", 2_263), + ("model.bin", 1_617_884_929), + ("preprocessor_config.json", 340), + ("tokenizer.json", 2_710_337), + ("vocabulary.json", 1_068_114), + ), + SIZE_LARGE, + min_vram_mb=4096, + min_ram_mb=6144, + ), +) + + +def _order(model: WhisperModel) -> tuple[int, int]: + return (SIZE_CLASSES.index(model.size_class), model.download_bytes) + + +def list_models() -> tuple[WhisperModel, ...]: + """Every model, cheapest first. + + Ordered by size class and then by download size, so the two large models + are listed turbo-first: it is the one to reach for when large-v3 is too + much, and putting it after would bury it under the 3 GB entry. + """ + return tuple(sorted(MODELS, key=_order)) + + +def get_model(model_id) -> WhisperModel | None: + """The model with this id, or None. + + Deliberately not raising: the id comes from settings, where a model that + was renamed or removed between versions is a normal state, not a fault. + """ + for model in MODELS: + if model.id == model_id: + return model + return None + + +def total_bytes(model_id) -> int | None: + """Bytes a fresh install of `model_id` costs, or None for an unknown id. + + None rather than 0, even though 0 would spare the caller a check. The + caller that matters here is the free-space gate, and an unknown id is + exactly what a settings file naming a model some later version dropped + looks like: answering 0 tells that gate "it fits" and starts a download of + something that does not exist. + """ + model = get_model(model_id) + return model.download_bytes if model else None + + +def size_class_i18n_key(size_class) -> str | None: + """The i18n key labelling a size class, or None for an unknown one.""" + return SIZE_CLASS_I18N_KEYS.get(size_class) diff --git a/client/core/transcription/model_store.py b/client/core/transcription/model_store.py new file mode 100644 index 000000000..71525b785 --- /dev/null +++ b/client/core/transcription/model_store.py @@ -0,0 +1,1024 @@ +"""Where the Whisper models live on disk, and how they get there. + +The catalogue (see model_catalog) says what a model is; this module is the only +thing that puts those bytes on a disk, checks them, and takes them away again. +These decisions are worth more than the code implementing them: + +* **The root is global, not per account.** WinZapp runs one account per + process, so anything under ``data_path()`` is downloaded once *per account* — + and large-v3 is 3 GB. The weights are read-only and hold nothing + account-specific, so ``default_models_dir()`` answers with ``global_dir()``. + Every other function takes the root as an argument, because part 5 lets the + user point it somewhere else entirely. + +* **Being global means being shared, so every mutation takes a lock.** Two + account processes asking for the same model would otherwise open the same + ``model.bin.part``, interleave their writes, both fail the digest, and each + one's cleanup would delete the file the other is still streaming into. + ``coord_locks.models_lock()`` is that lock; it is keyed on the *canonical* + root, which is also how two spellings of one directory are recognised as one + (Windows folds case and resolves junctions and ``subst`` drives — ``abspath`` + does not). + +* **A file under its final name is a promise.** "Is this model installed?" is + asked every time the model list is drawn and before every transcription, so + it can only afford existence plus an exact size — 3 GB of hashing cannot run + on that path. Which means a file carrying the catalogue's name and size + *will* be believed, so nothing may ever appear under its final name before it + has been verified — and the data has to be on the platter before the rename, + since NTFS journals the rename and not the bytes. + +* **The digest is computed while the bytes go past**, and re-seeded from the + ``.part`` already on disk when a transfer resumes. model.bin is up to 3 GB; + hashing it in a second pass would double the I/O of every download for a + number the transfer already had in its hands, and re-fetching it because a + user cancelled at 90% is how a feature gets used exactly once. + +* **Only the catalogue's own file names are ever deleted.** ``remove_model()`` + unlinks exactly those names and then calls ``os.rmdir()``, which refuses a + directory still holding anything else. Never ``shutil.rmtree()``: the root is + user-chosen, somebody will point it at a folder that already has their own + files in it, and a recursive delete of a user-chosen path only has to be + wrong once. + +**A note for the UI layer on the progress callbacks.** ``progress(done, total)`` +is reported per chunk, which is every 1 MB — around 3000 calls for large-v3, and +`total` means something different per call site: the whole model for +``download_model()``, model.bin alone for ``verify_model()``, and every file +being moved for ``move_models()``. Part 5 has to throttle before any +``wx.CallAfter`` or spoken percentage; forwarding these straight through would +flood the UI thread and have the screen reader read thousands of numbers. + +No user-facing text lives here either: failures travel as TranscriptionError +codes, and turning one into a sentence is the UI layer's job. +""" + +from __future__ import annotations + +import contextlib +import errno +import hashlib +import logging +import os +import shutil +import time +from dataclasses import dataclass + +import requests + +from app_paths import global_dir +from coord_locks import LockTimeout, canonical_dir, models_lock +from core import tls_trust +from core.transcription import errors, model_catalog + +# Subdirectory of the global data dir holding every model. +MODELS_DIRNAME = "whisper_models" + +_HF_FILE_URL = "https://huggingface.co/{repo}/resolve/{revision}/{filename}" + +# What a model directory currently holds. INCOMPLETE is what an interrupted +# download leaves behind, and it is deliberately distinct from ABSENT: the +# picker offers a download for one and a repair for the other, and only +# INCOMPLETE can name the files that are wrong. +STATE_ABSENT = "absent" +STATE_INCOMPLETE = "incomplete" +STATE_INSTALLED = "installed" + +_PART_SUFFIX = ".part" + +# Big enough that a 3 GB transfer is not three million iterations, small enough +# that cancelling between chunks still feels immediate on a slow connection. +_CHUNK_BYTES = 1024 * 1024 + +# (connect, read) — the same pair the portable Node.js download uses. The read +# timeout has to survive a stalled CDN; it is per read, not for the transfer. +_HTTP_TIMEOUT = (30, 300) + +# Room left on the volume beyond the transfer itself. Not politeness: the +# models directory normally shares a volume with messages.db, the media cache +# and log.log, all of which are being written while a multi-gigabyte download +# runs, so a download that "just fits" takes the rest of the app down with it. +_FREE_SPACE_SLACK_BYTES = 256 * 1024 * 1024 + +# How long to wait for another process to finish with the models directory, and +# in what slices. Sliced because NamedLock's own wait cannot be interrupted: +# between attempts is the only place a user's Cancel can be noticed, and being +# told "another window is busy" with a Cancel button that does nothing is worse +# than the wait itself. +# +# The user's Cancel is therefore the real bound on the wait, and the deadline +# below is only a backstop against a lock nobody will ever release. It is +# deliberately far past any plausible transfer: at five minutes it fired while +# another account was legitimately half way through a 3 GB download on a slow +# line, and reported it as a failure. +_LOCK_TIMEOUT_SECONDS = 12 * 60 * 60.0 +_LOCK_POLL_SECONDS = 2.0 + + +@dataclass(frozen=True) +class InstallState: + """What a root currently holds for one model. + + `missing` names the files that are absent *or* the wrong size — one + condition, not two, because the catalogue pins a revision and with it an + exact size for every file, so "present but 4 KB short" is an interrupted + download and nothing else. + """ + + state: str + missing: tuple[str, ...] = () + # Bytes of this model already on disk and correct — what a resumed download + # starts its progress bar at. + present_bytes: int = 0 + + +def default_models_dir() -> str: + """Where models live unless the user has chosen somewhere else. + + Global rather than per account: see the module docstring. + """ + return global_dir(MODELS_DIRNAME) + + +def model_dir(root, model_id) -> str: + """The directory holding one model's files under `root`.""" + return os.path.join(root, str(model_id)) + + +def file_url(model, filename) -> str: + """The Hugging Face URL for one of `model`'s files. + + Always ``/resolve//``, never ``/resolve/main/``. The sizes and the + model.bin digest in the catalogue were measured at that commit, so a URL + following the branch would start failing as MODEL_CORRUPTED the day + upstream republishes — and, far worse, would silently accept *different* + weights for any file whose size happened to be unchanged. + """ + return _HF_FILE_URL.format( + repo=model.repo, revision=model.revision, filename=filename + ) + + +def installation_state(root, model) -> InstallState: + """The cheap answer to "is this model usable?" — names and exact sizes. + + No hashing at all, on purpose: this runs whenever the model list is drawn + and before every transcription, and model.bin is up to 3 GB. verify_model() + is the expensive answer, for the caller with a reason to pay for it. + """ + directory = model_dir(root, model.id) + missing = [] + present_bytes = 0 + found_any = False + + for name, size in model.files: + try: + actual = os.path.getsize(os.path.join(directory, name)) + except OSError: + missing.append(name) + continue + found_any = True + if actual != size: + missing.append(name) + continue + present_bytes += size + + if not missing: + return InstallState(STATE_INSTALLED, (), present_bytes) + # "Nothing of it is here" is keyed on nothing of it being here, which + # includes the `.part` files of a transfer that was interrupted before its + # first file landed: zero correct bytes and a resumable 2 GB `.part` is an + # interrupted download, not an absence, and the two offer the user + # different buttons. + if not found_any and not _has_parts(directory, model): + return InstallState(STATE_ABSENT, tuple(missing), 0) + return InstallState(STATE_INCOMPLETE, tuple(missing), present_bytes) + + +def is_installed(root, model) -> bool: + """Whether every file of `model` is under `root` at its exact size.""" + return installation_state(root, model).state == STATE_INSTALLED + + +def list_installed(root) -> tuple[str, ...]: + """Ids of the complete models under `root`, in the catalogue's own order.""" + return tuple( + model.id for model in model_catalog.list_models() if is_installed(root, model) + ) + + +def list_unknown_dirs(root) -> tuple[str, ...]: + """Directory names under `root` that no catalogue entry claims. + + When a WinZapp update retires a model, whatever the user downloaded for it + stops being listed anywhere — and, since remove_model() refuses to delete + names it cannot look up in the catalogue, it also becomes undeletable from + inside the app. Up to 3 GB, invisible. This is what lets part 5 show those + folders and hand them to the user. + """ + known = {model.id for model in model_catalog.list_models()} + try: + entries = os.listdir(root) + except OSError: + return () + return tuple( + sorted( + name + for name in entries + if name not in known and os.path.isdir(os.path.join(root, name)) + ) + ) + + +def ensure_ready(root, model_id) -> str: + """The directory to load `model_id` from, or the right error code. + + What part 3 calls immediately before handing a path to CTranslate2. The + cheap check only: MODEL_NOT_INSTALLED and MODEL_CORRUPTED are two different + sentences for the user — download it, versus repair it — and choosing + between them must not cost 3 GB of hashing before a transcription that has + not even started. + """ + model = model_catalog.get_model(model_id) + if model is None: + # A settings file naming a model this version dropped. "Not installed" + # is both true and the one thing the UI can act on. + raise errors.TranscriptionError(errors.MODEL_NOT_INSTALLED, str(model_id)) + + state = installation_state(root, model) + if state.state == STATE_ABSENT: + raise errors.TranscriptionError(errors.MODEL_NOT_INSTALLED, model.id) + if state.state == STATE_INCOMPLETE: + raise errors.TranscriptionError( + errors.MODEL_CORRUPTED, + f"{model.id}: missing or wrong size: {', '.join(state.missing)}", + ) + return model_dir(root, model.id) + + +def free_bytes(path): + """Free bytes on the volume `path` lives on, or None if unmeasurable. + + Walks up to the nearest existing ancestor, because the models directory + does not exist yet the first time this is asked — and a "does it fit?" gate + answering "unknown" because the folder has not been created yet is a gate + that blocks the first download on every install. + """ + candidate = os.path.abspath(path) + while True: + try: + return int(shutil.disk_usage(candidate).free) + except OSError: + parent = os.path.dirname(candidate) + if parent == candidate: + return None + candidate = parent + + +def ensure_free_space(root, needed_bytes) -> None: + """Raise NO_DISK_SPACE unless `needed_bytes` fit under `root`, with slack. + + A volume nothing could be measured on passes: refusing the download because + disk_usage() failed would make transcription unavailable on a setup that + works perfectly well. + """ + free = free_bytes(root) + if free is None: + logging.info( + "[transcription] free space at %s is unknown; allowing the transfer", root + ) + return + if free < needed_bytes + _FREE_SPACE_SLACK_BYTES: + raise errors.TranscriptionError( + errors.NO_DISK_SPACE, + f"{root}: {needed_bytes} bytes needed plus " + f"{_FREE_SPACE_SLACK_BYTES} of slack, {free} free", + ) + + +def download_model(model, root, progress=None, should_cancel=None, session=None): + """Fetch every missing file of `model` into `root`, verified. Returns its dir. + + `progress(done, total)` counts the **whole model**, not the file in flight: + the user wants one bar for "downloading large-v3", not six of them in a row. + + This repairs an **incomplete** model — files missing or the wrong size — + and only that. Anything already there at its catalogued size is trusted on + the size alone, exactly as installation_state() trusts it, so a model whose + files are all the right size and whose weights are wrong is not repaired + here: this call would fetch nothing and report success. repair_model() is + the way out of that one, and verify_model() is what detects it. + + An interrupted file is picked up where it stopped, by asking for a byte + range: model.bin is 95-99% of a model, so restarting it is restarting + everything. + + `should_cancel()` is consulted between chunks and while waiting for the + shared models lock; `session` exists so the download can be driven from a + test without a network. + """ + # Sampled before the lock, which may create the root itself on the + # non-Windows fallback path. + root_created = not os.path.isdir(root) + try: + with _hold_models_lock(root, should_cancel): + return _download_locked( + model, root, root_created, progress, should_cancel, session + ) + except LockTimeout as exc: + raise errors.TranscriptionError(errors.MODELS_BUSY, f"{model.id}: {exc}") from exc + + +def repair_model(model, root, progress=None, should_cancel=None, session=None): + """Delete whatever is on disk for `model` and download it again. + + The only way out of the state download_model() cannot fix: every file the + right size, model.bin's digest wrong (a silent corruption, or weights from + a revision this catalogue never measured). installation_state() calls that + installed and a re-download then fetches nothing at all, so without this the + bad weights survive every attempt to replace them. + + Both halves run under one hold of the models lock — it is re-entrant within + the process — so no other process can start a download into the directory + between the delete and the fetch. + """ + try: + with _hold_models_lock(root, should_cancel): + remove_model(root, model.id) + return download_model( + model, + root, + progress=progress, + should_cancel=should_cancel, + session=session, + ) + except LockTimeout as exc: + raise errors.TranscriptionError(errors.MODELS_BUSY, f"{model.id}: {exc}") from exc + + +def verify_model(root, model, progress=None, should_cancel=None) -> None: + """The expensive check: every size, then model.bin's sha256. + + Separate from installation_state() because it reads up to 3 GB, which is + also why it takes a progress callback and a cancel check of its own: a user + who asked to verify a model has to be able to change their mind, and to be + told how far it got while they wait. + """ + directory = ensure_ready(root, model.id) + digest = _hash_file( + os.path.join(directory, "model.bin"), + model.model_bin_bytes, + progress, + should_cancel, + ) + if digest != model.model_bin_sha256: + raise errors.TranscriptionError( + errors.MODEL_CORRUPTED, + f"{model.id}: model.bin sha256 {digest}, expected {model.model_bin_sha256}", + ) + + +def remove_model(root, model_id, should_cancel=None) -> bool: + """Delete one model's files from `root` — and nothing else. True if any went. + + Only the names the catalogue lists (plus their `.part` leftovers) are + unlinked, and the directory itself goes through os.rmdir(), which refuses a + directory that still holds anything. See the module docstring for why this + is never shutil.rmtree(). + + False means "there was nothing to delete" and nothing else. Another process + holding the models directory raises MODELS_BUSY instead: part 5 wires this + to a delete button, and a user who is told nothing was removed has no way + to know whether their 3 GB is still there. `should_cancel` is honoured while + waiting for that lock, so the button's own Cancel works. + """ + model = model_catalog.get_model(model_id) + if model is None: + # An id the catalogue no longer knows is exactly the case where the + # names to delete are unknown, so nothing here can be removed safely. + # The folder is left alone; list_unknown_dirs() is how the user sees it. + logging.info("[transcription] not removing unknown model id %s", model_id) + return False + + try: + with _hold_models_lock(root, should_cancel): + directory = model_dir(root, model.id) + removed = False + for name, _size in model.files: + removed = _unlink(os.path.join(directory, name)) or removed + removed = _unlink(os.path.join(directory, name + _PART_SUFFIX)) or removed + _remove_empty_dir(directory) + return removed + except LockTimeout as exc: + raise errors.TranscriptionError(errors.MODELS_BUSY, f"{model.id}: {exc}") from exc + + +def move_models(old_root, new_root, progress=None, should_cancel=None): + """Move the models from `old_root` to `new_root`. Returns the ids moved. + + Copy, verify, then delete — one model at a time, and deliberately not a + per-file ``shutil.move()``. A move empties the source as it fills the + destination, so an interruption half way through leaves *both* roots + incomplete and the user with no usable model at all; copying into `.part` + names, publishing them only once every size matches, and deleting the + source afterwards means each model is whole in exactly one of the two roots + at every instant. The price is both copies on disk at once, which is what + the free-space gate is measured against — and why a same-volume move is + tried as a single rename first, where it is instant and costs nothing. + + The copy is chunked here rather than handed to shutil.copy2() so that it can + be cancelled and can report progress: a 3 GB copy is a minute or more of + silence, which for a screen-reader user is indistinguishable from a hang. + + Incomplete models travel too, and arrive as incomplete as they left: left + behind, they would sit in a directory no screen in part 5 ever looks at + again. (Their `.part` files ride along only on the rename path, which moves + the whole directory; the copy path is per catalogued file, and the leftovers + go with the source.) + """ + if canonical_dir(old_root) == canonical_dir(new_root): + # Two spellings of one directory. Windows folds case and resolves + # junctions and `subst` drives; os.path.abspath does neither, so + # "...\\Whisper_Models" and "...\\whisper_models" would pass a string + # comparison, be found "already at the destination" — because they ARE + # the destination — and be deleted as duplicates of themselves. + logging.info("[transcription] models root unchanged (%s); nothing to move", + new_root) + return () + + root_created = not os.path.isdir(new_root) + # Locked in a fixed order, so two processes moving in opposite directions + # cannot each hold the other's root. + first, second = sorted((old_root, new_root), key=canonical_dir) + try: + with _hold_models_lock(first, should_cancel), _hold_models_lock( + second, should_cancel + ): + return _move_locked( + old_root, new_root, root_created, progress, should_cancel + ) + except LockTimeout as exc: + raise errors.TranscriptionError(errors.MODELS_BUSY, str(exc)) from exc + + +# ── Internals ──────────────────────────────────────────────────────────────── + + +@contextlib.contextmanager +def _hold_models_lock(root, should_cancel=None): + """Hold the cross-process models lock, still answering the Cancel button. + + Waited for in short slices rather than one long block: NamedLock's own wait + is not interruptible, and between attempts is the only place a user who has + been told "another window is downloading" can give up. The cancel check sits + on the LockTimeout branch alone, so an uncontended lock is not slowed by it + and a contended one notices the first cancellation at t≈2 s — which is the + responsiveness this is for, not a deadline anyone can measure. + + LockTimeout is left to the caller, which knows which of its codes a failed + wait belongs to; for every caller here that is MODELS_BUSY. + """ + deadline = time.monotonic() + _LOCK_TIMEOUT_SECONDS + while True: + # The lock FILE goes in the global dir, never inside the models root: + # the flock fallback creates it, and a file of ours inside a folder the + # user chose is both litter and the reason an empty-root cleanup would + # stop working. + lock = models_lock(root, global_dir(), timeout=_LOCK_POLL_SECONDS) + try: + lock.acquire() + break + except LockTimeout: + _check_cancel(should_cancel) + if time.monotonic() >= deadline: + raise + try: + yield lock + finally: + lock.release() + + +def _download_locked(model, root, root_created, progress, should_cancel, session): + """download_model()'s body, with the models lock already held.""" + directory = model_dir(root, model.id) + total = model.download_bytes + done_bytes = 0 + pending = [] + + for name, size in model.files: + if _file_complete(os.path.join(directory, name), size): + done_bytes += size + continue + pending.append((name, size)) + + _report(progress, done_bytes, total) + if not pending: + # A complete model can still be sitting next to the `.part` of an older + # attempt at a file that has since arrived — up to 3 GB of dead weight + # that nothing else on this path would ever sweep. + _remove_parts(directory, model) + return directory + + # Before a single byte is fetched: the point of the gate is to fail while + # the user can still pick a smaller model, not half way into the big one. + # What a resume will not re-fetch is already on the disk, so it is not + # counted again. + ensure_free_space( + root, total - done_bytes - _resumable_bytes(directory, pending) + ) + + created_dir = not os.path.isdir(directory) + owned_session = session is None + # Through tls_trust, so a machine whose HTTPS is intercepted locally (an + # antivirus, a proxy) can download the models at all — see that module. + session = tls_trust.create_session() if owned_session else session + try: + try: + os.makedirs(directory, exist_ok=True) + for name, size in pending: + _check_cancel(should_cancel) + logging.info( + "[transcription] downloading %s/%s (%d bytes)", model.id, name, size + ) + done_bytes = _download_file( + session, model, directory, name, size, + done_bytes, total, progress, should_cancel, + ) + except Exception as exc: + failure = _as_transcription_error(exc, model.id, errors.MODEL_DOWNLOAD_FAILED) + # A prefix the server disowned is the one thing that must not be + # resumed from; every other interruption — a cancellation, a + # dropped connection — leaves a valid prefix, and keeping it is + # what makes cancelling at 90% cost 10% next time. + if failure.code == errors.MODEL_CORRUPTED: + _remove_parts(directory, model) + if created_dir: + _remove_empty_dir(directory) + if root_created: + # Nothing of the app's may be left in a folder the user chose + # and the app then failed to use. + _remove_empty_dir(root) + raise failure + finally: + if owned_session: + session.close() + + logging.info("[transcription] model %s is installed at %s", model.id, directory) + return directory + + +def _move_locked(old_root, new_root, root_created, progress, should_cancel): + """move_models()'s body, with both roots' locks already held.""" + # Every model with anything of it under the old root, complete or not. + plans = [] + for model in model_catalog.list_models(): + present = _present_files(old_root, model) + if present: + plans.append((model, present)) + + total = sum(size for _model, present in plans for _name, size in present) + done = 0 + _report(progress, done, total) + + # Filled in as each model lands, so that a failure part way through can + # still say which ones crossed: every model is whole in exactly one root + # either way, but "three of your five are in the new folder" is the only + # thing part 5 can tell the user, and a bare error code loses it. + moved = [] + try: + _move_each( + plans, old_root, new_root, done, total, progress, should_cancel, moved + ) + except Exception as exc: + if root_created: + # Nothing of the app's may be left in a folder the user chose and + # the app then failed to fill. + _remove_empty_dir(new_root) + failure = _as_transcription_error(exc, new_root, errors.MODEL_MOVE_FAILED) + carried = errors.TranscriptionError( + failure.code, + f"{failure.detail}; moved before the failure: {', '.join(moved) or 'none'}", + ) + # Also as data, not only as log text: `detail` is log-facing by design, + # so a UI that had to parse it would be reading a string this module + # promises nothing about. + carried.moved = tuple(moved) + raise carried from exc + + # The old root itself is left alone even when it ends up empty: it may be a + # folder the user made and pointed WinZapp at, and moving files out of a + # directory is not permission to delete it. + return tuple(moved) + + +def _move_each(plans, old_root, new_root, done, total, progress, should_cancel, moved): + """Move each planned model, in catalogue order, appending ids to `moved`. + + The caller owns that list so that it survives an exception raised half way + through — a returned value would not. + """ + for model, present in plans: + _check_cancel(should_cancel) + model_bytes = sum(size for _name, size in present) + source = model_dir(old_root, model.id) + destination = model_dir(new_root, model.id) + + if canonical_dir(source) == canonical_dir(destination): + # One directory under two names: it is already where it is going, + # and above all there is nothing here to delete. + done += model_bytes + _report(progress, done, total) + moved.append(model.id) + continue + + if is_installed(new_root, model): + # A complete copy is already at the destination, so the source one + # is pure duplication of up to 3 GB — which is what the user asked + # to be rid of by moving the folder in the first place. + remove_model(old_root, model.id) + done += model_bytes + _report(progress, done, total) + moved.append(model.id) + continue + + if not os.path.exists(destination) and _try_rename(source, destination): + done += model_bytes + _report(progress, done, total) + moved.append(model.id) + logging.info("[transcription] renamed model %s into %s", model.id, new_root) + continue + + # Only the copy path needs room: the rename above moved nothing. + ensure_free_space(new_root, model_bytes) + done = _copy_model( + model, present, old_root, new_root, done, total, progress, should_cancel + ) + remove_model(old_root, model.id) + moved.append(model.id) + logging.info("[transcription] copied model %s to %s", model.id, new_root) + + +def _present_files(root, model): + """[(name, size on disk)] for the catalogued files actually in `root`. + + Sizes come from the disk rather than from the catalogue because a + half-downloaded file has to move too, and has to arrive as the same half — + the destination must end in the state the source was in, or the move turns + an incomplete model into a corrupted-looking one. + """ + directory = model_dir(root, model.id) + present = [] + for name, _size in model.files: + try: + present.append((name, os.path.getsize(os.path.join(directory, name)))) + except OSError: + continue + return present + + +def _try_rename(source, destination) -> bool: + """Move a whole model directory with one rename, or say it is not possible. + + Same-volume is the common case (the user picks another folder on the same + disk), and there this is atomic, instant and free — without it, moving 5 GB + from D:\\a to D:\\b would be refused on a disk with 5.4 GB free for a + transfer that needs no space at all. Any OSError means "not this way": + EXDEV across volumes, but also a destination that will not take it. + """ + try: + os.makedirs(os.path.dirname(destination), exist_ok=True) + os.rename(source, destination) + return True + except OSError as exc: + logging.info( + "[transcription] cannot rename %s to %s (%s); copying instead", + source, destination, exc, + ) + return False + + +def _copy_model(model, present, old_root, new_root, done, total, progress, should_cancel): + """Copy one model's files from one root to the other, verified by size.""" + source = model_dir(old_root, model.id) + directory = model_dir(new_root, model.id) + created_dir = not os.path.isdir(directory) + try: + os.makedirs(directory, exist_ok=True) + for name, size in present: + _check_cancel(should_cancel) + if _file_complete(os.path.join(directory, name), size): + done += size + _report(progress, done, total) + continue + done = _write_part( + directory, + name, + _read_chunks(os.path.join(source, name)), + size, + # No hashing on a local copy: the source was verified when it + # was downloaded, and re-reading 3 GB to prove that a + # filesystem copied it correctly would double the move. + None, + done, + total, + progress, + should_cancel, + ) + except Exception as exc: + _remove_parts(directory, model) + if created_dir: + _remove_empty_dir(directory) + raise _as_transcription_error(exc, model.id, errors.MODEL_MOVE_FAILED) + return done + + +def _file_complete(path, expected_bytes) -> bool: + """Whether `path` exists at exactly `expected_bytes`.""" + try: + return os.path.getsize(path) == expected_bytes + except OSError: + return False + + +def _has_parts(directory, model) -> bool: + """Whether an interrupted transfer left any `.part` of `model` here.""" + return any( + os.path.exists(os.path.join(directory, name + _PART_SUFFIX)) + for name, _size in model.files + ) + + +def _resumable_bytes(directory, pending) -> int: + """Bytes of `pending` already on disk as a resumable `.part`.""" + return sum( + _resume_offset(os.path.join(directory, name + _PART_SUFFIX), size) + for name, size in pending + ) + + +def _resume_offset(part_path, expected_bytes) -> int: + """How much of `part_path` a byte range may pick up from. + + Only a strict prefix qualifies. A `.part` at or past the expected size is + not a prefix of anything — a server that sent too much, or a leftover from + a revision this catalogue never measured — and resuming from it would build + a file of exactly the right length out of the wrong bytes, which nothing + but the digest would catch, and only for model.bin. + """ + try: + size = os.path.getsize(part_path) + except OSError: + return 0 + return size if 0 < size < expected_bytes else 0 + + +def _download_file(session, model, directory, name, expected_bytes, + done_bytes, total, progress, should_cancel) -> int: + """One file of `model`, streamed into place. Returns the new byte count.""" + part_path = os.path.join(directory, name + _PART_SUFFIX) + # Only model.bin has a digest to check against: it is the sole LFS file in + # these repositories, and the one where a silent corruption costs a + # multi-gigabyte re-download to discover. + expected_sha256 = model.model_bin_sha256 if name == "model.bin" else None + + # Only a file with a digest may be resumed, and that is the whole rule. + # For the auxiliary files the only check is `written == expected_bytes`, + # which a `.part` that is NOT a prefix of the current file satisfies + # exactly: splice a stale 1 KB head onto the rest and the result has the + # catalogued size, so installation_state, is_installed, verify_model and + # ensure_ready all call the model healthy — verify_model included, because + # it only hashes model.bin — and CTranslate2 then dies on the tokenizer + # with nothing but "internal error" to show the user. A user's own + # cancellation always leaves a valid prefix, but a power cut or a kill does + # not (fsync only runs at the end of _write_part, so NTFS can leave a + # `.part` longer than the durable data, with a tail of zeros), and neither + # does a catalogue revision that changed between the interruption and the + # resume. model.bin is self-healing under the same accident because the + # digest catches it and the failure sweeps the `.part`; the others have + # nothing to catch it with. The cost of the rule is nil: on large-v3, + # model.bin is 3,087,284,237 of 3,090,835,702 bytes, so this keeps 99.885% + # of what resuming exists to keep and removes the class entirely. + # + # The prefix is hashed *before* the request, so the range asked for is the + # range actually accounted for. Hashing it afterwards would leave the + # digest and the offset free to disagree whenever the local read fell short. + resume_from = _resume_offset(part_path, expected_bytes) if expected_sha256 else 0 + digest, resume_from = _seed_from_part( + part_path, resume_from, expected_sha256, should_cancel + ) + + headers = {"Range": f"bytes={resume_from}-"} if resume_from else None + response = session.get( + file_url(model, name), stream=True, timeout=_HTTP_TIMEOUT, headers=headers + ) + try: + response.raise_for_status() + if resume_from and not _range_honoured(response, resume_from): + # The range was ignored and the whole file is coming: appending it + # to what is already there would produce a file too long, or — with + # a stale prefix — the right length and the wrong bytes. + logging.info( + "[transcription] %s/%s: range not honoured (status %s), starting over", + model.id, name, getattr(response, "status_code", "?"), + ) + digest, resume_from = _seed_from_part(part_path, 0, expected_sha256, None) + return _write_part( + directory, + name, + response.iter_content(chunk_size=_CHUNK_BYTES), + expected_bytes, + expected_sha256, + done_bytes, + total, + progress, + should_cancel, + resume_from=resume_from, + digest=digest, + ) + finally: + response.close() + + +def _range_honoured(response, resume_from) -> bool: + """Whether the answer really begins where the request asked it to. + + Status 206 says *a* range is coming, not *which* one. Neither Hugging Face + nor its CDN gets this wrong, and after the digest-only resume rule the one + file that resumes would catch a wrong offset by its hash anyway — but the + comparison is one line and closes the hole instead of arguing about it. A + 206 with no Content-Range at all (which is not legal) reads as "not + honoured", so the file starts over: always safe. + """ + if getattr(response, "status_code", None) != 206: + return False + headers = getattr(response, "headers", None) or {} + value = str(headers.get("Content-Range", "")).strip() + if value.lower().startswith("bytes"): + value = value[len("bytes"):].strip() + return value.startswith(f"{resume_from}-") + + +def _seed_from_part(part_path, resume_from, expected_sha256, should_cancel): + """(digest, offset) for a transfer resuming from `resume_from` bytes. + + Reading the prefix back is a local read of the bytes deliberately *not* + being fetched again — nothing against the gigabytes of network it saves — + and it is the only way a digest computed from the stream can survive an + interrupted transfer. Anything unexpected about the prefix answers (fresh + digest, 0): starting the file over is always safe, resuming from a prefix + that could not be read is not. + """ + fresh = hashlib.sha256() if expected_sha256 else None + if not resume_from: + return fresh, 0 + + digest = hashlib.sha256() if expected_sha256 else None + read = 0 + try: + with open(part_path, "rb") as fh: + while read < resume_from: + chunk = fh.read(min(_CHUNK_BYTES, resume_from - read)) + if not chunk: + break + _check_cancel(should_cancel) + if digest is not None: + digest.update(chunk) + read += len(chunk) + except OSError as exc: + logging.info("[transcription] cannot resume %s: %s", part_path, exc) + return fresh, 0 + + if read != resume_from: + return fresh, 0 + logging.info("[transcription] resuming %s at %d bytes", part_path, read) + return digest, read + + +def _write_part(directory, name, chunks, expected_bytes, expected_sha256, + done_bytes, total, progress, should_cancel, + resume_from=0, digest=None) -> int: + """Stream `chunks` into `.part`, verify them, and only then publish. + + The one place the "a file under its final name has been verified" invariant + is enforced, which is why the download and the move both go through it + rather than each writing their own files: two copies of this would drift, + and the way that drift shows up is a model that looks installed and that + CTranslate2 then refuses to load. + """ + part_path = os.path.join(directory, name + _PART_SUFFIX) + if digest is None and expected_sha256: + digest = hashlib.sha256() + written = resume_from + if resume_from: + _report(progress, done_bytes + written, total) + + # Appending only when there is a prefix this call has already accounted + # for; otherwise the file starts again from nothing. + with open(part_path, "ab" if resume_from else "wb") as fh: + for chunk in chunks: + _check_cancel(should_cancel) + if not chunk: + continue + fh.write(chunk) + if digest is not None: + digest.update(chunk) + written += len(chunk) + _report(progress, done_bytes + written, total) + fh.flush() + # NTFS journals the rename, not the data behind it. Without this, a + # power cut between the two can leave a file of exactly the right size + # full of zeros under the final name — which the install check believes + # forever, and which for the auxiliary files nothing would ever detect. + os.fsync(fh.fileno()) + + if written != expected_bytes: + raise errors.TranscriptionError( + errors.MODEL_CORRUPTED, + f"{name}: expected {expected_bytes} bytes, got {written}", + ) + if digest is not None: + actual = digest.hexdigest() + if actual != expected_sha256: + raise errors.TranscriptionError( + errors.MODEL_CORRUPTED, + f"{name}: sha256 {actual}, expected {expected_sha256}", + ) + + os.replace(part_path, os.path.join(directory, name)) + return done_bytes + written + + +def _read_chunks(path): + """`path` as a chunk iterator, shaped like requests' iter_content().""" + with open(path, "rb") as fh: + while True: + chunk = fh.read(_CHUNK_BYTES) + if not chunk: + return + yield chunk + + +def _hash_file(path, total_bytes, progress, should_cancel) -> str: + """sha256 of `path`, reported and cancellable as it goes.""" + digest = hashlib.sha256() + read = 0 + try: + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(_CHUNK_BYTES), b""): + _check_cancel(should_cancel) + digest.update(chunk) + read += len(chunk) + _report(progress, read, total_bytes) + except OSError as exc: + # A file that cannot be read is not a file that can be transcribed + # with, and the way out of it is the same as for a bad digest. + raise errors.TranscriptionError(errors.MODEL_CORRUPTED, f"{path}: {exc}") from exc + return digest.hexdigest() + + +def _check_cancel(should_cancel) -> None: + if should_cancel is not None and should_cancel(): + raise errors.TranscriptionError(errors.CANCELLED, "cancelled by the user") + + +def _report(progress, done, total) -> None: + if progress is not None: + progress(done, total) + + +def _as_transcription_error(exc, model_id, fallback): + """Map whatever went wrong onto the closed set of codes. + + Nothing escapes unclassified: a code the UI cannot translate is a code + I18n.t() renders as its own key name, which NVDA then reads out letter by + letter instead of a sentence. + """ + if isinstance(exc, errors.TranscriptionError): + return exc + if isinstance(exc, OSError) and exc.errno == errno.ENOSPC: + # The gate ran before the transfer, but a multi-gigabyte transfer takes + # long enough for something else on the machine to fill the volume. + return errors.TranscriptionError(errors.NO_DISK_SPACE, f"{model_id}: {exc}") + return errors.TranscriptionError(fallback, f"{model_id}: {exc}") + + +def _remove_parts(directory, model) -> None: + """Drop every `.part` this model could have left behind in `directory`.""" + for name, _size in model.files: + _unlink(os.path.join(directory, name + _PART_SUFFIX)) + + +def _unlink(path) -> bool: + try: + os.remove(path) + return True + except OSError: + return False + + +def _remove_empty_dir(directory) -> None: + """rmdir, which is a no-op on any directory that still holds a file.""" + try: + os.rmdir(directory) + except OSError: + pass diff --git a/client/languages/en-US.json b/client/languages/en-US.json index a736492a9..eb1c40b32 100644 --- a/client/languages/en-US.json +++ b/client/languages/en-US.json @@ -1036,5 +1036,30 @@ "contact_details_title": "Contact", "contact_pick_number": "This contact has more than one number. Choose one:", "contact_no_number": "This contact card carries no phone number.", - "search_contact_label": "&Search contact" + "search_contact_label": "&Search contact", + "transcription_size_fast": "fast", + "transcription_size_balanced": "balanced", + "transcription_size_accurate": "accurate", + "transcription_device_cuda_selected": "The transcription will run on the graphics card.", + "transcription_device_cpu_requested": "The transcription will run on the processor.", + "transcription_device_cuda_unavailable": "CUDA is not available. The transcription will run on the processor.", + "transcription_device_no_cuda_found": "No compatible graphics card was found. The transcription will run on the processor.", + "transcription_device_cuda_driver_error": "The graphics card could not be queried. The transcription will run on the processor.", + "transcription_error_model_not_installed": "The selected transcription model is not installed. Download the model before transcribing.", + "transcription_error_model_corrupted": "The model files are incomplete or corrupted. Download the model again.", + "transcription_error_no_disk_space": "There is not enough disk space to download the model.", + "transcription_error_cuda_unavailable": "CUDA is not available. Switch the device to the processor in the settings and try again.", + "transcription_error_insufficient_vram": "The graphics card does not have enough memory for this model. Choose a smaller model.", + "transcription_error_insufficient_ram": "The computer does not have enough memory for this model. Choose a smaller model.", + "transcription_error_unsupported_audio_format": "This audio format was not recognized and cannot be transcribed.", + "transcription_error_ffmpeg_failed": "The audio could not be converted for transcription.", + "transcription_error_audio_incomplete": "The audio file is incomplete or damaged.", + "transcription_error_media_not_downloaded": "The audio has not been downloaded yet. Wait for the download to finish and try again.", + "transcription_error_cancelled": "Transcription cancelled.", + "transcription_error_save_failed": "The transcription could not be saved.", + "transcription_error_backend_missing": "The transcription component is not installed in this copy of WinZapp.", + "transcription_error_model_download_failed": "The transcription model could not be downloaded. Check your connection and try again.", + "transcription_error_model_move_failed": "The models folder could not be moved. The models are still in the previous folder.", + "transcription_error_models_busy": "Another WinZapp window is using the models folder. Wait for it to finish and try again.", + "transcription_error_backend_error": "An internal error occurred during the transcription." } diff --git a/client/languages/es-ES.json b/client/languages/es-ES.json index 89acd90a9..9b349e44d 100644 --- a/client/languages/es-ES.json +++ b/client/languages/es-ES.json @@ -1036,5 +1036,30 @@ "contact_details_title": "Contacto", "contact_pick_number": "Este contacto tiene más de un número. Elige uno:", "contact_no_number": "Esta tarjeta de contacto no incluye ningún número de teléfono.", - "search_contact_label": "&Buscar contacto" + "search_contact_label": "&Buscar contacto", + "transcription_size_fast": "rápido", + "transcription_size_balanced": "equilibrado", + "transcription_size_accurate": "preciso", + "transcription_device_cuda_selected": "La transcripción se ejecutará en la tarjeta gráfica.", + "transcription_device_cpu_requested": "La transcripción se ejecutará en el procesador.", + "transcription_device_cuda_unavailable": "CUDA no está disponible. La transcripción se ejecutará en el procesador.", + "transcription_device_no_cuda_found": "No se encontró ninguna tarjeta gráfica compatible. La transcripción se ejecutará en el procesador.", + "transcription_device_cuda_driver_error": "No se pudo consultar la tarjeta gráfica. La transcripción se ejecutará en el procesador.", + "transcription_error_model_not_installed": "El modelo de transcripción elegido no está instalado. Descarga el modelo antes de transcribir.", + "transcription_error_model_corrupted": "Los archivos del modelo están incompletos o dañados. Descarga el modelo de nuevo.", + "transcription_error_no_disk_space": "No hay suficiente espacio en disco para descargar el modelo.", + "transcription_error_cuda_unavailable": "CUDA no está disponible. Cambia el dispositivo a procesador en la configuración e inténtalo de nuevo.", + "transcription_error_insufficient_vram": "La tarjeta gráfica no tiene memoria suficiente para este modelo. Elige un modelo más pequeño.", + "transcription_error_insufficient_ram": "El equipo no tiene memoria suficiente para este modelo. Elige un modelo más pequeño.", + "transcription_error_unsupported_audio_format": "No se reconoció el formato de este audio y no se puede transcribir.", + "transcription_error_ffmpeg_failed": "No se pudo convertir el audio para transcribirlo.", + "transcription_error_audio_incomplete": "El archivo de audio está incompleto o dañado.", + "transcription_error_media_not_downloaded": "El audio aún no se ha descargado. Espera a que termine la descarga e inténtalo de nuevo.", + "transcription_error_cancelled": "Transcripción cancelada.", + "transcription_error_save_failed": "No se pudo guardar la transcripción.", + "transcription_error_backend_missing": "El componente de transcripción no está instalado en esta copia de WinZapp.", + "transcription_error_model_download_failed": "No se pudo descargar el modelo de transcripción. Comprueba la conexión e inténtalo de nuevo.", + "transcription_error_model_move_failed": "No se pudo mover la carpeta de los modelos. Los modelos siguen en la carpeta anterior.", + "transcription_error_models_busy": "Otra ventana de WinZapp está usando la carpeta de modelos. Espera a que termine e inténtalo de nuevo.", + "transcription_error_backend_error": "Se produjo un error interno durante la transcripción." } diff --git a/client/languages/pl.json b/client/languages/pl.json index a1174a216..fd6a6c9c0 100644 --- a/client/languages/pl.json +++ b/client/languages/pl.json @@ -1036,5 +1036,30 @@ "contact_details_title": "Kontakt", "contact_pick_number": "Ten kontakt ma więcej niż jeden numer. Wybierz jeden:", "contact_no_number": "Ta wizytówka kontaktu nie zawiera numeru telefonu.", - "search_contact_label": "&Szukaj kontaktu" + "search_contact_label": "&Szukaj kontaktu", + "transcription_size_fast": "szybki", + "transcription_size_balanced": "zrównoważony", + "transcription_size_accurate": "dokładny", + "transcription_device_cuda_selected": "Transkrypcja zostanie wykonana przez kartę graficzną.", + "transcription_device_cpu_requested": "Transkrypcja zostanie wykonana przez procesor.", + "transcription_device_cuda_unavailable": "Technologia CUDA jest niedostępna. Transkrypcja zostanie wykonana przez procesor.", + "transcription_device_no_cuda_found": "Nie znaleziono zgodnej karty graficznej. Transkrypcja zostanie wykonana przez procesor.", + "transcription_device_cuda_driver_error": "Nie udało się odczytać informacji o karcie graficznej. Transkrypcja zostanie wykonana przez procesor.", + "transcription_error_model_not_installed": "Wybrany model transkrypcji nie jest zainstalowany. Pobierz model przed rozpoczęciem transkrypcji.", + "transcription_error_model_corrupted": "Pliki modelu są niekompletne lub uszkodzone. Pobierz model ponownie.", + "transcription_error_no_disk_space": "Na dysku nie ma wystarczającej ilości miejsca, aby pobrać model.", + "transcription_error_cuda_unavailable": "Technologia CUDA jest niedostępna. Zmień urządzenie na procesor w ustawieniach i spróbuj ponownie.", + "transcription_error_insufficient_vram": "Karta graficzna nie ma wystarczającej ilości pamięci dla tego modelu. Wybierz mniejszy model.", + "transcription_error_insufficient_ram": "Komputer nie ma wystarczającej ilości pamięci dla tego modelu. Wybierz mniejszy model.", + "transcription_error_unsupported_audio_format": "Format tego pliku audio nie został rozpoznany i nie można go przetranskrybować.", + "transcription_error_ffmpeg_failed": "Nie udało się przekonwertować pliku audio na potrzeby transkrypcji.", + "transcription_error_audio_incomplete": "Plik audio jest niekompletny lub uszkodzony.", + "transcription_error_media_not_downloaded": "Plik audio nie został jeszcze pobrany. Poczekaj na zakończenie pobierania i spróbuj ponownie.", + "transcription_error_cancelled": "Transkrypcja anulowana.", + "transcription_error_save_failed": "Nie udało się zapisać transkrypcji.", + "transcription_error_backend_missing": "Składnik transkrypcji nie jest zainstalowany w tej kopii programu WinZapp.", + "transcription_error_model_download_failed": "Nie udało się pobrać modelu transkrypcji. Sprawdź połączenie i spróbuj ponownie.", + "transcription_error_model_move_failed": "Nie udało się przenieść folderu z modelami. Modele pozostały w poprzednim folderze.", + "transcription_error_models_busy": "Inne okno programu WinZapp korzysta z folderu modeli. Poczekaj, aż skończy, i spróbuj ponownie.", + "transcription_error_backend_error": "Podczas transkrypcji wystąpił błąd wewnętrzny." } diff --git a/client/languages/pt-BR.json b/client/languages/pt-BR.json index 94ffecda6..1d50163b9 100644 --- a/client/languages/pt-BR.json +++ b/client/languages/pt-BR.json @@ -1036,5 +1036,30 @@ "contact_details_title": "Contato", "contact_pick_number": "Este contato tem mais de um número. Escolha um:", "contact_no_number": "Este cartão de contato não traz nenhum número de telefone.", - "search_contact_label": "&Buscar contato" + "search_contact_label": "&Buscar contato", + "transcription_size_fast": "rápido", + "transcription_size_balanced": "equilibrado", + "transcription_size_accurate": "preciso", + "transcription_device_cuda_selected": "A transcrição será executada pela placa de vídeo.", + "transcription_device_cpu_requested": "A transcrição será executada pelo processador.", + "transcription_device_cuda_unavailable": "CUDA não está disponível. A transcrição será executada pelo processador.", + "transcription_device_no_cuda_found": "Nenhuma placa de vídeo compatível foi encontrada. A transcrição será executada pelo processador.", + "transcription_device_cuda_driver_error": "Não foi possível consultar a placa de vídeo. A transcrição será executada pelo processador.", + "transcription_error_model_not_installed": "O modelo de transcrição escolhido não está instalado. Baixe o modelo antes de transcrever.", + "transcription_error_model_corrupted": "Os arquivos do modelo estão incompletos ou corrompidos. Baixe o modelo novamente.", + "transcription_error_no_disk_space": "Não há espaço em disco suficiente para baixar o modelo.", + "transcription_error_cuda_unavailable": "CUDA não está disponível. Mude o dispositivo para o processador nas configurações e tente novamente.", + "transcription_error_insufficient_vram": "A memória da placa de vídeo não é suficiente para este modelo. Escolha um modelo menor.", + "transcription_error_insufficient_ram": "A memória do computador não é suficiente para este modelo. Escolha um modelo menor.", + "transcription_error_unsupported_audio_format": "O formato deste áudio não foi reconhecido e não pode ser transcrito.", + "transcription_error_ffmpeg_failed": "Não foi possível converter o áudio para transcrever.", + "transcription_error_audio_incomplete": "O arquivo de áudio está incompleto ou danificado.", + "transcription_error_media_not_downloaded": "O áudio ainda não foi baixado. Aguarde o download terminar e tente novamente.", + "transcription_error_cancelled": "Transcrição cancelada.", + "transcription_error_save_failed": "Não foi possível salvar a transcrição.", + "transcription_error_backend_missing": "O componente de transcrição não está instalado nesta cópia do WinZapp.", + "transcription_error_model_download_failed": "Não foi possível baixar o modelo de transcrição. Verifique a conexão e tente novamente.", + "transcription_error_model_move_failed": "Não foi possível mover a pasta dos modelos. Os modelos continuam na pasta anterior.", + "transcription_error_models_busy": "Outra janela do WinZapp está usando a pasta dos modelos. Aguarde ela terminar e tente novamente.", + "transcription_error_backend_error": "Ocorreu um erro interno durante a transcrição." } diff --git a/client/languages/pt-PT.json b/client/languages/pt-PT.json index 474550903..1f7d7be60 100644 --- a/client/languages/pt-PT.json +++ b/client/languages/pt-PT.json @@ -1036,5 +1036,30 @@ "contact_details_title": "Contacto", "contact_pick_number": "Este contacto tem mais do que um número. Escolha um:", "contact_no_number": "Este cartão de contacto não traz nenhum número de telefone.", - "search_contact_label": "&Procurar contacto" + "search_contact_label": "&Procurar contacto", + "transcription_size_fast": "rápido", + "transcription_size_balanced": "equilibrado", + "transcription_size_accurate": "preciso", + "transcription_device_cuda_selected": "A transcrição será executada pela placa gráfica.", + "transcription_device_cpu_requested": "A transcrição será executada pelo processador.", + "transcription_device_cuda_unavailable": "O CUDA não está disponível. A transcrição será executada pelo processador.", + "transcription_device_no_cuda_found": "Não foi encontrada nenhuma placa gráfica compatível. A transcrição será executada pelo processador.", + "transcription_device_cuda_driver_error": "Não foi possível consultar a placa gráfica. A transcrição será executada pelo processador.", + "transcription_error_model_not_installed": "O modelo de transcrição escolhido não está instalado. Transfira o modelo antes de transcrever.", + "transcription_error_model_corrupted": "Os ficheiros do modelo estão incompletos ou danificados. Transfira o modelo novamente.", + "transcription_error_no_disk_space": "Não há espaço em disco suficiente para transferir o modelo.", + "transcription_error_cuda_unavailable": "O CUDA não está disponível. Mude o dispositivo para o processador nas definições e tente novamente.", + "transcription_error_insufficient_vram": "A memória da placa gráfica não é suficiente para este modelo. Escolha um modelo mais pequeno.", + "transcription_error_insufficient_ram": "A memória do computador não é suficiente para este modelo. Escolha um modelo mais pequeno.", + "transcription_error_unsupported_audio_format": "O formato deste áudio não foi reconhecido e não pode ser transcrito.", + "transcription_error_ffmpeg_failed": "Não foi possível converter o áudio para transcrever.", + "transcription_error_audio_incomplete": "O ficheiro de áudio está incompleto ou danificado.", + "transcription_error_media_not_downloaded": "O áudio ainda não foi transferido. Aguarde o fim da transferência e tente novamente.", + "transcription_error_cancelled": "Transcrição cancelada.", + "transcription_error_save_failed": "Não foi possível guardar a transcrição.", + "transcription_error_backend_missing": "O componente de transcrição não está instalado nesta cópia do WinZapp.", + "transcription_error_model_download_failed": "Não foi possível transferir o modelo de transcrição. Verifique a ligação e tente novamente.", + "transcription_error_model_move_failed": "Não foi possível mover a pasta dos modelos. Os modelos continuam na pasta anterior.", + "transcription_error_models_busy": "Outra janela do WinZapp está a usar a pasta dos modelos. Aguarde que termine e tente novamente.", + "transcription_error_backend_error": "Ocorreu um erro interno durante a transcrição." } diff --git a/client/ui/dialogs/node_download.py b/client/ui/dialogs/node_download.py index 56d807174..a3d4eb230 100644 --- a/client/ui/dialogs/node_download.py +++ b/client/ui/dialogs/node_download.py @@ -22,6 +22,7 @@ import wx from app_paths import resource_path +from core import tls_trust from node_download_config import ( NODE_FILENAME, NODE_SHASUMS_URL, @@ -139,7 +140,7 @@ def _finish_error(self, details: str = ""): def _download_zip(self, url: str, dest_path: str) -> bool: try: - response = requests.get(url, stream=True, timeout=(30, 300)) + response = tls_trust.get(url, stream=True, timeout=(30, 300)) response.raise_for_status() except requests.RequestException as exc: if not self._cancelled: @@ -181,7 +182,7 @@ def _download_zip(self, url: str, dest_path: str) -> bool: def _verify_checksum(self, zip_path: str) -> bool: self._set_status(self._i18n.t("node_download_verifying")) try: - resp = requests.get(_NODE_SHASUMS_URL, timeout=15) + resp = tls_trust.get(_NODE_SHASUMS_URL, timeout=15) resp.raise_for_status() except Exception as exc: if not self._cancelled: diff --git a/client/updater.py b/client/updater.py index 5ed743769..5ebd77d6c 100644 --- a/client/updater.py +++ b/client/updater.py @@ -21,10 +21,10 @@ import logging import ctypes import subprocess -import requests import wx from app_paths import _outer_exe_dir, _is_frozen, resource_path +from core import tls_trust from config import GITHUB_API_LATEST_RELEASE, GITHUB_API_LATEST_STABLE_RELEASE from version import __version__ @@ -59,7 +59,7 @@ def _verify_sha256sums(file_path: str, filename: str, sha256sums_url: str) -> "t ) return True, "" try: - resp = requests.get(sha256sums_url, timeout=15) + resp = tls_trust.get(sha256sums_url, timeout=15) resp.raise_for_status() except Exception as exc: return False, f"Failed to download SHA256SUMS.txt: {exc}" @@ -685,7 +685,7 @@ def _worker(self): os.close(zip_fd) logging.info("Auto-updater: Downloading ZIP from %s to %s", self._zip_url, zip_path) - resp = requests.get(self._zip_url, stream=True, timeout=60) + resp = tls_trust.get(self._zip_url, stream=True, timeout=60) resp.raise_for_status() total = int(resp.headers.get("content-length", 0)) @@ -887,7 +887,7 @@ def _alpha_enabled(self) -> bool: return False def _get_json(self, url: str, params: "dict | None" = None): - resp = requests.get( + resp = tls_trust.get( url, headers={"User-Agent": f"WinZapp/{__version__}"}, params=params, diff --git a/pytest.ini b/pytest.ini index 7d419de41..3f1461eba 100644 --- a/pytest.ini +++ b/pytest.ini @@ -15,3 +15,7 @@ markers = load: synthetic load tests with configurable scenario size docs: assertions about documentation matching the code; deselected by release.yml so a stale Markdown sentence can never delete a stable release + network: reaches a real remote host (the Hugging Face model URLs). SKIPPED BY + DEFAULT - set WINZAPP_RUN_NETWORK_TESTS=1 to run them. They are the safety + net for "no model may offer a broken link", so they are meant to be run by + hand whenever the model catalogue changes, not on every suite run. diff --git a/tests/test_coord_locks.py b/tests/test_coord_locks.py index da1e04700..99fabc24d 100644 --- a/tests/test_coord_locks.py +++ b/tests/test_coord_locks.py @@ -12,7 +12,9 @@ from coord_locks import ( app_settings_lock, + canonical_dir, LockTimeout, + models_lock, NamedLock, node_lock, node_port_lock, @@ -147,6 +149,40 @@ def test_factory_names_differ(tmp_path): updater_lock(gd).name, sessions_lock(gd).name, } + # The Whisper models directory is its own critical section. It is the one + # factory that does NOT share the (global_dir, timeout) shape of the others, + # and deliberately: the models root is user-chosen and movable, so the lock + # is KEYED on that root, while the lock FILE still has to live in the global + # dir — writing it inside the user's own folder would litter it and defeat + # model_store's "leave no empty directory of ours behind" cleanup. Hence two + # arguments here rather than one. + models_root = os.path.join(gd, "whisper_models") + assert models_lock(models_root, gd).name not in { + registry_lock(gd).name, + node_lock(gd).name, + updater_lock(gd).name, + sessions_lock(gd).name, + node_port_lock(gd).name, + app_settings_lock(gd).name, + } + + +def test_models_lock_keys_two_spellings_of_one_dir_together(tmp_path): + """Windows folds case and resolves junctions/subst drives; abspath does + not. Two spellings of one models directory must share one lock, or two + processes would write the same model.bin.part under different mutexes.""" + gd = str(tmp_path / "global") + root = str(tmp_path / "Whisper_Models") + other_spelling = str(tmp_path / "whisper_models") + assert canonical_dir(root) == canonical_dir(other_spelling) or os.name != "nt" + if canonical_dir(root) == canonical_dir(other_spelling): + assert models_lock(root, gd).name == models_lock(other_spelling, gd).name + # Different roots stay independent either way, and the lock file never + # lands inside the root it guards. + assert ( + models_lock(root, gd).name != models_lock(str(tmp_path / "elsewhere"), gd).name + ) + assert not models_lock(root, gd).lock_path.startswith(root) def test_same_global_dir_same_name(tmp_path): diff --git a/tests/test_tls_trust.py b/tests/test_tls_trust.py new file mode 100644 index 000000000..e0f365e2d --- /dev/null +++ b/tests/test_tls_trust.py @@ -0,0 +1,241 @@ +"""HTTPS verified the way the machine verifies it, not the way we shipped it. + +`requests` verifies against `certifi`'s baked-in CA list, which knows nothing +about the computer it runs on. On a machine whose TLS is intercepted locally — +an antivirus with HTTPS scanning on, a corporate proxy, an MDM profile — the +certificate presented is signed by a root that exists only in *Windows'* own +store, and every download fails with CERTIFICATE_VERIFY_FAILED. Measured on a +real install against huggingface.co, nodejs.org and api.github.com: all three +failed, all three worked once verification went through the system store. + +Which is three unrelated features breaking at once, for a reason none of them +explains to the user: the Whisper models (up to 3 GB, and without them there is +no transcription), the portable Node.js runtime WinZapp needs before it can talk +to WhatsApp at all, and the updater. + +The rules this file pins: + +* **Absence is never an error.** No truststore, or a Windows that refuses to + give a context, leaves an ordinary session behaving exactly as the code did + before. A certificate helper that could take the updater down on import + would be a worse bug than the one it fixes. +* **The proxy path counts too**, because a machine that intercepts TLS is very + often the same machine that routes through a proxy. +* **Every internet download goes through it.** The three modules that fetch + from outside `127.0.0.1` are checked by name — a fourth call site added later + and left on bare `requests.get` would fail for that user with no clue why. +""" + +import builtins +import logging +import os +import ssl + +import pytest +import requests + +from core import tls_trust +from core.transcription import errors, model_catalog, model_store + +# The three modules that fetch from the internet. Everything else in the app +# talks to the local WPPConnect server over plain HTTP on 127.0.0.1, where +# there is no certificate to verify and nothing here applies. +_DOWNLOADERS = ( + os.path.join("client", "updater.py"), + os.path.join("client", "ui", "dialogs", "node_download.py"), + os.path.join("client", "core", "transcription", "model_store.py"), +) + + +def _repo_root(): + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _no_truststore(monkeypatch): + """Make `import truststore` fail, as it does on an install without it.""" + real_import = builtins.__import__ + + def _watch(name, *args, **kwargs): + if name == "truststore": + raise ImportError("no truststore in this install") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _watch) + + +class TestSessionConstruction: + def test_https_is_verified_through_the_system_store(self): + session = tls_trust.create_session() + adapter = session.get_adapter("https://example.invalid/") + assert isinstance(adapter, tls_trust._SystemTrustAdapter) + + def test_plain_http_is_left_alone(self): + """There is nothing to verify on http://, and the local WPPConnect + server is reached that way — this must not touch it.""" + session = tls_trust.create_session() + adapter = session.get_adapter("http://127.0.0.1:6300/api/") + assert not isinstance(adapter, tls_trust._SystemTrustAdapter) + + def test_the_context_is_used_for_proxied_connections_too(self): + context = ssl.create_default_context() + adapter = tls_trust._SystemTrustAdapter(context) + manager = adapter.proxy_manager_for("http://proxy.invalid:8080") + assert manager.connection_pool_kw.get("ssl_context") is context + + def test_a_missing_truststore_is_not_an_error(self, monkeypatch): + """The whole point of the fallback: worse verification, never a crash.""" + _no_truststore(monkeypatch) + monkeypatch.setattr(tls_trust, "_reported", False) + + assert tls_trust.system_ssl_context() is None + session = tls_trust.create_session() + + assert isinstance(session, requests.Session) + # Still a usable session, with requests' own adapter behind it. + assert session.get_adapter("https://example.invalid/") is not None + assert not isinstance( + session.get_adapter("https://example.invalid/"), + tls_trust._SystemTrustAdapter, + ) + + def test_a_context_that_cannot_be_built_is_not_an_error_either(self, monkeypatch): + class _Broken: + @staticmethod + def SSLContext(*args, **kwargs): + raise ValueError("this Windows says no") + + monkeypatch.setitem(__import__("sys").modules, "truststore", _Broken) + monkeypatch.setattr(tls_trust, "_reported", False) + assert tls_trust.system_ssl_context() is None + + def test_an_adapter_that_cannot_be_installed_still_yields_a_session( + self, monkeypatch + ): + """The context is only half of it: building the adapter runs + HTTPAdapter.__init__ -> init_poolmanager, which hands `ssl_context` to + urllib3 — a pinned dependency that gets bumped like any other. A version + that renamed or validated that keyword would take the model download, + the Node download and the background update check down together, which + is the one failure this module exists to prevent.""" + + real_adapter = tls_trust._SystemTrustAdapter + + def _explode(_context, **_kwargs): + raise TypeError("urllib3 does not like this keyword any more") + + monkeypatch.setattr(tls_trust, "_SystemTrustAdapter", _explode) + session = tls_trust.create_session() + + assert session is not None + # Left on the stock adapter rather than a half-built one. + assert not isinstance( + session.get_adapter("https://example.invalid"), real_adapter + ) + session.close() + + +class TestLogging: + def test_which_path_was_taken_is_logged(self, monkeypatch, caplog): + """"Why does it work on his machine" is the question this answers.""" + monkeypatch.setattr(tls_trust, "_reported", False) + caplog.set_level(logging.INFO) + + tls_trust.create_session() + + lines = [r.getMessage() for r in caplog.records if "[tls]" in r.getMessage()] + assert len(lines) == 1 + + def test_it_is_logged_once_and_not_per_request(self, monkeypatch, caplog): + """A line per call would bury the log of a 3 GB download.""" + monkeypatch.setattr(tls_trust, "_reported", False) + caplog.set_level(logging.INFO) + + for _ in range(3): + tls_trust.create_session() + + lines = [r.getMessage() for r in caplog.records if "[tls]" in r.getMessage()] + assert len(lines) == 1 + + def test_the_fallback_says_why(self, monkeypatch, caplog): + _no_truststore(monkeypatch) + monkeypatch.setattr(tls_trust, "_reported", False) + caplog.set_level(logging.INFO) + + tls_trust.create_session() + + assert "no truststore in this install" in caplog.text + + +class TestGet: + def test_get_goes_through_a_session_and_closes_it(self, monkeypatch): + """`requests.get` builds and closes a session per call; so does this, + which is what keeps a caller's connection lifetime unchanged.""" + calls = [] + + class _Session: + def __init__(self): + self.closed = False + + def get(self, url, **kwargs): + calls.append((url, kwargs)) + return "response" + + def close(self): + self.closed = True + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + session = _Session() + monkeypatch.setattr(tls_trust, "create_session", lambda: session) + + assert tls_trust.get("https://example.invalid/x", timeout=15) == "response" + assert calls == [("https://example.invalid/x", {"timeout": 15})] + assert session.closed is True + + +class TestCallers: + @pytest.mark.parametrize("relative", _DOWNLOADERS) + def test_no_internet_download_is_left_on_the_bundled_ca_list(self, relative): + with open(os.path.join(_repo_root(), relative), encoding="utf-8") as handle: + source = handle.read() + assert "tls_trust" in source, f"{relative} does not use the trust layer" + assert "requests.get(" not in source, f"{relative} still calls requests.get" + assert "requests.Session()" not in source, ( + f"{relative} still builds a plain session" + ) + + def test_the_model_download_defaults_to_a_system_trust_session( + self, tmp_path, monkeypatch + ): + """model_store takes an injectable session, so this is its default. + + Driven all the way to a failure rather than stubbed at the seam: what + matters is that the session the download *actually* uses is the one + this module builds. + """ + created = [] + + class _DeadSession: + def get(self, *args, **kwargs): + raise requests.ConnectionError("no network in a test") + + def close(self): + pass + + def _create(): + created.append(True) + return _DeadSession() + + monkeypatch.setattr(model_store.tls_trust, "create_session", _create) + + with pytest.raises(errors.TranscriptionError) as caught: + model_store.download_model( + model_catalog.get_model("tiny"), str(tmp_path / "models") + ) + + assert caught.value.code == errors.MODEL_DOWNLOAD_FAILED + assert created == [True] diff --git a/tests/test_transcription_core.py b/tests/test_transcription_core.py new file mode 100644 index 000000000..0d41e9f34 --- /dev/null +++ b/tests/test_transcription_core.py @@ -0,0 +1,809 @@ +"""The pure core of local (Whisper) transcription: catalogue, device, errors. + +Everything here is a decision that is impossible to observe on the machine that +made it, which is why the decisions were written as functions over data in the +first place: + +* **The catalogue is a contract with Hugging Face.** The file list is not the + same for every repository — the Systran tiny..medium conversions ship + ``vocabulary.txt`` and no ``preprocessor_config.json``, large-v3 and the turbo + conversion ship ``vocabulary.json`` *and* ``preprocessor_config.json``. A + download built on one assumed shape leaves a model that looks complete and + that CTranslate2 refuses to load, which the user experiences as "it just says + internal error". + +* **sm_120 must never be given int8.** The CTranslate2 builds that support + Blackwell (>= 4.6.3, CUDA 12.8) are compiled with INT8 disabled for it, so + every int8 variant fails at model load on the reporter's own card. That is a + property of a machine no test runner has, so it is pinned as a property of + ``select_compute_type()`` instead. + +* **Asking for CUDA on a machine without it is not an error.** It falls back to + the CPU with a reason code the UI announces. A raised exception here would + replace a slower transcription with nothing at all. And the reason has to come + from ``cuda_probe_error``, never from the ``driver_error`` aggregate: the + aggregate is non-empty on every machine that has not installed the backend + yet, so reading it as a driver fault told users with no NVIDIA hardware at all + to go and reinstall a graphics driver. + +* **Memory is planned against what is free, not what is installed.** A 16 GB + machine running Chromium, WhatsApp Web and a screen reader is a 5 GB machine + for this purpose; deciding on the 16 buys a 3 GB download and then an + allocation error mid-transcription. Pinned on both devices, symmetrically. + +* **A code with no translation is spoken as the code.** ``I18n.t()`` is + ``translations.get(key, key)``, so a failure mode nobody translated has NVDA + read "transcription_error_ffmpeg_failed" out loud. The last two tests walk + every code there is, in every locale. +""" + +import ast +import dataclasses +import json +import os +import sys +import types + +import pytest + +from app_paths import resource_path +from core.transcription import device, errors, model_catalog + + +def _load(name): + with open(resource_path("languages", f"{name}.json"), "r", encoding="utf-8") as f: + return json.load(f) + + +LOCALES = sorted(_load("language_map")) + +# Repository, pinned revision, model.bin digest and the exact size of every +# file, as read from the Hugging Face API on 2026-09-03 — restated here rather +# than imported, so the catalogue is checked against the observation instead of +# against itself. Every size is a constant because the revision is pinned, and +# that is what lets "the wrong size" be a detectable state for all five files +# rather than only for the weights. +_EXPECTED_REPOS = { + "tiny": ( + "Systran/faster-whisper-tiny", + "d90ca5fe260221311c53c58e660288d3deb8d356", + "dcb76c6586fc06cbdac6dd21f14cfd129cc4cdd9dce19bf4ffa62e59cbe6e6d1", + {"config.json": 2_249, "model.bin": 75_538_270, + "tokenizer.json": 2_203_239, "vocabulary.txt": 459_861}, + ), + "base": ( + "Systran/faster-whisper-base", + "ebe41f70d5b6dfa9166e2c581c45c9c0cfc57b66", + "d01c3014881c9c6f3133c182f3d2887eb6ca1c789a7538c5c007196857a0a6a9", + {"config.json": 2_309, "model.bin": 145_217_532, + "tokenizer.json": 2_203_239, "vocabulary.txt": 459_861}, + ), + "small": ( + "Systran/faster-whisper-small", + "536b0662742c02347bc0e980a01041f333bce120", + "3e305921506d8872816023e4c273e75d2419fb89b24da97b4fe7bce14170d671", + {"config.json": 2_370, "model.bin": 483_546_902, + "tokenizer.json": 2_203_239, "vocabulary.txt": 459_861}, + ), + "medium": ( + "Systran/faster-whisper-medium", + "08e178d48790749d25932bbc082711ddcfdfbc4f", + "9b45e1009dcc4ab601eff815b61d80e60ce3fd8c74c1a14f4a282258286b51ae", + {"config.json": 2_257, "model.bin": 1_527_906_378, + "tokenizer.json": 2_203_239, "vocabulary.txt": 459_861}, + ), + "large-v3": ( + "Systran/faster-whisper-large-v3", + "edaa852ec7e145841d8ffdb056a99866b5f0a478", + "69f74147e3334731bc3a76048724833325d2ec74642fb52620eda87352e3d4f1", + {"config.json": 2_394, "model.bin": 3_087_284_237, + "preprocessor_config.json": 340, "tokenizer.json": 2_480_617, + "vocabulary.json": 1_068_114}, + ), + "large-v3-turbo": ( + "deepdml/faster-whisper-large-v3-turbo-ct2", + "4df90f75321148c3a29a9e2351b7ddf8f5b115a8", + "e76620f83d5f5b69efd3d87e3dc180c1bd21df9fbebacfd4335e5e1efcc018da", + {"config.json": 2_263, "model.bin": 1_617_884_929, + "preprocessor_config.json": 340, "tokenizer.json": 2_710_337, + "vocabulary.json": 1_068_114}, + ), +} + + +def _probe(**kwargs): + """A HardwareProbe with everything unknown unless the test says otherwise.""" + return device.HardwareProbe(**kwargs) + + +def _cuda_probe(capability=(8, 6), free_vram_mb=8192, **kwargs): + kwargs.setdefault("total_vram_mb", free_vram_mb) + return device.HardwareProbe( + cuda_available=True, + cuda_device_count=1, + compute_capability=capability, + free_vram_mb=free_vram_mb, + **kwargs, + ) + + +class TestModelCatalog: + def test_ids_are_unique(self): + ids = [m.id for m in model_catalog.MODELS] + assert sorted(ids) == sorted(set(ids)) + + def test_catalogue_is_exactly_the_verified_set(self): + assert {m.id for m in model_catalog.MODELS} == set(_EXPECTED_REPOS) + + @pytest.mark.parametrize("model_id", sorted(_EXPECTED_REPOS)) + def test_repo_and_files_match_hugging_face(self, model_id): + repo, _revision, _digest, files = _EXPECTED_REPOS[model_id] + model = model_catalog.get_model(model_id) + assert model.repo == repo + assert dict(model.files) == files + assert model.model_bin_bytes == files["model.bin"] + + @pytest.mark.parametrize("model_id", sorted(_EXPECTED_REPOS)) + def test_the_totals_are_the_exact_sum_of_the_files(self, model_id): + # The totals used to add a guessed couple of megabytes for the + # auxiliary files, and on the turbo conversion that guess was 181 KB + # UNDER the real figure — the wrong direction for a number that feeds a + # "does this fit?" gate and a free-space check. + _repo, _revision, _digest, files = _EXPECTED_REPOS[model_id] + model = model_catalog.get_model(model_id) + assert model.download_bytes == sum(files.values()) + assert model.disk_bytes == model.download_bytes + + @pytest.mark.parametrize("model_id", sorted(_EXPECTED_REPOS)) + def test_every_model_carries_the_verified_weights_digest(self, model_id): + # Pinned revision plus digest is what makes MODEL_CORRUPTED provable + # for the one file where a silent corruption costs a multi-gigabyte + # re-download to discover. + _repo, _revision, digest, _files = _EXPECTED_REPOS[model_id] + model = model_catalog.get_model(model_id) + assert model.model_bin_sha256 == digest + assert len(model.model_bin_sha256) == 64 + assert set(model.model_bin_sha256) <= set("0123456789abcdef") + + @pytest.mark.parametrize("model_id", sorted(_EXPECTED_REPOS)) + def test_every_model_is_pinned_to_the_verified_revision(self, model_id): + # Downloads resolve by sha, not by /main/: a revision published upstream + # must not silently replace weights an install already verified, nor + # invalidate the byte counts above. + _repo, revision, _digest, _files = _EXPECTED_REPOS[model_id] + model = model_catalog.get_model(model_id) + assert model.revision == revision + assert len(model.revision) == 40 + assert set(model.revision) <= set("0123456789abcdef") + + def test_every_file_can_be_checked_by_size_not_just_the_weights(self): + # errors.MODEL_CORRUPTED promises "missing or the wrong size", which is + # only true if every file has an exact expected size: a truncated + # tokenizer.json is as plausible-looking as a truncated model.bin, and + # fails the load just the same. + for model in model_catalog.MODELS: + sizes = dict(model.files) + assert len(sizes) == len(model.files), "duplicate file name" + for name, size in sizes.items(): + assert isinstance(size, int) and size > 0, name + + def test_a_truncated_file_changes_the_expected_total(self): + medium = dict(model_catalog.get_model("medium").files) + for damaged_file in ("model.bin", "tokenizer.json", "config.json"): + damaged = dict(medium) + damaged[damaged_file] -= 1 + assert sum(damaged.values()) != sum(medium.values()) + + def test_the_two_file_layouts_are_not_confused(self): + # The regression this guards: one shared file list for all six repos. + assert "vocabulary.txt" in dict(model_catalog.get_model("small").files) + assert "vocabulary.json" in dict(model_catalog.get_model("large-v3").files) + assert "preprocessor_config.json" not in dict( + model_catalog.get_model("medium").files + ) + assert "preprocessor_config.json" in dict( + model_catalog.get_model("large-v3-turbo").files + ) + + def test_every_model_ships_the_weights_and_a_tokenizer(self): + for model in model_catalog.MODELS: + names = dict(model.files) + assert "model.bin" in names + assert "tokenizer.json" in names + assert "config.json" in names + + def test_the_file_table_cannot_be_edited_in_place(self): + # A dict here would let one caller rewrite the expected sizes for every + # other caller in the process; a tuple of pairs cannot be. + assert isinstance(model_catalog.MODELS[0].files, tuple) + for entry in model_catalog.MODELS[0].files: + assert isinstance(entry, tuple) and len(entry) == 2 + + def test_list_models_is_ordered_by_size_class_then_bytes(self): + listed = model_catalog.list_models() + assert len(listed) == len(model_catalog.MODELS) + keys = [ + (model_catalog.SIZE_CLASSES.index(m.size_class), m.download_bytes) + for m in listed + ] + assert keys == sorted(keys) + + def test_the_cheapest_model_comes_first_and_the_costliest_last(self): + listed = model_catalog.list_models() + assert listed[0].id == "tiny" + assert listed[-1].id == "large-v3" + + def test_memory_requirements_never_decrease_with_size(self): + listed = model_catalog.list_models() + assert [m.min_vram_mb for m in listed] == sorted(m.min_vram_mb for m in listed) + assert [m.min_ram_mb for m in listed] == sorted(m.min_ram_mb for m in listed) + + def test_every_size_class_is_one_of_the_declared_ones(self): + for model in model_catalog.MODELS: + assert model.size_class in model_catalog.SIZE_CLASSES + + def test_unknown_id_returns_none_rather_than_raising(self): + # It comes from settings, where a model removed between versions is a + # normal state and must not crash the conversation window. + assert model_catalog.get_model("large-v4") is None + assert model_catalog.get_model(None) is None + assert model_catalog.get_model("") is None + + def test_total_bytes_matches_the_entry(self): + model = model_catalog.get_model("base") + assert model_catalog.total_bytes("base") == model.download_bytes + + def test_total_bytes_of_an_unknown_id_is_none_not_zero(self): + # 0 would be a trap: a settings file naming a model some later version + # dropped would tell the free-space gate "it fits" and start a download + # of something that does not exist. + assert model_catalog.total_bytes("large-v4") is None + assert model_catalog.total_bytes(None) is None + + def test_models_are_frozen(self): + # The catalogue is shared module state; a caller adjusting one entry + # would change it for every other caller in the process. + with pytest.raises(dataclasses.FrozenInstanceError): + model_catalog.MODELS[0].disk_bytes = 1 + + +class TestResolveDevice: + def test_cpu_preference_is_honoured_even_with_a_gpu_present(self): + chosen, reason = device.resolve_device(device.PREFERENCE_CPU, _cuda_probe()) + assert chosen == device.DEVICE_CPU + assert reason == device.REASON_CPU_REQUESTED + + def test_auto_takes_the_gpu_when_there_is_one(self): + chosen, reason = device.resolve_device(device.PREFERENCE_AUTO, _cuda_probe()) + assert chosen == device.DEVICE_CUDA + assert reason == device.REASON_CUDA_SELECTED + + def test_auto_falls_back_to_the_cpu_quietly(self): + chosen, reason = device.resolve_device(device.PREFERENCE_AUTO, _probe()) + assert chosen == device.DEVICE_CPU + assert reason == device.REASON_NO_CUDA_FOUND + + def test_cuda_preference_takes_the_gpu(self): + chosen, reason = device.resolve_device(device.PREFERENCE_CUDA, _cuda_probe()) + assert chosen == device.DEVICE_CUDA + assert reason == device.REASON_CUDA_SELECTED + + def test_cuda_without_cuda_is_a_fallback_not_an_exception(self): + chosen, reason = device.resolve_device(device.PREFERENCE_CUDA, _probe()) + assert chosen == device.DEVICE_CPU + assert reason == device.REASON_CUDA_UNAVAILABLE + + def test_cuda_with_a_driver_failure_says_so_specifically(self): + # This state does not come out of probe_hardware(), and re-probing + # cannot produce it either — a fresh probe would count the device again + # and land on CUDA_SELECTED. It takes a caller that demotes the card + # deliberately after a run failed on it, i.e. dataclasses.replace( + # probe, cuda_available=False, cuda_device_count=0, cuda_probe_error=…), + # which is what is built by hand here. + probe = _probe(cuda_probe_error="nvml: could not load nvml.dll", + driver_error="nvml: could not load nvml.dll") + chosen, reason = device.resolve_device(device.PREFERENCE_CUDA, probe) + assert chosen == device.DEVICE_CPU + assert reason == device.REASON_CUDA_DRIVER_ERROR + + def test_an_unrelated_probe_failure_is_not_a_driver_fault(self): + # driver_error is an aggregate of everything the probe could not do — + # the Windows memory API here. Reading it as "the graphics driver + # failed" sends the user off to reinstall a driver they may not have. + probe = _probe(driver_error="ram: GlobalMemoryStatusEx failed") + chosen, reason = device.resolve_device(device.PREFERENCE_CUDA, probe) + assert chosen == device.DEVICE_CPU + assert reason == device.REASON_CUDA_UNAVAILABLE + + def test_a_missing_backend_is_not_a_driver_fault(self): + # The state of EVERY install until the backend is installed: the + # aggregate carries the failed ctranslate2 import and nothing else. + probe = _probe(driver_error="ctranslate2: No module named ctranslate2") + _, reason = device.resolve_device(device.PREFERENCE_CUDA, probe) + assert reason == device.REASON_CUDA_UNAVAILABLE + _, auto_reason = device.resolve_device(device.PREFERENCE_AUTO, probe) + assert auto_reason == device.REASON_NO_CUDA_FOUND + + def test_a_driver_error_alongside_a_working_gpu_does_not_block_it(self): + # NVML failing only costs us the VRAM figure; CTranslate2 can still run. + probe = _cuda_probe(capability=None, free_vram_mb=None, total_vram_mb=None, + driver_error="nvml: nvmlInit_v2 failed") + chosen, _ = device.resolve_device(device.PREFERENCE_AUTO, probe) + assert chosen == device.DEVICE_CUDA + + def test_a_zero_device_count_is_not_cuda(self): + # ctranslate2 loaded but found nothing — "available" alone is not enough. + probe = device.HardwareProbe(cuda_available=True, cuda_device_count=0) + chosen, _ = device.resolve_device(device.PREFERENCE_AUTO, probe) + assert chosen == device.DEVICE_CPU + + @pytest.mark.parametrize("preference", [None, "", "gpu", "gpu ", 3]) + def test_an_unrecognised_preference_behaves_like_auto(self, preference): + # It is a settings value; a file from another version must not be able + # to make transcription unavailable. + assert device.resolve_device(preference, _cuda_probe())[0] == device.DEVICE_CUDA + assert device.resolve_device(preference, _probe())[0] == device.DEVICE_CPU + + @pytest.mark.parametrize("reason", sorted(device.DEVICE_REASON_I18N_KEYS)) + def test_every_reason_has_an_i18n_key(self, reason): + assert device.device_reason_i18n_key(reason).startswith("transcription_device_") + + def test_an_unknown_reason_still_resolves_to_a_real_key(self): + assert device.device_reason_i18n_key("something_new") == ( + device.DEVICE_REASON_I18N_KEYS[device.REASON_CPU_REQUESTED] + ) + + +class TestComputeType: + def test_cpu_is_int8(self): + assert device.select_compute_type(device.DEVICE_CPU, _probe()) == device.COMPUTE_INT8 + + @pytest.mark.parametrize("capability", [(12, 0), (12, 1), (13, 0)]) + def test_sm_120_and_newer_never_get_int8(self, capability): + # The whole reason this module exists: CTranslate2's sm_120 build has + # INT8 disabled, and every int8 variant dies at model load. + compute = device.select_compute_type( + device.DEVICE_CUDA, _cuda_probe(capability=capability) + ) + assert "int8" not in compute + assert compute == device.COMPUTE_FLOAT16 + assert device.gpu_supports_int8(capability) is False + + @pytest.mark.parametrize("capability", [(7, 0), (7, 5), (8, 6), (8, 9), (9, 0)]) + def test_modern_gpus_get_float16(self, capability): + assert device.select_compute_type( + device.DEVICE_CUDA, _cuda_probe(capability=capability) + ) == device.COMPUTE_FLOAT16 + + @pytest.mark.parametrize("capability", [(3, 5), (5, 0), (6, 1)]) + def test_pre_volta_gpus_fall_back_to_float32(self, capability): + assert device.select_compute_type( + device.DEVICE_CUDA, _cuda_probe(capability=capability) + ) == device.COMPUTE_FLOAT32 + + @pytest.mark.parametrize("candidate", ["int8", "int8_float16", "int8_bfloat16"]) + def test_the_gate_strips_every_int8_flavour_on_sm_120(self, candidate): + # The gate select_compute_type() actually runs its answer through, + # tested on the variants a future VRAM-saving change would reach for. + assert device._int8_safe(candidate, (12, 0)) == device.COMPUTE_FLOAT16 + assert device._int8_safe(candidate, None) == device.COMPUTE_FLOAT16 + assert device._int8_safe(candidate, (8, 6)) == candidate + + def test_the_cuda_path_actually_runs_through_the_gate(self, monkeypatch): + # The wiring, not the outcome. Every value-based assertion in this + # class stays green with the _int8_safe() call deleted, because the GPU + # path picks float16 either way today — so the one thing that keeps the + # gate on the path is checking that it is called. + seen = [] + + def _record(compute, capability): + seen.append((compute, capability)) + return device.COMPUTE_FLOAT16 + + monkeypatch.setattr(device, "_int8_safe", _record) + chosen = device.select_compute_type( + device.DEVICE_CUDA, _cuda_probe(capability=(12, 0)) + ) + assert seen == [(device.COMPUTE_FLOAT16, (12, 0))] + assert chosen == device.COMPUTE_FLOAT16 + + def test_the_cpu_path_does_not_need_the_gate(self, monkeypatch): + # The CPU answer is int8 and must stay int8: the gate is about GPU + # kernels, and running the CPU choice through it would silence the one + # place int8 is correct. + monkeypatch.setattr(device, "_int8_safe", lambda *_: "SHOULD NOT BE CALLED") + assert device.select_compute_type(device.DEVICE_CPU, _probe()) == ( + device.COMPUTE_INT8 + ) + + def test_the_gate_leaves_a_float_compute_type_alone(self): + assert device._int8_safe(device.COMPUTE_FLOAT16, (12, 0)) == device.COMPUTE_FLOAT16 + assert device._int8_safe(device.COMPUTE_FLOAT32, (6, 1)) == device.COMPUTE_FLOAT32 + + def test_an_unknown_capability_is_treated_as_modern_but_not_int8(self): + probe = _cuda_probe(capability=None) + assert device.select_compute_type(device.DEVICE_CUDA, probe) == device.COMPUTE_FLOAT16 + assert device.gpu_supports_int8(None) is False + + def test_no_gpu_path_ever_returns_an_int8_variant(self): + # Swept over VRAM as well as capability. A cramped card is exactly the + # condition under which someone would later make this prefer + # int8_float16 — the case _int8_safe() exists for — so a sweep that + # only varies capability would stay green with the gate deleted. + for major in range(3, 14): + for minor in (0, 5, 9): + for free_vram_mb in (256, 1024, 4096, 24576): + probe = _cuda_probe(capability=(major, minor), + free_vram_mb=free_vram_mb) + compute = device.select_compute_type(device.DEVICE_CUDA, probe) + assert "int8" not in compute or (major, minor) < (12, 0), ( + f"sm_{major}{minor} with {free_vram_mb} MB free got {compute}" + ) + + +class TestAutoSelectModel: + def test_picks_the_largest_model_the_gpu_can_hold(self): + chosen = device.auto_select_model( + _cuda_probe(free_vram_mb=8192), device.DEVICE_CUDA, installed_ids=() + ) + assert chosen == "large-v3" + + def test_never_picks_a_model_that_does_not_fit(self): + for free_vram_mb in (512, 1024, 2048, 4096, 6144, 8192, 24576): + probe = _cuda_probe(free_vram_mb=free_vram_mb) + chosen = device.auto_select_model(probe, device.DEVICE_CUDA, installed_ids=()) + if chosen is None: + continue + model = model_catalog.get_model(chosen) + assert model.min_vram_mb < free_vram_mb, ( + f"{chosen} needs {model.min_vram_mb} MB and only {free_vram_mb} MB is free" + ) + assert device.model_fits(model, free_vram_mb, device.DEVICE_CUDA) + + def test_nothing_fits_returns_none(self): + # The caller turns this into INSUFFICIENT_VRAM/RAM; guessing a model + # that cannot load would fail later and less clearly. + probe = _cuda_probe(free_vram_mb=256) + assert device.auto_select_model(probe, device.DEVICE_CUDA, installed_ids=()) is None + + def test_prefers_an_installed_model_over_a_bigger_download(self): + # 3 GB nobody asked for is worse than transcribing with what is here. + probe = _cuda_probe(free_vram_mb=24576) + chosen = device.auto_select_model(probe, device.DEVICE_CUDA, installed_ids=["base"]) + assert chosen == "base" + + def test_prefers_the_largest_installed_model_that_fits(self): + probe = _cuda_probe(free_vram_mb=24576) + chosen = device.auto_select_model( + probe, device.DEVICE_CUDA, installed_ids=["tiny", "small", "base"] + ) + assert chosen == "small" + + def test_an_installed_model_that_does_not_fit_is_skipped(self): + probe = _cuda_probe(free_vram_mb=2560) + chosen = device.auto_select_model( + probe, device.DEVICE_CUDA, installed_ids=["large-v3"] + ) + assert chosen != "large-v3" + assert model_catalog.get_model(chosen).min_vram_mb <= 2048 + + def test_the_cpu_path_is_measured_against_ram_not_vram(self): + # A machine with a big card and little RAM must not be handed large-v3 + # because the VRAM figure said so. + probe = device.HardwareProbe(total_ram_mb=4096, total_vram_mb=24576, + free_vram_mb=24576) + chosen = device.auto_select_model(probe, device.DEVICE_CPU, installed_ids=()) + assert model_catalog.get_model(chosen).min_ram_mb <= 3072 + + def test_unknown_memory_uses_an_installed_model_or_nothing(self): + probe = _probe(driver_error="nvml: could not load nvml.dll") + assert device.auto_select_model(probe, device.DEVICE_CPU, installed_ids=()) is None + assert device.auto_select_model( + probe, device.DEVICE_CPU, installed_ids=["medium", "tiny"] + ) == "tiny" + + def test_a_caller_supplied_catalogue_is_ordered_by_this_module(self): + # Reversed on purpose: "largest that fits" must not depend on the order + # the caller happened to hand over. + catalog = tuple(reversed(model_catalog.list_models())) + chosen = device.auto_select_model( + _cuda_probe(free_vram_mb=8192), device.DEVICE_CUDA, + installed_ids=(), catalog=catalog, + ) + assert chosen == "large-v3" + + def test_headroom_is_required_not_just_a_bare_fit(self): + # The rule spelled out rather than borrowed back from model_fits(): a + # model fits when the free memory covers its minimum plus 25%. 2048 MB + # free against a minimum of exactly 2048 MB is a miss — the minimum + # covers the model, not Chromium and the desktop beside it. + small = model_catalog.get_model("small") + assert small.min_vram_mb == 2048 + assert not device.model_fits(small, 2048, device.DEVICE_CUDA) + assert not device.model_fits(small, int(2048 * 1.25) - 1, device.DEVICE_CUDA) + assert device.model_fits(small, int(2048 * 1.25), device.DEVICE_CUDA) + + def test_no_chosen_model_ever_breaks_the_headroom_rule(self): + for free_vram_mb in (900, 1280, 2000, 2560, 5120, 7680, 24576): + chosen = device.auto_select_model( + _cuda_probe(free_vram_mb=free_vram_mb), device.DEVICE_CUDA, + installed_ids=(), + ) + if chosen is None: + continue + assert free_vram_mb >= model_catalog.get_model(chosen).min_vram_mb * 1.25 + + def test_free_vram_wins_over_total_vram(self): + probe = device.HardwareProbe( + cuda_available=True, cuda_device_count=1, compute_capability=(8, 6), + total_vram_mb=24576, free_vram_mb=1024, + ) + assert device.available_memory_mb(probe, device.DEVICE_CUDA) == 1024 + + def test_available_ram_wins_over_total_ram(self): + # 16 GB installed, 11 GB already taken by Chromium/WhatsApp Web, the + # screen reader and the user own work. Planning against the 16 buys a + # 3 GB download and then an allocation error. + probe = device.HardwareProbe(total_ram_mb=16384, available_ram_mb=5120) + assert device.available_memory_mb(probe, device.DEVICE_CPU) == 5120 + chosen = device.auto_select_model(probe, device.DEVICE_CPU, installed_ids=()) + assert chosen != "large-v3" + assert model_catalog.get_model(chosen).min_ram_mb * 1.25 <= 5120 + + def test_total_ram_is_the_fallback_when_available_is_unknown(self): + probe = device.HardwareProbe(total_ram_mb=16384) + assert device.available_memory_mb(probe, device.DEVICE_CPU) == 16384 + + +class TestProbeHardware: + def test_probing_never_raises_and_answers_with_a_probe(self): + # It loads a driver DLL and calls into the Windows memory API; on a + # runner with neither, every field is simply unknown. + probe = device.probe_hardware() + assert isinstance(probe, device.HardwareProbe) + assert isinstance(probe.cuda_available, bool) + assert probe.cuda_device_count >= 0 + + def test_a_failing_nvml_probe_still_returns_a_probe(self, monkeypatch): + # A runner with no GPU never reaches _probe_nvml() at all, so the + # failure has to be planted on both halves: a backend that reports a + # device, and a driver probe that then blows up on it. That is the real + # shape of the fault — the DLL loads and faults inside ctypes. + monkeypatch.setitem( + sys.modules, "ctranslate2", + types.SimpleNamespace(get_cuda_device_count=lambda: 1), + ) + + def _boom(): + raise OSError("nvml exploded") + + monkeypatch.setattr(device, "_probe_nvml", _boom) + probe = device.probe_hardware() + assert isinstance(probe, device.HardwareProbe) + assert probe.cuda_available is True + assert probe.compute_capability is None + # The GPU was counted and could not be described: this is the one case + # that may be announced as a driver fault. + assert "nvml exploded" in probe.cuda_probe_error + assert device.resolve_device(device.PREFERENCE_CUDA, probe)[1] == ( + device.REASON_CUDA_SELECTED + ) + + def test_a_failing_ram_probe_still_returns_a_probe(self, monkeypatch): + def _boom(): + raise OSError("GlobalMemoryStatusEx exploded") + + monkeypatch.setattr(device, "_probe_ram_mb", _boom) + probe = device.probe_hardware() + assert probe.total_ram_mb is None + assert probe.available_ram_mb is None + assert "GlobalMemoryStatusEx exploded" in probe.driver_error + + def test_a_missing_ctranslate2_still_returns_a_probe(self, monkeypatch): + import builtins + + real_import = builtins.__import__ + + def _refuse(name, *args, **kwargs): + if name == "ctranslate2": + raise ImportError("no ctranslate2 here") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _refuse) + probe = device.probe_hardware() + assert probe.cuda_available is False + # It lands in the aggregate, and nowhere near the driver-fault field. + assert "ctranslate2" in probe.driver_error + assert probe.cuda_probe_error is None + + def test_a_reported_nvml_failure_reaches_both_error_fields(self, monkeypatch): + # The common real case: a card is present, nvml.dll is not. The helper + # returns the reason rather than raising, and the reason has to land in + # cuda_probe_error (which may be announced) AND in the aggregate (which + # is what the log gets). + monkeypatch.setitem( + sys.modules, "ctranslate2", + types.SimpleNamespace(get_cuda_device_count=lambda: 1), + ) + monkeypatch.setattr( + device, "_probe_nvml", lambda: (None, None, None, "nvml: not found") + ) + probe = device.probe_hardware() + assert probe.cuda_probe_error == "nvml: not found" + assert "nvml: not found" in probe.driver_error + assert probe.total_vram_mb is None and probe.free_vram_mb is None + + def test_loading_nvml_reports_every_path_it_tried(self, monkeypatch): + # Without a card there is no DLL to load, so the failure path is the + # only one testable here — and it is the one that reaches the log. + def _refuse(path): + raise OSError(f"cannot load {path}") + + monkeypatch.setattr(device.ctypes, "CDLL", _refuse) + library, error = device._load_nvml() + assert library is None + # Keeping only the last failure left every card-less machine blaming + # the legacy NVSMI path, the least informative of the three. + assert "nvml.dll" in error + assert error.count("cannot load") == len(device._NVML_LIBRARY_PATHS) + + def test_the_nvml_search_starts_with_the_bare_name(self): + paths = device._NVML_LIBRARY_PATHS + assert len(paths) == 3 + assert paths[0] == "nvml.dll" + # Then the two absolute locations pynvml itself falls back to. + assert paths[1].lower().endswith(r"system32\nvml.dll") + assert "NVSMI" in paths[2] + + def test_a_probe_result_still_yields_a_usable_decision(self): + probe = device.probe_hardware() + chosen, reason = device.resolve_device(device.PREFERENCE_AUTO, probe) + assert chosen in (device.DEVICE_CPU, device.DEVICE_CUDA) + assert reason in device.DEVICE_REASON_I18N_KEYS + assert device.select_compute_type(chosen, probe) in ( + device.COMPUTE_INT8, device.COMPUTE_FLOAT16, device.COMPUTE_FLOAT32, + ) + + +class TestErrors: + def test_the_exception_carries_the_code(self): + exc = errors.TranscriptionError(errors.FFMPEG_FAILED, "ffmpeg exited with 1") + assert exc.code == errors.FFMPEG_FAILED + assert exc.i18n_key == "transcription_error_ffmpeg_failed" + + def test_the_technical_detail_is_optional(self): + exc = errors.TranscriptionError(errors.CANCELLED) + assert exc.detail is None + assert str(exc) == errors.CANCELLED + + def test_the_detail_reaches_the_log_line(self): + exc = errors.TranscriptionError(errors.BACKEND_ERROR, "CUDA out of memory") + assert exc.log_line == f"{errors.BACKEND_ERROR}: CUDA out of memory" + + def test_neither_str_nor_args_leaks_the_detail(self): + # wx.MessageBox(str(exc), ...) is an idiom this repository already uses + # (client/ui/media_viewer.py), so whatever __str__ returns is one + # careless handler away from a screen reader spelling out a file path. + exc = errors.TranscriptionError(errors.BACKEND_ERROR, r"C:\Users\name\file.ogg") + assert str(exc) == errors.BACKEND_ERROR + assert "file.ogg" not in str(exc) + assert exc.args == (errors.BACKEND_ERROR,) + assert "file.ogg" not in "".join(str(arg) for arg in exc.args) + + def test_the_log_line_is_just_the_code_without_a_detail(self): + assert errors.TranscriptionError(errors.CANCELLED).log_line == errors.CANCELLED + + def test_every_error_answers_moved_even_when_no_move_was_involved(self): + # move_models() attaches the ids it managed to move before failing, but + # only on the path that got that far. A caller reading exc.moved after a + # move would hit AttributeError on every other failure of the same call + # — the busy-folder one above all, which raises its own error. + assert errors.TranscriptionError(errors.MODELS_BUSY).moved == () + assert errors.TranscriptionError(errors.BACKEND_ERROR, "detail").moved == () + + def test_every_case_the_issue_names_has_a_code(self): + for code in ( + errors.MODEL_NOT_INSTALLED, errors.MODEL_CORRUPTED, errors.NO_DISK_SPACE, + errors.CUDA_UNAVAILABLE, errors.INSUFFICIENT_VRAM, errors.INSUFFICIENT_RAM, + errors.UNSUPPORTED_AUDIO_FORMAT, errors.FFMPEG_FAILED, + errors.AUDIO_INCOMPLETE, errors.MEDIA_NOT_DOWNLOADED, errors.CANCELLED, + errors.SAVE_FAILED, errors.BACKEND_MISSING, errors.MODEL_DOWNLOAD_FAILED, + errors.BACKEND_ERROR, + ): + assert code in errors.ERROR_CODES + + def test_codes_are_unique(self): + assert len(set(errors.ERROR_CODES)) == len(errors.ERROR_CODES) + + @pytest.mark.parametrize("code", errors.ERROR_CODES) + def test_every_code_maps_to_an_i18n_key(self, code): + key = errors.error_i18n_key(code) + assert key == f"transcription_error_{code}" + assert errors.ERROR_I18N_KEYS[code] == key + + def test_an_unknown_code_resolves_to_the_internal_error_message(self): + # I18n.t() falls back to the raw key name, so a made-up key would have + # the screen reader read "transcription_error_whatever" aloud. + assert errors.error_i18n_key("whatever") == errors.ERROR_I18N_KEYS[errors.BACKEND_ERROR] + assert errors.TranscriptionError(None).i18n_key in errors.ERROR_I18N_KEYS.values() + + +class TestTranslations: + """Same rule as tests/test_language_files_in_sync.py, aimed at these keys. + + The sync test would only notice a key missing from *some* locales; a key + missing from all five agrees with itself perfectly and still reaches the + user as its own name. + """ + + @pytest.mark.parametrize("locale", LOCALES) + def test_every_error_code_is_translated_everywhere(self, locale): + table = _load(locale) + missing = sorted( + errors.error_i18n_key(code) for code in errors.ERROR_CODES + if errors.error_i18n_key(code) not in table + ) + assert missing == [], f"{locale}.json would speak these key names aloud: {missing}" + + @pytest.mark.parametrize("locale", LOCALES) + def test_every_device_reason_is_translated_everywhere(self, locale): + table = _load(locale) + missing = sorted( + key for key in device.DEVICE_REASON_I18N_KEYS.values() if key not in table + ) + assert missing == [], f"{locale}.json is missing device reasons: {missing}" + + @pytest.mark.parametrize("locale", LOCALES) + def test_every_size_class_is_translated_everywhere(self, locale): + table = _load(locale) + missing = sorted( + key for key in model_catalog.SIZE_CLASS_I18N_KEYS.values() if key not in table + ) + assert missing == [], f"{locale}.json is missing size labels: {missing}" + + @pytest.mark.parametrize("locale", LOCALES) + def test_no_transcription_string_is_blank(self, locale): + table = _load(locale) + blank = sorted( + key for key, text in table.items() + if key.startswith("transcription_") and not text.strip() + ) + assert blank == [] + + def test_size_class_keys_cover_every_class_in_the_catalogue(self): + for model in model_catalog.MODELS: + assert model_catalog.size_class_i18n_key(model.size_class) + + def test_an_unknown_size_class_has_no_key(self): + assert model_catalog.size_class_i18n_key("enormous") is None + + +def test_the_backend_is_never_imported_at_module_level(): + """The app has to start on a machine with no faster-whisper installed. + + The menu item that offers to install it lives in the same process, so a + top-level import would make the feature unreachable exactly where it is + needed — and would take the conversation window down with it. Checked + against the source rather than sys.modules, because probe_hardware() legally + imports ctranslate2 inside a function and would mask the difference. + """ + from core.transcription import __file__ as package_file + + package_dir = os.path.dirname(package_file) + offenders = [] + for name in sorted(os.listdir(package_dir)): + if not name.endswith(".py"): + continue + with open(os.path.join(package_dir, name), "r", encoding="utf-8") as handle: + tree = ast.parse(handle.read()) + for node in tree.body: # top level only — nested imports are the rule + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom): + names = [node.module or ""] + else: + continue + for imported in names: + if imported.split(".")[0] in ("faster_whisper", "ctranslate2", "wx"): + offenders.append(f"{name}: {imported}") + assert offenders == [], f"imported at module level: {offenders}" diff --git a/tests/test_transcription_model_store.py b/tests/test_transcription_model_store.py new file mode 100644 index 000000000..1719929aa --- /dev/null +++ b/tests/test_transcription_model_store.py @@ -0,0 +1,1690 @@ +"""Getting the Whisper models onto disk without lying about what is there. + +Every failure this file pins has the same shape from the user's side: they ask +for a transcription and get an error, or nothing, after having waited for a +multi-gigabyte download. The mechanisms behind that are specific, and all of +them are cheap to reproduce with a handful of synthetic bytes: + +* **A half-written file under its final name is indistinguishable from a good + one.** "Is this model installed?" runs before every transcription and every + time the model list is drawn, so it can only afford names and exact sizes — + which means anything wearing the catalogue's name and size *will* be + believed. Cancel a download at the wrong moment, or leave a `.part` behind + under the wrong name, and the model looks installed while CTranslate2 refuses + to load it. So the tests below cancel, fail and interrupt mid-file and then + assert on the install check itself, not merely on the exception. + +* **The pinned revision is the entire integrity story.** The catalogue's sizes + and model.bin digest were measured at one commit; a URL that followed `main` + would fetch bytes nothing here can vouch for, and would accept different + weights for any file whose size happened to match. There is a test that fails + the moment a URL stops naming the revision. + +* **A "does it fit?" gate that runs late is not a gate.** NO_DISK_SPACE has to + be raised before the first byte, while the user can still choose a smaller + model. + +* **A user-chosen directory is not ours to empty.** Part 5 lets the user put + the models wherever they like, including a folder that already holds their + own files. `remove_model()` deletes the catalogue's own names and nothing + else, and the test that matters here is the one asserting a stranger's file + survived. + +* **Moving the folder must never leave both roots incomplete.** The move copies + and verifies before it deletes, so an interruption leaves every model whole + in exactly one of the two roots. + +The network is never touched: a fake session answers from a table of bytes. The +one test that does reach Hugging Face — the safety net for "no model may offer +a broken link" — is marked `network` and skipped unless +WINZAPP_RUN_NETWORK_TESTS is set. +""" + +import hashlib +import math +import os +import subprocess +import sys + +import pytest +import requests + +import app_paths +from coord_locks import LockTimeout +from core import tls_trust +from core.transcription import errors, model_catalog, model_store + +_NETWORK_OPT_IN_ENV = "WINZAPP_RUN_NETWORK_TESTS" + + +# ── Synthetic catalogue ────────────────────────────────────────────────────── + + +def _synthetic_model(model_id, weights): + """A catalogue entry of a few kilobytes, plus the bytes behind it. + + The real catalogue is 5.5 GB of weights sitting behind a network, and + everything in model_store is about names, sizes and digests — which these + bytes exercise exactly as well and in milliseconds. The file list keeps the + tiny..medium shape (`vocabulary.txt`, no `preprocessor_config.json`), since + the store never assumes a shape either way. + """ + contents = { + "config.json": b'{"model_type": "whisper"}', + "model.bin": weights, + "tokenizer.json": b'{"added_tokens": []}', + "vocabulary.txt": b"one\ntwo\nthree\n", + } + files = tuple((name, len(data)) for name, data in contents.items()) + total = sum(size for _name, size in files) + model = model_catalog.WhisperModel( + id=model_id, + repo=f"example-org/faster-whisper-{model_id}", + # Shaped like a real commit sha, so a test asserting the URL carries a + # 40-hex revision has something to bite on. + revision=("%s" % model_id).encode().hex().ljust(40, "0")[:40], + model_bin_sha256=hashlib.sha256(weights).hexdigest(), + files=files, + model_bin_bytes=len(weights), + download_bytes=total, + disk_bytes=total, + size_class=model_catalog.SIZE_SMALL, + min_vram_mb=1024, + min_ram_mb=2048, + ) + return model, contents + + +@pytest.fixture +def entry(monkeypatch): + """One synthetic model standing in for the whole catalogue. + + MODELS is patched rather than the model merely passed around, because + list_installed() and remove_model() look the id up in the catalogue + themselves — remove_model() deliberately refuses to delete anything it + cannot name from there. + """ + model, contents = _synthetic_model("alpha", b"A" * 4096) + monkeypatch.setattr(model_catalog, "MODELS", (model,)) + return model, contents + + +@pytest.fixture +def two_entries(monkeypatch): + """Two synthetic models, the second twice the size of the first.""" + first = _synthetic_model("alpha", b"A" * 4096) + second = _synthetic_model("beta", b"B" * 8192) + monkeypatch.setattr(model_catalog, "MODELS", (first[0], second[0])) + return first, second + + +def _bodies(model, contents, overrides=None): + """The fake session's URL -> bytes table, with per-file substitutions.""" + merged = dict(contents) + merged.update(overrides or {}) + return { + model_store.file_url(model, name): data for name, data in merged.items() + } + + +def _write_model(root, model, contents): + """Put a complete, valid model on disk without going through a download.""" + directory = model_store.model_dir(root, model.id) + os.makedirs(directory, exist_ok=True) + for name, data in contents.items(): + with open(os.path.join(directory, name), "wb") as fh: + fh.write(data) + return directory + + +def _names_in(directory): + return sorted(os.listdir(directory)) + + +# ── The fake network ───────────────────────────────────────────────────────── + + +class _FakeResponse: + def __init__(self, session, url, start=0, status_code=200, served_from=None): + self._session = session + self._url = url + self._start = start + self.status_code = status_code + self.headers = {} + if status_code == 206: + total = len(session.bodies[url]) + # What the server claims it is sending, which is not always where + # it actually starts — see _FakeSession(pretend_range=True). + claimed = start if served_from is None else served_from + self.headers["Content-Range"] = f"bytes {claimed}-{total - 1}/{total}" + self.closed = False + + def raise_for_status(self): + error = self._session.failures.get(self._url) + if error is not None: + raise error + + def iter_content(self, chunk_size=None): + self._session.chunk_sizes.append(chunk_size) + body = self._session.bodies[self._url][self._start:] + # Split into a fixed number of pieces rather than by chunk_size: the + # real chunk is 1 MB, so honouring it would mean either megabytes of + # synthetic weights or a single-chunk transfer that never exercises + # cancelling between chunks. The size actually asked for is pinned + # separately, by test_the_transfer_is_streamed_in_chunks. + step = max(1, math.ceil(len(body) / self._session.slices)) + for start in range(0, len(body), step): + chunk = body[start:start + step] + self._session.served += len(chunk) + yield chunk + + def close(self): + self.closed = True + + +class _FakeSession: + """The slice of requests.Session the store uses, answering from bytes. + + Records every request so a test can assert what was *not* fetched — which + is how "already complete files are skipped" and "the space gate runs before + the first byte" are checked. + """ + + def __init__(self, bodies, failures=None, slices=4, honour_range=True, + pretend_range=False): + self.bodies = dict(bodies) + self.failures = dict(failures or {}) + self.slices = slices + # Hugging Face serves byte ranges; a proxy or a mirror may not, and + # answering 200 to a Range request is the case that would concatenate + # a second copy onto the first if nothing noticed. `pretend_range` is + # the nastier one: a 206 label on a body that starts at zero. + self.honour_range = honour_range + self.pretend_range = pretend_range + self.requested = [] + self.streamed = [] + self.timeouts = [] + self.chunk_sizes = [] + self.ranges = [] + self.responses = [] + self.served = 0 + self.closed = False + + def get(self, url, stream=False, timeout=None, headers=None): + self.requested.append(url) + self.streamed.append(stream) + self.timeouts.append(timeout) + wanted = (headers or {}).get("Range") + self.ranges.append(wanted) + start, status, served_from = 0, 200, None + if wanted and self.honour_range: + start = int(wanted.split("=")[1].split("-")[0]) + status = 206 + elif wanted and self.pretend_range: + status = 206 # the label of a range, the body of the whole file + served_from = 0 + response = _FakeResponse( + self, url, start=start, status_code=status, served_from=served_from + ) + self.responses.append(response) + return response + + def close(self): + self.closed = True + + +class _CancelAfter: + """Cancels once `limit` bytes of the transfer have been reported. + + Counting reported bytes rather than callback invocations makes "cancel in + the middle of the big file" a fixed point of the test instead of a guess + about how many chunks the fake produced. + """ + + def __init__(self, limit): + self.limit = limit + self.done = 0 + + def progress(self, done, _total): + self.done = done + + def __call__(self): + return self.done >= self.limit + + +class _RecordingLock: + """Stands in for coord_locks.models_lock and records when it is held.""" + + def __init__(self, events, root): + self.events = events + self.root = root + + def acquire(self): + self.events.append(("acquire", self.root)) + + def release(self): + self.events.append(("release", self.root)) + + +def _recording_locks(monkeypatch): + events = [] + monkeypatch.setattr( + model_store, + "models_lock", + lambda root, lock_dir, timeout=None: _RecordingLock(events, root), + ) + return events + + +class _AlwaysBusyLock: + """A lock another process never lets go of.""" + + def __init__(self, root): + self.root = root + + def acquire(self): + raise LockTimeout(f"held by another process: {self.root}") + + def release(self): # pragma: no cover - never acquired + raise AssertionError("released a lock that was never acquired") + + +def _busy_locks(monkeypatch): + """Make every models lock unavailable, and stop waiting immediately. + + The real deadline is twelve hours — a backstop against a lock nobody will + release, not a policy — so a test that waited for it would be a test nobody + ever finishes. + """ + monkeypatch.setattr(model_store, "_LOCK_TIMEOUT_SECONDS", 0.0) + monkeypatch.setattr( + model_store, + "models_lock", + lambda root, lock_dir, timeout=None: _AlwaysBusyLock(root), + ) + + +def _link_directory(target, link): + """(created, detail) — a directory link by whatever route is permitted. + + os.symlink needs SeCreateSymbolicLinkPrivilege, which an ordinary Windows + account does not have; a junction needs nothing at all, resolves through + realpath exactly the same way, and is what somebody actually does to park a + 3 GB model on another drive. Skipping instead of linking is what left the + guard below deletable with the suite green. + """ + if sys.platform == "win32": + done = subprocess.run( + ["cmd", "/c", "mklink", "/J", link, target], + capture_output=True, text=True, + ) + return done.returncode == 0, (done.stdout or "") + (done.stderr or "") + try: + os.symlink(target, link, target_is_directory=True) + return True, "" + except OSError as exc: + return False, str(exc) + + +class _Progress: + """Collects every (done, total) pair the store reports.""" + + def __init__(self): + self.calls = [] + + def __call__(self, done, total): + self.calls.append((done, total)) + + +# ── Tests ──────────────────────────────────────────────────────────────────── + + +class TestUrls: + @pytest.mark.parametrize("model", model_catalog.list_models(), ids=lambda m: m.id) + def test_every_file_url_is_built_from_the_repo_and_revision(self, model): + for name, _size in model.files: + assert model_store.file_url(model, name) == ( + f"https://huggingface.co/{model.repo}/resolve/{model.revision}/{name}" + ) + + @pytest.mark.parametrize("model", model_catalog.list_models(), ids=lambda m: m.id) + def test_no_url_follows_a_branch_instead_of_the_pinned_revision(self, model): + """Swapping the revision for `main` is the regression this catches. + + It would keep working for as long as upstream leaves the files alone, + and then start serving bytes whose size and digest nothing here has + ever measured — silently accepting different weights for any file whose + size happened to be unchanged. + """ + for name, _size in model.files: + url = model_store.file_url(model, name) + assert "/resolve/main/" not in url + assert f"/resolve/{model.revision}/" in url + assert len(model.revision) == 40 + assert all(c in "0123456789abcdef" for c in model.revision) + + +class TestWhereModelsLive: + def test_the_default_root_is_global_not_per_account(self): + """A 3 GB model duplicated for every account is not a cost anyone + would accept, and WinZapp runs one account per process.""" + assert model_store.default_models_dir() == app_paths.global_dir( + model_store.MODELS_DIRNAME + ) + assert not model_store.default_models_dir().startswith( + app_paths.accounts_root() + ) + + def test_a_model_lives_in_its_own_subdirectory_of_the_root(self, entry, tmp_path): + model, _contents = entry + assert model_store.model_dir(str(tmp_path), model.id) == os.path.join( + str(tmp_path), model.id + ) + + +class TestInstallationState: + def test_an_empty_root_reports_absent(self, entry, tmp_path): + model, _contents = entry + state = model_store.installation_state(str(tmp_path), model) + assert state.state == model_store.STATE_ABSENT + assert set(state.missing) == {name for name, _ in model.files} + assert state.present_bytes == 0 + assert model_store.is_installed(str(tmp_path), model) is False + assert model_store.list_installed(str(tmp_path)) == () + + def test_a_complete_model_reports_installed(self, entry, tmp_path): + model, contents = entry + _write_model(str(tmp_path), model, contents) + state = model_store.installation_state(str(tmp_path), model) + assert state.state == model_store.STATE_INSTALLED + assert state.missing == () + assert state.present_bytes == model.disk_bytes + assert model_store.list_installed(str(tmp_path)) == (model.id,) + + def test_a_truncated_file_is_incomplete_not_installed(self, entry, tmp_path): + """The state an interrupted download leaves behind. Reporting it as + installed is what makes CTranslate2 fail with "internal error".""" + model, contents = entry + directory = _write_model(str(tmp_path), model, contents) + with open(os.path.join(directory, "tokenizer.json"), "wb") as fh: + fh.write(contents["tokenizer.json"][:-1]) + + state = model_store.installation_state(str(tmp_path), model) + assert state.state == model_store.STATE_INCOMPLETE + assert state.missing == ("tokenizer.json",) + assert model_store.is_installed(str(tmp_path), model) is False + assert model_store.list_installed(str(tmp_path)) == () + + def test_a_missing_file_is_incomplete_while_others_remain(self, entry, tmp_path): + model, contents = entry + directory = _write_model(str(tmp_path), model, contents) + os.remove(os.path.join(directory, "model.bin")) + + state = model_store.installation_state(str(tmp_path), model) + assert state.state == model_store.STATE_INCOMPLETE + assert state.missing == ("model.bin",) + assert state.present_bytes == model.disk_bytes - model.model_bin_bytes + + def test_a_lone_truncated_file_is_still_an_interrupted_download( + self, entry, tmp_path + ): + """Zero correct bytes is not the same as nothing being there — a + directory holding one bad file has something to repair.""" + model, _contents = entry + directory = model_store.model_dir(str(tmp_path), model.id) + os.makedirs(directory) + with open(os.path.join(directory, "model.bin"), "wb") as fh: + fh.write(b"A") + + state = model_store.installation_state(str(tmp_path), model) + assert state.state == model_store.STATE_INCOMPLETE + assert state.present_bytes == 0 + + def test_a_part_file_alone_is_an_interrupted_download(self, entry, tmp_path): + """Not ABSENT: the two states offer the user different buttons, and a + 2 GB `.part` waiting to be resumed is not nothing being there.""" + model, contents = entry + directory = model_store.model_dir(str(tmp_path), model.id) + os.makedirs(directory) + for name, data in contents.items(): + with open(os.path.join(directory, name + ".part"), "wb") as fh: + fh.write(data) + + state = model_store.installation_state(str(tmp_path), model) + assert state.state == model_store.STATE_INCOMPLETE + assert model_store.is_installed(str(tmp_path), model) is False + + +class TestFreeSpace: + def test_free_space_is_measured_from_the_nearest_existing_ancestor(self, tmp_path): + """The models directory does not exist yet the first time this is + asked, and a gate answering "unknown" there blocks every first + download.""" + missing = str(tmp_path / "not" / "created" / "yet") + assert model_store.free_bytes(missing) is not None + assert model_store.free_bytes(missing) > 0 + + def test_a_download_that_only_just_fits_is_refused(self, entry, tmp_path, monkeypatch): + model, _contents = entry + monkeypatch.setattr( + model_store, + "free_bytes", + lambda _path: model.download_bytes + model_store._FREE_SPACE_SLACK_BYTES - 1, + ) + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.ensure_free_space(str(tmp_path), model.download_bytes) + assert excinfo.value.code == errors.NO_DISK_SPACE + + def test_the_slack_is_enough_to_pass(self, entry, tmp_path, monkeypatch): + model, _contents = entry + monkeypatch.setattr( + model_store, + "free_bytes", + lambda _path: model.download_bytes + model_store._FREE_SPACE_SLACK_BYTES, + ) + model_store.ensure_free_space(str(tmp_path), model.download_bytes) + + def test_an_unmeasurable_volume_does_not_block_the_download(self, tmp_path, monkeypatch): + """Refusing because disk_usage() failed would make transcription + unavailable on a setup that works perfectly well.""" + monkeypatch.setattr(model_store, "free_bytes", lambda _path: None) + model_store.ensure_free_space(str(tmp_path), 10**12) + + +class TestDownload: + def test_a_download_writes_exactly_the_catalogued_files(self, entry, tmp_path): + model, contents = entry + session = _FakeSession(_bodies(model, contents)) + + directory = model_store.download_model(model, str(tmp_path), session=session) + + assert _names_in(directory) == sorted(contents) + for name, size in model.files: + with open(os.path.join(directory, name), "rb") as fh: + data = fh.read() + assert len(data) == size + assert data == contents[name] + assert model_store.is_installed(str(tmp_path), model) is True + assert model_store.list_installed(str(tmp_path)) == (model.id,) + + def test_every_file_is_fetched_from_its_pinned_url(self, entry, tmp_path): + model, contents = entry + session = _FakeSession(_bodies(model, contents)) + model_store.download_model(model, str(tmp_path), session=session) + assert sorted(session.requested) == sorted( + model_store.file_url(model, name) for name, _size in model.files + ) + + def test_the_transfer_is_streamed_in_chunks(self, entry, tmp_path): + """A non-streaming GET would pull 3 GB into memory before writing a + byte of it.""" + model, contents = entry + session = _FakeSession(_bodies(model, contents)) + model_store.download_model(model, str(tmp_path), session=session) + assert session.streamed == [True] * len(model.files) + assert session.chunk_sizes == [model_store._CHUNK_BYTES] * len(model.files) + assert session.timeouts == [model_store._HTTP_TIMEOUT] * len(model.files) + assert all(response.closed for response in session.responses) + + def test_progress_never_goes_backwards_and_ends_at_the_total(self, entry, tmp_path): + """One bar for the whole model, not six in a row — the caller is + drawing a gauge and announcing a percentage.""" + model, contents = entry + progress = _Progress() + session = _FakeSession(_bodies(model, contents)) + + model_store.download_model( + model, str(tmp_path), progress=progress, session=session + ) + + assert progress.calls + assert all(total == model.download_bytes for _done, total in progress.calls) + done = [d for d, _t in progress.calls] + assert done == sorted(done) + assert done[-1] == model.download_bytes + assert done[0] == 0 + + def test_the_digest_is_taken_from_the_stream_not_from_a_second_read( + self, entry, tmp_path, monkeypatch + ): + """Re-reading model.bin to hash it would double the I/O of every + download for a number the transfer already had in its hands.""" + model, contents = entry + reads = [] + monkeypatch.setattr( + model_store, "_hash_file", lambda *args, **kwargs: reads.append(args) + ) + session = _FakeSession(_bodies(model, contents)) + + model_store.download_model(model, str(tmp_path), session=session) + + assert reads == [] + + def test_a_file_of_the_wrong_size_is_corruption(self, entry, tmp_path): + model, contents = entry + session = _FakeSession( + _bodies(model, contents, {"model.bin": contents["model.bin"][:-8]}) + ) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.download_model(model, str(tmp_path), session=session) + + assert excinfo.value.code == errors.MODEL_CORRUPTED + assert model_store.is_installed(str(tmp_path), model) is False + assert not os.path.exists( + os.path.join(model_store.model_dir(str(tmp_path), model.id), "model.bin") + ) + + def test_the_right_number_of_wrong_bytes_is_still_corruption(self, entry, tmp_path): + """The one thing a size check cannot see, and the reason model.bin + carries a digest at all.""" + model, contents = entry + swapped = b"Z" * len(contents["model.bin"]) + assert len(swapped) == model.model_bin_bytes + session = _FakeSession(_bodies(model, contents, {"model.bin": swapped})) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.download_model(model, str(tmp_path), session=session) + + assert excinfo.value.code == errors.MODEL_CORRUPTED + assert model_store.is_installed(str(tmp_path), model) is False + # The size is right, so a file left under its final name would pass the + # install check forever — this is the case the `.part` rename order + # exists for. + assert not os.path.exists( + os.path.join(model_store.model_dir(str(tmp_path), model.id), "model.bin") + ) + + def test_a_failed_request_is_a_download_failure(self, entry, tmp_path): + model, contents = entry + session = _FakeSession( + _bodies(model, contents), + failures={ + model_store.file_url(model, "model.bin"): requests.HTTPError("404") + }, + ) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.download_model(model, str(tmp_path), session=session) + + assert excinfo.value.code == errors.MODEL_DOWNLOAD_FAILED + # The technical detail is for the log only; str() is what a careless + # wx.MessageBox would read out loud. + assert str(excinfo.value) == errors.MODEL_DOWNLOAD_FAILED + assert "404" in excinfo.value.log_line + assert model_store.is_installed(str(tmp_path), model) is False + + def test_a_failure_on_the_first_file_leaves_no_directory_behind( + self, entry, tmp_path + ): + model, contents = entry + session = _FakeSession( + _bodies(model, contents), + failures={ + model_store.file_url(model, "config.json"): requests.HTTPError("500") + }, + ) + + with pytest.raises(errors.TranscriptionError): + model_store.download_model(model, str(tmp_path), session=session) + + assert not os.path.exists(model_store.model_dir(str(tmp_path), model.id)) + + def test_no_part_file_survives_a_failure(self, entry, tmp_path): + model, contents = entry + session = _FakeSession( + _bodies(model, contents, {"model.bin": contents["model.bin"][:-8]}) + ) + + with pytest.raises(errors.TranscriptionError): + model_store.download_model(model, str(tmp_path), session=session) + + directory = model_store.model_dir(str(tmp_path), model.id) + assert [n for n in os.listdir(directory) if n.endswith(".part")] == [] + + def test_the_space_gate_runs_before_the_first_byte( + self, entry, tmp_path, monkeypatch + ): + """Failing half way into a 3 GB download is failing after the choice + the user could have made differently.""" + model, contents = entry + monkeypatch.setattr(model_store, "free_bytes", lambda _path: model.download_bytes) + session = _FakeSession(_bodies(model, contents)) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.download_model(model, str(tmp_path), session=session) + + assert excinfo.value.code == errors.NO_DISK_SPACE + assert session.requested == [] + assert not os.path.exists(model_store.model_dir(str(tmp_path), model.id)) + + def test_cancelling_mid_file_leaves_nothing_that_looks_installed( + self, entry, tmp_path + ): + """The whole point of the `.part` discipline: the install check trusts + a name and a size, so a cancelled transfer must never own one — while + the prefix itself is kept, under a name nothing believes, because that + is what the next attempt resumes from.""" + model, contents = entry + cancel = _CancelAfter(len(contents["config.json"]) + 1) + session = _FakeSession(_bodies(model, contents)) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.download_model( + model, + str(tmp_path), + progress=cancel.progress, + should_cancel=cancel, + session=session, + ) + + assert excinfo.value.code == errors.CANCELLED + directory = model_store.model_dir(str(tmp_path), model.id) + assert model_store.is_installed(str(tmp_path), model) is False + assert not os.path.exists(os.path.join(directory, "model.bin")) + assert os.path.getsize(os.path.join(directory, "model.bin.part")) > 0 + # The file that did finish stays under its own name. + assert "config.json" in _names_in(directory) + + def test_cancelling_before_anything_arrives_removes_the_new_directory( + self, entry, tmp_path + ): + model, contents = entry + session = _FakeSession(_bodies(model, contents)) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.download_model( + model, str(tmp_path), should_cancel=lambda: True, session=session + ) + + assert excinfo.value.code == errors.CANCELLED + assert not os.path.exists(model_store.model_dir(str(tmp_path), model.id)) + + def test_resuming_after_a_cancellation_refetches_only_what_is_missing( + self, entry, tmp_path + ): + """Re-downloading 3 GB because the user cancelled at 99% is the + difference between a feature and one nobody uses twice.""" + model, contents = entry + cancel = _CancelAfter(len(contents["config.json"]) + 1) + first = _FakeSession(_bodies(model, contents)) + with pytest.raises(errors.TranscriptionError): + model_store.download_model( + model, + str(tmp_path), + progress=cancel.progress, + should_cancel=cancel, + session=first, + ) + + second = _FakeSession(_bodies(model, contents)) + model_store.download_model(model, str(tmp_path), session=second) + + assert model_store.file_url(model, "config.json") not in second.requested + assert model_store.file_url(model, "model.bin") in second.requested + assert model_store.is_installed(str(tmp_path), model) is True + + def test_a_file_of_the_wrong_size_is_fetched_again_not_kept( + self, entry, tmp_path + ): + """Repairing an incomplete model is the same call as downloading it. + + "Already there" has to mean the exact size, not merely a file with that + name: a truncated model.bin left by a process that was killed would + otherwise be skipped by every future download, and the model would stay + broken with nothing offering to fix it. + """ + model, contents = entry + directory = model_store.model_dir(str(tmp_path), model.id) + os.makedirs(directory) + with open(os.path.join(directory, "model.bin"), "wb") as fh: + fh.write(contents["model.bin"][:100]) + session = _FakeSession(_bodies(model, contents)) + + model_store.download_model(model, str(tmp_path), session=session) + + assert model_store.file_url(model, "model.bin") in session.requested + assert model_store.is_installed(str(tmp_path), model) is True + model_store.verify_model(str(tmp_path), model) + + def test_a_completed_model_sweeps_a_stale_part_from_an_older_attempt( + self, entry, tmp_path + ): + """Nothing else on this path would: up to 3 GB of dead weight left by + an attempt whose file has since arrived by other means.""" + model, contents = entry + directory = _write_model(str(tmp_path), model, contents) + with open(os.path.join(directory, "model.bin.part"), "wb") as fh: + fh.write(b"A" * 128) + + model_store.download_model(model, str(tmp_path), session=_FakeSession({})) + + assert _names_in(directory) == sorted(contents) + + def test_a_second_download_of_a_complete_model_fetches_nothing( + self, entry, tmp_path + ): + model, contents = entry + model_store.download_model( + model, str(tmp_path), session=_FakeSession(_bodies(model, contents)) + ) + + progress = _Progress() + again = _FakeSession({}) + model_store.download_model( + model, str(tmp_path), progress=progress, session=again + ) + + assert again.requested == [] + assert progress.calls == [(model.download_bytes, model.download_bytes)] + assert model_store.is_installed(str(tmp_path), model) is True + + def test_an_interrupted_model_bin_resumes_from_a_byte_range( + self, entry, tmp_path + ): + """model.bin is 95-99% of a model, so "resuming costs the remainder" + is only true if the *file* resumes. Cancelling large-v3 at 90% used to + cost the whole 3 GB again.""" + model, contents = entry + cancel = _CancelAfter(len(contents["config.json"]) + 1) + first = _FakeSession(_bodies(model, contents)) + with pytest.raises(errors.TranscriptionError): + model_store.download_model( + model, + str(tmp_path), + progress=cancel.progress, + should_cancel=cancel, + session=first, + ) + part = os.path.join( + model_store.model_dir(str(tmp_path), model.id), "model.bin.part" + ) + carried = os.path.getsize(part) + assert 0 < carried < model.model_bin_bytes + + progress = _Progress() + second = _FakeSession(_bodies(model, contents)) + model_store.download_model( + model, str(tmp_path), progress=progress, session=second + ) + + assert f"bytes={carried}-" in second.ranges + assert second.served == ( + model.download_bytes - len(contents["config.json"]) - carried + ) + assert model_store.is_installed(str(tmp_path), model) is True + # The digest was re-seeded from the prefix rather than restarted, which + # is the only way a streamed hash can survive an interruption. + model_store.verify_model(str(tmp_path), model) + + done = [d for d, _t in progress.calls] + assert done == sorted(done) + assert done[-1] == model.download_bytes + + def test_a_server_that_ignores_the_range_starts_the_file_over( + self, entry, tmp_path + ): + """Answering 200 to a Range request means the whole file is coming; + appending it to the prefix would build a file out of duplicated bytes, + which only model.bin's digest would ever catch.""" + model, contents = entry + cancel = _CancelAfter(len(contents["config.json"]) + 1) + with pytest.raises(errors.TranscriptionError): + model_store.download_model( + model, + str(tmp_path), + progress=cancel.progress, + should_cancel=cancel, + session=_FakeSession(_bodies(model, contents)), + ) + + second = _FakeSession(_bodies(model, contents), honour_range=False) + model_store.download_model(model, str(tmp_path), session=second) + + assert second.ranges[0] is not None # it did ask + assert model_store.is_installed(str(tmp_path), model) is True + model_store.verify_model(str(tmp_path), model) + + def test_a_part_longer_than_the_file_is_not_resumed_from(self, entry, tmp_path): + """A prefix is only a prefix while it is shorter. A `.part` at or past + the expected size is a leftover from something else, and resuming from + it would produce the right length out of the wrong bytes.""" + model, contents = entry + directory = model_store.model_dir(str(tmp_path), model.id) + os.makedirs(directory) + with open(os.path.join(directory, "model.bin.part"), "wb") as fh: + fh.write(b"Z" * (model.model_bin_bytes + 10)) + session = _FakeSession(_bodies(model, contents)) + + model_store.download_model(model, str(tmp_path), session=session) + + assert session.ranges == [None] * len(model.files) + model_store.verify_model(str(tmp_path), model) + + def test_an_auxiliary_part_is_never_resumed_from(self, entry, tmp_path): + """Only a file with a digest may resume, and this is why. + + For config.json / tokenizer.json / vocabulary.* the only check is + `written == expected_bytes`, and a stale `.part` that is NOT a prefix + of the current file satisfies it exactly: the result has the catalogued + size, so installation_state, is_installed, ensure_ready — and + verify_model, which hashes only model.bin — all call the model healthy, + and CTranslate2 then dies on the tokenizer with nothing but "internal + error" for the user. A power cut or a kill leaves exactly that kind of + `.part`, since fsync only runs at the end of the file. + """ + model, contents = entry + directory = model_store.model_dir(str(tmp_path), model.id) + os.makedirs(directory) + stale = b"#" * 8 # the right length to splice, the wrong bytes + assert 0 < len(stale) < len(contents["tokenizer.json"]) + with open(os.path.join(directory, "tokenizer.json.part"), "wb") as fh: + fh.write(stale) + session = _FakeSession(_bodies(model, contents)) + + model_store.download_model(model, str(tmp_path), session=session) + + assert session.ranges == [None] * len(model.files) + with open(os.path.join(directory, "tokenizer.json"), "rb") as fh: + assert fh.read() == contents["tokenizer.json"] + + def test_a_206_that_starts_somewhere_else_is_not_appended( + self, entry, tmp_path + ): + """206 says *a* range is coming, not *which* one. A body that starts at + zero under a 206 label would be appended to the prefix and produce a + file too long — or, with a stale prefix, the right length and the wrong + bytes.""" + model, contents = entry + cancel = _CancelAfter(len(contents["config.json"]) + 1) + with pytest.raises(errors.TranscriptionError): + model_store.download_model( + model, + str(tmp_path), + progress=cancel.progress, + should_cancel=cancel, + session=_FakeSession(_bodies(model, contents)), + ) + + liar = _FakeSession( + _bodies(model, contents), honour_range=False, pretend_range=True + ) + model_store.download_model(model, str(tmp_path), session=liar) + + assert liar.ranges[0] is not None # it did ask for a range + assert model_store.is_installed(str(tmp_path), model) is True + model_store.verify_model(str(tmp_path), model) + + def test_the_bytes_reach_the_disk_before_the_rename( + self, entry, tmp_path, monkeypatch + ): + """NTFS journals the rename, not the data: without the flush a power + cut can leave a file of exactly the right size full of zeros under the + final name, which the install check then believes forever.""" + model, contents = entry + events = [] + real_fsync, real_replace = os.fsync, os.replace + monkeypatch.setattr(os, "fsync", lambda fd: (events.append("fsync"), real_fsync(fd))[1]) + monkeypatch.setattr( + os, "replace", lambda src, dst: (events.append("replace"), real_replace(src, dst))[1] + ) + + model_store.download_model( + model, str(tmp_path), session=_FakeSession(_bodies(model, contents)) + ) + + assert events == ["fsync", "replace"] * len(model.files) + + def test_the_download_is_held_under_the_shared_models_lock( + self, entry, tmp_path, monkeypatch + ): + """The root is global so every account process shares it: two of them + downloading large-v3 at once open the same `model.bin.part`, interleave + their writes and each one's cleanup deletes the other's file.""" + model, contents = entry + events = _recording_locks(monkeypatch) + + model_store.download_model( + model, + str(tmp_path), + progress=lambda done, total: events.append(("progress", done)), + session=_FakeSession(_bodies(model, contents)), + ) + + assert events[0] == ("acquire", str(tmp_path)) + assert events[-1] == ("release", str(tmp_path)) + assert any(event[0] == "progress" for event in events[1:-1]) + + def test_a_failure_does_not_leave_a_root_the_app_created( + self, entry, tmp_path, monkeypatch + ): + """The user picks that folder in part 5; failing to use it is no + reason to leave an empty directory of ours in their filesystem.""" + model, contents = entry + root = str(tmp_path / "brand" / "new") + session = _FakeSession( + _bodies(model, contents), + failures={model_store.file_url(model, "config.json"): requests.HTTPError("500")}, + ) + + with pytest.raises(errors.TranscriptionError): + model_store.download_model(model, root, session=session) + + assert not os.path.exists(root) + + def test_a_caller_supplied_session_is_left_open(self, entry, tmp_path): + """It belongs to the caller — closing it would break the next call.""" + model, contents = entry + session = _FakeSession(_bodies(model, contents)) + model_store.download_model(model, str(tmp_path), session=session) + assert session.closed is False + + +class TestVerifyModel: + def test_a_good_model_verifies(self, entry, tmp_path): + model, contents = entry + _write_model(str(tmp_path), model, contents) + model_store.verify_model(str(tmp_path), model) + + def test_verification_reports_progress_over_the_weights(self, entry, tmp_path): + model, contents = entry + _write_model(str(tmp_path), model, contents) + progress = _Progress() + + model_store.verify_model(str(tmp_path), model, progress=progress) + + assert progress.calls[-1] == (model.model_bin_bytes, model.model_bin_bytes) + + def test_swapped_weights_of_the_same_length_are_caught(self, entry, tmp_path): + model, contents = entry + directory = _write_model(str(tmp_path), model, contents) + with open(os.path.join(directory, "model.bin"), "wb") as fh: + fh.write(b"Z" * model.model_bin_bytes) + + # The cheap check still says installed, which is exactly why the + # expensive one exists. + assert model_store.is_installed(str(tmp_path), model) is True + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.verify_model(str(tmp_path), model) + assert excinfo.value.code == errors.MODEL_CORRUPTED + + def test_an_absent_model_is_not_installed_rather_than_corrupted( + self, entry, tmp_path + ): + """Two different answers for the user: download it, versus download it + again because what is there is broken.""" + model, _contents = entry + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.verify_model(str(tmp_path), model) + assert excinfo.value.code == errors.MODEL_NOT_INSTALLED + + def test_an_incomplete_model_is_corrupted(self, entry, tmp_path): + model, contents = entry + directory = _write_model(str(tmp_path), model, contents) + os.remove(os.path.join(directory, "vocabulary.txt")) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.verify_model(str(tmp_path), model) + assert excinfo.value.code == errors.MODEL_CORRUPTED + assert "vocabulary.txt" in excinfo.value.log_line + + def test_verification_can_be_cancelled(self, entry, tmp_path): + """Up to 3 GB of reading. A user who changed their mind has to be able + to say so and be believed.""" + model, contents = entry + _write_model(str(tmp_path), model, contents) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.verify_model( + str(tmp_path), model, should_cancel=lambda: True + ) + assert excinfo.value.code == errors.CANCELLED + + +class TestRemoveModel: + def test_only_the_catalogue_names_are_deleted(self, entry, tmp_path): + """Part 5 lets the user choose the models folder, and somebody will + choose one that already holds their own files. A shutil.rmtree() there + only has to be wrong once.""" + model, contents = entry + directory = _write_model(str(tmp_path), model, contents) + stranger = os.path.join(directory, "notes-of-my-own.txt") + with open(stranger, "wb") as fh: + fh.write(b"something the user put here") + + assert model_store.remove_model(str(tmp_path), model.id) is True + + assert _names_in(directory) == ["notes-of-my-own.txt"] + with open(stranger, "rb") as fh: + assert fh.read() == b"something the user put here" + assert model_store.is_installed(str(tmp_path), model) is False + + def test_an_emptied_directory_is_removed(self, entry, tmp_path): + model, contents = entry + directory = _write_model(str(tmp_path), model, contents) + model_store.remove_model(str(tmp_path), model.id) + assert not os.path.exists(directory) + + def test_leftover_part_files_go_too(self, entry, tmp_path): + model, contents = entry + directory = _write_model(str(tmp_path), model, contents) + with open(os.path.join(directory, "model.bin.part"), "wb") as fh: + fh.write(b"A" * 10) + + model_store.remove_model(str(tmp_path), model.id) + assert not os.path.exists(directory) + + def test_removing_what_is_not_there_is_not_an_error(self, entry, tmp_path): + model, _contents = entry + assert model_store.remove_model(str(tmp_path), model.id) is False + + def test_the_removal_is_held_under_the_shared_models_lock( + self, entry, tmp_path, monkeypatch + ): + """Deleting a model another process is downloading into is exactly + what the lock is for.""" + model, contents = entry + _write_model(str(tmp_path), model, contents) + events = _recording_locks(monkeypatch) + + assert model_store.remove_model(str(tmp_path), model.id) is True + + assert events == [("acquire", str(tmp_path)), ("release", str(tmp_path))] + + def test_an_unknown_id_deletes_nothing(self, entry, tmp_path): + """The names to delete are exactly what the catalogue no longer knows, + so there is nothing that can be removed safely.""" + directory = os.path.join(str(tmp_path), "retired-model") + os.makedirs(directory) + with open(os.path.join(directory, "model.bin"), "wb") as fh: + fh.write(b"A" * 16) + + assert model_store.remove_model(str(tmp_path), "retired-model") is False + assert _names_in(directory) == ["model.bin"] + + +class TestRepairModel: + """The one state a re-download cannot fix, and the only way out of it.""" + + def test_a_download_cannot_replace_weights_of_the_right_size( + self, entry, tmp_path + ): + model, contents = entry + directory = _write_model(str(tmp_path), model, contents) + with open(os.path.join(directory, "model.bin"), "wb") as fh: + fh.write(b"Z" * model.model_bin_bytes) + + idle = _FakeSession(_bodies(model, contents)) + model_store.download_model(model, str(tmp_path), session=idle) + + # Nothing was fetched and the call reported success: installation_state + # calls this installed, so every file is "already there". + assert idle.requested == [] + with pytest.raises(errors.TranscriptionError): + model_store.verify_model(str(tmp_path), model) + + def test_repair_deletes_first_and_downloads_everything_again( + self, entry, tmp_path + ): + model, contents = entry + directory = _write_model(str(tmp_path), model, contents) + with open(os.path.join(directory, "model.bin"), "wb") as fh: + fh.write(b"Z" * model.model_bin_bytes) + session = _FakeSession(_bodies(model, contents)) + + assert model_store.repair_model(model, str(tmp_path), session=session) == directory + + assert sorted(session.requested) == sorted( + model_store.file_url(model, name) for name, _size in model.files + ) + model_store.verify_model(str(tmp_path), model) + + def test_repair_reports_progress_over_the_whole_model(self, entry, tmp_path): + model, contents = entry + _write_model(str(tmp_path), model, contents) + progress = _Progress() + + model_store.repair_model( + model, + str(tmp_path), + progress=progress, + session=_FakeSession(_bodies(model, contents)), + ) + + assert progress.calls[-1] == (model.download_bytes, model.download_bytes) + + def test_repair_holds_the_lock_across_the_delete_and_the_download( + self, entry, tmp_path, monkeypatch + ): + """Between the two is the window another process could start its own + download into the directory this one just emptied.""" + model, contents = entry + _write_model(str(tmp_path), model, contents) + events = _recording_locks(monkeypatch) + + model_store.repair_model( + model, + str(tmp_path), + progress=lambda done, total: events.append(("progress", done)), + session=_FakeSession(_bodies(model, contents)), + ) + + assert events[0] == ("acquire", str(tmp_path)) + assert events[-1] == ("release", str(tmp_path)) + # The inner calls re-enter the same lock rather than taking a new one. + assert any(event[0] == "progress" for event in events[1:-1]) + + +class TestEnsureReady: + """What part 3 calls right before handing a path to CTranslate2. + + The two failures are different sentences for the user — download it, versus + repair it — and choosing between them may not cost 3 GB of hashing before a + transcription that has not started. + """ + + def test_a_complete_model_answers_with_its_directory(self, entry, tmp_path): + model, contents = entry + directory = _write_model(str(tmp_path), model, contents) + assert model_store.ensure_ready(str(tmp_path), model.id) == directory + + def test_an_absent_model_is_not_installed(self, entry, tmp_path): + model, _contents = entry + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.ensure_ready(str(tmp_path), model.id) + assert excinfo.value.code == errors.MODEL_NOT_INSTALLED + + def test_an_incomplete_model_is_corrupted(self, entry, tmp_path): + model, contents = entry + directory = _write_model(str(tmp_path), model, contents) + os.remove(os.path.join(directory, "config.json")) + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.ensure_ready(str(tmp_path), model.id) + assert excinfo.value.code == errors.MODEL_CORRUPTED + assert "config.json" in excinfo.value.log_line + + def test_an_id_the_catalogue_dropped_is_not_installed(self, entry, tmp_path): + """A settings file naming a model a later version retired. "Not + installed" is the one answer the UI can act on.""" + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.ensure_ready(str(tmp_path), "retired-model") + assert excinfo.value.code == errors.MODEL_NOT_INSTALLED + + def test_it_does_not_read_the_weights(self, entry, tmp_path, monkeypatch): + model, contents = entry + _write_model(str(tmp_path), model, contents) + monkeypatch.setattr( + model_store, "_hash_file", lambda *args, **kwargs: pytest.fail("hashed") + ) + model_store.ensure_ready(str(tmp_path), model.id) + + +class TestUnknownDirs: + """A model retired by an update becomes invisible and undeletable. + + remove_model() will not delete names the catalogue cannot look up, and + nothing lists it, so the user's 3 GB sits there with no screen mentioning + it. This is what part 5 shows. + """ + + def test_a_retired_model_directory_is_reported(self, entry, tmp_path): + model, contents = entry + _write_model(str(tmp_path), model, contents) + os.makedirs(os.path.join(str(tmp_path), "retired-model")) + + assert model_store.list_unknown_dirs(str(tmp_path)) == ("retired-model",) + + def test_files_beside_the_models_are_not_reported_as_directories( + self, entry, tmp_path + ): + with open(os.path.join(str(tmp_path), "notes-of-my-own.txt"), "wb") as fh: + fh.write(b"something the user put here") + assert model_store.list_unknown_dirs(str(tmp_path)) == () + + def test_a_root_that_does_not_exist_is_empty_not_an_error(self, tmp_path): + assert model_store.list_unknown_dirs(str(tmp_path / "nothing")) == () + + +class TestABusyModelsDirectory: + """Another window holding the folder is not a download failure. + + The models root is global so every account process shares it, and waiting + behind another window multi-gigabyte download is correct behaviour — but it + has to be *said* correctly. MODEL_DOWNLOAD_FAILED tells the user to check a + connection that is fine, and remove_model() answering False for it told them + nothing was deleted without saying whether their 3 GB is still there. + """ + + def test_a_download_says_the_folder_is_busy(self, entry, tmp_path, monkeypatch): + model, contents = entry + _busy_locks(monkeypatch) + session = _FakeSession(_bodies(model, contents)) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.download_model(model, str(tmp_path), session=session) + + assert excinfo.value.code == errors.MODELS_BUSY + assert session.requested == [] + + def test_a_removal_raises_instead_of_answering_nothing_was_there( + self, entry, tmp_path, monkeypatch + ): + model, contents = entry + _write_model(str(tmp_path), model, contents) + _busy_locks(monkeypatch) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.remove_model(str(tmp_path), model.id) + + assert excinfo.value.code == errors.MODELS_BUSY + # And nothing was touched, which is what a False return could not have + # distinguished itself from. + assert model_store.is_installed(str(tmp_path), model) is True + + def test_a_repair_says_the_folder_is_busy(self, entry, tmp_path, monkeypatch): + model, contents = entry + _write_model(str(tmp_path), model, contents) + _busy_locks(monkeypatch) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.repair_model( + model, str(tmp_path), session=_FakeSession(_bodies(model, contents)) + ) + + assert excinfo.value.code == errors.MODELS_BUSY + + def test_a_move_says_the_folder_is_busy(self, entry, tmp_path, monkeypatch): + model, contents = entry + old_root = str(tmp_path / "old") + _write_model(old_root, model, contents) + _busy_locks(monkeypatch) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.move_models(old_root, str(tmp_path / "new")) + + assert excinfo.value.code == errors.MODELS_BUSY + assert model_store.list_installed(old_root) == (model.id,) + + def test_waiting_for_the_lock_is_still_cancellable( + self, entry, tmp_path, monkeypatch + ): + """The Cancel button is the real bound on the wait — the deadline is a + backstop against a lock nobody will ever release, twelve hours out.""" + model, contents = entry + _write_model(str(tmp_path), model, contents) + monkeypatch.setattr( + model_store, + "models_lock", + lambda root, lock_dir, timeout=None: _AlwaysBusyLock(root), + ) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.remove_model( + str(tmp_path), model.id, should_cancel=lambda: True + ) + + assert excinfo.value.code == errors.CANCELLED + assert model_store.is_installed(str(tmp_path), model) is True + + +class TestCatalogueFileNames: + """Catalogue names are joined onto a user-chosen directory and deleted by + name, so a nested layout ("snapshots//model.bin", the shape these + repositories use internally) or a ".." would have remove_model() reach + outside the models root.""" + + @pytest.mark.parametrize("model", model_catalog.list_models(), ids=lambda m: m.id) + def test_every_catalogued_name_is_a_bare_basename(self, model): + for name, _size in model.files: + assert os.path.basename(name) == name + assert name not in (".", "..") + assert "/" not in name and "\\" not in name + + @pytest.mark.parametrize( + "bad", ["snapshots/abc/model.bin", "..\\model.bin", "..", "/model.bin", ""] + ) + def test_a_name_that_is_not_a_basename_is_refused(self, bad): + with pytest.raises(AssertionError): + model_catalog._model( + "synthetic", + "example-org/synthetic", + "0" * 40, + "0" * 64, + ((bad, 1), ("model.bin", 2)), + model_catalog.SIZE_SMALL, + min_vram_mb=1024, + min_ram_mb=2048, + ) + + +class TestMoveModels: + def test_every_installed_model_ends_up_at_the_new_root(self, two_entries, tmp_path): + (first, first_contents), (second, second_contents) = two_entries + old_root = str(tmp_path / "old") + new_root = str(tmp_path / "new") + _write_model(old_root, first, first_contents) + _write_model(old_root, second, second_contents) + + moved = model_store.move_models(old_root, new_root) + + assert moved == (first.id, second.id) + assert model_store.list_installed(new_root) == (first.id, second.id) + assert model_store.list_installed(old_root) == () + with open(os.path.join(new_root, second.id, "model.bin"), "rb") as fh: + assert fh.read() == second_contents["model.bin"] + + def test_the_move_reports_progress_to_the_total(self, two_entries, tmp_path): + (first, first_contents), (second, second_contents) = two_entries + old_root = str(tmp_path / "old") + new_root = str(tmp_path / "new") + _write_model(old_root, first, first_contents) + _write_model(old_root, second, second_contents) + progress = _Progress() + + model_store.move_models(old_root, new_root, progress=progress) + + total = first.disk_bytes + second.disk_bytes + assert all(t == total for _d, t in progress.calls) + done = [d for d, _t in progress.calls] + assert done == sorted(done) + assert done[-1] == total + + def test_an_interrupted_move_leaves_every_model_whole_somewhere( + self, two_entries, tmp_path, monkeypatch + ): + """The reason this is a copy-verify-delete and not a shutil.move() per + file: a move empties the source as it fills the destination, so an + interruption leaves both roots incomplete and the user with nothing + that loads.""" + (first, first_contents), (second, second_contents) = two_entries + old_root = str(tmp_path / "old") + new_root = str(tmp_path / "new") + _write_model(old_root, first, first_contents) + _write_model(old_root, second, second_contents) + # The copy path is the one with an invariant to keep; a same-volume + # move is a single rename and has no half way through. + monkeypatch.setattr(model_store, "_try_rename", lambda *_args: False) + cancel = _CancelAfter(first.disk_bytes + len(second_contents["config.json"]) + 1) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.move_models( + old_root, new_root, progress=cancel.progress, should_cancel=cancel + ) + + assert excinfo.value.code == errors.CANCELLED + assert model_store.list_installed(new_root) == (first.id,) + assert model_store.list_installed(old_root) == (second.id,) + second_dir = model_store.model_dir(new_root, second.id) + if os.path.isdir(second_dir): + assert [n for n in os.listdir(second_dir) if n.endswith(".part")] == [] + + def test_a_destination_that_cannot_be_written_keeps_the_source( + self, two_entries, tmp_path + ): + (first, first_contents), (second, second_contents) = two_entries + old_root = str(tmp_path / "old") + new_root = str(tmp_path / "new") + _write_model(old_root, first, first_contents) + _write_model(old_root, second, second_contents) + # A plain file where the first model's directory would go. + os.makedirs(new_root) + blocker = os.path.join(new_root, first.id) + with open(blocker, "wb") as fh: + fh.write(b"in the way") + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.move_models(old_root, new_root) + + # Its own code: after a failed move nothing is corrupted — the models + # are whole in the folder they were already in — and MODEL_CORRUPTED + # would tell the user to download 3 GB again when the way out is to + # pick a different folder. + assert excinfo.value.code == errors.MODEL_MOVE_FAILED + assert model_store.list_installed(old_root) == (first.id, second.id) + with open(blocker, "rb") as fh: + assert fh.read() == b"in the way" + + def test_a_model_already_at_the_destination_is_not_copied_again( + self, entry, tmp_path + ): + model, contents = entry + old_root = str(tmp_path / "old") + new_root = str(tmp_path / "new") + _write_model(old_root, model, contents) + _write_model(new_root, model, contents) + + moved = model_store.move_models(old_root, new_root) + + assert moved == (model.id,) + assert model_store.list_installed(new_root) == (model.id,) + assert not os.path.exists(model_store.model_dir(old_root, model.id)) + + def test_moving_a_root_onto_itself_does_nothing(self, entry, tmp_path): + model, contents = entry + root = str(tmp_path / "models") + _write_model(root, model, contents) + + assert model_store.move_models(root, root) == () + assert model_store.list_installed(root) == (model.id,) + + def test_a_destination_without_room_keeps_the_source_untouched( + self, entry, tmp_path, monkeypatch + ): + model, contents = entry + old_root = str(tmp_path / "old") + new_root = str(tmp_path / "new") + _write_model(old_root, model, contents) + monkeypatch.setattr(model_store, "_try_rename", lambda *_args: False) + monkeypatch.setattr(model_store, "free_bytes", lambda _path: model.disk_bytes) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.move_models(old_root, new_root) + + assert excinfo.value.code == errors.NO_DISK_SPACE + assert model_store.list_installed(old_root) == (model.id,) + assert not os.path.exists(new_root) + + def test_a_failure_part_way_still_says_which_models_crossed( + self, two_entries, tmp_path, monkeypatch + ): + """Every model stays whole in exactly one root either way — but "the + first one is in the new folder and the second is not" is the only thing + part 5 can tell the user, and a bare error code loses it.""" + (first, first_contents), (second, second_contents) = two_entries + old_root = str(tmp_path / "old") + new_root = str(tmp_path / "new") + _write_model(old_root, first, first_contents) + _write_model(old_root, second, second_contents) + monkeypatch.setattr(model_store, "_try_rename", lambda *_args: False) + # Room for the first model and none for the second. + answers = [10 ** 9, 0] + monkeypatch.setattr( + model_store, "free_bytes", lambda _path: answers.pop(0) if answers else 0 + ) + + with pytest.raises(errors.TranscriptionError) as excinfo: + model_store.move_models(old_root, new_root) + + assert excinfo.value.code == errors.NO_DISK_SPACE + assert excinfo.value.moved == (first.id,) + assert first.id in excinfo.value.log_line + assert model_store.list_installed(new_root) == (first.id,) + assert model_store.list_installed(old_root) == (second.id,) + + def test_a_same_volume_move_is_a_rename_and_needs_no_free_space( + self, entry, tmp_path, monkeypatch + ): + """Moving 5 GB to another folder on the same disk was refused with + 5.4 GB free, for a rename that is atomic, instant and costs nothing.""" + model, contents = entry + old_root = str(tmp_path / "old") + new_root = str(tmp_path / "new") + _write_model(old_root, model, contents) + monkeypatch.setattr(model_store, "free_bytes", lambda _path: 0) + + assert model_store.move_models(old_root, new_root) == (model.id,) + + assert model_store.list_installed(new_root) == (model.id,) + assert not os.path.exists(model_store.model_dir(old_root, model.id)) + + @pytest.mark.skipif( + sys.platform != "win32", + reason="two spellings are one directory only on a case-folding filesystem", + ) + def test_two_spellings_of_one_root_are_not_a_move(self, entry, tmp_path): + """The reproduced data-loss bug: os.path.abspath normalises separators + and "..", but it does not fold case and does not resolve junctions or + subst drives — the filesystem does. So the models were found already at + the destination, because they ARE the destination, and then deleted as + pure duplication of themselves: up to 3 GB, reported as a completed + move. canonical_dir() is the repository's existing answer to exactly + this, and coord_locks has carried it since multi-account shipped.""" + model, contents = entry + root = str(tmp_path / "Whisper_Models") + _write_model(root, model, contents) + other_spelling = str(tmp_path / "whisper_models") + + assert model_store.move_models(root, other_spelling) == () + + assert model_store.list_installed(root) == (model.id,) + assert model_store.list_installed(other_spelling) == (model.id,) + model_store.verify_model(root, model) + + def test_a_linked_model_directory_is_not_deleted_as_a_duplicate( + self, entry, tmp_path + ): + """Two roots can canonicalise apart and still share one model folder — + a user who linked a single 3 GB model onto another disk. The + already-at-the-destination branch reads the same files through the link, + answers "installed", and would then delete the source as pure + duplication of itself.""" + model, contents = entry + old_root = str(tmp_path / "old") + new_root = str(tmp_path / "new") + source = _write_model(old_root, model, contents) + os.makedirs(new_root) + created, detail = _link_directory( + source, model_store.model_dir(new_root, model.id) + ) + assert created, f"could not create a directory link: {detail}" + + assert model_store.move_models(old_root, new_root) == (model.id,) + + assert model_store.list_installed(new_root) == (model.id,) + assert model_store.list_installed(old_root) == (model.id,) + model_store.verify_model(old_root, model) + + def test_an_incomplete_model_travels_and_arrives_as_incomplete( + self, entry, tmp_path, monkeypatch + ): + """Left behind, it would sit in a directory no screen in part 5 ever + looks at again — invisible, and undeletable from inside the app.""" + model, contents = entry + old_root = str(tmp_path / "old") + new_root = str(tmp_path / "new") + directory = _write_model(old_root, model, contents) + os.remove(os.path.join(directory, "tokenizer.json")) + with open(os.path.join(directory, "vocabulary.txt"), "wb") as fh: + fh.write(contents["vocabulary.txt"][:-2]) # a half-written file + monkeypatch.setattr(model_store, "_try_rename", lambda *_args: False) + + assert model_store.move_models(old_root, new_root) == (model.id,) + + state = model_store.installation_state(new_root, model) + assert state.state == model_store.STATE_INCOMPLETE + assert sorted(state.missing) == ["tokenizer.json", "vocabulary.txt"] + assert _names_in(model_store.model_dir(new_root, model.id)) == [ + "config.json", "model.bin", "vocabulary.txt", + ] + assert not os.path.exists(model_store.model_dir(old_root, model.id)) + + def test_the_move_is_held_under_both_roots_locks( + self, entry, tmp_path, monkeypatch + ): + model, contents = entry + old_root = str(tmp_path / "old") + new_root = str(tmp_path / "new") + _write_model(old_root, model, contents) + events = _recording_locks(monkeypatch) + + model_store.move_models(old_root, new_root) + + # A set, not a list: the inner remove_model() re-enters the same lock + # on the real one, and how many times is not what this pins. + held = {root for action, root in events if action == "acquire"} + assert held == {old_root, new_root} + assert [action for action, _root in events[-2:]] == ["release", "release"] + + def test_the_old_root_itself_is_left_alone(self, entry, tmp_path): + """It may be a folder the user made and pointed WinZapp at; moving + files out of a directory is not permission to delete it.""" + model, contents = entry + old_root = str(tmp_path / "old") + new_root = str(tmp_path / "new") + _write_model(old_root, model, contents) + + model_store.move_models(old_root, new_root) + + assert os.path.isdir(old_root) + + +# ── The one test that reaches Hugging Face ─────────────────────────────────── + +_CATALOGUE_FILES = [ + (model, name, size) + for model in model_catalog.list_models() + for name, size in model.files +] +_CATALOGUE_IDS = [f"{model.id}-{name}" for model, name, _size in _CATALOGUE_FILES] + + +class TestTheCatalogueUrlsAreLive: + """Every URL the catalogue offers must serve exactly the bytes it claims. + + "No model may offer a broken link" is an explicit requirement, and it is + not something the fake session can answer: a wrong repo name, an unpublished + revision or a file renamed upstream all look perfectly fine from here. This + is the only test in the suite that talks to a remote host, so it is skipped + unless asked for — run it by hand whenever the catalogue changes. + """ + + pytestmark = [ + pytest.mark.network, + pytest.mark.skipif( + os.environ.get(_NETWORK_OPT_IN_ENV, "").strip() in ("", "0", "false", "False"), + reason=f"reaches huggingface.co - set {_NETWORK_OPT_IN_ENV}=1 to run it", + ), + ] + + @pytest.mark.parametrize("model, name, size", _CATALOGUE_FILES, ids=_CATALOGUE_IDS) + def test_the_url_serves_exactly_the_catalogued_size(self, model, name, size): + url = model_store.file_url(model, name) + # Through the same session the download itself uses, not a bare + # requests.head(): on a machine whose TLS is intercepted (an antivirus + # scanning HTTPS is enough) certifi does not know the injected root and + # every one of these fails with CERTIFICATE_VERIFY_FAILED — which reads + # exactly like the broken link this test exists to catch. Measured here: + # bare requests failed all 26 while curl fetched all 26 fine. + with tls_trust.create_session() as session: + # Accept-Encoding: identity because the question here is how many + # bytes the file HAS, not how many the server would send. requests + # advertises gzip/deflate/zstd by default, and Hugging Face answers + # a compressible JSON with no Content-Length at all — which is not a + # broken link, but reads as one. The LFS weights were unaffected + # (already compressed), so this only ever hid the small files. + response = session.head( + url, + allow_redirects=True, + timeout=30, + headers={"Accept-Encoding": "identity"}, + ) + + assert response.status_code == 200, f"{url} answered {response.status_code}" + # Followed to the CDN, an LFS file answers with the real Content-Length; + # answered by Hugging Face itself, the pointer's own length is returned + # and the real one comes back in x-linked-size. + reported = response.headers.get("Content-Length") or response.headers.get( + "x-linked-size" + ) + assert reported is not None, f"{url} reported no size at all" + assert int(reported) == size, f"{url} is {reported} bytes, catalogue says {size}" diff --git a/tests/test_updater_checksum.py b/tests/test_updater_checksum.py index dcf6a42d3..5cbbd9c5b 100644 --- a/tests/test_updater_checksum.py +++ b/tests/test_updater_checksum.py @@ -64,6 +64,16 @@ def test_empty_asset_list(self): class TestVerifySha256sums: + """The manifest fetch is patched on `updater.tls_trust`, not on `requests`. + + updater.py stopped calling `requests.get` directly when the downloads moved + onto core/tls_trust.py (HTTPS verified against the Windows certificate + store, so a locally intercepted connection can still reach GitHub). + Patching `updater.requests` here would now patch nothing at all — and a + test that quietly starts talking to the real network is worse than one that + fails. + """ + def test_no_manifest_url_fails_open(self, tmp_file): """Older releases published before this feature existed have no manifest at all — must not permanently block updating from them.""" @@ -74,7 +84,7 @@ def test_no_manifest_url_fails_open(self, tmp_file): def test_matching_checksum_passes(self, tmp_file, monkeypatch): expected = _sha256_of(tmp_file) manifest = f"{expected} WinZapp.zip\nsomeotherhash WinZappInstaller.exe\n" - monkeypatch.setattr(updater.requests, "get", lambda *a, **kw: _FakeResponse(manifest)) + monkeypatch.setattr(updater.tls_trust, "get", lambda *a, **kw: _FakeResponse(manifest)) ok, detail = updater._verify_sha256sums(tmp_file, "WinZapp.zip", "https://x/SHA256SUMS.txt") @@ -83,7 +93,7 @@ def test_matching_checksum_passes(self, tmp_file, monkeypatch): def test_mismatched_checksum_fails_closed(self, tmp_file, monkeypatch): manifest = "0000000000000000000000000000000000000000000000000000000000000000 WinZapp.zip\n" - monkeypatch.setattr(updater.requests, "get", lambda *a, **kw: _FakeResponse(manifest)) + monkeypatch.setattr(updater.tls_trust, "get", lambda *a, **kw: _FakeResponse(manifest)) ok, detail = updater._verify_sha256sums(tmp_file, "WinZapp.zip", "https://x/SHA256SUMS.txt") @@ -95,7 +105,7 @@ def test_missing_entry_for_filename_fails_closed(self, tmp_file, monkeypatch): but doesn't mention our filename at all — suspicious, not silently accepted.""" manifest = "abc123 SomeOtherFile.zip\n" - monkeypatch.setattr(updater.requests, "get", lambda *a, **kw: _FakeResponse(manifest)) + monkeypatch.setattr(updater.tls_trust, "get", lambda *a, **kw: _FakeResponse(manifest)) ok, detail = updater._verify_sha256sums(tmp_file, "WinZapp.zip", "https://x/SHA256SUMS.txt") @@ -105,7 +115,7 @@ def test_missing_entry_for_filename_fails_closed(self, tmp_file, monkeypatch): def test_manifest_fetch_failure_fails_closed(self, tmp_file, monkeypatch): def _raise(*a, **kw): raise Exception("network error") - monkeypatch.setattr(updater.requests, "get", _raise) + monkeypatch.setattr(updater.tls_trust, "get", _raise) ok, detail = updater._verify_sha256sums(tmp_file, "WinZapp.zip", "https://x/SHA256SUMS.txt") @@ -117,7 +127,7 @@ def test_ignores_asterisk_binary_mode_marker(self, tmp_file, monkeypatch): binary mode (e.g. " *WinZapp.zip") — must still match.""" expected = _sha256_of(tmp_file) manifest = f"{expected} *WinZapp.zip\n" - monkeypatch.setattr(updater.requests, "get", lambda *a, **kw: _FakeResponse(manifest)) + monkeypatch.setattr(updater.tls_trust, "get", lambda *a, **kw: _FakeResponse(manifest)) ok, _ = updater._verify_sha256sums(tmp_file, "WinZapp.zip", "https://x/SHA256SUMS.txt") From 32267f175935f8841eb6275024f203059e196a22 Mon Sep 17 00:00:00 2001 From: Gustavo Almeida Barrios Date: Thu, 3 Sep 2026 15:49:54 -0300 Subject: [PATCH 002/325] =?UTF-8?q?Transcrever=20de=20fato,=20sem=20que=20?= =?UTF-8?q?nada=20saia=20da=20m=C3=A1quina?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit O backend fica atrás de uma abstração porque a issue pede que o usuário possa escolher o mecanismo, e o whisper.cpp é candidato a segundo — mas a implementação é uma só, e o import dela mora dentro de função: o app precisa subir numa máquina sem o pacote instalado, já que é lá que a oferta de instalar aparece. O modelo é carregado do diretório local com local_files_only, e o tokenizer é conferido antes: sem ele o faster-whisper cai num Tokenizer.from_pretrained que vai à rede ignorando a flag, e "processamento completamente local" deixaria de ser verdade justamente onde ninguém olharia. Liberar o modelo passa por um contador de geração, porque a carga acontece fora do lock: sem isso, desistir de um modelo grande enquanto ele sobe fazia o app anunciar que a memória foi liberada e mantê-la presa até o fim da sessão. O filtro de voz fica ligado porque uma nota de voz termina em silêncio e o Whisper responde a silêncio inventando frases — mas quando o filtro não carrega, o resultado diz isso. Um ouvinte não distingue a frase inventada da real, e quem não pode conferir contra o áudio precisa ser avisado, não protegido por sorte. O áudio é preparado pelo ffmpeg que já vem no programa, recebido como parâmetro em vez de importado do main, e as quatro falhas possíveis chegam separadas: não baixado, formato não reconhecido, arquivo truncado e falha do ffmpeg. Um diagnóstico errado é pior que um genérico, então o que não dá para distinguir com confiança cai no genérico. O log técnico registra backend, modelo, idioma, dispositivo, tipo de cálculo, duração e tempo de processamento — nunca o que foi dito, nem o caminho do arquivo, cujo nome é o identificador da mensagem. Duas guardas cobram isso: uma varredura das chamadas de log e uma execução inteira conferindo o que saiu. Co-Authored-By: Claude Opus 5 --- build.py | 50 + client/core/transcription/audio_prep.py | 307 ++++ client/core/transcription/backend.py | 230 +++ .../transcription/faster_whisper_backend.py | 449 ++++++ client/core/transcription/job.py | 291 ++++ requirements.txt | 26 +- tests/test_transcription_backend.py | 1432 +++++++++++++++++ 7 files changed, 2784 insertions(+), 1 deletion(-) create mode 100644 client/core/transcription/audio_prep.py create mode 100644 client/core/transcription/backend.py create mode 100644 client/core/transcription/faster_whisper_backend.py create mode 100644 client/core/transcription/job.py create mode 100644 tests/test_transcription_backend.py diff --git a/build.py b/build.py index cfc35ef66..a676118ed 100644 --- a/build.py +++ b/build.py @@ -588,6 +588,41 @@ def pyinstaller_compile(): "winrt", "pyaudio", "aiosqlite", + # Local (offline) transcription of voice messages. faster_whisper + # reaches ctranslate2, av, tokenizers and huggingface_hub through + # imports PyInstaller's static analysis does not follow (huggingface_hub + # additionally resolves its own names through a lazy module __getattr__, + # which nothing static can follow at all), and the voice-activity + # filter's silero model is a *data* file in faster_whisper/assets, which + # --collect-all brings along: it collects each package's data files and + # binaries as well as its modules, where a bare --hidden-import would + # ship the code and leave the .onnx behind. + # + # It costs around 190 MB in the staged app, measured on the installed + # packages: av + av.libs ~68 MB, ctranslate2 ~60 MB, onnxruntime + # ~45 MB, the rest small. The payload is stored uncompressed in the + # installer, so that lands close to 1:1 — and it is not avoidable while + # the transcription is local, since the alternative is sending the + # user's audio to somebody's server, which is the thing this feature + # exists not to do. The models themselves are NOT bundled: they are + # downloaded on demand into the folder core/transcription/model_store + # owns. + # + # What is NOT in here, deliberately: the CUDA runtime. The ctranslate2 + # wheel ships ctranslate2.dll and a cuDNN *stub loader* only, and + # resolves cublas64_12.dll dynamically at run time — so a release built + # from this list runs the transcription on the CPU anywhere, and on the + # GPU only where a CUDA runtime is already present. Bundling it would + # be some 500 MB for every user; part 4 downloads it on demand, the way + # the models are downloaded, when the user turns NVIDIA acceleration on. + "faster_whisper", + "ctranslate2", + "av", + "tokenizers", + "huggingface_hub", + # HTTPS that verifies through the Windows certificate store — see + # client/core/tls_trust.py. Pure Python, and small. + "truststore", ] cmd = [ @@ -604,6 +639,21 @@ def pyinstaller_compile(): for pkg in collect_all: cmd += ["--collect-all", pkg] + # onnxruntime is reached only from inside faster-whisper's voice-activity + # filter, so PyInstaller has to be told it exists. --hidden-import would be + # the tidier spelling — pyinstaller-hooks-contrib already ships + # hook-onnxruntime.py for the provider DLLs, and --collect-all additionally + # drags in onnxruntime.transformers and .quantization, whose imports of + # torch/transformers/psutil are absent here and fill the build log with + # warnings. It stays --collect-all anyway, and the asymmetry is the point: + # if onnxruntime fails to import in a frozen build ONLY, the filter silently + # falls back (faster_whisper_backend._run) and Whisper starts inventing + # sentences in the silence at the end of a voice note — which a blind user + # cannot detect. Noise in a build log costs a developer a squint; the other + # way costs a user a fabricated message. Revisit once a frozen build has + # actually been exercised with a voice note and part 6 surfaces vad_used. + cmd += ["--collect-all", "onnxruntime"] + cmd += ["--paths", CLIENT_DIR] # In onefile mode, embed external resources as --add-data / --add-binary diff --git a/client/core/transcription/audio_prep.py b/client/core/transcription/audio_prep.py new file mode 100644 index 000000000..2b410fc9d --- /dev/null +++ b/client/core/transcription/audio_prep.py @@ -0,0 +1,307 @@ +"""Turning whatever WhatsApp stored into the one thing Whisper wants. + +Whisper's encoder consumes 16 kHz mono PCM, and a voice message can be OGG +Opus, OGG Vorbis, MP3, M4A, WAV or an MP4 with an audio track — so every run +starts with a conversion. ffmpeg does it, and ffmpeg's *path* is a parameter: +`MainWindow._find_api_ffmpeg()` already knows where the bundled binary is, and +a core module importing main.py to ask would be a cycle through the largest +file in the repository. The subprocess details (CREATE_NO_WINDOW, stderr kept +and decoded with `errors="replace"`) follow core/audio_transcode.py, which has +been converting audio here for far longer. + +Two decisions carry most of the weight: + +* **The four audio failures are four different sentences to the user**, so they + are distinguished as far as ffmpeg genuinely allows and no further. "There is + nothing to read" is decided before ffmpeg runs at all; a truncated file is + recognised from the markers only a truncated file produces; anything ffmpeg + simply could not parse is an unsupported format. Everything else is + FFMPEG_FAILED, on purpose: a wrong diagnosis sends a blind user to fix + something that is not broken, while the generic one at least says the audio + could not be converted. Truncation is checked *before* unrecognised, because + a file cut in half usually prints both kinds of line. + +* **ffmpeg's own log names the file it read, and that name is the message id.** + The issue forbids the log from carrying it, so the stderr is scrubbed of our + own paths before it can reach an error detail. The technical detail is worth + keeping; the name identifies the message and, through it, the conversation. + +The converted file is a temporary that is always removed — on success by the +caller (`prepared_audio()` is the context manager that does it), and on every +failure and cancellation by this module itself. It is written under a random +name rather than the message's, for the same privacy reason. +""" + +from __future__ import annotations + +import contextlib +import logging +import os +import subprocess +import sys +import tempfile +import time +import wave +from dataclasses import dataclass + +from core.transcription import errors + +TARGET_SAMPLE_RATE = 16000 +TARGET_CHANNELS = 1 + +# How often the conversion is checked on: it bounds how long a cancellation +# waits, and a wx user pressing Escape is the only thing waiting on it. +_POLL_SECONDS = 0.1 + +# A ceiling, not a schedule. Sized like audio_transcode.prepare_audio_for_ +# whatsapp()'s: generous per megabyte, because a phone recording an hour of +# audio is a real thing and a conversion killed at 90% costs the user the whole +# wait a second time. +_MIN_TIMEOUT_SECONDS = 120 +_MAX_TIMEOUT_SECONDS = 1800 +_TIMEOUT_BYTES_PER_SECOND = 512 * 1024 + +# Only a truncated file prints these. Kept apart from the unrecognised markers +# and checked first: a file cut in half often prints "Invalid data found" +# further down as well, and whichever table is checked second never wins. +_TRUNCATED_MARKERS = ( + "moov atom not found", + "truncat", + "partial file", + "premature end", + "unexpected end of file", + "incomplete frame", +) + +# ffmpeg read the bytes and could not make a stream of them. +_UNRECOGNISED_MARKERS = ( + "invalid data found when processing input", + "unknown format", + "could not find codec parameters", + "decoder not found", + "does not contain any stream", + "unsupported codec", +) + + +@dataclass(frozen=True) +class PreparedAudio: + """The converted file, and how long it turned out to be.""" + + path: str + duration_seconds: float + + +@contextlib.contextmanager +def prepared_audio(ffmpeg, source_path, should_cancel=None): + """`prepare_audio()` with the temporary file removed however it ends.""" + prepared = prepare_audio(ffmpeg, source_path, should_cancel=should_cancel) + try: + yield prepared + finally: + discard(prepared) + + +def prepare_audio(ffmpeg, source_path, should_cancel=None) -> PreparedAudio: + """Convert `source_path` to PCM 16 kHz mono, or raise the right code.""" + _check_cancel(should_cancel) + + try: + source_bytes = os.path.getsize(source_path) + except OSError: + source_bytes = 0 + if not source_bytes: + # No file, or a zero-byte placeholder: WhatsApp's own media download has + # not finished, or never started. Deliberately not FFMPEG_FAILED — the + # user's answer is to wait, not to report a broken installation. + raise errors.TranscriptionError( + errors.MEDIA_NOT_DOWNLOADED, f"{source_bytes} bytes on disk" + ) + + if not ffmpeg or not os.path.isfile(ffmpeg): + raise errors.TranscriptionError( + errors.FFMPEG_FAILED, "ffmpeg was not found next to the API" + ) + + # A random name, never the source's: the media file is named after the + # WhatsApp message id, and %TEMP% is not a private place. + handle, output_path = tempfile.mkstemp(prefix="winzapp-transcribe-", suffix=".wav") + os.close(handle) + try: + returncode, stderr_text = _run_ffmpeg( + ffmpeg, source_path, output_path, source_bytes, should_cancel + ) + if returncode != 0: + raise _classify_ffmpeg_failure(returncode, stderr_text) + duration = _wav_duration(output_path) + if duration <= 0: + # ffmpeg was happy and produced no samples: the container was + # readable and what it held was not. "Damaged" is the closest of + # the four, and the only one whose advice (this recording is not + # usable) is true here. + raise errors.TranscriptionError( + errors.AUDIO_INCOMPLETE, "the conversion produced no audio" + ) + except BaseException: + # Every exit but the successful one goes through here, cancellation + # included: a run abandoned half way must not leave a WAV of an hour of + # audio behind in %TEMP%. BaseException rather than Exception because a + # KeyboardInterrupt on a developer's machine leaves the same file. + _unlink(output_path) + raise + + logging.info( + "[transcription] prepared %.1fs of audio for transcription", duration + ) + return PreparedAudio(path=output_path, duration_seconds=duration) + + +def discard(prepared) -> None: + """Remove a prepared file. Safe to call twice, and on None.""" + if prepared is None: + return + _unlink(getattr(prepared, "path", None)) + + +def _run_ffmpeg(ffmpeg, source_path, output_path, source_bytes, should_cancel): + """(returncode, scrubbed stderr) — and a killed process on cancellation.""" + command = [ + ffmpeg, "-nostdin", "-y", + "-i", source_path, + "-vn", + "-ac", str(TARGET_CHANNELS), + "-ar", str(TARGET_SAMPLE_RATE), + "-c:a", "pcm_s16le", + "-f", "wav", + output_path, + ] + creationflags = 0 + if sys.platform == "win32" and hasattr(subprocess, "CREATE_NO_WINDOW"): + creationflags = subprocess.CREATE_NO_WINDOW + + deadline = time.monotonic() + _timeout_for(source_bytes) + # stderr into a temporary file rather than a pipe: this waits in slices so + # that a cancellation is noticed, instead of calling communicate(), and a + # pipe nobody is draining deadlocks ffmpeg as soon as the OS buffer fills — + # which a file with an odd stream fills with warnings. + with tempfile.TemporaryFile() as stderr_file: + try: + process = subprocess.Popen( + command, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=stderr_file, + creationflags=creationflags, + ) + except OSError as exc: + raise errors.TranscriptionError( + errors.FFMPEG_FAILED, f"{type(exc).__name__}: {exc}" + ) from exc + + try: + while True: + try: + returncode = process.wait(timeout=_POLL_SECONDS) + break + except subprocess.TimeoutExpired: + pass + if should_cancel is not None and should_cancel(): + raise errors.TranscriptionError( + errors.CANCELLED, "cancelled while converting the audio" + ) + if time.monotonic() > deadline: + raise errors.TranscriptionError( + errors.FFMPEG_FAILED, "the conversion timed out" + ) + except BaseException: + # Killed, not merely abandoned: an orphaned ffmpeg keeps the media + # file open (so nothing can clean it up) and keeps burning a core + # for as long as the app runs. + _kill(process) + raise + + stderr_file.seek(0) + stderr_text = stderr_file.read().decode("utf-8", errors="replace") + + return returncode, _scrub(stderr_text, source_path, output_path) + + +def _classify_ffmpeg_failure(returncode, stderr_text): + """The error a non-zero ffmpeg exit deserves.""" + message = (stderr_text or "").lower() + tail = (stderr_text or "").strip()[-800:] + if any(marker in message for marker in _TRUNCATED_MARKERS): + return errors.TranscriptionError(errors.AUDIO_INCOMPLETE, tail) + if any(marker in message for marker in _UNRECOGNISED_MARKERS): + return errors.TranscriptionError(errors.UNSUPPORTED_AUDIO_FORMAT, tail) + return errors.TranscriptionError(errors.FFMPEG_FAILED, f"rc={returncode}: {tail}") + + +def _wav_duration(path) -> float: + """Seconds in the converted file, read from its own header. + + Exact and free, where an ffprobe call would be a second process. A file we + wrote ourselves that will not open is our failure, not the source's, which + is why it is FFMPEG_FAILED rather than one of the audio codes. + """ + try: + with contextlib.closing(wave.open(path, "rb")) as handle: + rate = handle.getframerate() or TARGET_SAMPLE_RATE + return handle.getnframes() / float(rate) + except Exception as exc: + raise errors.TranscriptionError( + errors.FFMPEG_FAILED, f"unreadable output: {type(exc).__name__}: {exc}" + ) from exc + + +def _scrub(text, *paths) -> str: + """ffmpeg's log with our own file names taken out of it. + + ffmpeg echoes every path it was handed ("Input #0, ogg, from '...'"), and + the media file's name is the WhatsApp message id — which the issue forbids + the log to carry, because it identifies the message and, through it, the + conversation. The rest of the line is exactly the technical detail worth + keeping. + """ + scrubbed = text or "" + for path in paths: + if not path: + continue + for form in (path, os.path.basename(path)): + if form: + scrubbed = scrubbed.replace(form, "