diff --git a/ingest/bm25_indexer.py b/ingest/bm25_indexer.py index 4d2e82b..386938b 100644 --- a/ingest/bm25_indexer.py +++ b/ingest/bm25_indexer.py @@ -84,6 +84,7 @@ from rank_bm25 import BM25Okapi +from server.application_paths import ApplicationPaths from server.corpus import open_chunks_table, project_columns logger = logging.getLogger(__name__) @@ -93,7 +94,7 @@ # Paths and identity constants # --------------------------------------------------------------------------- -REPO_ROOT = Path(__file__).resolve().parent.parent +_APPLICATION_PATHS = ApplicationPaths.resolve() # The directory layout convention: ``/derived/bm25/v/``. # Confine the ``f"v{N}"`` literal to ``_bm25_version_dir`` below — every @@ -101,7 +102,7 @@ BM25_DIR_NAME = "bm25" BM25_INDEX_NAME = "bm25.pkl" BM25_CHUNK_IDS_NAME = "chunk_ids.json" -BM25_STATS_PATH = REPO_ROOT / "var" / "arxmcp" / "ops" / "bm25-stats.jsonl" +BM25_STATS_PATH = _APPLICATION_PATHS.ops / "bm25-stats.jsonl" def resolve_bm25_root( diff --git a/ingest/bulk_ingest.py b/ingest/bulk_ingest.py index 3043d4c..b40d511 100644 --- a/ingest/bulk_ingest.py +++ b/ingest/bulk_ingest.py @@ -17,6 +17,25 @@ active path), so it cannot accidentally pick up half-ingested data. +**What staging does not isolate:** the per-paper intermediate trees +(``corpus/parsed``, ``corpus/chunks``, ``corpus/embeddings``, +``corpus/preamble``) are shared by every dataset's runs, on purpose. +They are a per-paper cache, not a dataset's record. The parsed and raw +trees are downloads, the same whichever dataset reads them. The chunk and +embedding trees are what this code makes of those downloads: the same +paper and code give the same files, each set is replaced through a +temporary file and ``os.replace`` (two runs writing one paper concurrently +each leave a whole set), and nothing reuses a set without checking it. +The embedder re-encodes any paper whose sidecar records another chunker, +embedder or input-policy version or other chunks, and the rows a run +writes come from the embed step of that same run, never from the tree +alone. An aborted run that leaves another code version's sets behind +therefore costs a re-embed, not a wrong row. A tree per dataset would +re-embed every paper once per notebook and per staging run. The ops +files are per target where it matters: callers pass ``ops_dir``, +``log_path`` and ``failures_path``, and a notebook's run keeps them in +the notebook's ``ops/``. + **Scope at v1 (per research synthesis D1):** This module ships the SCAFFOLDING — orchestrator + CLI. The actual ingest of the 200K-paper corpus is a 1-2 day GPU run that requires a Bittorrent @@ -41,9 +60,11 @@ ``ops/parser-failures/bulk.jsonl``. Nougat PDF fallback is deferred (synthesis D2). -**Single-writer constraint** (`ingest/store.py:44-55`): the loop -is sequential at the write boundary. No parallel ``write_chunks`` -calls. +**One writer per dataset:** a run holds the target dataset's write +lock (:mod:`ingest.write_lock`) from its chunker-version check to its +index build, and the loop is sequential at the write boundary. A second +writer of the same dataset is refused at once, and the CLI exits with +:data:`ingest.write_lock.WRITE_LOCKED_EXIT_CODE`. **Chunker-version guard:** a run refuses, before its first write, a target whose arXiv rows another @@ -83,6 +104,7 @@ refuse_stale_chunker_rows, write_chunks, ) +from ingest.write_lock import dataset_write_lock, exits_when_write_locked from server.application_paths import ApplicationPaths logger = logging.getLogger(__name__) @@ -425,8 +447,9 @@ def run_bulk_ingest( """Run the full bulk-ingest loop. Returns the aggregate summary. The loop is **single-process sequential** at the write - boundary (``ingest.store.write_chunks`` is single-writer-per- - dataset). GPU batching happens inside ``embed_paper``. Naive + boundary, and the run holds the dataset's write lock throughout + (:class:`ingest.write_lock.DatasetWriteLockedError` if another + writer has it). GPU batching happens inside ``embed_paper``. Naive re-runs are safe: the embedder's per-paper sidecar carries the versions and each chunk's reuse identity, so already-processed papers short-circuit at the embed step regardless (a sidecar @@ -445,7 +468,33 @@ def run_bulk_ingest( ar5iv_cache_dir=ar5iv_cache_dir, parsed_dir=parsed_dir, ) + with dataset_write_lock(lancedb_staging_path): + return _run_locked( + paper_ids, + lancedb_staging_path=lancedb_staging_path, + ar5iv_cache_dir=ar5iv_cache_dir, + parsed_dir=parsed_dir, + failures_path=failures_path, + log_path=log_path, + ops_dir=ops_dir, + progress_interval=progress_interval, + limit=limit, + ) + +def _run_locked( + paper_ids: list[str], + *, + lancedb_staging_path: Path, + ar5iv_cache_dir: Path, + parsed_dir: Path, + failures_path: Path, + log_path: Path, + ops_dir: Path, + progress_interval: int, + limit: int | None, +) -> IngestSummary: + """The run itself, under the dataset write lock.""" # Never re-chunk in place across a chunker version. refuse_stale_chunker_rows(lancedb_staging_path) @@ -544,6 +593,7 @@ def _run_dry( # --------------------------------------------------------------------------- +@exits_when_write_locked def _cli(argv: list[str]) -> int: parser = argparse.ArgumentParser( description="arXMCP bulk ingest orchestrator (E11_S01).", diff --git a/ingest/derived_indexes.py b/ingest/derived_indexes.py index ef76576..d8db920 100644 --- a/ingest/derived_indexes.py +++ b/ingest/derived_indexes.py @@ -17,6 +17,8 @@ from dataclasses import dataclass from pathlib import Path +from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock + logger = logging.getLogger(__name__) @@ -29,6 +31,7 @@ class DerivedIndexSummary: theorem_name_papers: int +@holds_dataset_write_lock("lancedb_path") def build_derived_indexes(lancedb_path: str | Path) -> DerivedIndexSummary: """Build the BM25, definitions and theorem-name indexes of a dataset. @@ -59,6 +62,7 @@ def build_derived_indexes(lancedb_path: str | Path) -> DerivedIndexSummary: return summary +@exits_when_write_locked def _cli(argv: list[str]) -> int: import argparse diff --git a/ingest/embed_equations.py b/ingest/embed_equations.py index ff97a0a..a6560da 100644 --- a/ingest/embed_equations.py +++ b/ingest/embed_equations.py @@ -15,9 +15,9 @@ Same batch size (``EMBED_BATCH_DEFAULT = 32``), same dtype (``float32``), same L2 normalization invariant. -**Concurrency.** Single-writer-per-table contract. SQLite WAL it's -not — LanceDB merge_insert against the same table from two -processes can interleave. Document and serialize externally. +**Concurrency.** LanceDB merge_insert against the same table from two +processes can interleave, so :func:`embed_pending_equations` runs under +the dataset write lock (:mod:`ingest.write_lock`). """ from __future__ import annotations @@ -31,10 +31,12 @@ from ingest.embedder import EMBED_BATCH_DEFAULT, _encode_batch from ingest.index_equations import open_or_create_equations_table from ingest.schema import EQUATIONS_SCHEMA_V1 +from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock +from server.application_paths import ApplicationPaths logger = logging.getLogger(__name__) -REPO_ROOT = Path(__file__).resolve().parent.parent +_APPLICATION_PATHS = ApplicationPaths.resolve() def _embedding_input(row: dict[str, Any]) -> str: @@ -51,6 +53,7 @@ def _embedding_input(row: dict[str, Any]) -> str: return pl or ctx +@holds_dataset_write_lock("lancedb_path") def embed_pending_equations( lancedb_path: str | Path, batch_size: int = EMBED_BATCH_DEFAULT, @@ -152,6 +155,7 @@ def embed_pending_equations( # --------------------------------------------------------------------------- +@exits_when_write_locked def _cli(argv: list[str]) -> int: import argparse @@ -160,7 +164,7 @@ def _cli(argv: list[str]) -> int: ) parser.add_argument( "--lancedb-path", - default=str(REPO_ROOT / "var" / "arxmcp" / "index" / "lancedb"), + default=str(_APPLICATION_PATHS.lancedb), help="LanceDB dataset root (default: var/arxmcp/index/lancedb)", ) parser.add_argument( diff --git a/ingest/extract_equations.py b/ingest/extract_equations.py index d953fc9..3b11e4f 100644 --- a/ingest/extract_equations.py +++ b/ingest/extract_equations.py @@ -43,10 +43,10 @@ at 4000 characters (matches the handler-side ``latex_or_mathml`` Pydantic constraint). No sentence tokenization at v1. -**Concurrency.** Single-writer-per-paper contract; callers MUST -serialize concurrent invocations against the same LanceDB table. The -delete-then-insert pair (per-paper sweep) is two separate MVCC -commits; concurrent runs can interleave and orphan rows. +**Concurrency.** The delete-then-insert pair (per-paper sweep) is two +separate MVCC commits that concurrent runs could interleave, orphaning +rows, so :func:`extract_equations_for_paper` runs under the dataset +write lock (:mod:`ingest.write_lock`). """ from __future__ import annotations @@ -63,11 +63,13 @@ from ingest.identifiers import is_valid_arxiv_paper_id from ingest.index_equations import open_or_create_equations_table from ingest.schema import EQUATIONS_SCHEMA_V1 +from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock +from server.application_paths import ApplicationPaths logger = logging.getLogger(__name__) -REPO_ROOT = Path(__file__).resolve().parent.parent -PARSED_DIR = REPO_ROOT / "var" / "arxmcp" / "corpus" / "parsed" +_APPLICATION_PATHS = ApplicationPaths.resolve() +PARSED_DIR = _APPLICATION_PATHS.corpus / "parsed" #: Cap on the ``context_sentence`` column. Matches the handler-side #: ``latex_or_mathml`` Pydantic max_length so corpus and query rows @@ -349,6 +351,7 @@ def _table_has_any_rows_for_paper(tbl: Any, paper_id: str) -> bool: return True +@holds_dataset_write_lock("lancedb_path") def extract_equations_for_paper( paper_id: str, lancedb_path: str | Path, @@ -422,6 +425,7 @@ def extract_equations_for_paper( # --------------------------------------------------------------------------- +@exits_when_write_locked def _cli(argv: list[str]) -> int: import argparse @@ -431,7 +435,7 @@ def _cli(argv: list[str]) -> int: parser.add_argument("paper_id", help="arXiv paper id (e.g. 2401.00001)") parser.add_argument( "--lancedb-path", - default=str(REPO_ROOT / "var" / "arxmcp" / "index" / "lancedb"), + default=str(_APPLICATION_PATHS.lancedb), help="LanceDB dataset root (default: var/arxmcp/index/lancedb)", ) parser.add_argument( diff --git a/ingest/graph_ingest.py b/ingest/graph_ingest.py index 38dd9d2..8feba9b 100644 --- a/ingest/graph_ingest.py +++ b/ingest/graph_ingest.py @@ -101,6 +101,7 @@ import kuzu from ingest.kuzudb_schema import apply_schema +from server.application_paths import ApplicationPaths from tools.arxiv_fetch import ( DEFAULT_503_BACKOFF_SECONDS, MAX_503_BACKOFF_SECONDS, @@ -741,13 +742,13 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--checkpoint", type=Path, - default=Path("var/arxmcp/ops/graph-ingest-checkpoint.json"), + default=ApplicationPaths.resolve().ops / "graph-ingest-checkpoint.json", help="atomic-write checkpoint path (rewritten after each batch)", ) parser.add_argument( "--kuzudb", type=Path, - default=Path("var/arxmcp/index/kuzu"), + default=ApplicationPaths.resolve().kuzu, help="Kùzu database file (default: var/arxmcp/index/kuzu)", ) parser.add_argument( diff --git a/ingest/index_definitions.py b/ingest/index_definitions.py index 9171530..6f93996 100644 --- a/ingest/index_definitions.py +++ b/ingest/index_definitions.py @@ -38,17 +38,14 @@ The handler at :mod:`server.handlers.definitions` reads from the same table; this module is the writer half of that contract. -**CONCURRENCY:** :func:`index_definitions_for_paper` assumes a -**single-writer-per-paper** contract — callers MUST serialize -concurrent invocations for the same ``paper_id``. The +**CONCURRENCY:** :func:`index_definitions_for_paper` and +:func:`index_definitions_for_corpus` run under the dataset write lock +(:mod:`ingest.write_lock`), so a second writer of the dataset is +refused. Without it the delete-then-insert pair is two separate LanceDB MVCC commits, so two -concurrent runs for the same paper can interleave as +concurrent runs for the same paper could interleave as ``A.delete → B.delete → A.add → B.add`` (yielding duplicate rows) or -``A.add → B.delete → B.add`` (discarding A's writes). Different -papers may be indexed in parallel without coordination. The -production ingest driver landing in E11 must enforce this — either -via a per-paper file-system lock or a single-worker stage in the -pipeline DAG. +``A.add → B.delete → B.add`` (discarding A's writes). """ from __future__ import annotations @@ -67,6 +64,7 @@ DEFINITIONS_TABLE_NAME, LANCE_STORAGE_OPTIONS, ) +from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock if TYPE_CHECKING: from ingest.preamble_types import PreambleDoc @@ -386,6 +384,7 @@ def _ensure_scalar_indexes(tbl: Any) -> dict[str, bool]: return created +@holds_dataset_write_lock("lancedb_path") def index_definitions_for_paper( paper_id: str, lancedb_path: str | Path, @@ -441,6 +440,7 @@ def index_definitions_for_paper( +@holds_dataset_write_lock("lancedb_path") def index_definitions_for_corpus( lancedb_path: str | Path, paper_ids: list[str] | None = None, ) -> tuple[int, int]: @@ -462,6 +462,7 @@ def index_definitions_for_corpus( return total, len(paper_ids) +@exits_when_write_locked def _cli(argv: list[str]) -> int: import argparse diff --git a/ingest/index_equations.py b/ingest/index_equations.py index ec9d2e6..765cad6 100644 --- a/ingest/index_equations.py +++ b/ingest/index_equations.py @@ -18,11 +18,11 @@ predicate filters rows that need processing. **Concurrency.** Like the definitions indexer in -:mod:`ingest.index_definitions`, this writer assumes a -**single-writer contract** — concurrent invocations against the same -LanceDB table can interleave delete/insert pairs in ways that -duplicate or drop rows. The production ingest driver landing in E11 -must serialize the equation-tree population step. +:mod:`ingest.index_definitions`, concurrent invocations against the same +LanceDB table could interleave delete/insert pairs in ways that +duplicate or drop rows, so :func:`index_equation_trees` runs under the +dataset write lock (:mod:`ingest.write_lock`) and a second writer is +refused. """ from __future__ import annotations @@ -38,6 +38,7 @@ EQUATIONS_TABLE_NAME, LANCE_STORAGE_OPTIONS, ) +from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock from server.retrieval.equations import ( MATHML_TREE_NORMALIZER_VERSION, parse_mathml_to_tree, @@ -80,6 +81,7 @@ def open_or_create_equations_table(lancedb_path: str | Path) -> Any: ) +@holds_dataset_write_lock("lancedb_path") def index_equation_trees( lancedb_path: str | Path, *, rebuild: bool = False, ) -> dict[str, int]: @@ -173,6 +175,7 @@ def index_equation_trees( +@exits_when_write_locked def _cli(argv: list[str]) -> int: import argparse diff --git a/ingest/index_theorem_names.py b/ingest/index_theorem_names.py index 004100f..d35977a 100644 --- a/ingest/index_theorem_names.py +++ b/ingest/index_theorem_names.py @@ -18,11 +18,12 @@ parenthetical in the heading — that's a high-confidence signal. A future milestone may add lower-confidence body-text extraction. -**Concurrency.** Single-writer-per-table contract (synthesis D10). -Callers MUST serialize concurrent invocations of -:func:`index_theorem_names_for_paper` against the same SQLite -database; SQLite WAL tolerates concurrent readers but serializes -writers, and two interleaved per-paper passes can race the +**Concurrency.** :func:`build_theorem_names_index` runs under the write +lock of the dataset the index belongs to (:mod:`ingest.write_lock`). +Callers of :func:`index_theorem_names_for_paper` outside it must +serialize invocations against the same SQLite database; SQLite WAL +tolerates concurrent readers but serializes writers, and two interleaved +per-paper passes can race the delete + insert pair. """ @@ -32,6 +33,7 @@ from pathlib import Path from typing import Any +from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock from server.theorem_names_store import ( TheoremNamesStore, TheoremRow, @@ -193,6 +195,7 @@ def default_db_path(lancedb_path: str | Path) -> Path: +@holds_dataset_write_lock("lancedb_path") def build_theorem_names_index( lancedb_path: str | Path, db_path: str | Path | None = None, ) -> dict[str, int]: @@ -219,6 +222,7 @@ async def run() -> dict[str, int]: return asyncio.run(run()) +@exits_when_write_locked def _cli(argv: list[str]) -> int: import argparse diff --git a/ingest/inspire_ingest.py b/ingest/inspire_ingest.py index 25868d8..d9fb1f7 100644 --- a/ingest/inspire_ingest.py +++ b/ingest/inspire_ingest.py @@ -109,6 +109,7 @@ ) from ingest.identifiers import is_valid_arxiv_paper_id from ingest.kuzudb_schema import apply_schema +from server.application_paths import ApplicationPaths from tools.arxiv_fetch import ( DEFAULT_503_BACKOFF_SECONDS, MAX_503_BACKOFF_SECONDS, @@ -753,13 +754,13 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--checkpoint", type=Path, - default=Path("var/arxmcp/ops/inspire-ingest-checkpoint.json"), + default=ApplicationPaths.resolve().ops / "inspire-ingest-checkpoint.json", help="atomic-write checkpoint path (rewritten after each batch)", ) parser.add_argument( "--kuzudb", type=Path, - default=Path("var/arxmcp/index/kuzu"), + default=ApplicationPaths.resolve().kuzu, help="Kùzu database file (default: var/arxmcp/index/kuzu)", ) parser.add_argument( diff --git a/ingest/intra_paper_refs.py b/ingest/intra_paper_refs.py index 94e9057..a4a76bc 100644 --- a/ingest/intra_paper_refs.py +++ b/ingest/intra_paper_refs.py @@ -70,25 +70,28 @@ from ingest.graph_ingest import _merge_cite, save_checkpoint from ingest.identifiers import is_valid_arxiv_paper_id from ingest.kuzudb_schema import apply_schema +from server.application_paths import ApplicationPaths logger = logging.getLogger(__name__) +_APPLICATION_PATHS = ApplicationPaths.resolve() + #: LaTeXML emits ``\ref{label}`` as ````; #: we walk the parse tree for elements with class containing ``ltx_ref``. LTX_REF_CSS_CLASS = "ltx_ref" #: Default location of LaTeXML output (matches ``tools/fetch_seed.py``). -DEFAULT_PARSED_DIR = Path("var/arxmcp/corpus/parsed") +DEFAULT_PARSED_DIR = _APPLICATION_PATHS.corpus / "parsed" #: Default Kùzu DB directory. Matches the design notes + bootstrap; #: the brief's ``kuzudb/`` is documented drift (synthesis § 3). -DEFAULT_KUZUDB_PATH = Path("var/arxmcp/index/kuzu") +DEFAULT_KUZUDB_PATH = _APPLICATION_PATHS.kuzu #: Default LanceDB chunks-table path; mirrors ``ingest.store``. -DEFAULT_LANCEDB_PATH = Path("var/arxmcp/index/lancedb") +DEFAULT_LANCEDB_PATH = _APPLICATION_PATHS.lancedb #: Atomic-write checkpoint location. -DEFAULT_CHECKPOINT_PATH = Path("var/arxmcp/ops/intra-paper-refs-checkpoint.json") +DEFAULT_CHECKPOINT_PATH = _APPLICATION_PATHS.ops / "intra-paper-refs-checkpoint.json" #: Checkpoint flush cadence (papers / batch). CHECKPOINT_BATCH_SIZE = 100 diff --git a/ingest/kuzudb_schema.py b/ingest/kuzudb_schema.py index 53867a1..10cdb2c 100644 --- a/ingest/kuzudb_schema.py +++ b/ingest/kuzudb_schema.py @@ -30,6 +30,8 @@ import kuzu +from server.application_paths import ApplicationPaths + #: Schema version. Bump whenever ``SCHEMA_STATEMENTS`` mutates so #: callers (and downstream cache layers, e.g. ``cite_neighbors`` in #: E09_S03) can detect drift on an existing database. Any schema @@ -200,7 +202,7 @@ def main() -> int: parser.add_argument( "--kuzudb", type=Path, - default=Path("var/arxmcp/index/kuzu"), + default=ApplicationPaths.resolve().kuzu, help="file the Kùzu database will own (default: var/arxmcp/index/kuzu)", ) args = parser.parse_args() diff --git a/ingest/oai_delta.py b/ingest/oai_delta.py index 92e9a19..48f387a 100644 --- a/ingest/oai_delta.py +++ b/ingest/oai_delta.py @@ -84,10 +84,12 @@ refresh_indices, refuse_stale_chunker_rows, ) +from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock +from server.application_paths import ApplicationPaths logger = logging.getLogger(__name__) -REPO_ROOT = Path(__file__).resolve().parent.parent +_APPLICATION_PATHS = ApplicationPaths.resolve() #: OAI-PMH endpoint. arXiv migrated from #: ``http://export.arxiv.org/oai2`` to ``https://oaipmh.arxiv.org/oai`` @@ -131,25 +133,25 @@ #: ``last_resumption_token`` and, while a run is in flight, ``harvest_from`` #: so a mid-harvest crash can resume. DEFAULT_STATE_PATH: Path = ( - REPO_ROOT / "var" / "arxmcp" / "ops" / "oai-pmh-state.json" + _APPLICATION_PATHS.ops / "oai-pmh-state.json" ) #: Default delta log location. DEFAULT_DELTA_LOG_PATH: Path = ( - REPO_ROOT / "var" / "arxmcp" / "ops" / "oai-delta.log" + _APPLICATION_PATHS.ops / "oai-delta.log" ) #: Default parser-failures JSONL location (separate from the #: bulk-ingest path so each channel has its own audit trail). DEFAULT_DELTA_PARSER_FAILURES_PATH: Path = ( - REPO_ROOT / "var" / "arxmcp" / "ops" / "parser-failures" / "delta.jsonl" + _APPLICATION_PATHS.ops / "parser-failures" / "delta.jsonl" ) #: Sentinel flag file written when a run exceeds the 90-minute #: budget (synthesis D8). Mirrors the E10_S04 drift-detector #: pattern (``var/arxmcp/ops/drift-detected.flag``). DEFAULT_TIMEOUT_FLAG_PATH: Path = ( - REPO_ROOT / "var" / "arxmcp" / "ops" / "delta-timeout.flag" + _APPLICATION_PATHS.ops / "delta-timeout.flag" ) #: 90-minute budget in seconds. Synthesis §7 documents this is @@ -158,7 +160,7 @@ DEFAULT_BUDGET_SECONDS: float = 90 * 60.0 #: Default ops directory for sentinel files (ingest-summary.json etc.). -DEFAULT_OPS_DIR: Path = REPO_ROOT / "var" / "arxmcp" / "ops" +DEFAULT_OPS_DIR: Path = _APPLICATION_PATHS.ops #: Closes F1: total wall-clock cap on the 503-retry/backoff loop. #: Per design note 08:200, the delta loop pauses with exponential @@ -696,6 +698,7 @@ def _clear_budget_flag(flag_path: Path) -> None: # --------------------------------------------------------------------------- +@holds_dataset_write_lock("lancedb_staging_path", skip_if="dry_run") def run_delta( *, sets: tuple[str, ...] = DEFAULT_SETS, @@ -902,6 +905,7 @@ def run_delta( # --------------------------------------------------------------------------- +@exits_when_write_locked def _cli(argv: list[str]) -> int: # E14_S05 D5: defense-in-depth check for the ``ingest-paused`` # sentinel. The cron wrapper checks first, but manual diff --git a/ingest/preamble.py b/ingest/preamble.py index 6bd475d..84d2bb5 100644 --- a/ingest/preamble.py +++ b/ingest/preamble.py @@ -53,6 +53,7 @@ _validate_paper_id, ) from ingest.preamble_types import PreambleDoc +from server.application_paths import ApplicationPaths # Reuse the existing root-tex heuristic from the fetcher tooling rather than # re-implementing it. The function lives in tools/arxiv_fetch.py and the @@ -66,11 +67,11 @@ # Paths # --------------------------------------------------------------------------- -REPO_ROOT = Path(__file__).resolve().parent.parent -RAW_DIR = REPO_ROOT / "var" / "arxmcp" / "corpus" / "raw" -PREAMBLE_DIR = REPO_ROOT / "var" / "arxmcp" / "corpus" / "preamble" +_APPLICATION_PATHS = ApplicationPaths.resolve() +RAW_DIR = _APPLICATION_PATHS.corpus / "raw" +PREAMBLE_DIR = _APPLICATION_PATHS.corpus / "preamble" PREAMBLE_LOG_PATH = ( - REPO_ROOT / "var" / "arxmcp" / "ops" / "parser-failures" / "preamble.log" + _APPLICATION_PATHS.ops / "parser-failures" / "preamble.log" ) diff --git a/ingest/re_embed.py b/ingest/re_embed.py index 4c4308d..be57549 100644 --- a/ingest/re_embed.py +++ b/ingest/re_embed.py @@ -81,19 +81,21 @@ load_embed_record, write_chunks, ) +from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock +from server.application_paths import ApplicationPaths logger = logging.getLogger(__name__) -REPO_ROOT = Path(__file__).resolve().parent.parent +_APPLICATION_PATHS = ApplicationPaths.resolve() #: Default state file location. DEFAULT_STATE_PATH: Path = ( - REPO_ROOT / "var" / "arxmcp" / "ops" / "re-embed-state.json" + _APPLICATION_PATHS.ops / "re-embed-state.json" ) #: Default per-paper failures JSONL. DEFAULT_FAILURES_PATH: Path = ( - REPO_ROOT / "var" / "arxmcp" / "ops" / "parser-failures" / "re-embed.jsonl" + _APPLICATION_PATHS.ops / "parser-failures" / "re-embed.jsonl" ) @@ -708,6 +710,7 @@ def _subset_embed_record( # --------------------------------------------------------------------------- +@holds_dataset_write_lock("staging_lancedb_path", skip_if="dry_run") def run_re_embed( *, paper_ids: list[str] | None = None, @@ -981,6 +984,7 @@ def _append_failure( # --------------------------------------------------------------------------- +@exits_when_write_locked def _cli(argv: list[str]) -> int: parser = argparse.ArgumentParser( description="arXMCP partial re-embed driver (E11_S03).", @@ -1068,6 +1072,9 @@ def _cli(argv: list[str]) -> int: active_path = args.active_lancedb_path staging_path = args.lancedb_staging_path state_path = args.state_file + # A notebook's re-embed keeps its failures with the notebook, like its + # state, so one notebook's run never appends to another's log. + failures_path = DEFAULT_FAILURES_PATH if args.notebook is not None: # Late import: tools/ depends on ingest/, not the other way round # at module load. notebook_dir validates the slug and refuses a @@ -1078,6 +1085,7 @@ def _cli(argv: list[str]) -> int: active_path = active_path or nb / "lancedb" staging_path = staging_path or nb / "lancedb-staging" state_path = state_path or nb / "ops" / "re-embed-state.json" + failures_path = nb / "ops" / "parser-failures" / "re-embed.jsonl" active_path = active_path or DEFAULT_LANCEDB_PATH staging_path = staging_path or DEFAULT_LANCEDB_STAGING_PATH state_path = state_path or DEFAULT_STATE_PATH @@ -1099,6 +1107,7 @@ def _cli(argv: list[str]) -> int: active_lancedb_path=active_path, staging_lancedb_path=staging_path, state_path=state_path, + failures_path=failures_path, target_embedder_version=args.target_embedder_version, dry_run=args.dry_run, resume=args.resume, diff --git a/ingest/store.py b/ingest/store.py index b3917f4..d16a3b5 100644 --- a/ingest/store.py +++ b/ingest/store.py @@ -37,24 +37,17 @@ silently inserting a NULL-embedding row that would poison ANN results. -**Concurrency.** LanceDB's own MVCC layer serializes concurrent -writers at the dataset level — two simultaneous ``write_chunks`` calls -produce two separate dataset versions, never a corrupt half-write. No -additional filesystem-atomicity wrapper is needed beyond the directory -``mkdir(parents=True, exist_ok=True)``. - -**Single-writer assumption (F11 from the E04_S02 critique).** The -function is designed for a SINGLE writer per LanceDB dataset. Between -``merge_insert`` and ``_create_indices``, a concurrent writer-B -landing its own merge against the same dataset would shift -``tbl.version`` such that writer-A's returned integer points to -writer-B's post-merge state, not writer-A's own post-index state. -Callers running concurrent ingest from multiple processes against -the same dataset must serialize writes externally (e.g. a flock on -``/.write-lock``). The Tier-0 ingestion pipeline has -exactly one writer (the corpus driver), so this is a documented -constraint rather than an enforced one. Multi-writer support is an -E11 concern. +**Concurrency.** LanceDB's own MVCC layer keeps two simultaneous +commits from corrupting the table, but not from interleaving. + +**One writer per dataset, enforced.** Between ``merge_insert`` and the +marker write, a second writer's commit would shift ``tbl.version`` so that +the returned integer and the marker's counts describe neither write. +:func:`write_chunks` and :func:`delete_paper` therefore run under the +dataset's write lock (:mod:`ingest.write_lock`), and the drivers hold it +for their whole run; a second writer is refused at once with +:class:`ingest.write_lock.DatasetWriteLockedError`. A writer that goes +around the lock is caught by the write-audit gate, which names it. **MVCC handshake (E04_S02).** No symlink swaps. LanceDB version int IS the corpus_version. Writers use the current dataset; readers call @@ -117,6 +110,7 @@ LANCE_STORAGE_OPTIONS, EmbedRecord, ) +from ingest.write_lock import holds_dataset_write_lock, write_lock_path from server.application_paths import ApplicationPaths # NB: ``server.corpus.read_corpus_version`` is imported function-locally @@ -193,7 +187,9 @@ class WriteStats: # raised, so the post-mortem audit row in store-stats.jsonl pins the # failure surface. Empty string is the happy-path (and pre-e1) value. # Domain (when non-empty): "missing_marker" | "malformed_marker" | - # "count_mismatch_arithmetic" | "count_mismatch_swallow". + # "count_mismatch_arithmetic" | "count_mismatch_swallow" | + # "concurrent_writer" (another process committed to the dataset during + # the call, so neither count is wrong code). # The arithmetic-vs-swallow split mirrors the runbook's S5/S6 routing # (rect F3). gate_failure_reason: str = "" @@ -837,6 +833,9 @@ def _scan_paper_ids(tbl, chunk_count: int) -> set[str]: # --------------------------------------------------------------------------- +@holds_dataset_write_lock( + "lancedb_path", default=lambda: DEFAULT_LANCEDB_PATH +) def write_chunks( chunks: list[ChunkRecord], embeddings: EmbedRecord, @@ -1034,10 +1033,9 @@ def write_chunks( try: # F4 (corpus-integrity-observability-m1 critique): these counts are read # off the SAME `tbl` handle that pinned `dataset_version` above, under the - # single-writer-per-dataset model (module docstring, "MVCC handshake"). - # No write lands between L862 and here in-process, so the marker's - # `version` and its counts are coherent. A concurrent external writer is - # out of scope (E11). + # dataset write lock this call holds, so no lock-taking writer lands + # between the version read and here, and the marker's `version` and + # counts are coherent. One that bypasses the lock is named by the gate. chunk_count = tbl.count_rows() # corpus-integrity-observability-e3 (CAND-8): thread the total-row # count through WriteStats so callers can accumulate run-level @@ -1107,8 +1105,9 @@ def write_chunks( # (int overflow), and FM-10 (swallowed marker-write leaving a stale # prior marker OR no marker at all; the stale-marker production-common # path fires the count-mismatch arm). Out-of-scope: FM-4 caller - # arithmetic (m1 fix + m3 integration test), FM-5 TOCTOU (single-writer - # constraint), FM-6/FM-14 schema-version drift (deferred follow-on), + # arithmetic (m1 fix + m3 integration test), FM-5 TOCTOU (the dataset + # write lock; a bypassing writer is the concurrent-writer arm), + # FM-6/FM-14 schema-version drift (deferred follow-on), # FM-8 wrong path (config), FM-9 silent skip (logging), FM-11 sibling # marker writers (m3 follow-up F2-extension). # @@ -1161,6 +1160,14 @@ def write_chunks( # arithmetic regression; needs a code fix). Operators on # the 2am-page path get the routing decision in the # exception text alone — no separate `grep` required. + # A commit this call did not make shows as a marker naming + # another version or a table past the one written here. Only + # a writer bypassing the dataset write lock gets there. + latest_version = _latest_version(target_path) + concurrent = ( + re_read_marker.version != dataset_version + or latest_version != dataset_version + ) if marker_write_failed: stats.gate_failure_reason = "count_mismatch_swallow" routing_tag = "Routing: S5 (swallow + stale marker)" @@ -1172,6 +1179,20 @@ def write_chunks( "corpus-version.json marker' warning that the " "swallow emitted in this call)" ) + elif concurrent: + stats.gate_failure_reason = "concurrent_writer" + raise WapGateError( + f"WAP gate: corpus-version.json marker at {target_path} reports " + f"chunk_count={re_read_marker.chunk_count} but tbl.count_rows()=" + f"{fresh_count}. Cause: another writer committed to this " + f"dataset during this call (this call wrote version " + f"{dataset_version}; the marker names version " + f"{re_read_marker.version} and the table is at version " + f"{latest_version}). The rows and the writer code are not " + f"at fault. That writer bypassed the dataset write lock at " + f"{write_lock_path(target_path)}: find and stop it, then run " + f"`make reconcile` to rewrite the marker from the table." + ) else: stats.gate_failure_reason = "count_mismatch_arithmetic" routing_tag = "Routing: S6 (arithmetic regression)" @@ -1200,6 +1221,14 @@ def write_chunks( return dataset_version +def _latest_version(path: Path) -> int: + """The dataset's newest committed version, read through a fresh handle.""" + import lancedb # noqa: PLC0415 + + tbl = lancedb.connect(str(path)).open_table(CHUNKS_TABLE_NAME) + return int(getattr(tbl, "version", 0) or 0) + + def refresh_indices(lancedb_path: str | Path) -> int | None: """Rebuild the vector + scalar indices once, after a run of writes. @@ -1250,6 +1279,7 @@ def _marker_embed_record(path: Path) -> EmbedRecord | None: ) +@holds_dataset_write_lock("lancedb_path") def delete_paper(lancedb_path: str | Path, paper_id: str) -> int: """Remove a paper from a dataset: its chunks and its derived rows. @@ -1261,9 +1291,8 @@ def delete_paper(lancedb_path: str | Path, paper_id: str) -> int: still in ``chunks``, so nothing else would remove them. The BM25 index is rebuilt per version from ``chunks``. - This is a write, under the same single-writer rule as - :func:`write_chunks`: call it from the loop that owns the dataset, - never beside another writer. It runs its own event loop for the + This is a write under the dataset write lock, like + :func:`write_chunks`. It runs its own event loop for the theorem-name store, so it is for offline callers only. Vector indexes stay valid (deleted rows are masked) until the caller's end-of-run :func:`refresh_indices` rebuilds them. diff --git a/ingest/textbook_chunker.py b/ingest/textbook_chunker.py index 291b8ea..ab5ff7a 100644 --- a/ingest/textbook_chunker.py +++ b/ingest/textbook_chunker.py @@ -72,7 +72,6 @@ # Paths + constants # --------------------------------------------------------------------------- -REPO_ROOT = Path(__file__).resolve().parent.parent NOTEBOOKS_BASE = ApplicationPaths.resolve().notebooks #: Deliberately SEPARATE from ``ingest.chunk.CHUNKER_VERSION`` (the arXiv diff --git a/ingest/write_lock.py b/ingest/write_lock.py new file mode 100644 index 0000000..4f62fd5 --- /dev/null +++ b/ingest/write_lock.py @@ -0,0 +1,242 @@ +"""One writer per LanceDB dataset, enforced with an OS file lock. + +LanceDB's MVCC keeps two concurrent commits from corrupting a table, but +not from interleaving: a second writer that commits between one writer's +merge and its ``corpus-version.json`` write leaves a marker that describes +neither, and a driver's run-level checks (the chunker-version refusal, the +end-of-run index build) stop holding once another writer is in between. +So every ingest write entry point holds an exclusive lock for the dataset +while it writes. The lock is taken without waiting: a run can last days, +and a second writer queued behind it would only surprise the operator, so +it fails at once with :class:`DatasetWriteLockedError` and the CLIs exit +with :data:`WRITE_LOCKED_EXIT_CODE`. + +The lock file sits beside the dataset, ``/..write-lock``, +not inside it. A cutover renames dataset directories; a lock file inside +one would move with it while a writer still held it, and a second writer +would then lock a fresh file at the same path. The file is never removed, +for the same reason. + +The lock is advisory and per host: it keeps out writers that take it, on +filesystems where ``flock`` locks are shared across processes. A write +that goes around it is caught after the fact by the write-audit gate in +:func:`ingest.store.write_chunks`. + +Holding is reentrant within a thread, so a driver holds the lock for its +whole run and the writers it calls take it again without refusing +themselves. Another thread of the same process is refused like another +process. +""" + +from __future__ import annotations + +import contextlib +import functools +import inspect +import os +import socket +import sys +import threading +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Any, TypeVar + +#: Exit code of an ingest CLI refused because another writer holds the +#: dataset. Distinct from 1 (a failed paper), 2 (argparse's usage error) +#: and 3 (the chunker-version refusal): the remedy is to wait, not to fix. +WRITE_LOCKED_EXIT_CODE = 4 + +_F = TypeVar("_F", bound=Callable[..., Any]) + + +class DatasetWriteLockedError(RuntimeError): + """Another writer holds the dataset's write lock; this call wrote nothing.""" + + def __init__(self, lancedb_path: Path, lock_path: Path, holder: str) -> None: + self.lancedb_path = lancedb_path + self.lock_path = lock_path + self.holder = holder + super().__init__( + f"another ingest is writing the dataset at {lancedb_path} " + f"(held by {holder or 'an unidentified writer'}; lock file " + f"{lock_path}). A dataset takes one writer at a time and this " + "call wrote nothing. Wait for that run to finish, or stop it, " + "then run this again." + ) + + +def write_lock_path(lancedb_path: str | Path) -> Path: + """The lock file that serializes writers of the dataset at ``lancedb_path``.""" + path = Path(lancedb_path).resolve() + return path.parent / f".{path.name}.write-lock" + + +@dataclass +class _Held: + fd: int + owner: int + depth: int + + +_GUARD = threading.Lock() +_HELD: dict[str, _Held] = {} + + +if sys.platform == "win32": # pragma: no cover - exercised on Windows only + import msvcrt + + def _try_lock(fd: int) -> None: + os.lseek(fd, 0, os.SEEK_SET) + try: + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + except OSError as exc: + raise BlockingIOError(str(exc)) from exc + + def _unlock(fd: int) -> None: + os.lseek(fd, 0, os.SEEK_SET) + msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) + +else: + import fcntl + + def _try_lock(fd: int) -> None: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + + def _unlock(fd: int) -> None: + fcntl.flock(fd, fcntl.LOCK_UN) + + +def _read_holder(lock_path: Path) -> str: + try: + return lock_path.read_text(encoding="utf-8").strip() + except OSError: + return "" + + +def _describe_self() -> str: + command = " ".join(Path(a).name if i == 0 else a for i, a in enumerate(sys.argv)) + return f"pid {os.getpid()} on {socket.gethostname()}: {command}"[:500] + + +@contextlib.contextmanager +def dataset_write_lock(lancedb_path: str | Path) -> Iterator[Path]: + """Hold the dataset's write lock for the ``with`` body. + + Raises :class:`DatasetWriteLockedError` at once if another process, or + another thread of this one, holds it. Yields the lock file's path. + """ + lock_path = write_lock_path(lancedb_path) + key = str(lock_path) + me = threading.get_ident() + with _GUARD: + held = _HELD.get(key) + if held is not None and held.owner != me: + raise DatasetWriteLockedError( + Path(lancedb_path), lock_path, _read_holder(lock_path) + ) + if held is not None: + held.depth += 1 + if held is not None: + try: + yield lock_path + finally: + with _GUARD: + held.depth -= 1 + return + + lock_path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(lock_path, os.O_RDWR | os.O_CREAT, 0o644) + try: + _try_lock(fd) + except BlockingIOError: + os.close(fd) + raise DatasetWriteLockedError( + Path(lancedb_path), lock_path, _read_holder(lock_path) + ) from None + except BaseException: + os.close(fd) + raise + try: + # Name the holder for the next writer's refusal. Best-effort: the + # lock, not this text, is what keeps writers apart. + with contextlib.suppress(OSError): + os.ftruncate(fd, 0) + os.lseek(fd, 0, os.SEEK_SET) + os.write(fd, (_describe_self() + "\n").encode("utf-8")) + with _GUARD: + _HELD[key] = _Held(fd=fd, owner=me, depth=1) + try: + yield lock_path + finally: + with _GUARD: + _HELD.pop(key, None) + finally: + try: + _unlock(fd) + finally: + os.close(fd) + + +def holds_dataset_write_lock( + path_param: str, + *, + skip_if: str | None = None, + default: Callable[[], Path] | None = None, +) -> Callable[[_F], _F]: + """Run the decorated writer under the lock of the dataset it writes. + + ``path_param`` names the parameter holding the dataset path. When it is + ``None``, ``default()`` supplies the path the writer itself falls back + to. A true ``skip_if`` parameter (a dry run) takes no lock. + """ + + def decorate(fn: _F) -> _F: + signature = inspect.signature(fn) + + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + bound = signature.bind(*args, **kwargs) + bound.apply_defaults() + if skip_if is not None and bound.arguments[skip_if]: + return fn(*args, **kwargs) + path = bound.arguments[path_param] + if path is None and default is not None: + path = default() + if path is None: + raise ValueError(f"{fn.__qualname__}: {path_param} is required") + with dataset_write_lock(path): + return fn(*args, **kwargs) + + return wrapper # type: ignore[return-value] + + return decorate + + +def exits_when_write_locked(cli: Callable[..., int]) -> Callable[..., int]: + """Turn a write-lock refusal in an ingest CLI into its own exit code. + + A refusal, not a crash: one line on stderr, no traceback, and + :data:`WRITE_LOCKED_EXIT_CODE` so a cron mailer or a supervisor can + tell "another run has the dataset" from a failure. + """ + + @functools.wraps(cli) + def wrapper(*args: Any, **kwargs: Any) -> int: + try: + return cli(*args, **kwargs) + except DatasetWriteLockedError as exc: + print(f"error: {exc}", file=sys.stderr) + return WRITE_LOCKED_EXIT_CODE + + return wrapper + + +__all__ = [ + "WRITE_LOCKED_EXIT_CODE", + "DatasetWriteLockedError", + "dataset_write_lock", + "exits_when_write_locked", + "holds_dataset_write_lock", + "write_lock_path", +] diff --git a/ops/prune_versions.py b/ops/prune_versions.py index 33c03d7..dff8d2c 100644 --- a/ops/prune_versions.py +++ b/ops/prune_versions.py @@ -28,9 +28,10 @@ next write moves the marker and a later prune drops the old ones. A run with nothing to remove does not call ``optimize``. -**When to run it.** It commits to the dataset, so it obeys the -single-writer rule: never beside an ingest, re-embed, delta harvest or -cutover on the same dataset. A server started before the last write and +**When to run it.** It commits to the dataset, so it takes the dataset +write lock (:mod:`ingest.write_lock`) and is refused beside an ingest, +re-embed or delta harvest on the same dataset. A cutover does not take +the lock: never prune during one. A server started before the last write and not restarted is pinned to an older version, which survives only while it is younger than ``--keep-days``; restart servers before pruning with a short window. @@ -44,6 +45,8 @@ from datetime import UTC, datetime, timedelta from pathlib import Path +from ingest.write_lock import exits_when_write_locked, holds_dataset_write_lock + #: Versions younger than this are kept whatever they are. DEFAULT_KEEP_DAYS = 7.0 @@ -148,6 +151,7 @@ def plan_prune( ) +@holds_dataset_write_lock("lancedb_path") def prune_versions( lancedb_path: str | Path, *, @@ -182,6 +186,7 @@ def prune_versions( ) +@exits_when_write_locked def _cli(argv: list[str]) -> int: from ingest.store import DEFAULT_LANCEDB_PATH diff --git a/tests/test_dataset_write_lock.py b/tests/test_dataset_write_lock.py new file mode 100644 index 0000000..8cf2ae5 --- /dev/null +++ b/tests/test_dataset_write_lock.py @@ -0,0 +1,256 @@ +"""One writer per LanceDB dataset: a second writer is refused at once. + +The holder in these tests is a separate Python process, the case the lock +exists for (an operator's terminal run beside a console-started one). +""" + +from __future__ import annotations + +import subprocess +import sys +import threading +from collections.abc import Callable, Iterator +from pathlib import Path + +import lancedb +import pytest + +import ingest.store as store_mod +from ingest import bulk_ingest +from ingest.write_lock import ( + WRITE_LOCKED_EXIT_CODE, + DatasetWriteLockedError, + dataset_write_lock, +) +from ops import prune_versions as prune_mod +from tests._corpus_helpers import seed_corpus_multi_paper + +REPO_ROOT = Path(__file__).resolve().parents[1] + +_HOLDER = """ +import sys +from ingest.write_lock import dataset_write_lock +with dataset_write_lock(sys.argv[1]): + print("held", flush=True) + sys.stdin.read() +""" + + +@pytest.fixture +def hold_in_other_process() -> Iterator[Callable[[Path], subprocess.Popen]]: + procs: list[subprocess.Popen] = [] + + def hold(path: Path) -> subprocess.Popen: + proc = subprocess.Popen( + [sys.executable, "-c", _HOLDER, str(path)], + cwd=REPO_ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + text=True, + ) + procs.append(proc) + if proc.stdout.readline().strip() != "held": + raise RuntimeError("lock holder did not start") + return proc + + yield hold + for proc in procs: + proc.stdin.close() + proc.wait(timeout=30) + + +def _writers(tmp_path: Path) -> dict[str, Callable[[Path], object]]: + from ingest.derived_indexes import build_derived_indexes + from ingest.embed_equations import embed_pending_equations + from ingest.extract_equations import extract_equations_for_paper + from ingest.index_definitions import index_definitions_for_corpus + from ingest.index_equations import index_equation_trees + from ingest.index_theorem_names import build_theorem_names_index + from ingest.oai_delta import run_delta + from ingest.re_embed import run_re_embed + + def no_fetch(*_args, **_kwargs): + raise AssertionError("a refused run must not harvest") + + return { + "write_chunks": lambda ds: store_mod.write_chunks([], None, lancedb_path=ds), + "delete_paper": lambda ds: store_mod.delete_paper(ds, "2307.00001"), + "bulk_ingest": lambda ds: bulk_ingest.run_bulk_ingest( + ["2307.00001"], lancedb_staging_path=ds, + failures_path=tmp_path / "failures.jsonl", + log_path=tmp_path / "ingestion.log", ops_dir=tmp_path / "ops", + ), + "oai_delta": lambda ds: run_delta( + lancedb_staging_path=ds, state_path=tmp_path / "oai-state.json", + log_path=tmp_path / "delta.log", ops_dir=tmp_path / "ops", + timeout_flag_path=tmp_path / "timeout.flag", fetch_page=no_fetch, + ), + "re_embed": lambda ds: run_re_embed( + active_lancedb_path=tmp_path / "active", staging_lancedb_path=ds, + state_path=tmp_path / "re-embed-state.json", + failures_path=tmp_path / "re-embed-failures.jsonl", + ), + "prune_versions": lambda ds: prune_mod.prune_versions(ds), + "index_equation_trees": lambda ds: index_equation_trees(ds), + "index_definitions": lambda ds: index_definitions_for_corpus(ds), + "extract_equations": lambda ds: extract_equations_for_paper("2307.00001", ds), + "embed_equations": lambda ds: embed_pending_equations(ds), + "theorem_names": lambda ds: build_theorem_names_index(ds), + "derived_indexes": lambda ds: build_derived_indexes(ds), + } + + +_WRITER_NAMES = list(_writers(Path("unused"))) + + +@pytest.mark.parametrize("writer", _WRITER_NAMES) +def test_every_write_entry_point_is_refused_beside_another_writer( + tmp_path, hold_in_other_process, writer +): + dataset = tmp_path / "lancedb" + holder = hold_in_other_process(dataset) + + with pytest.raises(DatasetWriteLockedError) as excinfo: + _writers(tmp_path)[writer](dataset) + + message = str(excinfo.value) + assert "another ingest is writing the dataset" in message + assert f"pid {holder.pid}" in message + # Refused before any work: no table, no harvest state, no sentinel. + assert not (dataset / "chunks.lance").exists() + assert not (tmp_path / "oai-state.json").exists() + assert not (tmp_path / "ops").exists() + + +def _bulk_cli(tmp_path: Path, dataset: Path) -> int: + ids = tmp_path / "ids.txt" + ids.write_text("2307.00001\n", encoding="utf-8") + return bulk_ingest._cli( + ["--paper-ids-file", str(ids), "--lancedb-staging-path", str(dataset)] + ) + + +def _prune_cli(_tmp_path: Path, dataset: Path) -> int: + return prune_mod._cli(["--lancedb-path", str(dataset)]) + + +@pytest.mark.parametrize("cli", [_bulk_cli, _prune_cli], ids=["bulk", "prune"]) +def test_cli_exits_with_its_own_code_when_the_dataset_is_held( + tmp_path, hold_in_other_process, capsys, cli +): + dataset = tmp_path / "lancedb" + hold_in_other_process(dataset) + + assert cli(tmp_path, dataset) == WRITE_LOCKED_EXIT_CODE + + err = capsys.readouterr().err + assert err.startswith("error: another ingest is writing the dataset") + assert "Traceback" not in err + + +def test_lock_is_free_once_the_holder_exits(tmp_path, hold_in_other_process): + dataset = tmp_path / "lancedb" + holder = hold_in_other_process(dataset) + holder.stdin.close() + holder.wait(timeout=30) + + seed_corpus_multi_paper(dataset, n_papers=1, chunks_per_paper=2) + + assert lancedb.connect(str(dataset)).open_table("chunks").count_rows() == 2 + + +def test_lock_is_released_when_the_writer_raises(tmp_path): + dataset = tmp_path / "lancedb" + with pytest.raises(ValueError), dataset_write_lock(dataset): + raise ValueError("writer failed") + + probe = subprocess.run( + [sys.executable, "-c", _HOLDER, str(dataset)], + cwd=REPO_ROOT, input="", capture_output=True, text=True, timeout=60, + ) + assert probe.returncode == 0, probe.stderr + assert probe.stdout.strip() == "held" + + +def test_dry_run_takes_no_lock(tmp_path, hold_in_other_process, capsys): + dataset = tmp_path / "lancedb" + hold_in_other_process(dataset) + + summary = bulk_ingest.run_bulk_ingest( + ["2307.00001"], lancedb_staging_path=dataset, dry_run=True, + ar5iv_cache_dir=tmp_path / "ar5iv", parsed_dir=tmp_path / "parsed", + ) + + assert summary.papers_skipped == 1 + + +def test_another_dataset_is_not_blocked(tmp_path, hold_in_other_process): + hold_in_other_process(tmp_path / "a" / "lancedb") + + seed_corpus_multi_paper(tmp_path / "b" / "lancedb", n_papers=1, chunks_per_paper=2) + + table = lancedb.connect(str(tmp_path / "b" / "lancedb")).open_table("chunks") + assert table.count_rows() == 2 + + +def test_renaming_the_dataset_does_not_free_its_path(tmp_path, hold_in_other_process): + # A cutover renames dataset directories while nothing should be + # writing; a writer that is still running keeps the path locked. + dataset = tmp_path / "lancedb" + dataset.mkdir() + hold_in_other_process(dataset) + dataset.rename(tmp_path / "lancedb-previous") + dataset.mkdir() + + with pytest.raises(DatasetWriteLockedError): + seed_corpus_multi_paper(dataset, n_papers=1, chunks_per_paper=2) + + +def test_holder_thread_reenters_and_other_threads_are_refused(tmp_path): + dataset = tmp_path / "lancedb" + refused: list[BaseException] = [] + + def other_thread() -> None: + try: + with dataset_write_lock(dataset): + pass + except DatasetWriteLockedError as exc: + refused.append(exc) + + with dataset_write_lock(dataset): + # The driver holds the lock; the writes it makes take it again. + seed_corpus_multi_paper(dataset, n_papers=2, chunks_per_paper=2) + thread = threading.Thread(target=other_thread) + thread.start() + thread.join(timeout=30) + + assert len(refused) == 1 + assert lancedb.connect(str(dataset)).open_table("chunks").count_rows() == 4 + + +def test_wap_gate_names_a_writer_that_bypassed_the_lock(tmp_path, monkeypatch): + dataset = tmp_path / "lancedb" + seed_corpus_multi_paper(dataset, n_papers=1, chunks_per_paper=3) + real_marker = store_mod.write_corpus_version_marker + + def marker_then_foreign_commit(target_path, **kwargs): + real_marker(target_path, **kwargs) + # Another process, not taking the lock, commits in between and + # stamps its own marker, as every writer does. + foreign = lancedb.connect(str(target_path)).open_table("chunks") + foreign.delete("paper_id = '2307.00001'") + real_marker( + target_path, **{**kwargs, "version": foreign.version, + "chunk_count": foreign.count_rows(), "paper_count": 0}, + ) + + monkeypatch.setattr( + store_mod, "write_corpus_version_marker", marker_then_foreign_commit + ) + + with pytest.raises(store_mod.WapGateError) as excinfo: + seed_corpus_multi_paper(dataset, n_papers=1, chunks_per_paper=3) + + message = str(excinfo.value) + assert "another writer committed to this dataset" in message + assert "write lock" in message + assert "S6" not in message + assert "arithmetic" not in message diff --git a/tests/test_derived_indexes.py b/tests/test_derived_indexes.py index 319243c..4d112fd 100644 --- a/tests/test_derived_indexes.py +++ b/tests/test_derived_indexes.py @@ -59,8 +59,11 @@ def test_a_staged_dataset_carries_its_own_indexes(tmp_path, monkeypatch): summary = build_derived_indexes(staging) - # Nothing lands beside the dataset but the build-stats log. - assert {p.name for p in tmp_path.iterdir()} == {"lancedb-staging", "preamble", "ops"} + # No index lands beside the dataset: only the build-stats log and the + # dataset's write lock, which must not move when the dataset is renamed. + assert {p.name for p in tmp_path.iterdir()} == { + "lancedb-staging", "preamble", "ops", ".lancedb-staging.write-lock", + } bm25 = staging / "derived" / "bm25" / f"v{summary.corpus_version}" assert sorted(json.loads((bm25 / "chunk_ids.json").read_text())) == [ f"{pid}#c0" for pid in PAPERS diff --git a/tests/test_ingest_paths.py b/tests/test_ingest_paths.py new file mode 100644 index 0000000..36d7862 --- /dev/null +++ b/tests/test_ingest_paths.py @@ -0,0 +1,89 @@ +"""Where ingest writes: every path follows ARXMCP_DATA_DIR, and a notebook's +run keeps its sentinel with the notebook.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +import ingest.bulk_ingest as bulk +from ingest.bulk_ingest import PaperOutcome +from tools import _notebook_common, notebook_ingest + +REPO_ROOT = Path(__file__).resolve().parents[1] + +_PROBE = """ +import importlib, json, pkgutil +from pathlib import Path +import ingest + +found = {} +for info in pkgutil.iter_modules(ingest.__path__): + module = importlib.import_module(f"ingest.{info.name}") + for name, value in vars(module).items(): + if isinstance(value, Path) and not name.startswith("__"): + found[f"{info.name}.{name}"] = str(value) +print(json.dumps(found)) +""" + + +def test_every_ingest_data_path_follows_the_data_dir(tmp_path): + root = tmp_path / "data" + env = {k: v for k, v in os.environ.items() if not k.startswith("ARXMCP_")} + env["ARXMCP_DATA_DIR"] = str(root) + + result = subprocess.run( + [sys.executable, "-c", _PROBE], cwd=tmp_path, + env={**env, "PYTHONPATH": str(REPO_ROOT)}, + capture_output=True, text=True, timeout=300, check=False, + ) + + assert result.returncode == 0, result.stderr + paths = json.loads(result.stdout.strip().splitlines()[-1]) + # The corpus trees, the datasets and the ops files are all there. + for key in ( + "preamble.RAW_DIR", "preamble.PREAMBLE_DIR", "extract_equations.PARSED_DIR", + "chunker.CHUNKS_DIR", "embedder.EMBEDDINGS_DIR", "store.DEFAULT_LANCEDB_PATH", + "oai_delta.DEFAULT_STATE_PATH", "oai_delta.DEFAULT_OPS_DIR", + "re_embed.DEFAULT_STATE_PATH", "bm25_indexer.BM25_STATS_PATH", + "intra_paper_refs.DEFAULT_PARSED_DIR", "bulk_ingest.DEFAULT_OPS_DIR", + ): + assert key in paths + outside = { + key: value for key, value in paths.items() + if not Path(value).resolve().is_relative_to(root.resolve()) + } + assert outside == {} + + +def test_notebook_ingest_keeps_its_ingest_summary_with_the_notebook( + tmp_path, monkeypatch +): + base = tmp_path / "notebooks" + nb = base / "demo" + nb.mkdir(parents=True) + (nb / "papers.txt").write_text("2303.07061\n", encoding="utf-8") + monkeypatch.setattr(_notebook_common, "NOTEBOOKS_BASE", base) + monkeypatch.setattr( + bulk, "ingest_one_paper", + lambda paper_id, **_kw: PaperOutcome(paper_id, failure_reason="no_parsed_html"), + ) + shared = bulk.DEFAULT_OPS_DIR / "ingest-summary.json" + shared_before = shared.read_bytes() if shared.is_file() else None + + assert notebook_ingest.run("demo") == 1 + + summary = json.loads((nb / "ops" / "ingest-summary.json").read_text("utf-8")) + assert summary["papers_processed"] == 1 + assert summary["papers_failed"] == 1 + assert (shared.read_bytes() if shared.is_file() else None) == shared_before + + +@pytest.fixture(autouse=True) +def _no_contact_email(monkeypatch): + monkeypatch.delenv("ARXMCP_CONTACT_EMAIL", raising=False) diff --git a/tools/notebook_ingest.py b/tools/notebook_ingest.py index 7a93720..136494b 100644 --- a/tools/notebook_ingest.py +++ b/tools/notebook_ingest.py @@ -37,6 +37,7 @@ from ingest.bm25_indexer import resolve_bm25_root from ingest.bulk_ingest import run_bulk_ingest from ingest.store import STALE_CHUNKER_EXIT_CODE, StaleChunkerRowsError +from ingest.write_lock import exits_when_write_locked from tools._notebook_common import ( NotebookError, notebook_dir, @@ -117,8 +118,9 @@ def run(slug: str) -> int: ops_dir = nb_dir / "ops" ops_dir.mkdir(parents=True, exist_ok=True) - # Per-notebook log path under ops/ — keeps notebook ingest logs - # separate from the global var/arxmcp/ops/ingestion.log. + # Per-notebook log, failures and ingest-summary.json under the + # notebook's ops/, so one notebook's run never overwrites the summary + # the shared corpus's /metrics gauges read. log_path = ops_dir / "ingestion.log" failures_path = ops_dir / "parser-failures.jsonl" @@ -131,6 +133,7 @@ def run(slug: str) -> int: lancedb_staging_path=lancedb_path, log_path=log_path, failures_path=failures_path, + ops_dir=ops_dir, ) print( f"bulk_ingest: total={summary.papers_total} " @@ -194,6 +197,7 @@ def _build_arg_parser() -> argparse.ArgumentParser: return parser +@exits_when_write_locked def main(argv: list[str] | None = None) -> int: parser = _build_arg_parser() args = parser.parse_args(argv)