diff --git a/.github/workflows/container-test.yml b/.github/workflows/container-test.yml index bd1b06479..b752c12d2 100644 --- a/.github/workflows/container-test.yml +++ b/.github/workflows/container-test.yml @@ -154,6 +154,12 @@ jobs: timeout-minutes: 180 strategy: fail-fast: false + # The account runs 20 jobs at once. A change that picks every image + # starts about 66 rows: they held the runners for up to two hours while + # the E2E run of the same commit waited 54 minutes for runners (runs + # 44721158887 and 44720913761 on the geiresx fork). At 8 at a time + # every row still runs, only later. + max-parallel: 8 matrix: var: ${{ fromJSON(needs.plan.outputs.images) }} steps: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 47eab7d07..8b2072512 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -97,10 +97,6 @@ jobs: rustup toolchain install "$channel" --profile minimal rustc --version - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - workspaces: rust - - name: Build og working-directory: rust run: | diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6ef1df5dd..7f1af299c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -117,6 +117,14 @@ env: # of picked cases runs fewer when they fit (scripts/ci/e2e-shards.sh count). The cases took 230 # minutes in one job on 2026-11-02 (run 41390434697). E2E_SHARDS: 6 + # Times that replace scripts/ci/e2e-case-times.tsv's for this run + # (scripts/ci/e2e-shards.sh). prs-3-panel-measure measures the 1000 Genomes + # panel only on a run dispatched on a branch other than main (measure_on in + # its file), where it took 2684 to 3452 s (median 2900, runs 44488552622, + # 44616213787 and 44790003535 on the geiresx fork); elsewhere it takes 0 s, + # as the table says. Without this its shard got a full share of other cases + # on top and finished last. Set here, so every job of a run splits alike. + E2E_SECONDS: ${{ github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main' && 'prs-3-panel-measure=2900' || '' }} concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} @@ -397,19 +405,24 @@ jobs: { echo "### E2E split: base and ${n} shards, $(wc -l < "${RUNNER_TEMP}/plan.tsv") shard cases" echo + if [ -n "${E2E_SECONDS:-}" ]; then echo "Seconds this run gives in place of the table's: ${E2E_SECONDS}"; echo; fi echo "| Job | Cases | Measured minutes |" echo "|---|---|---|" - awk -F'\t' 'FILENAME == ARGV[1] { if ($0 !~ /^#/) t[$1] = $3; next } - FILENAME == ARGV[2] { n["base"]++; s["base"] += ($1 in t) ? t[$1] : 300; next } - { n[$1]++; s[$1] += ($2 in t) ? t[$2] : 300 } + awk -F'\t' 'FILENAME == ARGV[1] { t[$1] = $2; next } + FILENAME == ARGV[2] { n["base"]++; s["base"] += t[$1]; next } + { n[$1]++; s[$1] += t[$2] } END { printf "| base | %d | %.0f |\n", n["base"], s["base"] / 60 for (k = 1; k in n; k++) printf "| shard %d | %d | %.0f |\n", k, n[k], s[k] / 60 }' \ - scripts/ci/e2e-case-times.tsv <(scripts/ci/e2e-shards.sh base) "${RUNNER_TEMP}/plan.tsv" + <(scripts/ci/e2e-shards.sh seconds) <(scripts/ci/e2e-shards.sh base) "${RUNNER_TEMP}/plan.tsv" } | tee -a "$GITHUB_STEP_SUMMARY" # og for the legs (rust/, docs/rust.md): built once, as scripts/setup.sh # --og builds it, and handed to each leg as the artifact og-e2e, so no leg - # pulls the build image or compiles. + # pulls the build image or compiles. Only og-linux-x86_64, the one the + # runners run (OG_ARCHES, rust/og.sh): Guard's rust job and the Rust + # workflow's images job still build all four on every pull request that + # changes rust/. The base leg sets up while this builds and waits for the + # artifact only before its first case. og: needs: e2e-scope if: >- @@ -420,6 +433,9 @@ jobs: (github.event_name == 'workflow_dispatch' && inputs.job == 'e2e')) }} runs-on: ubuntu-latest timeout-minutes: 20 + env: + OG_ARCHES: x86_64 + OG_DARWIN_ARCHES: '' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -443,14 +459,17 @@ jobs: # the artifact e2e-base; the shard legs start once it passed, unpack that # artifact at the same paths and run their own cases. A shard does not take # a runner before the base leg is done: in a busy queue, shards that start - # first and wait would hold runners the base legs of every run need. Both + # first and wait would hold runners the base legs of every run need. The + # base leg does not wait for the job og either: it frees its disk, sets up + # Docker, Java and Nextflow and pulls the images while og builds, and waits + # for the artifact og-e2e only then (step "Wait for og"). Both # jobs run the same steps (the YAML anchor leg-steps); matrix.job tells # them apart. Only the base leg saves the image cache. The checks are named # "e2e base" and "e2e shard N"; the required check is the job e2e below, # which sums them up. e2e-base: - needs: [e2e-scope, e2e-plan, og] - if: ${{ !cancelled() && needs.e2e-plan.result == 'success' && needs.og.result == 'success' }} + needs: [e2e-scope, e2e-plan] + if: ${{ !cancelled() && needs.e2e-plan.result == 'success' }} name: ${{ matrix.job == 'base' && 'e2e base' || format('e2e shard {0}', matrix.job) }} runs-on: ubuntu-latest # The base leg waits up to 75 minutes for a fixture a pull request bumps @@ -464,24 +483,15 @@ jobs: job: [base] env: JOB: ${{ matrix.job }} + # The one og binary of the job og: rust/og.sh check --required and the + # cases' scripts/setup.sh ask for that one only. + OG_ARCHES: x86_64 + OG_DARWIN_ARCHES: '' steps: &leg-steps - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - # og from the job og, checked against this checkout's rust/, and the - # oracles bin/og's shadow mode runs (OG_SHADOW above). - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: og-e2e - path: bin/.og - - - name: og and the shadow oracles - run: | - chmod 755 bin/.og/og-linux-* - rust/og.sh check --required - scripts/ci/oracle-diff.sh materialise - # The same cases take about a third longer on some runners than on # others; the CPU line lets a timing claim name the hardware. - name: Runner CPU @@ -696,11 +706,56 @@ jobs: grep -qxE 'callability-1-catalog|cn-intervals-1-catalog|followups-ci-3-sv-rule-regions' "$list" && echo "data_catalogs=true" >> "$GITHUB_OUTPUT" echo "list=${list}" >> "$GITHUB_OUTPUT" - # Before the base outputs: the base leg pulled the same images. + # Before og and the base outputs, so the pulls overlap the wait for + # them. The base leg's cases would pull the same images first thing. - name: Pull the images the cases use - if: matrix.job != 'base' run: scripts/ci/e2e-run.sh --pull-images + # The artifact og-e2e of the job og, which the base leg did not wait + # for: a shard leg, which starts after the base leg passed, finds it at + # once. Stops the leg when the job og ended without it. + - name: Wait for og + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ github.run_id }} + run: | + t0=$(date +%s) done_at="" + while :; do + n=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts?name=og-e2e" --jq '.total_count' 2> /dev/null) || n="" + if [ "${n:-0}" -gt 0 ]; then + echo "og-e2e is there after $(( $(date +%s) - t0 )) s of waiting" | tee -a "$GITHUB_STEP_SUMMARY" + exit 0 + fi + state=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/jobs?filter=latest&per_page=100" \ + --jq '.jobs[] | select(.name == "og") | "\(.status) \(.conclusion)"' 2> /dev/null) || state="" + case $state in + "completed success") + # The upload ends before the job does; give the listing a minute. + done_at=${done_at:-$(date +%s)} + if [ $(( $(date +%s) - done_at )) -gt 60 ]; then + echo "::error::The job og passed, but this run has no og-e2e artifact."; exit 1 + fi ;; + completed*) echo "::error::The job og ended ${state#completed }: the cases have no og."; exit 1 ;; + esac + if [ $(( $(date +%s) - t0 )) -gt 3600 ]; then + echo "::error::No og-e2e artifact after 60 minutes (job og: ${state:-not found})."; exit 1 + fi + sleep 20 + done + + # og from the job og, checked against this checkout's rust/, and the + # oracles bin/og's shadow mode runs (OG_SHADOW above). + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: og-e2e + path: bin/.og + + - name: og and the shadow oracles + run: | + chmod 755 bin/.og/og-linux-* + rust/og.sh check --required + scripts/ci/oracle-diff.sh materialise + # The base leg uploads e2e-base only once its cases all passed, and this # leg starts only after it passed. A rerun of failed legs finds the # artifact of the earlier attempt. A broken transfer is tried again. @@ -1130,10 +1185,12 @@ jobs: job: ${{ fromJSON(needs.e2e-plan.outputs.jobs) }} env: JOB: ${{ matrix.job }} + OG_ARCHES: x86_64 + OG_DARWIN_ARCHES: '' steps: *leg-steps # The required check. Green when e2e-scope finds nothing to run; otherwise - # green only when the plan and every leg succeeded, every picked case + # green only when the plan, og and every leg succeeded, every picked case # reported exactly one pass (one table of every case goes to the summary) # and each parity check whose cases ran passed once. Red when e2e-scope # did not succeed. A leg that was skipped, failed or cut by its time limit @@ -1142,7 +1199,7 @@ jobs: # cancelled with it and reads cancelled, not failed: nothing ran to fail. # A cancelled check blocks a merge as a red one does. e2e: - needs: [e2e-scope, e2e-plan, e2e-base, e2e-shards] + needs: [e2e-scope, e2e-plan, og, e2e-base, e2e-shards] if: >- ${{ !cancelled() && ( github.event_name == 'pull_request' || @@ -1181,6 +1238,7 @@ jobs: GH_TOKEN: ${{ github.token }} RUN_ID: ${{ github.run_id }} PLAN: ${{ needs.e2e-plan.result }} + OG: ${{ needs.og.result }} BASE: ${{ needs.e2e-base.result }} LEGS: ${{ needs.e2e-shards.result }} SHARDS: ${{ needs.e2e-plan.outputs.shards }} @@ -1193,10 +1251,10 @@ jobs: tr ' ' '\n' <<< "$SELECT" > "${RUNNER_TEMP}/select.txt" export E2E_SELECT="${RUNNER_TEMP}/select.txt" fi - if [ "$PLAN" != success ] || [ "$BASE" != success ] || [ "$LEGS" != success ]; then + if [ "$PLAN" != success ] || [ "$OG" != success ] || [ "$BASE" != success ] || [ "$LEGS" != success ]; then # The run was not cancelled (then this job would not run), so a # job that ended cancelled was cut by its time limit. - echo "::error::The plan job ended ${PLAN:-unknown}, the base leg ${BASE:-unknown} and the shard legs ${LEGS:-unknown}; e2e needs all three to succeed. A job that ended cancelled in a run that was not cancelled hit its time limit." + echo "::error::The plan job ended ${PLAN:-unknown}, the job og ${OG:-unknown}, the base leg ${BASE:-unknown} and the shard legs ${LEGS:-unknown}; e2e needs all four to succeed. A job that ended cancelled in a run that was not cancelled hit its time limit." rc=1 fi dir="${RUNNER_TEMP}/reports" diff --git a/.github/workflows/freshness.yml b/.github/workflows/freshness.yml index 435058219..334a05810 100644 --- a/.github/workflows/freshness.yml +++ b/.github/workflows/freshness.yml @@ -144,11 +144,6 @@ jobs: rustup toolchain install "$channel" --profile minimal rustc --version - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - workspaces: rust - key: freshness - - name: Build og working-directory: rust run: cargo build --release --locked -p og --no-default-features @@ -209,11 +204,6 @@ jobs: rustup toolchain install "$channel" --profile minimal rustc --version - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - workspaces: rust - key: freshness - - name: Build og working-directory: rust run: cargo build --release --locked -p og --no-default-features @@ -267,11 +257,6 @@ jobs: rustup toolchain install "$channel" --profile minimal rustc --version - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - workspaces: rust - key: freshness - - name: Build og working-directory: rust run: cargo build --release --locked -p og --no-default-features diff --git a/.github/workflows/guard.yml b/.github/workflows/guard.yml index 953d99d1c..0dfed60f7 100644 --- a/.github/workflows/guard.yml +++ b/.github/workflows/guard.yml @@ -151,11 +151,6 @@ jobs: rustup toolchain install "$channel" --profile minimal rustc --version - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - workspaces: rust - key: fake-docker - - name: Build og into bin/.og working-directory: rust run: | @@ -225,10 +220,6 @@ jobs: rustup toolchain install "$channel" --profile minimal rustc --version - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - workspaces: rust - - name: Build og working-directory: rust run: OG_TREE_HASH=$(./og.sh tree-hash) cargo build --release --locked -p og @@ -358,9 +349,15 @@ jobs: rustup toolchain install "$channel" --profile minimal -c rustfmt -c clippy rustc --version + # The one Guard job a cache speeds up (clippy and the tests, 1 to 2 + # minutes). It saves on main only: a branch reads main's entry, and the + # per-branch entries of every job filled the 10 GB the E2E image and + # data caches need. The jobs that only build og --release have no + # cache: it did not make that build faster. - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: rust + save-if: ${{ github.ref == 'refs/heads/main' }} - name: rustfmt run: cargo fmt --all --check @@ -1009,10 +1006,6 @@ jobs: rustup toolchain install "$channel" --profile minimal rustc --version - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - workspaces: rust - - name: Build og working-directory: rust run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index adaf29e20..3f179fad7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -300,3 +300,65 @@ jobs: exit 1 fi echo "OK: All internal links valid" + + # Every data file the repository ships has a recorded source and licence: + # assets/SOURCES.tsv has a row for each file under assets/, each file + # carries the header phrase its row names, and NOTICE names every + # third-party row (og dev licences check, rust/crates/og-dev/src/licences.rs). + data-sources: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install the toolchain rust/rust-toolchain.toml pins + working-directory: rust + run: | + channel=$(sed -n 's/^channel = "\(.*\)"$/\1/p' rust-toolchain.toml) + rustup toolchain install "$channel" --profile minimal + rustc --version + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + workspaces: rust + + # A debug build without BAM reading: the check needs no htslib and no + # optimised code. + - name: Build og + working-directory: rust + run: cargo build --locked -p og --no-default-features + + - name: Check that every bundled data file has a recorded source and licence + run: rust/target/debug/og dev licences check + + # A check that cannot fail is no check: a new asset with no row, and an + # asset with its source line stripped, must both fail it. + - name: Controls, an unregistered asset and a stripped source line must fail + run: | + og=rust/target/debug/og + log=$(mktemp) + + planted=assets/lint-control.tsv + printf 'a\tb\n' > "$planted" + rc=0; "$og" dev licences check > "$log" 2>&1 || rc=$? + rm -f "$planted" + cat "$log" + if [ "$rc" -ne 1 ] || ! grep -qF "${planted} has no row in assets/SOURCES.tsv" "$log"; then + echo "ERROR: control 1, an asset with no row passed (exit ${rc})"; exit 1 + fi + echo "Control 1: the unregistered ${planted} was caught." + + target=assets/sv_rule_regions.tsv + sed -i '/^# Sources and terms:/d' "$target" + if git diff --quiet -- "$target"; then + echo "ERROR: control 2 stripped nothing from ${target}"; exit 1 + fi + rc=0; "$og" dev licences check > "$log" 2>&1 || rc=$? + git checkout -- "$target" + cat "$log" + if [ "$rc" -ne 1 ] || ! grep -qF "${target}: its opening # lines lack \"Sources and terms:" "$log"; then + echo "ERROR: control 2, ${target} without its source line passed (exit ${rc})"; exit 1 + fi + echo "Control 2: ${target} without its source line was caught." diff --git a/.github/workflows/nextflow.yml b/.github/workflows/nextflow.yml index d7f8985cd..68283ed4f 100644 --- a/.github/workflows/nextflow.yml +++ b/.github/workflows/nextflow.yml @@ -211,11 +211,6 @@ jobs: rustup toolchain install "$channel" --profile minimal rustc --version - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - if: steps.scope.outputs.run == 'true' - with: - workspaces: rust - - name: Build og if: steps.scope.outputs.run == 'true' working-directory: rust diff --git a/.github/workflows/personal-data.yml b/.github/workflows/personal-data.yml index 81ace8050..bee6ec8ed 100644 --- a/.github/workflows/personal-data.yml +++ b/.github/workflows/personal-data.yml @@ -35,11 +35,6 @@ jobs: rustup toolchain install "$channel" --profile minimal rustc --version - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 - with: - workspaces: rust - key: personal-data - - name: Build og working-directory: rust run: cargo build --release --locked -p og --no-default-features diff --git a/.github/workflows/public-evidence.yml b/.github/workflows/public-evidence.yml index 2667f64f4..e241536af 100644 --- a/.github/workflows/public-evidence.yml +++ b/.github/workflows/public-evidence.yml @@ -1,8 +1,9 @@ name: Public evidence -# Runs two opt-in steps whole, on public data only (GIAB's HG002 and the -# public archaic and ancient panels), and writes what they measured to the -# job summary. Not a gate. +# Runs three opt-in steps whole, on public data only (GIAB's HG002 and the +# public archaic and ancient panels, 1000 Genomes men with published Y +# haplogroups), and writes what they measured to the job summary. No pull +# request runs it, so none of its jobs gates a merge. # # archaic-sites the 1000 Genomes sites file setup.sh --archaic reads, # downloaded once (its md5 checked) and indexed, for @@ -32,6 +33,15 @@ name: Public evidence # then step 44 on HG002, each with the peak memory of # its containers (the cgroup's memory.peak), the peak # disk use and the time. The ancient input off skips it. +# y-benchmark step 37 on the chrY of 1000 Genomes men (NYGC 30x +# CRAMs) with published Y haplogroups (Poznik et al. +# 2016), and og dev evidence y-bench: each tool's +# call (Yleaf, og y tree-place) against the published +# one on the current YFull tree. Its gate fails when a +# tool puts fewer than Y_MIN of the men on the +# published branch or below it; the same calls handed +# to other men are the red control and must fail it. +# The full 58-man table is in docs/37-y-haplogroup.md. # # It runs by hand only (workflow_dispatch). Public repo: the GitHub-hosted # runner is free here. @@ -58,6 +68,14 @@ on: description: "Autosomes for the archaic-same-person job, separated by spaces or commas, or none" type: string default: none + y_benchmark: + description: "Run the y-benchmark job" + type: boolean + default: true + y_samples: + description: "1000 Genomes men for the y-benchmark job, separated by spaces or commas (3 or more, from different haplogroups: the red control hands each man's calls to another; the docs table has 58)" + type: string + default: HG01890 HG02588 NA18952 HG01583 HG00530 HG00107 permissions: contents: read @@ -69,9 +87,21 @@ concurrency: env: NCBI_REF: https://ftp.ncbi.nlm.nih.gov/genomes/all/GCA/000/001/405/GCA_000001405.15_GRCh38/seqs_for_alignment_pipelines.ucsc_ids GIAB_HG002: https://ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/release/AshkenazimTrio/HG002_NA24385_son/NISTv4.2.1/GRCh38/HG002_GRCh38_1_22_v4.2.1_benchmark + # y-benchmark: Poznik et al. 2016's calls (column 3, the most derived + # SNP seen), the index of the NYGC 30x CRAMs, and chrY of the reference + # those CRAMs were aligned to (its M5 is the CRAM header's). + Y_CALLS: https://ftp.1000genomes.ebi.ac.uk/vol1/ftp/release/20130502/supporting/chrY/1000Y.sample_level_haplogroup_calls.ver5.txt + Y_CALLS_SHA256: a7975573a1aa2d708ee09d1a11aa3d6feb4efe6ac3db3db5bdba5069d512df42 + Y_HC_INDEX: https://ftp.1000genomes.ebi.ac.uk/vol1/ftp/data_collections/1000G_2504_high_coverage/1000G_2504_high_coverage.sequence.index + Y_HC_INDEX_SHA256: 0ac5fdda0b60b6575b1867827f53e39d723f915b325b10bf023b2dc3d71c08f2 + Y_REF: https://ftp.1000genomes.ebi.ac.uk/vol1/ftp/technical/reference/GRCh38_reference_genome/GRCh38_full_analysis_set_plus_decoy_hla.fa + Y_REF_CHRY_MD5: ce3e31103314a704255f3cd90369ecce + Y_MIN: "0.95" jobs: archaic-sites: + # Only the archaic jobs read the sites file. + if: inputs.hg002 || inputs.same_person_chromosomes != 'none' runs-on: ubuntu-latest timeout-minutes: 120 outputs: @@ -745,3 +775,142 @@ jobs: name: ancient-footprint path: ${{ env.EV }}/measure if-no-files-found: ignore + + y-benchmark: + if: inputs.y_benchmark + runs-on: ubuntu-latest + timeout-minutes: 240 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Build og as scripts/setup.sh --og does (og dev evidence) + run: | + scripts/setup.sh --og + rust/og.sh check --required + + - name: Yleaf's data, the current tree, chrY of the reference and the published calls + run: | + set -euo pipefail + . ./versions.env + D="${RUNNER_TEMP}/yb"; G="${D}/genome" + mkdir -p "${G}/reference" + echo "D=${D}" >> "$GITHUB_ENV" + echo "G=${G}" >> "$GITHUB_ENV" + scripts/setup.sh --yleaf-data "$G" + curl -fsSL --retry 5 -o "${D}/published.txt" "$Y_CALLS" + echo "${Y_CALLS_SHA256} ${D}/published.txt" | sha256sum -c + curl -fsSL --retry 5 -o "${D}/hc.index" "$Y_HC_INDEX" + echo "${Y_HC_INDEX_SHA256} ${D}/hc.index" | sha256sum -c + # The image has no CA bundle: htslib reads the runner's. + docker run --rm -u "$(id -u):$(id -g)" -v /etc/ssl/certs:/etc/ssl/certs:ro -v "${G}/reference:/r" \ + "$SAMTOOLS_IMAGE" sh -c 'samtools faidx "$1" chrY > /r/chrY.fa && samtools faidx /r/chrY.fa' _ "$Y_REF" + m5=$(docker run --rm -v "${G}/reference:/r:ro" "$SAMTOOLS_IMAGE" samtools dict /r/chrY.fa | grep -o 'M5:[0-9a-f]*') + [ "$m5" = "M5:${Y_REF_CHRY_MD5}" ] || { echo "chrY of ${Y_REF} has ${m5}, not M5:${Y_REF_CHRY_MD5}"; exit 1; } + + - name: Step 37 on each man's chrY, from his 30x CRAM + env: + SAMPLES: ${{ inputs.y_samples }} + run: | + set -euo pipefail + . ./versions.env + export GENOME_DIR="$G" REF_FASTA=reference/chrY.fa + read -ra want <<< "$(printf '%s' "$SAMPLES" | tr ',\r\n' ' ')" + [ "${#want[@]}" -ge 3 ] || { echo "y_samples: name 3 men or more (the red control hands each man's calls to another)"; exit 1; } + # The red control hands each man the calls of the man half the list + # away. It can fail only when most of those pairs sit on different + # lines, so check that on the tree before any CRAM is read: each + # man's published branch, then his partner's against it. + TREE="${G}/reference/yleaf-${YLEAF_DATA_VERSION}/data/ytree/current_tree.json" + printf 'sample\tpublished\ttool\tcall\n' > "${D}/pub.tsv" + for s in "${want[@]}"; do + [[ "$s" =~ ^(HG|NA)[0-9]{5}$ ]] || { echo "y_samples: '${s}' is not a 1000 Genomes sample id"; exit 1; } + pub=$(awk -v s="$s" '$1 == s {print $3}' "${D}/published.txt") + [ -n "$pub" ] || { echo "${s}: no published call"; exit 1; } + printf '%s\t%s\tpublished\t\n' "$s" "$pub" >> "${D}/pub.tsv" + done + bin/og dev evidence y-bench "$TREE" "${D}/pub.tsv" "${D}/pub.out.tsv" 0 > /dev/null + awk -F'\t' -v OFS='\t' 'NR == 1 {print "sample", "published", "tool", "call"; next} {s[++n] = $1; p[n] = $2; b[n] = $5} + END {for (i = 1; i <= n; i++) print s[i], p[i], "partner", b[(i - 1 + int(n / 2)) % n + 1]}' "${D}/pub.out.tsv" > "${D}/partner.tsv" + bin/og dev evidence y-bench "$TREE" "${D}/partner.tsv" "${D}/partner.out.tsv" 0 > /dev/null + awk -F'\t' -v min="$Y_MIN" 'NR > 1 {n++; if ($7 == "exact" || $7 == "below" || $7 == "ancestor") {k++; print "y_samples: " $1 " (" $2 ") is on one line with his partner, " $6}} + END {if (k / n >= min) {printf "y_samples: %d of %d men share a line with the man whose calls the red control hands them; name men from different haplogroups\n", k, n; exit 1}}' "${D}/partner.out.tsv" + # col FILE NAME: the value of column NAME in the one data row of FILE + col() { awk -F'\t' -v n="$2" 'NR == 1 {for (i = 1; i <= NF; i++) c[$i] = i; next} NR == 2 {print $c[n]}' "$1"; } + # slice URL MAN: MAN's chrY reads from his CRAM at URL, as a BAM with + # its index. EBI fails a read now and then (a .crai that does not + # parse, a region that still fails after htslib's own reconnects), + # so try 4 times, 1, 2 and 4 minutes apart, then stop the job. Each + # try starts clean: samtools keeps the .crai it fetched in /g and + # would read a bad copy again. A retry reads the same file again and + # nothing else: the calls and the gate below do not change. + slice() { + local crai="${G}/${1##*/}.crai" bam="${G}/$2/aligned/$2_sorted.bam" n + for n in 1 2 3 4; do + rm -f "$crai" "$bam" "${bam}.bai" + if docker run --rm -u "$(id -u):$(id -g)" -v /etc/ssl/certs:/etc/ssl/certs:ro -v "${G}:/g" -w /g "$SAMTOOLS_IMAGE" \ + sh -c 'samtools view -b -T reference/chrY.fa -o "$2/aligned/$2_sorted.bam" "$1" chrY && samtools index "$2/aligned/$2_sorted.bam"' \ + _ "$1" "$2"; then + rm -f "$crai" + return 0 + fi + [ "$n" -lt 4 ] || break + echo "::warning::$2: reading chrY from $1 failed (try $n of 4); next try in $((60 << (n - 1))) s" + sleep $((60 << (n - 1))) + done + echo "::error::$2: reading chrY from $1 failed 4 times; no calls are made for him, so the job stops" + return 1 + } + printf 'sample\tpublished\ttool\tcall\n' > "${D}/calls.tsv" + for s in "${want[@]}"; do + pub=$(awk -v s="$s" '$1 == s {print $3}' "${D}/published.txt") + url=$(awk -F'\t' -v s="$s" '$10 == s {print $1}' "${D}/hc.index" | sed 's#^ftp://#https://#') + [ -n "$pub" ] && [ -n "$url" ] || { echo "${s}: no published call or no 30x CRAM"; exit 1; } + mkdir -p "${G}/${s}/aligned" "${G}/${s}/indexcov" + slice "$url" "$s" + # A chrY slice has no autosomes for indexcov to read the sex from: + # the men are men in the 1000 Genomes panel, so the row says male. + printf '#family_id\tsample_id\tpaternal_id\tmaternal_id\tsex\tphenotype\tCNchrX\tCNchrY\n%s\t%s\t-9\t-9\t1\t-9\t1.0\t1.0\n' \ + "$s" "$s" > "${G}/${s}/indexcov/indexcov-indexcov.ped" + scripts/37-y-haplogroup.sh "$s" + O="${G}/${s}/y_haplogroup" + printf '%s\t%s\tYleaf %s\t%s\n' "$s" "$pub" "$YLEAF_DATA_VERSION" "$(col "${O}/${s}_y_haplogroup.txt" Hg)" >> "${D}/calls.tsv" + printf '%s\t%s\tog y tree-place\t%s\n' "$s" "$pub" "$(col "${O}/${s}_y_tree.txt" Hg)" >> "${D}/calls.tsv" + rm -rf "${G:?}/${s}" + done + cat "${D}/calls.tsv" + + - name: The calls against the published ones (the gate), and the red control + run: | + set -euo pipefail + . ./versions.env + TREE="${G}/reference/yleaf-${YLEAF_DATA_VERSION}/data/ytree/current_tree.json" + { + echo "### Y placement against Poznik et al. 2016 (YFull YTree ${YTREE_VERSION})" + echo "" + bin/og dev evidence y-bench "$TREE" "${D}/calls.tsv" "${D}/bench.tsv" "$Y_MIN" + echo "" + echo "Gate: every tool puts at least ${Y_MIN} of the men on the published branch or below it." + } >> "$GITHUB_STEP_SUMMARY" + cat "$GITHUB_STEP_SUMMARY" + # Red control: each tool's calls handed to the man half the list + # away must fail the gate, and fail it as the gate. + awk -F'\t' -v OFS='\t' 'NR == 1 {print; next} {s[$3, ++n[$3]] = $1; p[$3, n[$3]] = $2; c[$3, n[$3]] = $4; if (!($3 in seen)) {seen[$3]; t[++k] = $3}} + END {for (j = 1; j <= k; j++) {m = n[t[j]]; for (i = 1; i <= m; i++) print s[t[j], i], p[t[j], i], t[j], c[t[j], (i - 1 + int(m / 2)) % m + 1]}}' \ + "${D}/calls.tsv" > "${D}/rotated.tsv" + if bin/og dev evidence y-bench "$TREE" "${D}/rotated.tsv" "${D}/rotated.out.tsv" "$Y_MIN" 2> "${D}/rotated.err"; then + echo "red control: the rotated calls passed the gate"; exit 1 + fi + cat "${D}/rotated.err" + grep -q '^og dev evidence: y-bench gate: ' "${D}/rotated.err" || { echo "red control: failed, but not on the gate"; exit 1; } + + - name: Upload the calls and their results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: y-benchmark + path: | + ${{ env.D }}/calls.tsv + ${{ env.D }}/bench.tsv + if-no-files-found: ignore diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 67949cd6f..34c8d9fdb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -125,7 +125,12 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # The runner's own binary only (rust/og.sh): pycompat runs og on the + # host. The images job above builds all four. - name: Build og with scripts/setup.sh --og + env: + OG_ARCHES: x86_64 + OG_DARWIN_ARCHES: '' run: | scripts/setup.sh --og bin/og version diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ac2445c2..8cf6c20a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,28 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n ## Unreleased +- The E2E run dispatched on a batch of pull requests finishes sooner, with every case and check it ran before. The shard split now weighs `prs-3-panel-measure` at its measured 2900 seconds on a run dispatched on a branch (`E2E_SECONDS`), where it measures the 1000 Genomes panel, so its shard no longer gets a full share of other cases on top. The base leg sets up while `og` builds, and the job `og` builds only the Linux x86_64 binary the legs run (`OG_ARCHES`, `OG_DARWIN_ARCHES` in `rust/og.sh`); Guard's `rust` job still builds all four on every pull request, as does the Rust `images` job when it runs. The nightly Rust run, which is the `streams` job only, now builds only `og-linux-x86_64`, the binary it runs. Container Test runs at most 8 smoke rows at once, so a change to every image no longer takes the runners the E2E legs wait for. The jobs that only build `og --release` lose their Rust cache, which did not speed them up, and Guard's `rust` job saves its cache on main only, so the E2E image and data caches fit the 10 GB budget. +- A step you selected that produced nothing now says so. A report-only Nextflow step (TelomereHunter, VNtyper, the CHIP genes and the others in `nextflow.config`) is dropped when it fails, after one retry when the exit was a memory kill. Its section had no file, so the report listed it as not run, the same as a step nobody asked for. `HTML_REPORT` now records the run's `--tools` list, and `og summary collect` lists selected sections with no file under Steps Not Run as "(selected, no result)", with a note naming the tool and the step. Y haplogroup, Y tree placement, and Y-STR markers get this label only for samples declared male. `summary.json` gets a new top-level list, `selected_no_result`, with those section keys. It is present only when the run recorded its tool list, so a bash run's summary is unchanged. The section's state stays `missing`. +- One section that cannot be written no longer stops the whole report. Step 24, the Nextflow `HTML_REPORT` module and `scripts/generate-report.sh` ran `bin/render_report.py`, and a section whose data it could not read stopped it with a Python error: the run got no text report and no HTML report at all. The three now run `og report render` (`rust/crates/og-render`), so no Python runs for the reports. A section that cannot be written says so in its own place, `[could not be read: ...]` in the text report and an Unreadable card in the HTML report, and every other section is still written; `og report render` prints one `WARNING` line on stderr that counts and names those parts, and still exits 0. A panic inside a section writer counts the same way, so the release build now unwinds on a panic (`rust/Cargo.toml`); a panic anywhere else still aborts og at once, through the panic hook og installs first. A search index or charts file that cannot be made is written empty and named in the same `WARNING` line. On the 164 report-render parity summaries (154 before, and 10 new ones that reach each state of the mosaic sex-loss, raw-data export and structural-variant sections) both reports are byte for byte the Python's, whole, with two changes on purpose: a gene the ACMG card names is listed once in the closing gene list, not twice (the `acmg_sf` section repeats the card the clinical section holds), and a run whose BAM has no unplaced unmapped reads says the viral screen was not assessed instead of "no read". The checks of `tests/test_render_report.py` are cargo tests (`rust/og/tests/report_checks.rs`, and `rust/crates/og-render/tests/section_failure.rs` for a section that panics), and the Python file is gone. `bin/render_report.py`, `bin/report_charts.py` and `bin/results_index.py` stay only for the Python live site. +- Nextflow's `docker` profile now runs each task as you, as the step scripts do, not as root. Before, every task ran with `--user root:root`, so a file a task wrote outside its work directory, beside an input under `GENOME_DIR`, stayed root's (Nextflow's `docker.fixOwnership` gives back only the work directory). Tasks now run with `-u` and your UID and GID, with `HOME=/tmp`. `CPSR` and `CNVPYTOR` still run as root, with `HOME=/root` as before: PCGR has not been shown to run unprivileged, which is why step 17 runs it as root too, and the CNVpytor module copies its resources into its image. A file those two write outside their work directory stays root's. The ids are written as bash's `${UID}:${GROUPS}`, which have no space. The PRS module, which hands the run's options on to pgsc_calc, now also drops a user flag with its whole value: a config's common `-u $(id -u):$(id -g)` gave pgsc_calc a stray `-g)` before. pgsc_calc's containers now also get `HOME=/tmp`. A sample folder written by an older run can hold root-owned files that a task running as you cannot overwrite: [troubleshooting](docs/troubleshooting.md#permission-denied-writing-to-mounted-volumes) gives the `chown` that takes it back. `tests/test_prs_runopts.sh` checks the profile's user and the PRS module's user flag drop, and the e2e case `prs-runopts-and-trace-1` checks the docker run line of every task of a stub run. +- The README and Getting started now say what to install before the first command to copy. The README's quick start opens with a "Before you begin" list (Docker, bash 4.4, Java 17, the pinned Nextflow, 16 GB of RAM with swap for step 25, and the disk), and its clone line checks out the newest release tag, so `setup.sh` downloads the `og` binaries instead of building them. Getting started has one ordered install path each for Linux, an Intel Mac, an Apple Silicon Mac and Windows with WSL2, with Homebrew first on a Mac, and each ends in the same four-command check. `tests/test_install_docs_order.sh` keeps that order. - The reads drawing on a live finding page now says what it is before it is shown. It is the same nine bars on every finding, but its label read "The reads at " above it and only a caption after it said it was a sketch, so it looked like the person's own reads. The label now reads "Where to look at the reads: . An illustration, not your reads.", and the link to the real reads in the genome viewer stays. In the public demo, which has no reads, every offer of reads said so only after the click: the finding page's link, a card's "See the reads at" link, a chart mark's link and a search hit's "reads at " link opened a page saying the demo has none, and the map's card button and reads pill opened a panel saying so. They now read "Reads: not in the demo": the links are plain text, and the map's buttons still open the panel that says why. On any site, the map's reads panel, which opens under the map and often below the screen, now scrolls into view when it opens. The browser check `rust/crates/og-browser/tests/rv_live_reads_sketch_label.rs` builds the demo and checks all of it. No option, output or file name changes. +- `og live serve` serves the live site from the one static `og` binary, in Rust: the home, run and family pages, `state.json` and its headlines, the event stream, the run files, the map, `/metrics` and the search, charts, ops and re-check plugins, with the same policy on every answer. It has no login yet, so it serves loopback only and refuses the password, TLS and proxy options with exit 2; `scripts/owngenome-live.sh` still starts the Python server ([Watch a Run Live](docs/live-site.md)). `bin/owngenome_recheck.py`, the shim that ran `og recheck`, is gone: call `bin/og recheck`. The Python checks of the site that `og live serve` now answers moved to `cargo test -p og-live`. +- Step 37's two Y placements are now measured against published haplogroups. On 58 men of the 1000 Genomes Project, with the haplogroups Poznik et al. 2016 published and each man's chrY from his 30x CRAM, Yleaf and `og y tree-place` each put all 58 on the published branch or below it, and named the same branch for 44. `docs/37-y-haplogroup.md` has the table, how the men were chosen and each man's calls. A new verb, `og dev evidence y-bench`, places the published calls and the tools' calls on the current YFull tree and scores them. The public-evidence workflow has a new job, `y-benchmark`: it runs step 37 again on 6 of the men, fails when a tool puts fewer than 95% of them on the published branch or below it, and checks that the same calls given to other men fail. The outvote rule of `og y tree-place` is unchanged: it fired for one man, whose placement is on his published line, so the data give no reason to change it. The `archaic-sites` job of that workflow now runs only when an archaic job needs it. +- The Guard check no longer fails at random on the SMN table tests of `og-cn`. Two of those tests ran at the same time in one folder, and the first to end removed it while the other still wrote its table there. Each test table now gets a folder of its own. +- Every data file the repository ships now has a recorded source and licence, and CI fails on one that has none. Before, nothing tied a table under `assets/` to where it came from, so a new file could ship third-party data with no credit and no terms. `assets/SOURCES.tsv` lists every file under `assets/` with its source, version, licence and the header phrase it must carry; the new top-level `NOTICE` names each third-party file and its terms. `og dev licences check` fails when a file under `assets/` has no row, a row matches no file, a file lacks the header phrase its row names, or `NOTICE` does not name a third-party row. Lint runs it on every push and pull request, with two controls that must fail: an asset with no row, and `assets/sv_rule_regions.tsv` with its source line removed. A change that adds a file under `assets/` now adds its row in the same pull request. +- A pypgx `Indeterminate` call no longer counts as a call. Step 27 (`og pgx cpic-report`) read the word as a diplotype, so a gene PharmCAT could not call got a warning that pypgx "called Indeterminate", and the comparison table listed it as a pypgx call or a mismatch. Step 36 (`og pgx outside-calls`) and the summary's CYP2D6 block read a diplotype with one `Indeterminate` allele, such as `*1/Indeterminate`, as a call that could disagree with Cyrius. Now `Indeterminate`, in any case and as any allele, is no call in step 27, in step 36 and in the summary's CYP2D6 block. The comparison table's Match and Called_by columns change for such a gene. The pypgx block of `summary.json` counts a gene in `genes_called` only when its diplotype is a call by the rule step 32's Status column uses, so `*1/Indeterminate`, a lone allele and several diplotypes are no longer counted. +- Every drug of a combination product on the medication list is now checked. Step 27 read only the first word of a line, so `olanzapine/fluoxetine` was checked as olanzapine alone, and `naltrexone/bupropion 8/90 mg` was listed as not in the inhibitor table although bupropion is a strong CYP2D6 inhibitor. Now each word of a line counts: whitespace, commas, semicolons, `/`, `+` and brackets split it, punctuation at either end of a word is dropped, and the word `and`, doses, units and frequencies are dropped. A hyphenated word counts whole and by each part of three or more characters that starts with a letter, so `olanzapine-fluoxetine` is checked for fluoxetine while the `co` of `co-trimoxazole`, the `L` of `L-dopa` and the `5` of `5-HTP` never count as drugs. So `Contrave (naltrexone/bupropion) 8/90 mg` is checked for bupropion and `Epilim (sodium valproate)` for valproate. The phenoconversion takes the strongest inhibitor among all of them, a line is "Not in the inhibitor table" only when none of its drugs is, and a combination's second drug is marked `ON THE MEDICATION LIST` in the drug contraindications too. + +- Step 25 now says when your genetic ancestry sits between two reference groups. pgsc_calc flags its placement as low-confidence only when the top random-forest probability is under 0.5, so a sample at 0.55 EUR and 0.45 AFR got the EUR percentile with no caveat. When the two most probable groups of step 26 each reach 0.2, the step log names both groups and says each percentile ranks you among one of them only, and summary.json's PRS ancestry gains `mixed` and `second_population`, with `low_confidence` `True` for such a sample, so the report's low-confidence line shows. The ancestry table and the percentiles do not change. Unit tests check that `og prs table` reports the sample's pgsc_calc percentile at the table's precision. The reported percentiles for 1,000 simulated people pass a uniformity check when the reference is from the same group. The check fails when the simulated reference mean shifts by half a standard deviation. Part of #411 (lin-44, lin-45). +- Step 42's traits readings no longer say more than their studies found. A genotype with no -13910T read "lactase activity usually falls after childhood", which a reader could take as proof of lactose intolerance; it now says it does not prove that, and the lactase notes say the rows hold the main known persistence alleles, not every one. A new row reads the Ethiopian persistence allele -14009G (rs869051967, Jones et al. 2013). The weaker -22018 marker no longer counts as a persistence allele, and its readings say to use it only when the -13910 row has no genotype. Other readings changed to match their evidence. ALDH2 no longer says acetaldehyde is "cleared normally". One MC1R copy no longer predicts poor tanning, and the MC1R rows cite Flanagan et al. 2000 and say their findings are those of the people studied. The APOE rows no longer say one site "defines" e4 or e2, and they name the rare e1. FUT2 says other variants can still make a non-secretor and that the allele's frequency differs between populations. The vitamin D row compares its Framingham means with supplements and season, as its study did. UGT1A1 no longer calls one copy a carrier, and names rs887829 as the *80 tag of *28. Two CYP2A6*2 site copies now read as *2/*2 or *2 with a deleted copy (*4), which the row cannot tell apart. The FUT2 and UGT1A1 rows are now marked `ancestry` yes. In the genotype rules, the GC readings name the protein type only: they drop the protein levels of Powe 2013, whose monoclonal assay reads the types differently (Denburg et al. 2016). The APOE note says its odds ratios come from clinic and autopsy studies, with their age and sex limits, and are not a lifetime risk; the APOE line's trait now reads "APOE alleles (e2, e3, e4, and the rare e1)". The HFE note gives how many people with a risk genotype became ill in later cohorts (Allen 2008, Gurrin 2009, Beutler 2002). The traits table has 37 rows, one more than before; its columns are unchanged. The step 42 docs and the output contract page say that the genotype-rules table's APOL1 `result` never shows G2, and point scripts at `summary.json` for the result with G2. The `og dev traits-sources` tests no longer build a 135 Mb chr2 to check one base; they use a 150-base contig. Part of #411 (trait-08, trait-10 to trait-13, trait-17 to trait-21, trait-24, trait-27, trait-68, trait-69). - The live site's family page and its pedigree drawing are now built by `og live family`, in Rust. `bin/owngenome_live_family.py` asks `og` for a family's state, its `family.json`, the list of files it serves and `pedigree.svg`, and no longer reads the family's tables itself; `bin/pedigree_svg.py` and `bin/family_report.py` are gone. The page shows the same thing: `family.json` and the drawing were compared with the old code on the demo family and the test families, field by field and number by number, and matched. The live site therefore needs `og` (`scripts/setup.sh --og` installs it); without it a family's `family.json` answers 500, its `pedigree.svg` answers 503, and the run list marks each family unreadable. A family file that cannot be read still gives no page and an unreadable family. - `setup.sh --ancestry-fine-panel` no longer runs plink2 and a supervised ADMIXTURE at install to fit each reference group's allele frequencies. When every reference person's group is given, the frequencies ADMIXTURE climbs to are each group's own allele frequencies, so the new `og ancestry-fine freqs` counts them exactly from the LD-pruned VCF, clamped to ADMIXTURE's bounds (1e-5 and 1 - 1e-5). On a reduced panel of 1,954 SNVs and 2,866 reference people, no frequency differs from ADMIXTURE's by more than 0.0004, the columns come in the same order, `ref.bim` and `ref.fam` are byte for byte what plink2 wrote, and the fit takes 3 seconds. Step 26b still projects your genome with `admixture -P` on these frequencies, so its results do not change. `model/` keeps `ref.bim`, `ref.fam`, `continent.P`, `region.P`, `levels.tsv` and `paint_map.tsv` in the same formats. It gains `ref.geno`, the reference people's genotypes for the projection's bootstrap, and loses `ref.bed`, which nothing read. `og ancestry-fine freqs --exclude FILE` leaves people out of the fit. `og ancestry-fine order`, which read the column order from ADMIXTURE's `.Q`, is gone: the order is now the order in which the groups first appear in the VCF, as ADMIXTURE numbers them. The next `--ancestry-fine-panel` run fits the model again in seconds and keeps the panel's genotypes. - Step 40 (the 23andMe and AncestryDNA export) no longer writes a weak reference call as a genotype. A reference block or a `0/0` record with `GQ` under 20, or with a filter other than `PASS`, `.` or `RefCall`, was written as two reference bases, so a block with no reads behind it read as a confident `AA`. Such a site is now `--` (`0 0`), and the summary counts it under a new reason, `no_call: weak reference call (GQ below 20, or filtered)`. A call without `GQ` and every variant call read as before. A gVCF of several samples is now read only for the column `--sample` names, and refused when none or two match; before, the step read the first column, which could be another person's. A gVCF of one sample is read as before. No option or file name changes. Part of #411 (plat-66, plat-67, plat-70, D10). +- A live page that cannot load its data now says so, with a Try again button. A person page whose `results.json` failed read as a finished report with no results. A home page whose `runs.json` failed, or got no answer, showed only the server's plain list of runs. A family page whose `family.json` held no family (`{}`) stayed blank. Each page now names what could not be loaded, and Try again asks for it again. A person page that cannot read its run's state, or reads a state with no plugin data, stays closed as before, but its heading reads "Results not loaded" instead of "Results about a child", it says it could not read the run's state and keeps its results closed, and it names no role. Before the run's first state arrives, the wire, run health and re-check panels read "Waiting for the run's data." instead of saying that there is nothing yet. - The local genome viewer runs on `og` and no longer needs `python3` on the host. `scripts/owngenome-view.sh` starts `og view serve` (through `bin/og`, or `OG`), which serves the same page, config and byte ranges on 127.0.0.1 as `assets/viewer/serve.py` did. Two requests it answered before are now refused. A locus whose end comes before its start (`chr1:13000-12000`) stops the script with exit 2 and the reason, instead of opening a view igv.js cannot show. A request with a `Forwarded`, `X-Forwarded-*` or `X-Real-IP` header gets 403: the viewer has no password, so a reverse proxy in front of it would hand the sample to anyone who reaches the proxy; an ssh port forward still works. The default SV track is now Manta's diploid SVs (step 4), else step 22's consensus, which keeps only the calls two callers agree on. `docs/viewer.md` says that every account on the machine can reach 127.0.0.1. No option of the script, file or output name changes; `og view serve` takes the flags `serve.py` took, plus `--assets`, the page's folder. The live site's viewer also opens Manta's diploid SVs first; the rest of it is unchanged until its server moves to `og`. Part of #411 (plat-11, plat-12). - Step 22's SV consensus no longer counts a record a caller did not call as a second caller. Before, every PASS record went to SURVIVOR, so a PASS Delly record genotyped `0/0`, a Delly call whose sample filter (`FT`) says `LowQual`, or a CNVpytor segment no quality rule had read sat beside a Manta-only call and made it read "2 callers". A new verb, `og sv survivor-prep --caller NAME IN OUT`, now writes each caller's input for both the script and the Nextflow `SURVIVOR_PREP`, in place of the bcftools and awk they each carried. A record votes when its FILTER and its sample `FT` are `PASS` or `.` and its genotype is not reference only (`0/0`, `0|0`, haploid `0`, `0/.`); a no-call (`./.`) and a sites-only record still vote. A CNVpytor call votes when its e-value is below 0.01 and its share of MAPQ 0 reads below 0.5, the rule step 05 already applies when it adds CNVpytor's calls; the VCF step 22 makes from CNVpytor's table now carries the table's e-value, `q0` and `pN` as `pytorP1`, `pytorQ0` and `pytorPN`, so the rule reads both inputs. A header without `GT` or without these tags no longer matters. Step 18's closing hint and `docs/18-cnvpytor.md` gave e-value below 0.05 and size over 1 kb; they now give the same rule. What changes in the output: `sv_merged/inputs/.vcf` holds only the records that vote, and the consensus can hold fewer records. No file, column or option is renamed or removed. Part of #411 (sv-05, sv-06, sv-09, sv-10, sv-11, sv-40). +- A class 4 or 5 SV gets support from more than one caller only from the step 22 consensus event that lists its own record: same contig and SV type, its caller voting in `SUPP_VEC`, and its original record ID in that caller's `FORMAT/ID`. Before, the nearest consensus record within 1,000 bp lent its callers, whatever its type or filter, so a filtered call, a call of the other type or an unrelated call nearby made a Manta-only call read "2 callers". A row that no event lists shows as single caller, when the consensus names every vote of its caller. When the consensus cannot tell, the support is now "not known": `Support` and `Support_callers` are empty in `_sv_class4_5_checked.tsv`, and `support` is `null` in `summary.json`. That happens when the row has no record ID, its caller is not a consensus column (`sv_vcf` rows of an `SV_VCF` run), its ID is in two events, or a vote of its caller has no readable ID (CNVpytor calls when step 22 made CNVpytor's VCF from its table). `og sv add-callers` now keeps Delly's `RDCN` on each Delly call it adds (`FORMAT GT:RDCN` and the `##FORMAT=` header line), so `Depth_support` reads the depth that step checked: `yes` for a call over 1,000 bp it kept. A Delly call over 1,000 bp with no `RDCN`, from a VCF not written by Delly 2.x, now reads `unknown` instead of `yes` and keeps its class; step 05's log line counts it apart from `not checked`. No column, option or file name changes. Part of #411 (sv-01, sv-08, sv-03) and follow-up to #542. - The callability section of `summary.json` now says, for each gene a step of its own calls, whether that step ran. A gene such as SMN1, CYP2D6, HBA1, LPA or RHD that step 16b did not read is listed under `not_read_owned` with its owner, and it read as handed to that step even when the step never ran for the sample, so its coverage stood alone as the answer. Each entry gains two fields: `owner_step` (`35`, `32`, `46` or `41`) and `owner_status` (`ran`; `not run` when that step wrote no file of its own for the gene in the sample folder, a Parascopy gene being judged by its own table in `assets/paralog_loci.tsv`; or `from an earlier run` when that file is older than the last run and the step was not `ok` in it, the rule that makes a section stale). Fields are only added; none is renamed or removed, and the reports do not show them yet. The share keeps its four decimals (0.8988 stays 0.8988). Part of #411 (clin-25). - Step 10 (telomere content) always reads the GRCh38 chromosome bands now. When `reference/cytoBand.hg38.txt` was missing, `scripts/10-telomere-hunter.sh` warned and ran TelomereHunter without `-b`, so it sorted reads by its own hg19 band ends on GRCh38 positions; the Nextflow module did the same whenever `--cytoband` was not set, which is the default. The script now installs the copy the repository ships (`assets/reference/cytoBand.hg38.txt`) before it runs, and the pipeline passes that copy when `--cytoband` is not set. The step also moves from TelomereHunter 1.1.0, Python 2.7 code, to its maintained successor TelomereHunter2 1.0.12 (`TELOMEREHUNTER_IMAGE`, `quay.io/biocontainers/telomerehunter2`). On the image test's input both give the same `tel_content` (5347.255643 with 1,100 planted telomeric reads, 5337.585253 with 1,098), so earlier values stay comparable. THREADS now sets its worker count (`-c`), and the container gets a higher open-file limit, which it needs on a BAM with many contigs. Output change: `telomere///_summary.tsv` keeps its name, but its columns follow TelomereHunter2 (`tel_content` third, `intratelomeric_reads`, `tel_read_count`, `read_lengths`, plus three read-class counts). `summary.json` and the reports read both layouts by name and do not change. The docs no longer describe the value as an age measure: `tests/test_telomere_wording.sh` fails when a file that describes step 10 does. Part of #411 (trait-46, trait-47, trait-48, trait-49). - Step 25 and the Nextflow `PRS_SUMMARY` withhold a score's percentile when the genotypes pgsc_calc scored list variants only. Every site such a VCF leaves out adds nothing to the sum, so the sum leans away from the panel's sums, which come from complete genotypes, and the percentile was not comparable. Each score keeps its raw sum. Its `Percentile` is `NA`, its `Effect` is empty, and a new column before `Input`, `Percentile_Note`, says `percentile withheld: the input lists variants only`. The decision reads the genotypes, not the kind of input: at least one genotyped position in five must be a reference call (0/0), the rule step 26b applies to the panel's sites. Genotypes from a gVCF and chip VCFs from `scripts/chip-to-vcf.sh`, which list their reference calls, keep their percentiles. `og prs table` takes the genotypes with a new option, `--target VCF`, and prints the NOTE that step 25 printed before; the NOTE now follows the genotypes too, so a gVCF without reference blocks at the score positions gets it. `og summary collect` passes the note on as `percentile_note`. When every percentile was withheld, both reports say "Percentile withheld: the input lists variants only" and why, and the summary's `not_assessed` list says the percentiles were withheld, in place of the lines for a run without the panel, which told the user to install a panel the run had used. The output gains one column; no column, option or file name changes. @@ -22,6 +38,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - Lint's ShellCheck job now checks every tracked shell script, not only `scripts/`: the e2e cases and their library, the fake-docker and parity tests, `tests/*.sh`, `bin/og` and the fake tools, 482 files in all, at every ShellCheck level. The 265 findings these files hold today are listed in `tests/shellcheck-baseline.txt` and still pass; any other finding fails lint. A listed finding is matched by its file, its code and the text of its line, so it keeps matching when lines above it move, and a new line with the same mistake still fails. Each run plants an unquoted `$1` in a new test script and on a new line of a listed script, and fails if either passes or if the file search misses the new script or `bin/og`. The `scripts/` check at warning level stays as it was. Part of #411 (plat-10). - `setup.sh` no longer says "Setup complete!" after a download failed. Before, a failed or skipped install (AnnotSV's data, a VEP cache, the CPSR bundle, somalier's sites, the STRchive catalog, the tandem-repeat map, gnomAD's chrM sites, a pinned data file, the sequence dictionary or og) printed a warning, then the same banner and exit status 0 as a full setup. Now setup keeps a list of what it could not install. When the list is empty it prints "Setup complete!" and exits 0 as before. Otherwise it prints "Setup finished: core ready, N items missing", lists each item with the command that retries it (`--annotsv-data`, `--vep-cache`, `--clinical-data`, `--sample-qc-data`, `--strchive-data`, `--og`, or setup again), and exits 3. A script that runs `setup.sh` and checks its status now sees a partial setup as one. `--no-clinical-data` still ends complete: skipping the clinical data is a choice. [Getting started](docs/getting-started.md) lists the three exit statuses. - `og lookup` no longer trusts an index older than its VCF for a chromosome the index does not list. A gVCF or annotated VCF replaced by hand after its `.tbi` or `.csi` was built kept the old index, and a chromosome the new file added was missing from it, so a variant called there read as "no call". The lookup now compares the two modification times, as the blood-type step does. When the index is older and does not list the queried chromosome, it reads the whole file and prints `NOTE: the index ... is older than ... and does not list chrN; reading the whole file`. An older index that lists the chromosome is still read: in a Nextflow output directory the small index is often copied before its large VCF, so it carries the earlier time. A fresh index is used as before, and no answer or output file changes. +- A genotype with a negative allele number (`GT -1/-1`) no longer reads as the last alternate allele. VCF numbers alleles from 0, the reference, upward; step 40 (the 23andMe and AncestryDNA export) and `og lookup` counted a negative number from the end of the allele list, as the Python they replace did. With `REF A` and `ALT T`, step 40 wrote `TT` and `og lookup` answered `T/T`. Step 40 now writes `--` (`0 0`) and counts the site under `no_call: unreadable genotype` (one whose `-1` fell on `<*>` was counted under `symbolic allele`, and such a record is no longer counted in `reference_checked`). Such a record also no longer hides the reference block after it as if it were a called deletion. `og lookup` now reads any negative number as a no-call (`.`), as it already read a number past the last allele; a number below minus the allele count used to stop it with an error. Genotypes with no negative number read as before. Every other reader now treats any index outside the allele list the same way, past the last allele included. In step 40, an index past the last allele no longer hides the reference block after a longer REF; a part that is not a number still does. `og lookup` counts a record under "bases have a genotype call" only when some part of its GT names an allele. Fine-grained ancestry drops a SNV whose GT names no allele instead of reading `-1` as the last ALT, and the traits phase reader skips such a record, so that site reads as phase unknown. No option, column or file name changes. - `validate-setup.sh` now checks the bases of the reference, not only its names and lengths. It reads the MD5 of each sequence (`M5`) from the `.dict` setup makes from the FASTA (it does not read the FASTA itself) and compares the 195 names, lengths and MD5s with values pinned for NCBI's no-ALT analysis set and for its masked copy. A FASTA whose `.dict` shows the same names and lengths and one other base (one changed base of chr21, in the test) used to pass; it now fails. A masked FASTA without `_maskedGRC` in its name fails too, as does a `.dict` that does not list the `.fai`'s sequences; a `.dict` older than the FASTA gets a warning (so does a FASTA changed after its `.dict` was made), and a reference with other names or lengths gets an `[INFO]` line. When a BAM's `@SQ` lines carry `M5` (Picard, GATK and many vendors write it; the pipeline's own aligners do not), each one is compared with the `.dict`, and a BAM aligned to other bases under the same names fails, naming the first sequence that differs; when they all match, the masked-reference check trusts them over the file names in the `@PG` lines, unless those lines name both references or the `.dict` is older than the FASTA. New `[OK]`, `[WARN]`, `[FAIL]` and `[INFO]` lines; no option or file changes. Part of #411 (plat-52). - `setup.sh` checks somalier's sites and VerifyBamID2's panel (step 33) again when they are already there. It reported any file of that name as present; one with another sha256 now gets a warning and is downloaded again, and offline the install fails instead. Part of #411 (plat-72). - The ALT depth A/B (`scripts/ci/alt-depth-ab.sh`, `docs/realignment.md`) now says that it maps with a split minimap2 index, not step 02's single index, and that this can overstate the loss where an ALT copy differs a little from the primary. The script comment said the split gave the MAPQ of one index. Docs and comments only. Part of #411 (plat-53). @@ -237,6 +254,7 @@ Each release's notes. From 1.0.0 on, a major or minor release has hand-written n - The walk of the sample folder leaves out `/nextflow`. `run-all.sh` launches Nextflow there, and its `work/` holds every task's files and staged links. No section reads a file there. With 200,000 files under it, the collect took 67 MB walked and 14 MB skipped. - The sections other than the sample VCF are read one at a time, while the sample VCF is read on its own thread, so the peak memory is near the biggest section's and not the sum of all of them. With every side table 50 times its demo size, the peak footprint fell from 119-130 MB to 105-112 MB (the Python: 100-102 MB), with the same CPU time. - Below 50 kb, the genome map offers the run's reads in an inline viewer. R, See the reads, a finding's map card, and the Reads gauge open the panel. Moving the map moves the viewer. Back follows the map's history, and resizing the viewer keeps its requested locus until you move inside the viewer. Esc closes it and returns focus to the map. Closing the panel clears its previous locus, title, and link to the full viewer. Closed findings have no reads button. Only the live viewer allows same-origin framing. Other routes keep their framing protection, and the standalone viewer still blocks frames. Alignments stay visible through a 50 kb window. +- The genome viewer's page is dark like the rest of OwnGenome, on the local viewer and on the live site. A heading names the sample, and a line says what the tracks are, that the search box takes coordinates and not gene names, and that reads show at a few kilobases or less. igv.js keeps its white tracks in a framed panel, which scrolls sideways on a phone instead of widening the page. When the viewer cannot start, the message is announced to screen readers, says what to check, and has a Try again button. - Step 41 (blood groups) reads every VCF record that covers a marker site, and their order no longer changes the answer. Before, the first record at a position won. An all-sites VCF writes a reference record and then the indel at one position (`T . 0/0`, then `T TC 0/1` at ABO c.261), so a person with an A or B allele read O. A split multi-allelic record with the SNV first did the same. No column, file or option changes; only the values of the affected sites do. What changes: - A call of the marker's ALT now wins over the other records at the position. Another allele a second record calls there is named in the note, as when one record holds both. - Distinct ALTs listed by one null marker add their copies across split records. JK_01N06 and KEL_02N01 now give the same evidence and null/null phenotype before and after splitting. Repeated calls of the same ALT count once. Conflicting dosages for one ALT, or totals above two copies, give a no call with an incompatible-dosage note. diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..6999eba11 --- /dev/null +++ b/NOTICE @@ -0,0 +1,786 @@ +OwnGenome +========= + +OwnGenome is free software under the GNU General Public License, version 3 +or (at your option) any later version (GPL-3.0-or-later). The full text is +in LICENSE. + +Third-party data +---------------- + +The repository ships some data that comes, in whole or in part, from third +parties. Each file below names its source in its own header, or in the file +named beside it. assets/SOURCES.tsv records the source, version, licence and +header of every data file the repository ships; `og dev licences check`, run +by the Lint workflow, fails when a file under assets/ has no row there, when +a file lacks the header its row names, or when this file does not name a +third-party row. The licence texts are in +assets/licences/ and in the sections at the end of this notice. + +assets/acmg_sf.tsv + ACMG SF v3.3 secondary-findings genes: Lee et al., Genet Med + 2025;27(8):101454, doi:10.1016/j.gim.2025.101454, Table 1. The gene list + is facts; the table_text column holds short cited quotations of a + factual reporting rule. + +assets/bloodtype_markers.tsv + Blood-group variants as packaged by RBCeq2 2.4.4 (database v2.5.1), + Copyright 2025 Australian Red Cross Lifeblood, MIT licence + (assets/licences/RBCeq2-LICENSE.txt). Names follow the ISBT nomenclature. + +assets/callability_genes.tsv + NIST Genome in a Bottle CMRG v1.00 and GRCh38 stratifications v3.6, not + subject to US copyright (assets/licences/NIST-GIAB-NOTICE.txt); + GENCODE 50, open access. + +assets/carrier_genes.tsv + Gene spans from GENCODE release 50, open access. Gene lists from Gregg et + al., Genet Med 2021, ACOG Committee Opinions 690 and 691, and Orphadata + (en_product6 and en_product9_ages of 2026-06-23), CC BY 4.0, + https://www.orphadata.com. + +assets/chip_genes_grch38.bed + Gene records of GENCODE release 50, open access, no restriction on use. + +assets/licences/ + The licence texts of the sources in this list, each under its own terms. + +assets/locityper_loci.bed + Timofey Prodanov, Locityper loci database, + https://doi.org/10.5281/zenodo.15879519, CC BY 4.0. Changed: six loci + kept, two joined. + +assets/ownhealth/genome-facts.v1.json +assets/ownhealth/sensitive_genes.tsv + Copied byte for byte from OwnHealth, https://github.com/GeiserX/OwnHealth, + under AGPL-3.0-or-later. The licence text is below. + assets/ownhealth/VENDORED.tsv records the commit and SHA-256 of each file. + +assets/par_grch38.bed + The GRCh38 pseudoautosomal regions of the Genome Reference Consortium, + https://www.ncbi.nlm.nih.gov/grc/human (docs/03-variant-calling.md). The + file has no header because its readers take every line as an interval. + +assets/parascopy_models_v1.7.txt + Timofey Prodanov, Parascopy precomputed data, + https://doi.org/10.5281/zenodo.15019940, CC BY 4.0. Changed: only the + model regions are kept, one per line, overlapping copies merged. + +assets/reference/cytoBand.hg38.txt + UCSC Genome Browser chromosome bands for hg38, freely usable for any + purpose. Credit: the UCSC Genome Browser Group, University of + California, Santa Cruz (assets/reference/README.md). + +assets/sv_rule_regions.tsv + TRExplorer catalog, Copyright (c) 2024 Broad Institute, MIT licence + (assets/licences/TRExplorer-LICENSE.txt); GENCODE, open access; HGNC and + NCBI assembly regions, no restriction on use. + +assets/ystr_panel.tsv + The marker order, names and panels of FamilyTreeDNA's Y-111 test, + reproduced only so that counts line up with FamilyTreeDNA results; no + FamilyTreeDNA text is copied. FamilyTreeDNA is a trademark of Gene by + Gene, Ltd., which does not endorse this table. Regions from + HipSTR-references under the MIT licence (text below). Motifs are from + NIST STRBase. + +tests/fixtures/consumer_export/sites.tsv.gz + 23andMe v5 site list from 23andme2vcf, Copyright (c) 2015 Rob Long, MIT + licence (assets/licences/23andme2vcf-LICENSE.txt); NCBI dbSNP b157, no + restriction on use (tests/fixtures/consumer_export/README.md). + +The og binaries +--------------- + +The og binaries also link third-party code. Each release ships their notices +in THIRD-PARTY-LICENSES.txt, written by `og dev licences notices`. + +HipSTR-references MIT licence +----------------------------- + +MIT License + +Copyright (c) 2017 HipSTR-Tool + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +OwnHealth AGPL licence +---------------------- + + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/README.md b/README.md index e2e53b432..56dc22b30 100644 --- a/README.md +++ b/README.md @@ -42,23 +42,38 @@ This pipeline takes raw sequencing data (FASTQ/BAM/VCF) from any vendor and runs ## Quick start +### Before you begin + +The machine needs these before the first command: + +- **Docker** 20.10 or later, running. On a Mac or on Windows that is Docker Desktop. +- **bash** 4.4 or later. Linux has it. A Mac ships bash 3.2, so install a newer one with Homebrew. +- **Java** 17 or later and **Nextflow**, the release that `NEXTFLOW_VERSION` in `versions.env` pins. `run-all.sh` starts the Nextflow pipeline, so a full run needs both. +- **RAM:** 16 GB at least, 32 GB recommended. A 16 GB machine needs swap for step 25 ([step 25, Memory](https://geiserx.github.io/OwnGenome/25-prs/#memory)). +- **Disk:** 500 GB free per sample from a BAM, about 700 GB from FASTQ, on the disk that holds `GENOME_DIR` ([disk](https://geiserx.github.io/OwnGenome/hardware-requirements/#disk-space-breakdown)). + +[Install the requirements](https://geiserx.github.io/OwnGenome/getting-started/#install-the-requirements) has the commands in order for Linux, an Intel Mac, an Apple Silicon Mac and Windows with WSL2, and one check that all four are in place. + +### Run + Put the reads at `$GENOME_DIR//fastq/_R1.fastq.gz` and `_R2.fastq.gz` (a BAM or a VCF works too), then: ```bash -git clone https://github.com/GeiserX/OwnGenome.git && cd OwnGenome # main; `git checkout ` for a release +git clone https://github.com/GeiserX/OwnGenome.git && cd OwnGenome +git checkout "$(git tag --list 'v[0-9]*' --sort=-v:refname | head -n 1)" # the newest release export GENOME_DIR=/path/to/your/data SAMPLE=your_name ./scripts/setup.sh $GENOME_DIR # the GRCh38 reference, ClinVar, the images and og, once ./scripts/validate-setup.sh $SAMPLE ./scripts/run-all.sh $SAMPLE male # or female ``` -`run-all.sh` starts the Nextflow pipeline, so it needs Java 17 or later and the Nextflow release `versions.env` pins (`NEXTFLOW_VERSION`) beside Docker and bash 4.4; a rerun reuses every finished step. [Full run](https://geiserx.github.io/OwnGenome/getting-started/#full-run) has the install line and the switches, and each step also runs on its own as a script. Optional databases are in [reference setup](https://geiserx.github.io/OwnGenome/00-reference-setup/). Try a small public dataset with the [quick test](https://geiserx.github.io/OwnGenome/quick-test/). +On a release tag, `setup.sh` downloads that release's `og` binaries. On `main` it builds `og` in a container instead, which takes longer. A rerun of `run-all.sh` reuses every finished step. [Full run](https://geiserx.github.io/OwnGenome/getting-started/#full-run) has the switches, and each step also runs on its own as a script. Optional databases are in [reference setup](https://geiserx.github.io/OwnGenome/00-reference-setup/). Try a small public dataset with the [quick test](https://geiserx.github.io/OwnGenome/quick-test/). ## Documentation The full documentation is at **https://geiserx.github.io/OwnGenome/**, one page per pipeline step included. -- [Getting started](https://geiserx.github.io/OwnGenome/getting-started/): prerequisites, platform notes (macOS, WSL2, Unraid), the full run with `run-all.sh` and its switches, the FASTQ, BAM, VCF and ORA entry paths step by step, directory layout +- [Getting started](https://geiserx.github.io/OwnGenome/getting-started/): prerequisites, the install steps for each platform, platform notes (macOS, WSL2, Unraid), the full run with `run-all.sh` and its switches, the FASTQ, BAM, VCF and ORA entry paths step by step, directory layout - [Quick test](https://geiserx.github.io/OwnGenome/quick-test/): verify the setup on public data before your own - [Hardware and storage requirements](https://geiserx.github.io/OwnGenome/hardware-requirements/): download sizes, per-step runtime, memory and disk figures - [Reference data setup](https://geiserx.github.io/OwnGenome/00-reference-setup/): the GRCh38 reference (the no-ALT analysis set) and every database @@ -87,3 +102,5 @@ Your genome data is sensitive personal information. This pipeline runs locally a ## License [GPL-3.0-or-later](LICENSE). OwnGenome is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. Contributions come in under the same terms ([CONTRIBUTING.md](CONTRIBUTING.md#license)). + +Some data files the repository ships come from third parties, under their own terms. [NOTICE](NOTICE) lists them, and [assets/SOURCES.tsv](assets/SOURCES.tsv) records the source, version and licence of every data file under `assets/`. The Lint check fails on a data file with no row there. diff --git a/ROADMAP.md b/ROADMAP.md index 4bac3dd4c..65f9c7c56 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -104,7 +104,7 @@ The next release, and the first stable one. From 1.0.0 on, options, config keys, - [x] **GLNexus joint genotyping** — step 38 joint-calls the family's DeepVariant gVCFs with GLnexus ([docs/multi-sample.md](docs/multi-sample.md#family-analysis-step-38-opt-in)) - [x] **Trio analysis support** — step 38 checks the declared relationships with somalier, phases each child of a complete trio by inheritance (WhatsHap), and writes de novo candidates when both parents are sequenced and compound heterozygotes in trans or in cis ([docs/multi-sample.md](docs/multi-sample.md#family-analysis-step-38-opt-in)) - [x] **Interactive HTML dashboard** — step 24's single-page report: seven chapters with a table of contents, charts, a closing list of every gene and rsID the results name, and one section printable on its own; the live site adds a search box over the same list ([docs/24-html-report.md](docs/24-html-report.md)) -- [x] **Database refresh** — `setup.sh --refresh clinvar` fetches the latest ClinVar in place, a monthly issue lists which databases and images have a newer release, and `bin/owngenome_recheck.py` says which results a newer ClinVar or catalog would change ([docs/updating.md](docs/updating.md), [docs/recheck.md](docs/recheck.md)) +- [x] **Database refresh** — `setup.sh --refresh clinvar` fetches the latest ClinVar in place, a monthly issue lists which databases and images have a newer release, and `bin/og recheck` says which results a newer ClinVar or catalog would change ([docs/updating.md](docs/updating.md), [docs/recheck.md](docs/recheck.md)) - [x] **Progress dashboard** — a live web page, not a terminal UI: the step map, the time left as a range, and alerts before a task runs out of memory or disk ([docs/live-site.md](docs/live-site.md), [docs/run-state.md](docs/run-state.md), [docs/alerts.md](docs/alerts.md)) ## Planned — Multi-sample & joint analysis diff --git a/assets/SOURCES.tsv b/assets/SOURCES.tsv new file mode 100644 index 000000000..738a4e212 --- /dev/null +++ b/assets/SOURCES.tsv @@ -0,0 +1,65 @@ +# assets/SOURCES.tsv: where every data file the repository ships comes from, and under which terms. +# `og dev licences check` (rust/crates/og-dev/src/licences.rs, run by the Lint workflow) fails when a +# file under assets/ has no row, a file is in two rows, a row matches no file, a header phrase is not in +# its file, or NOTICE does not name a third-party row. A change that adds a file under assets/ adds its +# row here in the same pull request; one that adds third-party data also names it in NOTICE. +# Columns, tab separated, none empty: +# path the file, relative to the repository root; a folder when it ends with / (every file below). +# Every file under assets/ needs a row; a row elsewhere (a test fixture) is checked the same +# way, but files outside assets/ need none. +# source where the data comes from: who made it and where it was taken. A source that starts with +# "OwnGenome" is our own work (written here, or curated here from facts the source names); +# any other row is a third party's, and NOTICE must name its path. +# version the release, commit or date the data was taken at; - for our own files. +# licence the terms the data comes under: an SPDX id, or the source's own words for data with no +# licence (public domain, open access, short quotations). Never -, unknown or empty. +# header a phrase the file's opening # lines must carry word for word (line breaks and runs of +# blanks count as one space), several split by |; - when the file carries no header +# (a binary file, JSON, a format whose readers take every line, a byte-for-byte copy). +path source version licence header +assets/acmg_sf.tsv Lee et al., ACMG SF v3.3 list for reporting of secondary findings, Genet Med 2025;27(8):101454 (doi:10.1016/j.gim.2025.101454), Table 1; HGNC ids; Ensembl GRCh38 positions ACMG SF v3.3, read 2026-10-27 gene list: facts; table_text: short cited quotations of a factual reporting rule Source: Lee et al., ACMG SF v3.3 | short cited quotations of a factual reporting rule +assets/bloodtype_markers.tsv RBCeq2 2.4.4 (database v2.5.1), Australian Red Cross Lifeblood; ISBT nomenclature; dbSNP and Ensembl GRCh38 positions RBCeq2 2.4.4 MIT (assets/licences/RBCeq2-LICENSE.txt); dbSNP and Ensembl: no restriction on use used under RBCeq2's MIT licence, Copyright 2025 Australian Red Cross Lifeblood +assets/callability_genes.tsv NIST Genome in a Bottle CMRG v1.00 and GRCh38 genome stratifications v3.6; GENCODE 50; HGNC; NCBI GRCh38 assembly report CMRG v1.00, stratifications v3.6, GENCODE 50 NIST data: not subject to US copyright (assets/licences/NIST-GIAB-NOTICE.txt); GENCODE: open access Sources and terms: GIAB CMRG v1.00 and genome stratifications v3.6, from NIST, not subject to US copyright +assets/carrier_genes.tsv GENCODE release 50 gene spans; Gregg et al., Genet Med 2021 (ACMG tier 3 gene tables); ACOG Committee Opinions 690 and 691; Orphadata en_product6 and en_product9_ages GENCODE 50; Orphadata 2026-06-23 Orphadata: CC-BY-4.0; GENCODE: open access; gene lists: facts GENCODE release 50 | Orphadata en_product6 and en_product9_ages of 2026-06-23, CC BY 4.0 +assets/chip_genes.tsv OwnGenome, curated from the papers in the pmids column; Ensembl MANE Select codons and exons - GPL-3.0-or-later Hand-curated from the papers in the pmids column +assets/chip_genes_grch38.bed GENCODE release 50 basic annotation gene records, https://www.gencodegenes.org/human/release_50.html GENCODE 50 open access, no restriction on use (EMBL-EBI terms of use) Source: GENCODE release 50 +assets/construct_variants.tsv OwnGenome, curated from ClinVar (canonical SPDI, E-utilities) and the papers it cites ClinVar queried 2026-11-04 GPL-3.0-or-later ClinVar's alleles, from its canonical SPDI +assets/deep_intronic_sites.tsv OwnGenome, curated from ClinVar (canonical SPDI, E-utilities) and the papers it cites ClinVar queried 2026-11-04 GPL-3.0-or-later ClinVar's alleles on the plus strand, from its canonical SPDI +assets/everything.tsv OwnGenome - GPL-3.0-or-later - +assets/families.example.tsv OwnGenome, an example with made-up names - GPL-3.0-or-later - +assets/genotype_rules.tsv OwnGenome, curated from the primary papers in the pmids column - GPL-3.0-or-later Hand-curated from the primary papers in the pmids column; no text is taken from SNPedia +assets/imprinted_regions.tsv OwnGenome, curated from the papers it cites - GPL-3.0-or-later Factual reference rows, curated by OwnGenome +assets/licences/ the licensors each licence text names as each text says each text's own terms - +assets/live/ OwnGenome - GPL-3.0-or-later - +assets/locityper_loci.bed Timofey Prodanov, Locityper loci database, https://doi.org/10.5281/zenodo.15879519 Zenodo 15879519 CC-BY-4.0 Source: Timofey Prodanov, Locityper loci database | licensed CC BY 4.0 +assets/mei_sites.tsv OwnGenome, curated from ClinVar, PubMed and Ensembl checked 2026-10-27 GPL-3.0-or-later Checked 2026-10-27 against ClinVar, PubMed and Ensembl +assets/mito_confirmed.tsv OwnGenome, written by scripts/ci/build-mito-confirmed.sh from ClinVar and the primary papers - GPL-3.0-or-later from ClinVar (variation id, gene, HGVS) and the primary papers +assets/monitoring/ OwnGenome - GPL-3.0-or-later - +assets/output_contract.tsv OwnGenome - GPL-3.0-or-later - +assets/ownhealth/VENDORED.tsv OwnGenome - GPL-3.0-or-later - +assets/ownhealth/cpic_genes.tsv OwnGenome, from the CPIC API guideline list queried 2026-11-17 GPL-3.0-or-later from the CPIC API +assets/ownhealth/genome-facts.v1.json OwnHealth, https://github.com/GeiserX/OwnHealth, data/genome-facts/genome-facts.v1.json, copied byte for byte (assets/ownhealth/VENDORED.tsv) commit 117071ef325cd4e3c0bc3f6ade0aeef580a3b3aa AGPL-3.0-or-later (OwnHealth's licence) - +assets/ownhealth/grch38_refget.tsv OwnGenome, written by scripts/ci/build-refget-table.sh from NCBI's RefSeq GRCh38 chromosome FASTA GCF_000001405.26 GPL-3.0-or-later NCBI's RefSeq GRCh38 chromosome FASTA (GCF_000001405.26) +assets/ownhealth/sensitive_genes.tsv OwnHealth, https://github.com/GeiserX/OwnHealth, data/genome/sensitive_genes.tsv, copied byte for byte (assets/ownhealth/VENDORED.tsv) commit 117071ef325cd4e3c0bc3f6ade0aeef580a3b3aa AGPL-3.0-or-later (OwnHealth's licence) - +assets/par_grch38.bed Genome Reference Consortium, GRCh38 PAR1 and PAR2 coordinates, https://www.ncbi.nlm.nih.gov/grc/human (docs/03-variant-calling.md) GRCh38 facts (coordinates), no restriction on use - +assets/paralog_loci.tsv OwnGenome - GPL-3.0-or-later - +assets/paraphase_genes.tsv OwnGenome - GPL-3.0-or-later - +assets/parascopy_models_v1.7.txt Timofey Prodanov, Parascopy precomputed data, https://doi.org/10.5281/zenodo.15019940 1.7 CC-BY-4.0 Source: Timofey Prodanov, Parascopy precomputed data | licensed CC BY 4.0 +assets/pedigree.example.tsv OwnGenome, an example with made-up names - GPL-3.0-or-later - +assets/pgs_scores.tsv OwnGenome, a list of PGS Catalog score ids with the trait and evaluations the catalog records - GPL-3.0-or-later The PGS Catalog scores +assets/pgx_contraindications.tsv OwnGenome, curated from the CPIC API and FDA drug labels (DailyMed) CPIC API queried 2026-11-03 GPL-3.0-or-later CPIC API pair_view, queried 2026-11-03 +assets/pgx_genes.tsv OwnGenome - GPL-3.0-or-later - +assets/pgx_inhibitors.tsv OwnGenome, curated from the FDA table of clinical CYP inhibitors read 2026-11-03 GPL-3.0-or-later as the FDA's table of clinical CYP inhibitors classes the drug +assets/reference/README.md OwnGenome - GPL-3.0-or-later - +assets/reference/cytoBand.hg38.txt UCSC Genome Browser, hg38 cytoBand.txt.gz, chr1-22, X and Y (assets/reference/README.md) retrieved 2026-11-10 freely usable for any purpose (UCSC hg38 database README) - +assets/schema_input.json OwnGenome - GPL-3.0-or-later - +assets/steps.tsv OwnGenome - GPL-3.0-or-later - +assets/str_extra_loci.json OwnGenome, two loci written in ExpansionHunter's catalog form - GPL-3.0-or-later - +assets/str_penetrance_ranges.tsv OwnGenome, curated from the papers each row cites - GPL-3.0-or-later - +assets/stub/ OwnGenome, empty or synthetic stand-ins for the stub runs - GPL-3.0-or-later - +assets/sv_rule_regions.tsv TRExplorer catalog (Broad Institute); GENCODE; HGNC; NCBI assembly regions as scripts/ci/build-sv-rule-regions.sh pins MIT (assets/licences/TRExplorer-LICENSE.txt); GENCODE: open access; HGNC and NCBI: no restriction on use Sources and terms: the TRExplorer catalog, Copyright (c) 2024 Broad Institute, MIT licence +assets/test_samplesheet.csv OwnGenome - GPL-3.0-or-later - +assets/traits.tsv OwnGenome, curated from the primary papers in the pmids column - GPL-3.0-or-later Hand-curated from the primary papers in the pmids column; no text is taken from SNPedia +assets/viewer/ OwnGenome - GPL-3.0-or-later - +assets/ystr_panel.tsv FamilyTreeDNA Y-111 marker order and names (Y-STR Results Guide); HipSTR-references all_annot_markers.hg38.fixed.bed; NIST STRBase motifs HipSTR-references commit cc9275210a10911721015ce908f30a12e4ea44ca, read 2026-10-25 marker order and names: reproduced for interoperability, no FamilyTreeDNA text copied; HipSTR-references: MIT (NOTICE); STRBase: NIST, not subject to US copyright No FamilyTreeDNA text is copied | at commit cc9275210a10911721015ce908f30a12e4ea44ca (MIT) +tests/fixtures/consumer_export/sites.tsv.gz 23andme2vcf's 23andMe v5 site list (Rob Long); NCBI dbSNP b157 (tests/fixtures/consumer_export/README.md) dbSNP b157 23andme2vcf: MIT (assets/licences/23andme2vcf-LICENSE.txt); dbSNP: no restriction on use - diff --git a/assets/genotype_rules.tsv b/assets/genotype_rules.tsv index 8110a12ab..e87c66fc8 100644 --- a/assets/genotype_rules.tsv +++ b/assets/genotype_rules.tsv @@ -4,8 +4,8 @@ # other database. GRCh38: ref is the GRCh38 base and alt the other allele the rule reads, both on # the plus strand as in dbSNP's GRCh38 record, also for a gene on the minus strand. # scripts/ci/check-rule-alleles.sh checks every site against dbSNP; og traits check --table -# assets/traits.tsv --rules assets/genotype_rules.tsv checks the rest (tests/test_traits_report.py -# runs both). +# assets/traits.tsv --rules assets/genotype_rules.tsv checks the rest +# (rust/crates/og-steps/tests/genome_traits.rs runs both). # Four kinds of line, tab separated, each with its own columns: # rule gene trait needs_phase ancestry note pmids # site rsid chrom pos ref alt (one per site, in the order the haplotypes use) @@ -20,7 +20,7 @@ # the reading ends with that assumption, and read-backed phase, when one block holds the sites, # confirms it or reads the pair as other_haplotype. # ancestry: yes when the reading depends on ancestry (the note says how). -rule APOE APOE APOE alleles (e2, e3, e4) no yes Odds ratios from a meta-analysis of people of European descent (Farrer 1997); the e4 effect was weaker in African American and Hispanic people and stronger in Japanese people. Step 25's Alzheimer's score (PGS000334) includes the same two sites; docs/42-traits.md says why step 06 does not list APOE. 8346443,9343467 +rule APOE APOE APOE alleles (e2, e3, e4, and the rare e1) no yes Odds ratios of Alzheimer's disease against e3/e3 from a meta-analysis of clinic and autopsy studies of people of European descent (Farrer 1997), adjusted for age: odds ratios of those case-control studies, not a lifetime risk. The e4 effect was seen at every age from 40 to 90 but was smaller after 70, and the risk of a genotype differed between men and women. The e4 effect was weaker in African American and Hispanic people and stronger in Japanese people. The two sites also make the rare e1, whose pairs are not read. Step 25's Alzheimer's score (PGS000334) includes the same two sites; docs/42-traits.md says why step 06 does not list APOE. 8346443,9343467 site APOE rs429358 chr19 44908684 T C site APOE rs7412 chr19 44908822 C T haplotype APOE e1 C,T @@ -65,7 +65,7 @@ combination SERPINA1 M/Z PiMZ One Z copy: a carrier, with an intermediate alpha- combination SERPINA1 S/S PiSS Two S copies: a mildly lowered level, not usually a deficiency that raises the risk of disease. combination SERPINA1 S/Z PiSZ One S and one Z copy: a moderate deficiency, with a raised risk of emphysema, above all in smokers. Worth a serum level and a talk with a doctor. combination SERPINA1 Z/Z PiZZ Two Z copies: severe alpha-1 antitrypsin deficiency, with a raised risk of early emphysema and of liver disease. Worth a serum level and a talk with a doctor. -rule HFE HFE HFE haemochromatosis genotype (C282Y, H63D) no yes The two HFE missense changes of Feder 1996: C282Y (rs1800562), found homozygous in 83% of 178 patients with hereditary haemochromatosis, and H63D (rs1799945). The two sites are about 2 kb apart, too far for most read pairs to phase: heterozygous at both reads as C282Y/H63D, one change on each copy, and the reading says it assumes that. Many people with a risk genotype never build up iron; ferritin and transferrin saturation tell. The secondary-findings list reports C282Y homozygotes too (assets/acmg_sf.tsv); this line is the reading of every genotype. C282Y is most common in people of northern European descent. 8696333 +rule HFE HFE HFE haemochromatosis genotype (C282Y, H63D) no yes The two HFE missense changes of Feder 1996: C282Y (rs1800562), found homozygous in 83% of 178 patients with hereditary haemochromatosis, and H63D (rs1799945). The two sites are about 2 kb apart, too far for most read pairs to phase: heterozygous at both reads as C282Y/H63D, one change on each copy, and the reading says it assumes that. Many people with a risk genotype never build up iron; ferritin and transferrin saturation tell. How many become ill comes from later cohorts: in a Melbourne cohort of people of northern European descent aged 40 to 69, followed for 12 years, iron-overload disease was documented in 28% of male and 1% of female C282Y homozygotes (Allen 2008) and in 1 of 82 male and none of 95 female C282Y/H63D compound heterozygotes (Gurrin 2009); a US screening of 41,038 clinic attendees put frank haemochromatosis at under 1% of homozygotes (Beutler 2002). The secondary-findings list reports C282Y homozygotes too (assets/acmg_sf.tsv); this line is the reading of every genotype. C282Y is most common in people of northern European descent. 8696333,18199861,19554541,11812557 site HFE rs1800562 chr6 26092913 G A site HFE rs1799945 chr6 26090951 C G haplotype HFE wt G,C @@ -77,18 +77,18 @@ combination HFE wt/H63D wt/H63D One H63D copy: a common carrier type that rarely combination HFE C282Y/C282Y C282Y/C282Y Two C282Y copies: the genotype of most people with HFE haemochromatosis. Not everyone with it builds up iron; ferritin and transferrin saturation tell, and iron overload is treatable when found early. combination HFE C282Y/H63D C282Y/H63D One C282Y and one H63D copy: a compound heterozygote, with a lower risk of iron overload than two C282Y copies. Ferritin and transferrin saturation tell. combination HFE H63D/H63D H63D/H63D Two H63D copies: rarely leads to iron overload on its own. -rule GC GC Vitamin D-binding protein type (Gc1F, Gc1S, Gc2) no yes rs7041 (Asp432Glu) and rs4588 (Thr436Lys) together give the three common vitamin D-binding protein types: Gc1F (Asp, Thr), Gc1S (Glu, Thr) and Gc2 (Asp, Lys). GC lies on the minus strand, so the alleles here are the complement of the codon bases. In Powe 2013 the two sites explained 79.4% of the variation in the protein level and 9.9% of that of total 25-hydroxyvitamin D. Gc1F was the most common type in black participants and Gc1S in white ones. The monoclonal assay that study used was questioned in later letters, so the protein levels are those of that assay. A protein type, not a vitamin D deficiency. 24256378 +rule GC GC Vitamin D-binding protein type (Gc1F, Gc1S, Gc2) no yes rs7041 (Asp432Glu) and rs4588 (Thr436Lys) together give the three common vitamin D-binding protein types: Gc1F (Asp, Thr), Gc1S (Glu, Thr) and Gc2 (Asp, Lys). GC lies on the minus strand, so the alleles here are the complement of the codon bases. In Powe 2013 the two sites explained 9.9% of the variation in total 25-hydroxyvitamin D, and Gc1F was the most common type in black participants and Gc1S in white ones. That study also reported protein levels that differed widely by type, but its monoclonal assay reads the types differently: with mass spectrometry or a polyclonal assay, genotype explained at most 9% of the variation in the protein level, against 85% with the monoclonal assay (Denburg 2016). So the readings name the type and give no protein level. A protein type, not a vitamin D deficiency. 24256378,27250744 site GC rs7041 chr4 71752617 A C site GC rs4588 chr4 71752606 G T haplotype GC Gc1F A,G haplotype GC Gc1S C,G haplotype GC Gc2 A,T -combination GC Gc1F/Gc1F Gc1F/Gc1F Two Gc1F copies: in Powe 2013 this group had the lowest vitamin D-binding protein level (mean 93 micrograms per millilitre). -combination GC Gc1F/Gc1S Gc1F/Gc1S One Gc1F and one Gc1S copy: in Powe 2013 this group had an intermediate protein level (mean 285 micrograms per millilitre). -combination GC Gc1F/Gc2 Gc1F/Gc2 One Gc1F and one Gc2 copy: Powe 2013 gave protein levels for the homozygous groups and Gc1F/Gc1S only. -combination GC Gc1S/Gc1S Gc1S/Gc1S Two Gc1S copies: in Powe 2013 this group had the highest vitamin D-binding protein level (mean 468 micrograms per millilitre). -combination GC Gc1S/Gc2 Gc1S/Gc2 One Gc1S and one Gc2 copy: Powe 2013 gave protein levels for the homozygous groups and Gc1F/Gc1S only. -combination GC Gc2/Gc2 Gc2/Gc2 Two Gc2 copies: in Powe 2013 this group had an intermediate protein level (mean 190 micrograms per millilitre). +combination GC Gc1F/Gc1F Gc1F/Gc1F Two Gc1F copies: a protein type, not a level of the protein or of vitamin D. +combination GC Gc1F/Gc1S Gc1F/Gc1S One Gc1F and one Gc1S copy: a protein type, not a level of the protein or of vitamin D. +combination GC Gc1F/Gc2 Gc1F/Gc2 One Gc1F and one Gc2 copy: a protein type, not a level of the protein or of vitamin D. +combination GC Gc1S/Gc1S Gc1S/Gc1S Two Gc1S copies: a protein type, not a level of the protein or of vitamin D. +combination GC Gc1S/Gc2 Gc1S/Gc2 One Gc1S and one Gc2 copy: a protein type, not a level of the protein or of vitamin D. +combination GC Gc2/Gc2 Gc2/Gc2 Two Gc2 copies: a protein type, not a level of the protein or of vitamin D. rule CAD9p21 CDKN2B-AS1 Chromosome 9p21 heart disease risk haplotype no yes Two tags of the 9p21 risk haplotype near CDKN2A and CDKN2B: rs10757278 (G, Helgadottir 2007, myocardial infarction) and rs1333049 (C, the strongest coronary artery disease signal of the WTCCC 2007 scan). McPherson 2007 found the same interval, with no link to cholesterol, blood pressure or diabetes. The two are read together as tags of one haplotype; heterozygous at both reads as one risk copy, and the reading says it assumes both risk alleles sit on that copy. A copy with only one of the two risk alleles is not read. The studies were of people of European descent. 17478679,17554300,17478681 site CAD9p21 rs10757278 chr9 22124478 A G site CAD9p21 rs1333049 chr9 22125504 G C diff --git a/assets/live/core.js b/assets/live/core.js index 3d3f697ae..9c10f1f43 100644 --- a/assets/live/core.js +++ b/assets/live/core.js @@ -160,6 +160,9 @@ var originalFetch = window.fetch.bind(window); var records = {}, current = null, ownsPage = false, eventSources = [], shared = {}, cleaned = false; var CLOSED_TEXT = "This child's report stays closed until every result to act on in adult life is opened on purpose."; + // A page whose run state could not be read stays closed too (it fails closed), and names no role. Its results + // may still be readable and counted below, so the text speaks of the state only. + var UNREAD_TEXT = "This page could not read this run's state, so it keeps its results closed."; var TRANSFER = 'og-report-opening', NONCE = 'og-report-opening:'; var findingPath = /\/finding\//.test(location.pathname) ? location.pathname : null; // The opening ticket of the navigation that replaced the last document (transfer below): read and removed at @@ -297,7 +300,7 @@ // opened act's own record, in the order it was opened. PREFIX keeps element ids apart on a page with two children. function controls(rec, changed, prefix) { var p = projection(rec), box = el('section', { 'class': 'report-gate' }), n = (rec.acts || []).length; - add(box, el('p', { 'class': 'lede' }, CLOSED_TEXT)); + add(box, el('p', { 'class': 'lede' }, rec.unread ? UNREAD_TEXT : CLOSED_TEXT)); add(box, el('p', {}, rec.acts ? n + (n === 1 ? ' result' : ' results') + ' to act on' : 'The results cannot be read yet. Reload to try again.')); function row(a, open) { var art = el('article', { id: prefix + 'f-' + a.handle, 'data-finding': a.handle, 'class': 'finding' }); @@ -339,7 +342,7 @@ if (findingPath) { history.replaceState(null, '', runPath() + location.hash); } normalizeHash(current); var wrap = el('main', { 'class': 'wrap', id: 'report-gate' }); - add(wrap, el('h1', {}, 'Results about a child')); + add(wrap, el('h1', {}, current.unread ? 'Results not loaded' : 'Results about a child')); add(wrap, controls(current, function () { if (shut(current)) { renderGate(); } else { transfer(); } }, '')); add(document.body, wrap); root.setAttribute('data-pgp-status', 'ready'); @@ -396,12 +399,15 @@ var replaying = root.getAttribute('data-mode') === 'replay'; // live.js reads a replay's frames.json with no options; finding.js reads it, and state.json, as JSON (NO_STORE). var stateRead = sharedJSON(url + (replaying ? 'frames.json' : 'state.json'), replaying && !findingPath ? undefined : NO_STORE); - var results = findingPath ? record(url, undefined, true) : null, pre = null; + var results = findingPath ? record(url, undefined, true) : null, pre = null, unread = false; return Promise.all([stateRead, parsed]).then(function (x) { var raw = x[0]; var state = replaying ? raw && raw.frames && raw.frames.slice(-1)[0] : raw; state = replaying ? state && state.state : state; var fam = state && state.plugins ? state.plugins.family : undefined; + // A state that cannot be read, or has no plugins, keeps the gate closed (a possible child) and marks the + // record unread: the same condition that makes it a possible child for want of data. + unread = !state || !state.plugins; var child = !state || !state.plugins || !!(fam && (fam.unknown || (fam.kind === 'member' && fam.child !== false))); var entry = fam && fam.kind === 'member' && (fam.families || [])[0]; if (child && entry) { @@ -411,7 +417,7 @@ return child; }).then(function (child) { if (!child && !results) { return { url: url, child: false, acts: [], opened: {}, order: [], perm: [] }; } - return (results || record(url, child, true, pre)).then(function (rec) { rec.child = child; return rec; }); + return (results || record(url, child, true, pre)).then(function (rec) { rec.child = child; rec.unread = unread; return rec; }); }).then(function (rec) { current = records[url] = rec; return consume(current); @@ -419,7 +425,7 @@ ownsPage = shut(current); if (ownsPage) { renderGate(); } }).catch(function () { - current = { child: true, acts: null, opened: {}, order: [], perm: [] }; ownsPage = true; renderGate(); + current = { child: true, unread: true, acts: null, opened: {}, order: [], perm: [] }; ownsPage = true; renderGate(); }); } // A family's children with a run here, by the one rule (childIn, bin/owngenome_live.py child_of): family.json diff --git a/assets/live/family-page.js b/assets/live/family-page.js index 4781493a8..6d9566360 100644 --- a/assets/live/family-page.js +++ b/assets/live/family-page.js @@ -470,10 +470,11 @@ var folds = {}; Array.prototype.forEach.call(main.querySelectorAll('details[data-child]'), function (x) { folds[x.getAttribute('data-child')] = x.open; }); var y = window.scrollY; - main.textContent = ''; + // The page is built first and replaces the shown one only once it is whole: a redraw that throws keeps it. var page = el('section', { 'class': 'family-page', 'data-plugin': 'family', 'aria-label': 'Family ' + f.state.name }); if (f.state.error) { add(page, el('p', {}, f.state.error)); + main.textContent = ''; add(main, page); root.setAttribute('data-pgp-status', 'ready'); return; @@ -499,6 +500,7 @@ add(page, top); add(page, related(f), couples(f), children(f), sideBySide(f)); if ((f.state.plan || []).length || f.state.reports.length) { add(page, more(f)); } + main.textContent = ''; add(main, page); Array.prototype.forEach.call(main.querySelectorAll('details[data-child]'), function (x) { var k = x.getAttribute('data-child'); @@ -524,10 +526,12 @@ } function fetchFamily() { return U.getJSON('family.json').then(function (f) { - if (!f) { failed(); return; } + // An answer with no family state ({} or a wrong file) is no family: say so, as for an error answer. + if (!f || !f.state) { failed(); return; } var k = JSON.stringify(f); // Each child's results are asked again with the family: a result to act on can appear while a run goes. - if (k !== drawnKey) { drawnKey = k; draw(f); } + // drawnKey is set once draw() returns, so a first draw that throws reaches failed(). + if (k !== drawnKey) { draw(f); drawnKey = k; } }).catch(failed); } function start() { diff --git a/assets/live/home.js b/assets/live/home.js index fed41b05c..206a51f14 100644 --- a/assets/live/home.js +++ b/assets/live/home.js @@ -302,14 +302,30 @@ root.setAttribute('data-pgp-status', 'ready'); } - // While a run goes, the page asks again every 30 seconds, as the family page does. + // While a run goes, the page asks again every 30 seconds, as the family page does. A failed read (an error + // answer, no answer at all) is asked again in 30 seconds too; while nothing is drawn the page says so, with a + // retry, above the server's plain list of runs. A drawn page stays as it is. + var drawn = false, timer = null; + function again() { clearTimeout(timer); timer = setTimeout(start, 30000); } + function failed() { + root.setAttribute('data-pgp-status', 'offline'); + again(); + if (drawn || document.getElementById('home-failed')) { return; } + var main = document.getElementById('home'); + var b = el('button', { type: 'button', 'class': 'more' }, 'Try again'); + var box = add(el('div', { 'class': 'load-failed', id: 'home-failed' }), el('p', {}, 'The list of people could not be loaded.'), b); + b.addEventListener('click', function () { box.parentNode.removeChild(box); start(); }); + main.insertBefore(box, main.firstChild); + } function start() { + clearTimeout(timer); U.getJSON(root.getAttribute('data-data') || '/runs.json').then(function (d) { - if (!d) { return; } + if (!d) { failed(); return; } draw(d); + drawn = true; // A stopped run reads running again once it writes, so keep looking at it too. - if ((d.runs || []).some(function (r) { return r.state === 'running' || r.state === 'stopped'; })) { setTimeout(start, 30000); } - }).catch(function () { root.setAttribute('data-pgp-status', 'offline'); setTimeout(start, 30000); }); + if ((d.runs || []).some(function (r) { return r.state === 'running' || r.state === 'stopped'; })) { again(); } + }).catch(failed); } if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', start); } else { start(); } }()); diff --git a/assets/live/index.html b/assets/live/index.html index d740c1eda..c18c61900 100644 --- a/assets/live/index.html +++ b/assets/live/index.html @@ -70,7 +70,7 @@

Step

Click a step to see its tasks.

-

The wire

  • Nothing finished yet.
+

The wire

  • Waiting for the run's data.
diff --git a/assets/live/live.css b/assets/live/live.css index f810abbde..f6549f8c8 100644 --- a/assets/live/live.css +++ b/assets/live/live.css @@ -262,6 +262,8 @@ table.rows th { text-align: left; font: 600 10.5px/1 var(--sans); letter-spacing table.rows td { padding: 10px 12px 10px 0; border-bottom: 1px solid var(--hair); vertical-align: top; color: var(--ink-2); overflow-wrap: break-word; min-width: 4ch; font-variant-numeric: tabular-nums; } .more { margin-top: 8px; } +/* A page whose data did not load: the sentence and its Try again button (person.js, home.js). */ +.load-failed { margin: 16px 0; } .caveat { margin-top: 12px; font-size: 13px; line-height: 1.55; color: var(--ink-2); border-top: 1px solid var(--hair-2); padding-top: 9px; } .caveat::before { content: "Caveat"; display: block; font: 600 10.5px/1 var(--sans); letter-spacing: .12em; text-transform: uppercase; color: var(--brass); margin-bottom: 7px; } .caveat p { margin: 0 0 6px; max-width: 75ch; } diff --git a/assets/live/person.js b/assets/live/person.js index 455a69e28..f9b1852fc 100644 --- a/assets/live/person.js +++ b/assets/live/person.js @@ -21,6 +21,8 @@ // 'none': in no family; 'pending' or 'failed': in one whose roles are not known (yet); 'known'. var familyStatus = 'none'; var openingSeeded = false; + // The last read of results.json failed while no results are drawn: the report says so, with a retry. + var unloaded = false; var shown = {}; // act openings transferred from the gate, plus the independent APOE switch var genomeMap = null, resultsKey = null; // Figure 1 (map.js), and the results it was built for var panel = document.getElementById('results'); @@ -729,6 +731,12 @@ } function drawSections() { panel.textContent = ''; + if (!results && unloaded) { + var b = el('button', { type: 'button', 'class': 'more', id: 'results-retry' }, 'Try again'); + b.addEventListener('click', retry); + add(panel, add(el('div', { 'class': 'load-failed', id: 'results-failed' }), el('p', {}, 'The results could not be loaded.'), b)); + return; + } if (results && results.note && !(results.chapters || []).length) { add(panel, el('p', { 'class': 'empty' }, results.note)); return; @@ -1063,12 +1071,33 @@ fetchedFor = key; U.getJSON('results.json').then(function (r) { if (key !== fetchedFor) { return; } - if (!(r && (r.chapters || r.note))) { fetchedFor = null; return; } + // Results carry their chapters as a list, or only a note; chapters of any other shape would throw while + // drawing, after results was set, and leave a broken report. + if (!(r && (Array.isArray(r.chapters) || (!r.chapters && r.note)))) { notLoaded(key); return; } + unloaded = false; results = r; resultsKey = key; redraw(true); route(); - }).catch(function () { if (key === fetchedFor) { fetchedFor = null; } }); + }).catch(function () { notLoaded(key); }); + } + // A failed read (an error answer, no answer, or one that is not results) is asked again on the next state. + // While nothing is drawn the report says it could not load them: a finished run never reads as a report with + // no results. Results already drawn stay as they are. data-pgp-status stays live.js's: it tells whether the + // run's state arrived, which it did. + function notLoaded(key) { + if (key !== fetchedFor) { return; } + fetchedFor = null; + if (results) { return; } + unloaded = true; + drawSections(); + } + function retry() { + unloaded = false; + fetchedFor = null; + panel.textContent = ''; + add(panel, el('p', { 'class': 'dim' }, 'Loading the results.')); + load(); } // Every fold of the report as the reader left it, open or closed, by a key that names the same fold in // the next drawing: its data-card (a finding's caveat "More" carries its finding's id), else the element diff --git a/assets/live/plugins/ops.js b/assets/live/plugins/ops.js index b399503f5..afeeb570d 100644 --- a/assets/live/plugins/ops.js +++ b/assets/live/plugins/ops.js @@ -28,7 +28,9 @@ if (!box) { return; } while (box.firstChild) { box.removeChild(box.firstChild); } box.appendChild(el('h2', null, 'Run health')); - var o = last && last.plugins && last.plugins.ops; + // Before the first state arrives the card says it waits, never that there is nothing. + if (!last) { box.appendChild(el('p', 'muted', 'Waiting for the run\'s data.')); return; } + var o = last.plugins && last.plugins.ops; if (!o) { box.appendChild(el('p', 'muted', 'No numbers yet.')); return; } if (o.error) { box.appendChild(el('p', 'tone-attention', o.error)); return; } diff --git a/assets/live/plugins/recheck.js b/assets/live/plugins/recheck.js index 19275e7e1..6a6819304 100644 --- a/assets/live/plugins/recheck.js +++ b/assets/live/plugins/recheck.js @@ -8,7 +8,8 @@ 'use strict'; if (!window.PGP) { return; } var NS = 'http://www.w3.org/2000/svg'; - var banner = null, list = null, shownKey = null; + // shownKey starts undefined, so the first state draws the panel even when it has no re-check (null). + var banner = null, list = null, shownKey; function el(tag, attrs, text) { var e = document.createElement(tag); @@ -19,7 +20,7 @@ function panel(box) { box.appendChild(el('h2', {}, 'Database re-check')); - banner = el('p', { 'class': 'muted' }, 'No re-check yet: bin/owngenome_recheck.py writes one (docs/recheck.md).'); + banner = el('p', { 'class': 'muted' }, 'Waiting for the run\'s data.'); list = el('ul', { 'class': 'plain recheck-steps' }); box.appendChild(banner); box.appendChild(list); @@ -58,7 +59,7 @@ shownKey = key; list.textContent = ''; if (!key) { - banner.textContent = 'No re-check yet: bin/owngenome_recheck.py writes one (docs/recheck.md).'; + banner.textContent = 'No re-check yet: bin/og recheck writes one (docs/recheck.md).'; document.documentElement.removeAttribute('data-owngenome-recheck'); return; } diff --git a/assets/pgx_contraindications.tsv b/assets/pgx_contraindications.tsv index 4aee4d508..2de673460 100644 --- a/assets/pgx_contraindications.tsv +++ b/assets/pgx_contraindications.tsv @@ -14,8 +14,9 @@ # drugs does not follow). # Gene: the HGNC symbol, as ClinVar's GENEINFO writes it. # Drug: the drug as CPIC names it. -# Names: the first words a medication list may use for it, comma-separated, -# lower case; a drug on the list is marked in the report. +# Names: the words a medication list may use for it, comma-separated, lower +# case. Every word of a medication line is matched, the second drug of a +# combination product too; a drug on the list is marked in the report. # CPIC_level: the pair's CPIC level, printed as provisional. # Note: what the label says. # Label_source: the FDA label (DailyMed) the note quotes. diff --git a/assets/steps.tsv b/assets/steps.tsv index aefdc50b1..d300d8f51 100644 --- a/assets/steps.tsv +++ b/assets/steps.tsv @@ -9,7 +9,7 @@ # reference, such as building an index, left out); always below hours. # reads: the run_manifest.tsv keys (bin/write_manifest.sh) of the databases and pinned data the step reads # itself (pgs:* is every PGS score file; key@self: only a table of the step's own that no other step reads -# changes). bin/owngenome_recheck.py marks a step whose key moved, then every step downstream of it through +# changes). bin/og recheck marks a step whose key moved, then every step downstream of it through # upstream, then the reports; the Nextflow workflow checks that a stub rerun with a new ClinVar file redoes # exactly the steps it names. # tests/test_run_state.py checks this table against run-all.sh, main.nf, modules/local and collect_summary.py. diff --git a/assets/traits.tsv b/assets/traits.tsv index bea62616a..8ff57406b 100644 --- a/assets/traits.tsv +++ b/assets/traits.tsv @@ -4,12 +4,13 @@ # hom_ref, het, hom_alt: the reading for ref/ref, ref/alt and alt/alt. ancestry: yes when the # reading depends on ancestry (the note says how). og traits check checks the schema. chrom pos rsid gene ref alt trait hom_ref het hom_alt ancestry note pmids -chr2 135851076 rs4988235 MCM6 G A Lactase persistence (digesting milk sugar as an adult) No copy of -13910T, the European lactase persistence allele: lactase activity usually falls after childhood, unless another row below carries a persistence allele. One copy of -13910T: lactase activity usually stays into adulthood. Two copies of -13910T: lactase activity usually stays into adulthood. yes Lactase persistence is dominant: one persistence allele at any of the five lactase rows is usually enough. -13910T explains it in Europeans; in Africa other alleles do (rows -14010, -13915, -13907), so read the five rows together. 11788828 -chr2 135859184 rs182549 MCM6 C T Lactase persistence (digesting milk sugar as an adult) No copy of -22018A, which travels with the -13910T persistence allele in Europeans. One copy of -22018A; read it with row rs4988235, which predicts persistence better. Two copies of -22018A; read it with row rs4988235, which predicts persistence better. yes A weaker marker than -13910: it matched lactase non-persistence in 229 of 236 cases of the original study, -13910 in all of them. Lactase persistence is dominant: one persistence allele at any of the five lactase rows is usually enough. -13910T explains it in Europeans; in Africa other alleles do (rows -14010, -13915, -13907), so read the five rows together. 11788828 -chr2 135851176 rs145946881 MCM6 C G Lactase persistence (digesting milk sugar as an adult) No copy of -14010C, an African lactase persistence allele. One copy of -14010C: lactase activity likely stays into adulthood. Two copies of -14010C: lactase activity likely stays into adulthood. yes Found in Tanzanian, Kenyan and Sudanese people; it raises the activity of the lactase promoter in cell assays. Lactase persistence is dominant: one persistence allele at any of the five lactase rows is usually enough. -13910T explains it in Europeans; in Africa other alleles do (rows -14010, -13915, -13907), so read the five rows together. 17159977 -chr2 135851081 rs41380347 MCM6 A C Lactase persistence (digesting milk sugar as an adult) No copy of -13915G, an African lactase persistence allele. One copy of -13915G: lactase activity likely stays into adulthood. Two copies of -13915G: lactase activity likely stays into adulthood. yes Found in Tanzanian, Kenyan and Sudanese people; it raises the activity of the lactase promoter in cell assays. Lactase persistence is dominant: one persistence allele at any of the five lactase rows is usually enough. -13910T explains it in Europeans; in Africa other alleles do (rows -14010, -13915, -13907), so read the five rows together. 17159977 -chr2 135851073 rs41525747 MCM6 G C Lactase persistence (digesting milk sugar as an adult) No copy of -13907G, an African lactase persistence allele. One copy of -13907G: lactase activity likely stays into adulthood. Two copies of -13907G: lactase activity likely stays into adulthood. yes Found in Tanzanian, Kenyan and Sudanese people; it raises the activity of the lactase promoter in cell assays. Lactase persistence is dominant: one persistence allele at any of the five lactase rows is usually enough. -13910T explains it in Europeans; in Africa other alleles do (rows -14010, -13915, -13907), so read the five rows together. 17159977 -chr12 111803962 rs671 ALDH2 G A Alcohol flush (breaking down acetaldehyde) No ALDH2*2: acetaldehyde from alcohol is cleared normally. One ALDH2*2 copy (Glu504Lys). The inactive form is dominant: flushing, a fast heartbeat and nausea after alcohol are likely. Two ALDH2*2 copies: almost no ALDH2 activity; alcohol causes strong flushing and sickness. yes ALDH2*2 is common in East Asian people; the study typed livers of Japanese people. 2562960 +chr2 135851076 rs4988235 MCM6 G A Lactase persistence (digesting milk sugar as an adult) No copy of -13910T, the European lactase persistence allele. In people of European descent lactase activity then usually falls after childhood. This alone does not prove lactose intolerance: other persistence alleles exist, the African ones of the rows below among them, and this table does not hold every one. One copy of -13910T: lactase activity usually stays into adulthood. Two copies of -13910T: lactase activity usually stays into adulthood. yes Lactase persistence is dominant: one copy of a persistence allele is usually enough. -13910T explains it in Europeans; in Africa and the Middle East other alleles do (rows -14010, -13915, -13907 and -14009). These rows hold the main known persistence alleles, not every one, so a genotype with none of them does not prove lactase non-persistence. 11788828 +chr2 135859184 rs182549 MCM6 C T Lactase persistence (digesting milk sugar as an adult) No copy of -22018A, a marker that travels with the -13910T persistence allele in Europeans. One copy of -22018A, a marker that travels with -13910T in Europeans, not a persistence allele itself. Row rs4988235 reads persistence; only when that row has no genotype does this copy hint at one -13910T copy. Two copies of -22018A, a marker that travels with -13910T in Europeans, not a persistence allele itself. Row rs4988235 reads persistence; only when that row has no genotype do these copies hint at -13910T. yes A weaker marker than -13910: it matched lactase non-persistence in 229 of 236 cases of the original study, -13910 in all of them. It does not count as a persistence allele: read it only when row rs4988235 has no genotype, and then only as this weaker hint at -13910T. Lactase persistence is dominant: one copy of a persistence allele is usually enough. -13910T explains it in Europeans; in Africa and the Middle East other alleles do (rows -14010, -13915, -13907 and -14009). These rows hold the main known persistence alleles, not every one, so a genotype with none of them does not prove lactase non-persistence. 11788828 +chr2 135851176 rs145946881 MCM6 C G Lactase persistence (digesting milk sugar as an adult) No copy of -14010C, an African lactase persistence allele. One copy of -14010C: lactase activity likely stays into adulthood. Two copies of -14010C: lactase activity likely stays into adulthood. yes Found in Tanzanian, Kenyan and Sudanese people; it raises the activity of the lactase promoter in cell assays. Lactase persistence is dominant: one copy of a persistence allele is usually enough. -13910T explains it in Europeans; in Africa and the Middle East other alleles do (rows -14010, -13915, -13907 and -14009). These rows hold the main known persistence alleles, not every one, so a genotype with none of them does not prove lactase non-persistence. 17159977 +chr2 135851081 rs41380347 MCM6 A C Lactase persistence (digesting milk sugar as an adult) No copy of -13915G, an African lactase persistence allele. One copy of -13915G: lactase activity likely stays into adulthood. Two copies of -13915G: lactase activity likely stays into adulthood. yes Found in Tanzanian, Kenyan and Sudanese people; it raises the activity of the lactase promoter in cell assays. Lactase persistence is dominant: one copy of a persistence allele is usually enough. -13910T explains it in Europeans; in Africa and the Middle East other alleles do (rows -14010, -13915, -13907 and -14009). These rows hold the main known persistence alleles, not every one, so a genotype with none of them does not prove lactase non-persistence. 17159977 +chr2 135851073 rs41525747 MCM6 G C Lactase persistence (digesting milk sugar as an adult) No copy of -13907G, an African lactase persistence allele. One copy of -13907G: lactase activity likely stays into adulthood. Two copies of -13907G: lactase activity likely stays into adulthood. yes Found in Tanzanian, Kenyan and Sudanese people; it raises the activity of the lactase promoter in cell assays. Lactase persistence is dominant: one copy of a persistence allele is usually enough. -13910T explains it in Europeans; in Africa and the Middle East other alleles do (rows -14010, -13915, -13907 and -14009). These rows hold the main known persistence alleles, not every one, so a genotype with none of them does not prove lactase non-persistence. 17159977 +chr2 135851175 rs869051967 MCM6 A C Lactase persistence (digesting milk sugar as an adult) No copy of -14009G, an Ethiopian lactase persistence allele. One copy of -14009G: lactase activity likely stays into adulthood. Two copies of -14009G: lactase activity likely stays into adulthood. yes Found in an Ethiopian cohort tested for lactose digestion, where it went with digesting lactose; it raises the activity of a lactase promoter construct in cell assays. Lactase persistence is dominant: one copy of a persistence allele is usually enough. -13910T explains it in Europeans; in Africa and the Middle East other alleles do (rows -14010, -13915, -13907 and -14009). These rows hold the main known persistence alleles, not every one, so a genotype with none of them does not prove lactase non-persistence. 23993196 +chr12 111803962 rs671 ALDH2 G A Alcohol flush (breaking down acetaldehyde) No ALDH2*2 copy: the flush this variant causes is not expected. Other causes of alcohol flushing or sensitivity are not read here. One ALDH2*2 copy (Glu504Lys). The inactive form is dominant: flushing, a fast heartbeat and nausea after alcohol are likely. Two ALDH2*2 copies: almost no ALDH2 activity; alcohol causes strong flushing and sickness. yes ALDH2*2 is common in East Asian people; the study typed livers of Japanese people. 2562960 chr4 99318162 rs1229984 ADH1B T C Alcohol metabolism (ADH1B) Two copies of ADH1B*2 (His48), a fast form of alcohol dehydrogenase. In a study of Chinese men it was less common among people with alcoholism. One copy of ADH1B*2 (His48), a fast form of alcohol dehydrogenase. No ADH1B*2: the usual ADH1B*1 (Arg48) form. yes The reference genome carries ADH1B*2, so here a match to the reference is the less common genotype in Europeans and Africans. ADH1B*2 is common in East Asia. The study names the gene by its old name, ADH2. 2014795 chr15 74749576 rs762551 CYP1A2 C A Caffeine metabolism (CYP1A2 inducibility) C/C: in smokers, caffeine was broken down more slowly than with A/A; in non-smokers the genotypes did not differ. One A allele: in smokers the caffeine breakdown was close to C/C; in non-smokers the genotypes did not differ. A/A, the high-inducibility genotype: in smokers caffeine was broken down faster; in non-smokers the genotypes did not differ. no Tested in 185 non-smokers and 51 smokers of European descent. 10233211 chr15 74735539 rs2472297 CYP1A1-CYP1A2 C T Coffee consumption No copy of the T allele linked to drinking more coffee. One T copy: on average about 0.2 cups of coffee a day more among coffee drinkers. Two T copies: on average about 0.4 cups of coffee a day more among coffee drinkers. no An average effect in coffee drinkers from Iceland, the Netherlands, Germany and the USA; it says little about one person. 21357676 @@ -26,17 +27,17 @@ chr15 27985172 rs1800407 OCA2 C T Eye colour (modifier) No OCA2 419Q. One OCA2 4 chr5 33951588 rs16891982 SLC45A2 C G Hair, skin and eye pigmentation 374Leu on both copies: in people of European descent this genotype went with dark hair, skin and eyes. 374Leu on one copy and 374Phe on the other. 374Phe on both copies, the common form in Europeans: lighter hair, skin and eyes than with 374Leu. yes The allele frequencies differ strongly between populations; the association was measured in people of European descent. 15714523 chr15 48134287 rs1426654 SLC24A5 A G Skin pigmentation Thr111 on both copies, the form nearly fixed in Europeans; it goes with lighter skin. Thr111 on one copy and Ala111 on the other. Ala111 on both copies, the ancestral form common in African and East Asian populations. yes The reference genome carries Thr111. In admixed populations each Thr111 copy went with lighter skin. 16357253 chr6 396321 rs12203592 IRF4 C T Freckles and sun sensitivity No copy of the T allele linked to freckles and sun-sensitive skin. One T copy: freckles, sun-sensitive skin and brown rather than blond hair are more likely. Two T copies: freckles, sun-sensitive skin and brown rather than blond hair are more likely. yes Studied in people of European ancestry, where the T allele is found. 18483556,24267888 -chr16 89919709 rs1805007 MC1R C T Red hair and fair skin (MC1R R151C) No R151C. One R151C copy, a red-hair variant of MC1R: fair skin that tans poorly is more likely. Two R151C copies: red hair and fair skin that tans poorly are likely. no Red hair usually needs a red-hair MC1R variant on both copies of the gene: the same one twice, or two different ones (rows rs1805007, rs1805008, rs1805009). One copy at two of these rows can mean one on each copy; the phase is not known here. MC1R variants are necessary but not always sufficient for red hair. 9302268,7581459 -chr16 89919736 rs1805008 MC1R C T Red hair and fair skin (MC1R R160W) No R160W. One R160W copy, a red-hair variant of MC1R: fair skin that tans poorly is more likely. Two R160W copies: red hair and fair skin that tans poorly are likely. no Red hair usually needs a red-hair MC1R variant on both copies of the gene: the same one twice, or two different ones (rows rs1805007, rs1805008, rs1805009). One copy at two of these rows can mean one on each copy; the phase is not known here. MC1R variants are necessary but not always sufficient for red hair. 9302268,7581459 -chr16 89920138 rs1805009 MC1R G C Red hair and fair skin (MC1R D294H) No D294H. One D294H copy, a red-hair variant of MC1R: fair skin that tans poorly is more likely. Two D294H copies: red hair and fair skin that tans poorly are likely. no Red hair usually needs a red-hair MC1R variant on both copies of the gene: the same one twice, or two different ones (rows rs1805007, rs1805008, rs1805009). One copy at two of these rows can mean one on each copy; the phase is not known here. MC1R variants are necessary but not always sufficient for red hair. 9302268,7581459 +chr16 89919709 rs1805007 MC1R C T Red hair and fair skin (MC1R R151C) No R151C. One R151C copy, a red-hair variant of MC1R. Red hair usually needs a variant on both copies, but in a British study one R151C copy already raised the chance of red hair, and one variant copy went with skin that burns more easily and with freckles: averages of the people studied, not a prediction for you. Two R151C copies: red hair and fair skin that tans poorly are likely. no Red hair usually needs a red-hair MC1R variant on both copies of the gene: the same one twice, or two different ones (rows rs1805007, rs1805008, rs1805009). One copy at two of these rows can mean one on each copy; the phase is not known here. In a study of red-haired British families and unrelated people (Flanagan 2000), red hair was usually recessive and the effect grew with the number of variant copies. These are findings of the people studied, not rules: MC1R variants are not always enough for red hair (in a study of 25 red-haired and 62 other twins, some twin pairs with the same MC1R alleles differed in hair colour), and some red-haired people carry only one variant copy, or other MC1R variants than these three. 9302268,7581459,11030758 +chr16 89919736 rs1805008 MC1R C T Red hair and fair skin (MC1R R160W) No R160W. One R160W copy, a red-hair variant of MC1R. Red hair usually needs a variant on both copies. In a British study one variant copy went on average with skin that burns more easily and with freckles, a smaller effect than two: averages of the people studied, not a prediction for you. Two R160W copies: red hair and fair skin that tans poorly are likely. no Red hair usually needs a red-hair MC1R variant on both copies of the gene: the same one twice, or two different ones (rows rs1805007, rs1805008, rs1805009). One copy at two of these rows can mean one on each copy; the phase is not known here. In a study of red-haired British families and unrelated people (Flanagan 2000), red hair was usually recessive and the effect grew with the number of variant copies. These are findings of the people studied, not rules: MC1R variants are not always enough for red hair (in a study of 25 red-haired and 62 other twins, some twin pairs with the same MC1R alleles differed in hair colour), and some red-haired people carry only one variant copy, or other MC1R variants than these three. 9302268,7581459,11030758 +chr16 89920138 rs1805009 MC1R G C Red hair and fair skin (MC1R D294H) No D294H. One D294H copy, a red-hair variant of MC1R. Red hair usually needs a variant on both copies. In a British study one variant copy went on average with skin that burns more easily and with freckles, a smaller effect than two: averages of the people studied, not a prediction for you. Two D294H copies: red hair and fair skin that tans poorly are likely. no Red hair usually needs a red-hair MC1R variant on both copies of the gene: the same one twice, or two different ones (rows rs1805007, rs1805008, rs1805009). One copy at two of these rows can mean one on each copy; the phase is not known here. In a study of red-haired British families and unrelated people (Flanagan 2000), red hair was usually recessive and the effect grew with the number of variant copies. These are findings of the people studied, not rules: MC1R variants are not always enough for red hair (in a study of 25 red-haired and 62 other twins, some twin pairs with the same MC1R alleles differed in hair colour), and some red-haired people carry only one variant copy, or other MC1R variants than these three. 9302268,7581459,11030758 chr2 108897145 rs3827760 EDAR A G Hair thickness (EDAR V370A) 370Val on both copies, the usual form outside East Asia. One 370Ala copy, which goes with thicker head hair. 370Ala on both copies: thicker head hair is likely. yes 370Ala arose in Asia after the split from Europeans and is common in East Asia; the study calls it Asian-specific. 18065779 chr1 11796321 rs1801133 MTHFR G A MTHFR enzyme activity (C677T), an enzyme note, not health advice 677C on both copies: the usual MTHFR enzyme. One 677T copy: in lab tests the enzyme is somewhat less active and less stable to heat. This is common and is no reason to change diet or supplements. 677T on both copies: in lab tests lower MTHFR activity and, on average, somewhat higher blood homocysteine. It is not a diagnosis and no reason for treatment. no Common (about 38% of chromosomes in the original study carry 677T). The American College of Medical Genetics found too little evidence for MTHFR testing and advises against it in the work-up of thrombophilia or recurrent pregnancy loss, and for relatives of a carrier (Hickey et al. 2013). 7647779,23288205 -chr19 48703417 rs601338 FUT2 G A Secretor status (FUT2) Secretor: FUT2 works, and blood group antigens are in saliva and other body fluids. Secretor: one working FUT2 copy is enough. Non-secretor, as about 20% of people are: no blood group antigens in saliva. In a challenge study nobody with this genotype was infected by Norwalk virus. no The stop codon is W143X in the original numbering. Other FUT2 variants, not in this table, can also make a person a non-secretor. 7876235,12692541 -chr4 71742666 rs2282679 VDBP (GC) T G Vitamin D level (25-hydroxyvitamin D) No copy of the G allele linked to lower vitamin D. One G copy: on average lower 25-hydroxyvitamin D. Two G copies: on average the lowest 25-hydroxyvitamin D (in one cohort 64.6 against 82.6 nmol/L for T/T). no A genome-wide study of 33,996 people of European descent; sun, diet and supplements matter more. 20541252 -chr2 233759924 rs887829 UGT1A1 C T Bilirubin level (Gilbert syndrome genotype) No copy of the T allele. One T copy: a carrier; the Gilbert genotype needs two. Two T copies: the genotype of Gilbert syndrome (about 10% of UK Biobank participants), with higher bilirubin. Only 3% of people with it had the diagnosis recorded. no A marker strongly associated with Gilbert syndrome, measured in people of European ancestry. Step 07 (PharmCAT) reports UGT1A1 for drugs. 19414484,37456363 -chr19 40848628 rs1801272 CYP2A6 A T Nicotine breakdown (CYP2A6*2) No CYP2A6*2 (Leu160His). One CYP2A6*2 copy: that copy makes an inactive enzyme. Two CYP2A6*2 copies: no CYP2A6 activity from these copies, so nicotine is cleared slowly. no CYP2A6*2 is rare (1 to 3% in Finnish, Spanish and Swedish people). Other CYP2A6 alleles, deletions included, are not read here; step 32 (pypgx) calls CYP2A6 from the BAM. 2322567,9827545 +chr19 48703417 rs601338 FUT2 G A Secretor status (FUT2) No copy of the W143X stop: a secretor, with blood group antigens in saliva and other body fluids, unless another FUT2 variant, not read here, stops the gene. One W143X copy: a secretor, since one working FUT2 copy is enough, unless the other copy carries another FUT2 variant, not read here. Non-secretor: no blood group antigens in saliva. About 20% of the people the original study typed had this genotype. In a challenge study nobody with this genotype was infected by Norwalk virus. yes The stop codon is W143X in the original numbering (c.461G>A, often written c.428G>A). How common it is differs between populations, and in some, other FUT2 alleles, weak secretor alleles among them, are the common ones; they are not read here. Step 41 (blood groups, opt-in with TOOLS=...,bloodtype) reads more FUT2 null alleles; weak secretor alleles are not read there either. 7876235,12692541 +chr4 71742666 rs2282679 VDBP (GC) T G Vitamin D level (25-hydroxyvitamin D) No copy of the G allele linked to lower vitamin D. One G copy: on average lower 25-hydroxyvitamin D. Two G copies: on average the lowest 25-hydroxyvitamin D (in the Framingham Heart Study cohort 64.6 against 82.6 nmol/L for T/T). no A genome-wide study of 33,996 people of European descent. In its Framingham Heart Study cohort the mean was 82.6 nmol/L with T/T and 64.6 with G/G, against 83.4 and 74.7 with and without vitamin D supplements, and 61.6 in winter, 77.4 in spring and autumn and 95.8 in summer; the study calls the genotype gaps comparable to those of supplements and nearly as large as one change of season. In its discovery cohorts the variants at the three loci it confirmed explained up to 1 to 4% of the differences between people. 20541252 +chr2 233759924 rs887829 UGT1A1 C T Bilirubin level (UGT1A1, a Gilbert syndrome marker) No copy of the T allele. One T copy. In the bilirubin studies each T copy went with a higher bilirubin level on average; the UK Biobank study counted only two T copies as the genotype of Gilbert syndrome. Two T copies: the genotype a UK Biobank study used for Gilbert syndrome (about 10% of its participants), with higher bilirubin. Only 3% of people with it had the diagnosis recorded. A marker, not a diagnosis. yes rs887829 T is UGT1A1*80 (c.-364C>T). In people of European ancestry it travels with *28, the extra TA repeat of the UGT1A1 promoter that lowers the enzyme and is the common cause of Gilbert syndrome there: the UK Biobank study calls the two in complete linkage disequilibrium. Both studies were of people of European ancestry; in other populations the link and the common Gilbert alleles can differ. Other UGT1A1 variants are not read here. With STR_EXTRA_LOCI=yes, step 09 reads the UGT1A1 promoter TA repeat (*28) itself. Step 07 (PharmCAT) reports UGT1A1 for drugs. 19414484,37456363 +chr19 40848628 rs1801272 CYP2A6 A T Nicotine breakdown (CYP2A6*2) No CYP2A6*2 (Leu160His). One CYP2A6*2 copy: that copy makes an inactive enzyme. T/T at this site: two CYP2A6*2 copies, or one *2 copy with the other copy of the gene deleted (*4), which reads the same here. This row cannot tell the two apart, and it cannot say how fast nicotine is cleared: step 32 (pypgx) calls the CYP2A6 diplotype and copy number from the BAM. no CYP2A6*2 is rare (1 to 3% in Finnish, Spanish and Swedish people). Other CYP2A6 alleles, deletions included, are not read here; step 32 (pypgx) calls CYP2A6 from the BAM. 2322567,9827545 chr6 160589086 rs10455872 LPA A G Lp(a) level (LPA rs10455872) No copy of the G allele linked to a high Lp(a) level. One G copy: in the study it went with a higher Lp(a) level and coronary disease (odds ratio 1.70). Two G copies: a higher Lp(a) level is likely; in the study the G allele went with coronary disease (odds ratio 1.70). yes Lp(a) is a mostly inherited risk factor for coronary disease. With the other LPA row: one risk allele at either went with an odds ratio of 1.51 for coronary disease, two or more with 2.57, and both alleles went with a smaller apo(a) (fewer KIV-2 repeats; step 46 counts them). Studied in people of European descent; a sequencing study found partly different genetic determinants of Lp(a) in people of African ancestry. The actionable test is a blood Lp(a) measurement: a 2022 European Atherosclerosis Society consensus advises measuring it at least once in every adult. 20032323,36036785,29973585 chr6 160540105 rs3798220 LPA T C Lp(a) level (LPA rs3798220, Ile4399Met) No copy of the C allele (4399Met) linked to a high Lp(a) level. One C copy (4399Met): in the study it went with a higher Lp(a) level and coronary disease (odds ratio 1.92). Two C copies (4399Met): a higher Lp(a) level is likely; in the study the C allele went with coronary disease (odds ratio 1.92). yes Lp(a) is a mostly inherited risk factor for coronary disease. With the other LPA row: one risk allele at either went with an odds ratio of 1.51 for coronary disease, two or more with 2.57, and both alleles went with a smaller apo(a) (fewer KIV-2 repeats; step 46 counts them). Studied in people of European descent; a sequencing study found partly different genetic determinants of Lp(a) in people of African ancestry. The actionable test is a blood Lp(a) measurement: a 2022 European Atherosclerosis Society consensus advises measuring it at least once in every adult. 20032323,36036785,29973585 -chr19 44908684 rs429358 APOE T C APOE allele (rs429358, Cys112Arg) Cys112 on both copies: no e4 change. Arg112 on one copy, the change that defines e4. Arg112 on both copies, the change that defines e4. yes Read with the APOE line, which combines rs429358 and rs7412 into the e2, e3 and e4 alleles. e4 is a risk factor for late-onset Alzheimer's disease, not a diagnosis. 8346443,9343467 -chr19 44908822 rs7412 APOE C T APOE allele (rs7412, Arg158Cys) Arg158 on both copies: no e2 change. Cys158 on one copy, the change that defines e2. Cys158 on both copies, the change that defines e2. yes Read with the APOE line, which combines rs429358 and rs7412 into the e2, e3 and e4 alleles. e4 is a risk factor for late-onset Alzheimer's disease, not a diagnosis. 8346443,9343467 +chr19 44908684 rs429358 APOE T C APOE allele (rs429358, Cys112Arg) Cys112 on both copies: no e4 change. Arg112 on one copy, the e4 change. With Arg158 at rs7412 on that copy it makes e4, with Cys158 the rare e1: the APOE line reads the two sites together. Arg112 on both copies, the e4 change. With Arg158 at rs7412 a copy is e4, with Cys158 the rare e1, so this is e4/e4, e1/e4 or e1/e1: the APOE line reads the two sites together. yes Read with the APOE line, which combines rs429358 and rs7412 into the e2, e3 and e4 alleles and the rare e1. e4 is a risk factor for late-onset Alzheimer's disease, not a diagnosis. 8346443,9343467 +chr19 44908822 rs7412 APOE C T APOE allele (rs7412, Arg158Cys) Arg158 on both copies: no e2 change. Cys158 on one copy, the e2 change. With Cys112 at rs429358 on that copy it makes e2, with Arg112 the rare e1: the APOE line reads the two sites together. Cys158 on both copies, the e2 change. With Cys112 at rs429358 a copy is e2, with Arg112 the rare e1, so this is e2/e2, e1/e2 or e1/e1: the APOE line reads the two sites together. yes Read with the APOE line, which combines rs429358 and rs7412 into the e2, e3 and e4 alleles and the rare e1. e4 is a risk factor for late-onset Alzheimer's disease, not a diagnosis. 8346443,9343467 chr7 117530975 rs78655421 CFTR G A CFTR R117H, read with the CFTR poly-T tract No R117H copy. One R117H copy (Arg117His). What it means depends on the poly-T tract on the same copy: read it with the CFTR TG and poly-T line of the repeat expansions (step 09, STR_EXTRA_LOCI=yes). Two R117H copies (Arg117His). What they mean depends on the poly-T tracts: read them with the CFTR TG and poly-T line of the repeat expansions (step 09, STR_EXTRA_LOCI=yes). no A CFTR variant read here for the poly-T reading, not as a carrier result: step 06 (ClinVar) lists it with its classification. R117H with 5T on the same copy is a CF-causing allele of variable penetrance; with 7T it more often gives a milder CFTR-related disorder or none. 32404922 diff --git a/assets/viewer/index.html b/assets/viewer/index.html index baf434b63..332276551 100644 --- a/assets/viewer/index.html +++ b/assets/viewer/index.html @@ -7,18 +7,46 @@ Genome viewer -
OwnGenome viewer, sample ?: served from this computer only. - Type a locus such as chr1:12000-13000 in the box.
-
-
+
+

Reads of this sample

+

OwnGenome viewer: served from this computer only.

+
+
+

The tracks show the reference, then the sample's reads and its small and structural variants, those that were found. + Type coordinates such as chr1:12000-13000 in the search box: gene names do not work, because looking + one up would need igv.org, which the viewer never calls. Reads show once the view spans a few kb or less.

+ +
+
diff --git a/assets/viewer/viewer.js b/assets/viewer/viewer.js index 1bdd3f89c..3da2aef35 100644 --- a/assets/viewer/viewer.js +++ b/assets/viewer/viewer.js @@ -18,11 +18,23 @@ root.dataset.pgpBlocked = blocked.join(" | "); }); + // The error box (role="alert") says what failed, what to check, and offers Try again, which + // loads the page again. function fail(why) { root.dataset.pgpStatus = "error: " + why; const box = document.getElementById("error"); - box.textContent = "The viewer could not start: " + why; - box.style.display = "block"; + const what = document.createElement("p"); + what.textContent = "The viewer could not start: " + why; + const next = document.createElement("p"); + next.textContent = "Check that owngenome-view.sh, or the live site, is still running, then try again."; + const retry = document.createElement("button"); + retry.type = "button"; + retry.textContent = "Try again"; + retry.addEventListener("click", function () { + window.location.reload(); + }); + box.replaceChildren(what, next, retry); + box.hidden = false; } function locusFromHash() { diff --git a/bin/owngenome_live_recheck.py b/bin/owngenome_live_recheck.py index 5b235d76a..ecfee7d41 100644 --- a/bin/owngenome_live_recheck.py +++ b/bin/owngenome_live_recheck.py @@ -1,6 +1,6 @@ """owngenome_live_recheck.py: the database re-check on the live page, a plugin of bin/owngenome_live.py. -bin/owngenome_recheck.py writes /recheck/state.json: which databases the run +bin/og recheck writes /recheck/state.json: which databases the run used have a newer release, which steps that would change, and the last rerun. This plugin lets the signed-in browser fetch that file (/runs//recheck/state.json) and puts a short form of it in state.json diff --git a/bin/owngenome_recheck.py b/bin/owngenome_recheck.py deleted file mode 100755 index 053167eec..000000000 --- a/bin/owngenome_recheck.py +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env python3 -"""Deprecated: runs `bin/og recheck` with the same arguments. Kept for crontabs until 1.1.0.""" -import os, sys -sys.stderr.write("owngenome_recheck.py is deprecated: call bin/og recheck (docs/recheck.md)\n") -og = os.path.join(os.path.dirname(os.path.abspath(__file__)), "og"); os.execv(og, [og, "recheck", *sys.argv[1:]]) diff --git a/bin/render_report.py b/bin/render_report.py index a339af688..fa3496b4f 100755 --- a/bin/render_report.py +++ b/bin/render_report.py @@ -997,7 +997,7 @@ def section_keys(sid): def recheck_line(r): - """One line of run facts from the last database re-check (bin/owngenome_recheck.py).""" + """One line of run facts from the last database re-check (og recheck).""" newer = [i for i in r.get("items") or [] if i.get("class") not in ("no-newer-release", None)] head = f"Database re-check: {r.get('checked_utc') or 'unknown date'}: " if not newer: @@ -3365,7 +3365,7 @@ def terms_appendix(index, parts): browser's find (Ctrl+F) searches it; the live site's search box filters the same index.""" terms = index.get("terms") or {} - # No card title: the chapter title above names it (and the cards' titles are tests/test_render_report.py's list). + # No card title: the chapter title above names it (and the cards' titles are rust/og/tests/report_checks.rs's list). body = ['
', '

Every gene symbol and rsID a section of this report names, with the sections that ' "name it and how many times. A gene listed here is a gene the results mention, not a finding: open the " diff --git a/docs/05-annotsv.md b/docs/05-annotsv.md index 664e61cd1..fc4be32a3 100644 --- a/docs/05-annotsv.md +++ b/docs/05-annotsv.md @@ -120,7 +120,7 @@ AnnotSV gives class 4 or 5 to some calls that are not a rare deletion: a call on Some class 4 and 5 calls are common or unreliable for a reason AnnotSV does not weigh. The table keeps them, with `ACMG_class` 3, AnnotSV's class in `AnnotSV_class` and the reason in `Rule_detail`. The reports list such a call apart from the class 4 and 5 section, with AnnotSV's class, the rule and `Rule_detail`, and leave it out of the section's count. -- **No read-depth support** (`no_depth_support`). A real deletion lowers the read depth and a real duplication raises it. `Depth_support` reads the record's own value: duphold's `DHFFC` below 0.7 for a deletion and `DHBFC` above 1.3 for a duplication, the thresholds of [step 15](15-duphold.md) (Pedersen and Quinlan 2019, PMID 31222198); for a Delly call over 1,000 bp without them, Delly's `RDCN`, judged by the rule `og sv add-callers` applies when it adds a Delly call: fewer copies than normal for a deletion and more for a duplication, where normal is 1 on a male's chrX and chrY outside the pseudoautosomal regions and 2 everywhere else, PARs included. Step 05 and `ANNOTSV` pass the sample's sex and `assets/par_grch38.bed` for this. With no sex given, 1 and 2 both count as normal on chrX and chrY outside the PARs, so there a deletion needs 0 or 1 copies and a duplication 2 or more. A CNVpytor call is a read-depth call, so it has support. A Delly call step 05 added to AnnotSV's input keeps only `FORMAT/GT`, so its `RDCN` does not reach the table; over 1,000 bp it says `yes`, because `og sv add-callers` already set aside every such call whose `RDCN` does not show the change. An `RDCN` that is not a whole number of copies (`1.5`, `inf`) or is negative says nothing about the change, so it reads `not checked` here, as a missing one does. `RDCN` itself is read here only for `SV_VCF` input, where the caller's FORMAT fields survive. `no` moves the call to class 3. A record with no depth value says `not checked` and keeps its class: that happens for a Delly call of 1,000 bp or less, when step 15 did not run (the step then annotates Manta's calls and says so), or for `SV_VCF` input without these fields. Step 15 already drops the Manta calls its depth contradicts, so in a default run this check mostly confirms; it catches an `SV_VCF` file duphold annotated but did not filter. +- **No read-depth support** (`no_depth_support`). A real deletion lowers the read depth and a real duplication raises it. `Depth_support` reads the record's own value: duphold's `DHFFC` below 0.7 for a deletion and `DHBFC` above 1.3 for a duplication, the thresholds of [step 15](15-duphold.md) (Pedersen and Quinlan 2019, PMID 31222198); for a Delly call over 1,000 bp without them, Delly's `RDCN`, judged by the rule `og sv add-callers` applies when it adds a Delly call: fewer copies than normal for a deletion and more for a duplication, where normal is 1 on a male's chrX and chrY outside the pseudoautosomal regions and 2 everywhere else, PARs included. Step 05 and `ANNOTSV` pass the sample's sex and `assets/par_grch38.bed` for this. With no sex given, 1 and 2 both count as normal on chrX and chrY outside the PARs, so there a deletion needs 0 or 1 copies and a duplication 2 or more. A CNVpytor call is a read-depth call, so it has support. A Delly call that step 05 adds to AnnotSV's input keeps `FORMAT/GT` and Delly's `RDCN`: `og sv add-callers` writes `GT:RDCN`, with `.` for a value that is not a whole number. Over 1,000 bp such a call reads `yes`, because `og sv add-callers` set aside every call whose `RDCN` does not show the change. A Delly call over 1,000 bp with no `RDCN` value, from a VCF not written by Delly 2.x, reads `unknown` and keeps its class: nothing checked its read depth. The step's log line counts these calls apart from `not checked`. An `RDCN` that is not a whole number of copies (`1.5`, `inf`) or is negative says nothing about the change, so it reads `not checked`. With `SV_VCF` input the caller's own FORMAT fields reach the table, `RDCN` included. `no` moves the call to class 3. Other calls with no depth value show `not checked` and keep their class: that happens for a Delly call of 1,000 bp or less, when step 15 did not run (the step then annotates Manta's calls and says so), or for `SV_VCF` input without these fields. Step 15 already drops the Manta calls its depth contradicts, so in a default run this check mostly confirms; it catches an `SV_VCF` file duphold annotated but did not filter. - **Inside a coding tandem repeat** (`tandem_repeat`). An SV whose two breakpoints lie within 10 bp of a tandem repeat that overlaps a MANE Select coding exon changes the repeat's length. Short-read SV callers place such breakpoints loosely and do not size repeats, and AnnotSV can read the change as a frameshift. A length change of a repeat is a question for a repeat genotyper (the repeats of [step 9](09-str-expansions.md)'s catalog are read there), not an SV call. ACMG's PM4 and BP3 (Richards 2015, PMID 25741868) already weigh in-frame changes inside a repeat differently from other length changes. - **The IG and TR loci** (`ig_tr_locus`). An SV more than half inside IGH, IGK, IGL, TRA (with TRD inside it), TRB or TRG. B and T cells cut DNA out of these loci when they rearrange their receptor genes (V(D)J recombination), so DNA from blood carries such deletions in a share of its cells without any inherited change. - **The MHC** (`mhc`). An SV more than half inside the MHC, GRCh38 chr6:28,510,120-33,480,577. Its haplotypes differ by tens of kilobases between people, and gnomAD-SV marks every SV over half inside IGH or the MHC as low confidence (FILTER `IGH_MHC_OVERLAP`, Collins 2020, PMID 32461652). CYP21A2 lies in the MHC; short reads cannot call it here, and [step 35b](35-paralogs.md) reads it from PacBio HiFi reads. @@ -128,7 +128,14 @@ Some class 4 and 5 calls are common or unreliable for a reason AnnotSV does not The regions are in `assets/sv_rule_regions.tsv`, one row each (`chrom`, `start` 0-based, `end`, `rule`, `action`, `name`, `source`): 2,409 coding tandem repeats, the six IG and TR loci, the MHC and 10 pseudogenes. `scripts/ci/build-sv-rule-regions.sh` writes it from four pinned downloads, each checked against its sha256: TRExplorer v2.1's GRCh38 tandem repeat catalog (the file upstream rebuilt in place on 2026-10-30, sha256 `df9c23c4…`; every repeat of 50 bp or more on a MANE Select coding exon of GENCODE 50), GENCODE 50's IG and TR genes, the MHC row of the GRCh38 assembly regions report, and HGNC's complete set of 2026-10-06 (each pseudogene whose alias or previous symbol is the approved symbol of a gene with a GenCC curation). The file is about 250 KB and ships with the repository, so the step downloads nothing. Its header names each source's terms: the TRExplorer catalog is MIT, copyright 2024 Broad Institute, with its notice in `assets/licences/TRExplorer-LICENSE.txt`; GENCODE is open access, and HGNC and the NCBI assembly regions carry no restriction on use. The e2e case `followups-ci-3-sv-rule-regions` rebuilds it with `scripts/ci/build-sv-rule-regions.sh --check`, which fails when one byte differs, so a hand edit or a drifted pin cannot ship. Step 05 mounts it into the AnnotSV image and the Nextflow `ANNOTSV` gets it as an input. -How many callers saw each SV comes from step 22's consensus (`sv_merged/_sv_consensus.vcf.gz`), which runs after this step, and beside `ANNOTSV` in Nextflow. So the reports (`og summary collect`) join it when they are written: an SV matches the consensus record of the same type whose two breakpoints lie within 1,000 bp of its own, step 22's distance, and its support is that record's `SUPP` with the callers of `SUPP_VEC`. An SV in no consensus record was made by one caller and shows as "single caller". Without the consensus file the support is "not known", never 1. `og sv postfilter --consensus` adds the same `Support` and `Support_callers` columns to the table. +The reports (`og summary collect`) read each SV's support from step 22's consensus (`sv_merged/_sv_consensus.vcf.gz`), which runs after this step, and beside `ANNOTSV` in Nextflow. A checked row is a member of a consensus event when the event has the same contig and SV type and lists the row's own record ID under the row's caller. `SUPP_VEC` marks which caller columns voted, and `FORMAT/ID` holds each voter's original record ID (SURVIVOR 1.0.7 writes a `:` in it as `_`). A member's support is the number of callers that voted, with their names. A row that no event lists shows as "single caller", when the consensus names every vote of its caller. The support is "not known", never 1, when the consensus cannot tell: + +- the row has no record ID (`.`); +- its caller is not a column of the consensus (an `SV_VCF` run labels its rows `sv_vcf`); +- its ID is a member of two events; +- or the consensus holds a vote of its caller without a readable ID. This covers the CNVpytor calls when step 22 made CNVpytor's VCF from its table, which writes no record IDs. It also covers a consensus without `FORMAT/ID`; SURVIVOR 1.0.7 always writes it, so only a file made another way lacks it. + +Without the consensus file the support is "not known" too. `og sv postfilter --consensus` adds the same `Support` and `Support_callers` columns to the table, both empty when the support is not known. Both reports list every class 4 and 5 SV in a section of their own, with its support, the exon-edge flag and the frequency. The bash report (`scripts/24-html-report.sh`, which `run-all.sh` runs) reads the table from the sample folder. The Nextflow `HTML_REPORT` gets the table from `ANNOTSV` and, with `survivor_merge` in `--tools`, the consensus, so a report written by `nextflow run` alone has the same section. A report of a run without AnnotSV lists it under the steps not run. diff --git a/docs/24-html-report.md b/docs/24-html-report.md index 1771ef211..fbfd553d1 100644 --- a/docs/24-html-report.md +++ b/docs/24-html-report.md @@ -10,7 +10,7 @@ The pipeline produces output across many directories in different formats (VCF, ## Tool -`og summary collect` (`rust/crates/og-summary`, run by `bin/render_report.py`) reads every step's output once into `${SAMPLE}/summary.json`, and `bin/render_report.py` renders the report from it. `scripts/generate-report.sh` renders the text report from the same summary, so the two reports cannot disagree about a count. Both are Python standard library only. +`og report render` (`rust/crates/og-render`) runs `og summary collect` (`rust/crates/og-summary`), which reads every step's output once into `${SAMPLE}/summary.json`, and renders the report from it. `scripts/generate-report.sh` renders the text report from the same summary, so the two reports cannot disagree about a count. No Python runs for either report. A section that cannot be written (its data cannot be read, or its writer stops on an internal error) says so in its own place, `[could not be read: ...]` in the text report and an Unreadable card in the HTML report, and the rest of the report is still written; `og report render` then prints one `WARNING` line on stderr that counts and names those parts, and still exits 0. ## Docker Image @@ -43,7 +43,7 @@ Two more files, when present: ## How the report is laid out -The report is one HTML file in seven chapters, in this order, each opening with the question it answers. The text report (`scripts/generate-report.sh`) has the same chapters in the same order. `CHAPTERS` in `bin/render_report.py` is the one table both read. +The report is one HTML file in seven chapters, in this order, each opening with the question it answers. The text report (`scripts/generate-report.sh`) has the same chapters in the same order. `CHAPTERS` in `rust/crates/og-render/src/chapters.rs` is the one table both read. | Chapter | Question | Sections | |---|---|---| @@ -58,8 +58,8 @@ The report is one HTML file in seven chapters, in this order, each opening with After them, "What this report leaves out" lists the steps not run and what no step assesses. The report closes with "Genes and rsIDs in these results": one line per gene symbol or rsID the results name, with a link to each section that names it and how many variants there (below). - **Contents.** A list on the left (on top on a narrow screen) names every chapter and section shown, each section with a dot for its state: blue complete, yellow stale or part not run (a section of several steps where one did not run), red failed, gray not run. A section with no card (a step such as step 39 that shows nothing when it did not run) is left out of both. -- **Each section** has an anchor (`#clinvar`, `#cpic`, ...). Under its first card title a line gives the step, the state, the newest file date and a link to the section itself. The state badge is never green: a step that ran is not a finding. The caveat each card already printed (a `*_NOTE` sentence of `bin/render_report.py`) now sits right under that card's title, before the numbers. The PRS caption keeps its ancestry caveat inside its own sentence. -- **Five charts** show how sure each finding is, each in its section and right below that section's caveat (the PRS chart below the caption whose sentence holds the caveat). They are inline SVG drawn by `bin/report_charts.py` from the same summary, with no script: hover a mark for its value, click it to jump to the table or section it comes from. +- **Each section** has an anchor (`#clinvar`, `#cpic`, ...). Under its first card title a line gives the step, the state, the newest file date and a link to the section itself. The state badge is never green: a step that ran is not a finding. The caveat each card already printed (a `*_NOTE` sentence of `rust/crates/og-render/src/notes.rs`) now sits right under that card's title, before the numbers. The PRS caption keeps its ancestry caveat inside its own sentence. +- **Five charts** show how sure each finding is, each in its section and right below that section's caveat (the PRS chart below the caption whose sentence holds the caveat). They are inline SVG drawn by `og report render` (`rust/crates/og-render/src/charts`) from the same summary, with no script: hover a mark for its value, click it to jump to the table or section it comes from. - **ClinVar** (`#clinvar`): one bar per review star, 4 to 0, and a grey bar for the low-confidence calls (GQ or DP below the call floor), which the star counts leave out. Yellow and grey only, never green. With no hit, a grey frame says that no hit is not an all-clear. - **Chromosomes** (`#roh`): chromosomes 1 to 22, X and Y of GRCh38 with the runs of homozygosity of 5 MB or more and a tick per ClinVar hit, grey for a low-confidence call. A hit on chrM or on another contig is not drawn; the line under the chart counts it. A tick for one of the first 50 hits links to the hits table, a later one to the ClinVar section. It also draws an outlined span for each AnnotSV class 4 or 5 SV and a bar under the chromosome for each mosaic chromosomal alteration MoChA reports, each linking to its section; an SV or a MoChA call on chrM, on another contig or with no position is counted in the same line under the chart. Every mark stays inside its chromosome: a span that runs past the chromosome's end is cut there, and a mark that starts past it (a file from another assembly) is not drawn and is counted in that line. - **chrM** (`#mito`): each heteroplasmic site by position and allele fraction, the D-loop poly-C length variants as grey squares apart from them. @@ -68,7 +68,7 @@ After them, "What this report leaves out" lists the steps not run and what no st - **Links to the step's own files**, relative to the report, when the summary says they exist: the PharmCAT HTML report, the CPIC recommendations per gene, the CPSR HTML report (or the `cpsr/` folder that holds it) and the Exomiser ranking table. The report sits in the sample folder beside each step's folder, in the bash layout and in a Nextflow `--outdir`. MultiQC and the couple report are not linked: the summary does not record them, and the couple report sits in the family folder. - **Nothing is fetched.** A Content-Security-Policy (`default-src 'none'; img-src data:; style-src 'unsafe-inline'`) lets the browser load nothing but the file itself and its inline pictures, and there is no JavaScript. The only addresses off this computer are the haplogroup links (FamilyTreeDNA, YFull, the docs) and the project link in the footer, each marked "opens a website"; the FamilyTreeDNA and YFull ones add that the address carries the haplogroup. The genome viewer links go to `127.0.0.1`, this computer. - **A Viewer column on every findings table.** The ClinVar hits, the secondary-findings genes, the mitochondrial variants, the known deep intronic sites, the cDNA scan, the AnnotSV class 4 and 5 SVs and the calls moved to class 3, slivar's pairs in trans, the mosaic alterations and Exomiser's ranked variants each end with a Viewer column: one link per row, to that row's position (a range for an SV, a region or a pair). The link opens the reads there in the [genome viewer](viewer.md) once `scripts/owngenome-view.sh ` runs; a note under the table says so. The column is new, so every other cell keeps its text; it replaces the single line of links that sat under three of these tables. A row with no position has an empty Viewer cell. On the [live site](live-site.md) the served report points the same links at the site's own viewer of the run. -- **Genes and rsIDs in these results.** The closing list is built by `bin/results_index.py`. It holds every gene symbol and rsID that a section in state complete or stale names in `summary.json`, by the same rule `scripts/owngenome-lookup.sh` uses for "in the reports" (`names_of` in `bin/genome_lookup.py`, which `og lookup` ports in `rust/crates/og-lookup`; `og report index` ports the list itself, and `rust/crates/og-render/tests/index_search.rs` checks the page search against it). A section's locus name counts as a term too, when it holds an upper-case letter: a repeat expansion (HTT, FMR1, CFTR_TG), a Locityper locus (HLA-DRB1, FCGR) and a copy-number locus are indexed by their `locus` field, so a term is not always a gene symbol. An expansion record with no REPID or VARID takes its position as its locus (`chrX:147912050`), and that counts as a term only when it holds an upper-case letter (a chrX, chrY or chrM position). It adds the genes of the clinical filter (step 23) and variant prioritization (step 31), whose cards only count them: from their summary tables on the bash path, from the VCFs on the Nextflow path. The two give the same genes when a variant's most severe consequence is on the transcript the report ranks first (MANE, else canonical); the step 31 table keeps the gene of the most severe consequence, so where an overlapping gene's transcript carries the worse one, the two forms name different genes. A ClinVar hit's rsIDs come from step 13's VEP output (`Existing_variation`), because the hits file's ID column holds ClinVar's own id; without step 13 a hit has none. The Nextflow `HTML_REPORT` process does not get step 13's VEP output, so its index has no ClinVar rsIDs; step 24, which `run-all.sh` runs over the output folder afterwards, adds them. A failed, missing or unreadable section adds nothing. The browser's find (Ctrl+F) searches the list; there is still no script. Apart from those locus names a position is not in it, and the search does not answer a position: ask `scripts/owngenome-lookup.sh` for a genotype ([genotype lookup](lookup.md)). +- **Genes and rsIDs in these results.** The closing list is built by `og report index` (`rust/crates/og-render/src/index.rs`), which `og report render` runs. It holds every gene symbol and rsID that a section in state complete or stale names in `summary.json`, by the same rule `scripts/owngenome-lookup.sh` uses for "in the reports" (`names_of` in `bin/genome_lookup.py`, which `og lookup` ports in `rust/crates/og-lookup`; `og report index` ports the list itself, and `rust/crates/og-render/tests/index_search.rs` checks the page search against it). A section's locus name counts as a term too, when it holds an upper-case letter: a repeat expansion (HTT, FMR1, CFTR_TG), a Locityper locus (HLA-DRB1, FCGR) and a copy-number locus are indexed by their `locus` field, so a term is not always a gene symbol. An expansion record with no REPID or VARID takes its position as its locus (`chrX:147912050`), and that counts as a term only when it holds an upper-case letter (a chrX, chrY or chrM position). It adds the genes of the clinical filter (step 23) and variant prioritization (step 31), whose cards only count them: from their summary tables on the bash path, from the VCFs on the Nextflow path. The two give the same genes when a variant's most severe consequence is on the transcript the report ranks first (MANE, else canonical); the step 31 table keeps the gene of the most severe consequence, so where an overlapping gene's transcript carries the worse one, the two forms name different genes. A ClinVar hit's rsIDs come from step 13's VEP output (`Existing_variation`), because the hits file's ID column holds ClinVar's own id; without step 13 a hit has none. The Nextflow `HTML_REPORT` process does not get step 13's VEP output, so its index has no ClinVar rsIDs; step 24, which `run-all.sh` runs over the output folder afterwards, adds them. A failed, missing or unreadable section adds nothing. The browser's find (Ctrl+F) searches the list; there is still no script. Apart from those locus names a position is not in it, and the search does not answer a position: ask `scripts/owngenome-lookup.sh` for a genotype ([genotype lookup](lookup.md)). - **Dark on screen, light on paper.** - **Print one section.** Open a section from the contents (or its "link to this section"), then print: only that section prints, with the header (sample, generation date, pipeline commit, ClinVar file date, ClinVar in CPSR's bundle, HLA database), the list of what is not assessed and the disclaimer. A chapter prints the same way from its title in the contents. Without an anchor in the address the whole report prints, as it does in a browser without CSS `:has()`. @@ -76,7 +76,7 @@ After them, "What this report leaves out" lists the steps not run and what no st `${GENOME_DIR}/${SAMPLE}/${SAMPLE}_report.html`, `${GENOME_DIR}/${SAMPLE}/summary.json`, `${GENOME_DIR}/${SAMPLE}/${SAMPLE}_charts.json` and `${GENOME_DIR}/${SAMPLE}/${SAMPLE}_search_index.json`. -`${SAMPLE}_search_index.json` is the list the report closes with, as JSON: `schema_version` 1, `sample`, `generated_utc` and `terms`, where each term (a gene in upper case, an rsID in lower case) maps to one item per section that names it: the section key, its report anchor, title, step, state, source file, a count of mentions (a variant counts once per section) and the first 20 mentions, each with its text and locus. The Nextflow `HTML_REPORT` process writes the same file beside its report, without the ClinVar hits' rsIDs (see above). `bin/results_index.py query --index BRCA2` answers one query at a shell. The file names genes of the results, so keep it on this machine like the report. On the live site its plugin (`bin/owngenome_live_search.py`, `assets/live/plugins/search.js`) serves the file only behind the site's password. The page fetches it once, and again only when the report is written anew, then filters it in the browser as you type. No request carries the query, so it is in no log. A prefix lists the terms that start with it (`BRCA` gives `BRCA1` and `BRCA2`), and a position gets a pointer to `scripts/owngenome-lookup.sh`. +`${SAMPLE}_search_index.json` is the list the report closes with, as JSON: `schema_version` 1, `sample`, `generated_utc` and `terms`, where each term (a gene in upper case, an rsID in lower case) maps to one item per section that names it: the section key, its report anchor, title, step, state, source file, a count of mentions (a variant counts once per section) and the first 20 mentions, each with its text and locus. The Nextflow `HTML_REPORT` process writes the same file beside its report, without the ClinVar hits' rsIDs (see above). When the index cannot be built, the file is written with no term, the report has no closing list, and the step's `WARNING` line names it (`terms`). `bin/results_index.py query --index BRCA2` answers one query at a shell. The file names genes of the results, so keep it on this machine like the report. On the live site its plugin (`bin/owngenome_live_search.py`, `assets/live/plugins/search.js`) serves the file only behind the site's password. The page fetches it once, and again only when the report is written anew, then filters it in the browser as you type. No request carries the query, so it is in no log. A prefix lists the terms that start with it (`BRCA` gives `BRCA1` and `BRCA2`), and a position gets a pointer to `scripts/owngenome-lookup.sh`. `${SAMPLE}_charts.json` holds the five charts as data and as SVG (`schema_version`, `sample`, `generated_utc`, `charts`), each with its section, its caveat and one entry per mark (value, range, locus, detail). The Nextflow `HTML_REPORT` process writes the same file beside its report. One input differs: Nextflow also passes the samplesheet's sex, so when no run status or manifest records a sex, its PRS chart marks the sex-specific scores against that sex and step 24's does not. It holds findings, so treat it like the report: keep it on this machine. On the live site its plugin (`bin/owngenome_live_charts.py`, `assets/live/plugins/charts.js`) serves this file only behind the site's password and shows the same charts; a click on a mark opens a panel with its detail, its caveat, a `viewer/#` link that opens the reads in the site's genome viewer of the run (`bin/owngenome_live_viewer.py`), and a link to its section of the report. @@ -144,6 +144,8 @@ A step that is skipped or fails leaves the previous run's output on disk. When ` A section with no file whose step is recorded `failed` in `logs/run_status.tsv` says **Failed**, with the step, in both reports (the CYP2D6 rows too), and is listed under Steps Not Run with "(step N failed)". A section whose step never ran still says **Not run**. +The Nextflow report also knows which steps the run selected: `HTML_REPORT` writes the `--tools` list as the `run tools` row of the manifest it collects from (not the published one). A section with no file whose step was selected is listed under Steps Not Run with "(selected, no result)", its note says which `--tools` name selected it and which step wrote nothing, and its key is in the list `selected_no_result` of `summary.json`, in summary order. This is how a report-only step that failed, and that the pipeline then dropped, stays visible. `og summary collect` maps each tool to its sections through `assets/steps.tsv`. Its state stays `missing`. A run with no `run tools` row, such as the bash runner's, has no `selected_no_result` key, and its sections read as before. A section that is no step of its own (the carrier list, the couple screen, the ACMG SF card and the read count) is not listed. Y haplogroup, Y tree placement, and Y-STR markers get this label only for samples declared male. For a sample with no BAM, other selected BAM-only steps still read "(selected, no result)" when they have no output. +

![The step 24 HTML report: the contents on the left, the header, the Quality chapter with the quality control, hard-to-read genes and variant calling cards, then the Clinical chapter's ClinVar screening card with its caveat and the chart of hits by review stars](images/demo-html-report.png){ loading=lazy }
The report for DEMO-001, an invented sample. Every number is made up. The picture stops above the ClinVar hits table, the first table that names genes, so the rest of the chapters and the disclaimer are not shown.
@@ -178,4 +180,4 @@ start ${GENOME_DIR}/${SAMPLE}/${SAMPLE}_report.html - Steps that were not run show "Not run" and are listed under Steps Not Run — this is expected. A step recorded as failed shows "Failed" instead - The report contains health findings: the ClinVar table lists your pathogenic and likely pathogenic variants, and other sections summarise pharmacogenomics, CPSR and the other steps. It holds no raw reads or full VCF, but share it only as you would share a medical record - Re-run this script anytime to update the report after running additional steps -- The Nextflow `HTML_REPORT` module runs the same `bin/render_report.py` on the outputs of its run's QC, ClinVar, PharmCAT, CPIC, CPSR, clinical filter, slivar, ROH and mitochondrial haplogroup steps, so both reports print the same numbers for the same file. Its slivar card has no compound-het variant and gene counts: the Nextflow slivar module writes no `_compound_hets.tsv`. For every section on a Nextflow run, run this script with `GENOME_DIR` set to the `--outdir` +- The Nextflow `HTML_REPORT` module runs the same `og report render` on the outputs of its run's QC, ClinVar, PharmCAT, CPIC, CPSR, clinical filter, slivar, ROH and mitochondrial haplogroup steps, so both reports print the same numbers for the same file. Its slivar card has no compound-het variant and gene counts: the Nextflow slivar module writes no `_compound_hets.tsv`. For every section on a Nextflow run, run this script with `GENOME_DIR` set to the `--outdir` diff --git a/docs/25-prs.md b/docs/25-prs.md index 8ef1b061d..c7c437ff7 100644 --- a/docs/25-prs.md +++ b/docs/25-prs.md @@ -179,7 +179,7 @@ The panel is pgsc_calc's 1000 Genomes database, `pgsc_1000G_v1.tar.zst` (`PGSC_P | Memory | 7.3 to 7.6 GB at the peak for the whole machine; the largest task, 6.9 to 7.1 GB (plink2 scoring the panel's own samples, and the panel's variant filter) | | Time | 20 to 30 minutes for the run, on 4 CPUs | -That run scored PGS000018 alone and fits a 16 GB machine; the default scores need more ([Memory](#memory)). It does not fit the 14 GB of disk a GitHub-hosted runner guarantees, and the download alone outlasts the pull request test budget, so the panel stays opt-in (`setup.sh --ancestry-panel`), and the tests on every pull request project the fixture sample onto the PGS Catalog's 268 MB synthetic panel instead (`tests/e2e/prs-2-ancestry.sh`). The 1000 Genomes run above repeats on an E2E run dispatched on a branch other than main (or with `PGSC_MEASURE=1`): it adds about 40 minutes, which a dispatched run of main keeps for its other cases. +That run scored PGS000018 alone and fits a 16 GB machine; the default scores need more ([Memory](#memory)). It does not fit the 14 GB of disk a GitHub-hosted runner guarantees, and its download takes 13 to 31 minutes on a runner, so the panel stays opt-in (`setup.sh --ancestry-panel`), and the tests on every pull request project the fixture sample onto the PGS Catalog's 268 MB synthetic panel instead (`tests/e2e/prs-2-ancestry.sh`). The 1000 Genomes run above repeats on an E2E run dispatched on a branch other than main and not traced (or with `PGSC_MEASURE=1`): it adds about 40 minutes, which a dispatched run of main keeps for its other cases. ## Interpreting Results @@ -240,7 +240,7 @@ Call the sample again with step 3 to get the gVCF. Imputation (step 14) is not n - A raw sum is NOT a percentile. Without the panel there is no percentile at all. - A percentile is NOT a probability. Being at the 90th percentile does not mean a 90% chance of the condition. - They are NOT comparable across conditions. -- The percentile compares you with the reference samples most similar to you, not with your own family or community. For a group the panel represents poorly, the percentile is less reliable; the ancestry table says when the population match is low-confidence. +- The percentile compares you with the reference samples most similar to you, not with your own family or community. For a group the panel represents poorly, the percentile is less reliable; the ancestry table says when the population match is low-confidence. When your ancestry sits between two groups (each at a probability of 0.2 or more in [step 26](26-ancestry.md#interpreting-results)), the step prints a note that names both groups and says the percentile ranks you among one of them only, and the report reads the ancestry match as low-confidence. A percentile among a group your ancestry does not match is off: people from a reference group whose mean sits 0.5 standard deviations away from their own get percentiles far from uniform. - They are NOT stable across scoring file versions. A new harmonised file is a new baseline. ## Limitations @@ -249,6 +249,7 @@ Call the sample again with step 3 to get the gVCF. Imputation (step 14) is not n - A score captures only the genetic component. Lifestyle, environment, and family history are often more predictive. - Breast cancer and prostate cancer are scored for everyone, but get a percentile only for the sex they were built for, or when no sex was given. - The test fixture is a few slices of one genome: its sums, match rates and percentiles show the code paths work, not what a real genome gives. Real percentiles, and the time and memory of the four new scores on a whole genome, have to come from a real run. +- Unit tests check that `og prs table` reads the sample's percentile from pgsc_calc output containing both sample and reference panel rows. The table formats the percentile to one decimal place. In a simulation, 1,000 people are ranked among 2,000 reference people drawn from the same group's allele frequencies using pgsc_calc-style ranks. The reported percentiles pass a uniformity check (Kolmogorov-Smirnov p > 0.01). Shifting the simulated reference mean by 0.5 standard deviations makes that check fail. Calibration on real panels remains outside the scope of the simulation. - Chip data matches far fewer score variants than a genome; most scores then fall under pgsc_calc's 75% match rate and get no sum (see [Using chip data](chip-data-guide.md)). ## Notes diff --git a/docs/26-ancestry.md b/docs/26-ancestry.md index 9a14893ed..714e3ab64 100644 --- a/docs/26-ancestry.md +++ b/docs/26-ancestry.md @@ -71,11 +71,12 @@ The panel's extraction, QC and PCA come on top of step 25's run; see the measure - **population** is the reference group whose genetic ancestry is most similar to yours. It is a statement about similarity to five continental groups of the 1000 Genomes Project, not about identity, nationality or ethnicity. - **probability_** are the random forest's probabilities. A low top probability (`population_low_confidence` is `True`) means you sit between groups or far from all of them; the step 25 percentile is then less reliable. +- When the two most probable groups each have a probability of 0.2 or more, you sit between them. pgsc_calc sets `population_low_confidence` only when the top probability is under 0.5, so a sample at 0.55 and 0.45 reads `False` there. Step 25 then prints a note that names both groups and says each percentile ranks you among `population` only. In `summary.json`, the PRS section's `ancestry` says `mixed` (`True` or `False`) and `second_population` (the most probable group other than `population`), and its `low_confidence` is `True` for such a sample, so the report reads the ancestry match as low-confidence. - **PC1 to PC10** are your coordinates on the panel's principal components. They can be plotted against the panel samples in `sample_popsimilarity.txt.gz`. ## Limitations -- Five continental groups only. Mixed ancestry is shown as the single most similar group, with its probabilities; this step does not estimate admixture fractions or local ancestry. +- Five continental groups only. Mixed ancestry is shown as the single most similar group, with its probabilities; this step does not estimate admixture fractions or local ancestry. The probabilities say how the random forest split its vote, not what share of your genome comes from each group. - Groups the 1000 Genomes Project does not sample well are placed less reliably. - Fractions, regional groups and a chromosome painting are [step 26b](26b-ancestry-fine.md), against an open panel of 80 populations. It has limits of its own: a regional name is a cluster we name, splits within Iberia are weak for every method, Greek and Southern Italian ancestry overlap, and what it supports well is continental, and in places country-level, similarity. diff --git a/docs/27-cpic-lookup.md b/docs/27-cpic-lookup.md index bd94b0d9e..362748bb7 100644 --- a/docs/27-cpic-lookup.md +++ b/docs/27-cpic-lookup.md @@ -41,10 +41,10 @@ MEDICATIONS=/path/to/medications.txt ./scripts/27-cpic-lookup.sh your_name 2. Runs `og pgx cpic-report` in the Python image. It reads the gene calls (PharmCAT 3.x flat `genes` map, the 2.x map nested by source, and the older list are all read) and writes the phenotype table. 3. A gene counts as non-normal only when its phenotype names a changed metabolism or function (poor, intermediate, rapid or ultrarapid metabolizer; decreased, increased, poor or no function), a positive HLA test, a G6PD deficiency, malignant hyperthermia susceptibility, an increased MT-RNR1 risk, an ivacaftor-responsive CFTR, or a VKORC1 `-1639` genotype with the A allele (`-1639 GA` or `-1639 AA`, the warfarin dose-reduction case). A called gene whose phenotype is anything else (`n/a`, `no phenotype assigned`, the reference VKORC1 genotype `-1639 GG`, `Uncertain Susceptibility`, `Indeterminate`) is `unclassified`: it is listed in the section "Called Genes Without a Function Phenotype", is not counted, and gets no drug list. One exception: PharmCAT can leave a gene `Indeterminate` at gene level while the annotations of its drugs assign it a phenotype (for example a gene with unphased heterozygous positions plus positions missing from the VCF). When the drug annotations PharmCAT gives that gene (those for any of its possible diplotypes when there are any) all name the same phenotype, other than `Indeterminate`, `No Result` or `n/a`, the gene takes it, with the note `[gene level Indeterminate; phenotype from PharmCAT's drug annotations]` in the phenotype table and the text, and its status follows that phenotype. An annotation with no phenotype for the gene is not counted. Several possible diplotypes must also get the same drug guidance, because the drug list shows the first one's. With no annotation that names a phenotype, with annotations that disagree (one that says `Indeterminate` disagrees too), or with possible diplotypes whose guidance differs, it stays `Indeterminate` and `unclassified`. PharmCAT's own example report has 3 non-normal genes and 7 unclassified ones. For each gene with a non-normal phenotype it lists the drugs from the report's `drugs` section: CPIC's recommendation for the called diplotype per drug first, then the drugs DPWG or the FDA name. PharmCAT lists an annotation for every diplotype the sample may have, so only the ones for the called diplotype (or, without diplotype labels, its phenotype) are shown. When the report names no drug for the gene, it falls back to the gene's `relatedDrugs`, then to a small static table, and finally prints a line saying the gene is not in the drug table, so a gene is never skipped silently. 4. When PharmCAT lists more than one possible diplotype for a gene and their phenotypes differ (positions missing from the VCF leave it unable to choose), the gene is `ambiguous`: it is listed in its own section with the possible phenotypes and no drug guidance, never as its first diplotype. -5. With the pypgx summary it writes the comparison table and, for a gene PharmCAT reports as not called or ambiguous but pypgx called (CYP2D6 is the usual one), a warning in the recommendations naming every drug PharmCAT links to that gene. When pypgx's call is normal the line is a note instead. +5. With the pypgx summary it writes the comparison table and, for a gene PharmCAT reports as not called or ambiguous but pypgx called (CYP2D6 is the usual one; a pypgx `Indeterminate`, in any case or as one allele such as `*1/Indeterminate`, is no call), a warning in the recommendations naming every drug PharmCAT links to that gene. When pypgx's call is normal the line is a note instead. 6. With step 36's consensus table it writes a section "Calls From Other Tools": each gene passed to PharmCAT as an outside call, from which tool, and whether PharmCAT's report shows it as one (`callSource` `OUTSIDE`); and each gene held back, with the reason and what each caller said. For a held-back CYP2D6 it names the drugs CYP2D6 affects and says no guidance is given for them, and the pypgx-only warning of point 5 is not printed for it: one caller's call is what the consensus refused. Genes PharmCAT reports as outside calls carry `[outside call]` in the gene results. An outside call of CYP2D6, HLA-A or HLA-B in PharmCAT's report that the table does not pass on, with the same diplotype, comes from an earlier step 36 run that step 07 has not replaced: it gets no drug guidance, is listed as not called, and a warning says to rerun step 07. When the table is missing (a caller step ran again since) but PharmCAT's report has outside calls of those genes, the step passes the missing table, so none of them is confirmed. MT-RNR1's outside call comes from step 20's chrM calls through step 07, never from step 36: the table never lists it, so it is not checked against it and keeps its drug guidance. -7. With a medication list, each line is matched by its first word (a space, comma, semicolon, slash or plus ends a word, so `lopinavir/ritonavir` is matched as lopinavir; leading commas, semicolons, slashes and pluses are ignored, and a line of only those is skipped) against `assets/pgx_inhibitors.tsv`: the FDA's strong and moderate inhibitors of CYP2D6 and CYP2C19, its moderate CYP2C9 inhibitors (it lists no strong one) and its strong CYP3A inhibitors, one row each for CYP3A4 and CYP3A5, as the FDA's table of clinical CYP inhibitors classes them, each row with its source. A ritonavir-boosted combination the FDA classes as one (`lopinavir and ritonavir`) has a row under ritonavir and one under its partner. For CYP2D6 the step applies CPIC's rule from its opioid guideline: "for patients on strong CYP2D6 inhibitors, the CYP2D6 activity score is adjusted to 0 and the predicted phenotype is a poor metabolizer", and for a moderate inhibitor "the activity score is multiplied by 0.5 and then converted to the predicted phenotype" (CPIC's bands: 0 poor, under 1.25 intermediate, 1.25 to 2.25 normal, above 2.25 ultrarapid). The activity score is the one PharmCAT gives the called diplotype. With several inhibitors of one gene the strongest counts. The result is written as `Poor Metabolizer (phenoconverted by paroxetine)` beside the genotype phenotype, never in its place, with a `WARNING:` line in the section "Medications and Phenoconversion", so the summary carries it. Nothing is converted for a gene that is not called, ambiguous or unclassified, or for a moderate inhibitor when the report gives no activity score: the line says why. CPIC has no activity score for CYP2C19, and gives no rule that converts the phenotype of CYP2C9, CYP3A4 or CYP3A5, so an inhibitor of one of these genes gets a note and a `not computed: ...` value, with no converted phenotype. For a poor metabolizer of these genes (the most common CYP3A5 result) the note does not say the inhibitor can lower the activity further. A line that names no drug of the table is listed as "Not in the inhibitor table", never dropped. A list that names no drug converts nothing and leaves the phenotype table as it is without a list. An unreadable list or table is reported in that section; the rest of the report is written as usual. The FDA's table lists no strong or moderate CYP2B6 inhibitor, so CYP2B6 has no rows. -8. With step 06's ClinVar hits, each hit classed `Pathogenic` or `Likely_pathogenic` (one part of `CLNSIG`; a conflicting classification or `risk_factor` alone does not count) in a gene of `assets/pgx_contraindications.tsv` adds a line in the section "Drug Contraindications From Disease Variants" for each drug the table names for the gene, with the label's note, the provisional CPIC level and the label; a drug whose name is on your medication list is marked `ON THE MEDICATION LIST`. The table's `Report_if` column says when the line is a `WARNING:`. For POLG, OTC and HPRT1 (`any`) one variant gives it. For the recessive urea cycle genes ASL, ASS1, CPS1 and NAGS (`biallelic`) it needs a genotype with both copies changed (`1/1`, a hemizygous `1`, or `1/.` and `./1`, which step 06 writes for the two alleles of a `1/2` site: the missing copy holds the other allele, not the reference) or two distinct variants in the gene, and the line says which. The hits are not phased, so two variants on the same copy still give the warning. One heterozygous variant in such a gene makes a carrier: it gives a `NOTE:` line instead, which says so, and only `WARNING:` lines reach the summary. The table holds the CPIC gene-drug pairs at level A, B or A/B (CPIC API, queried 2026-11-03) whose gene is a disease gene PharmCAT does not call, whose drug the variant makes risky and whose risk a drug label names: POLG with valproic acid (the valproate label contraindicates it in POLG mitochondrial disorders), ASL, ASS1, CPS1, NAGS and OTC with valproic acid (the label contraindicates it in urea cycle disorders), and HPRT1 with mycophenolic acid (the label says to avoid it in hereditary HGPRT deficiency). Every one of these pairs is provisional in CPIC: no CPIC guideline covers it, and the line says so. SCN1A with carbamazepine and phenytoin is left out: no label or guideline names the gene, and SCN1A also has gain-of-function disease variants, for which a warning against these drugs does not follow. Without the hits table the section lists the table's genes as not checked. +7. With a medication list, every drug a line names is matched (whitespace, commas, semicolons, slashes, pluses and brackets split a line, and punctuation at either end of a word is dropped; the word `and` is dropped, and so are doses, units and frequencies such as `20`, `mg` and `daily`; a word with a hyphen is checked whole and then by each part that starts with a letter, has three or more characters and is no dose word, so a hyphenated name in a table still matches and a prefix, a lone letter or a number is never taken for a drug; so `olanzapine/fluoxetine` and `olanzapine-fluoxetine` are checked for olanzapine and for fluoxetine, `Contrave (naltrexone/bupropion) 8/90 mg` for contrave, naltrexone and bupropion, `co-trimoxazole` for co-trimoxazole and trimoxazole but never `co`, `L-dopa` for l-dopa and dopa, and a line of only separators is skipped) against `assets/pgx_inhibitors.tsv`: the FDA's strong and moderate inhibitors of CYP2D6 and CYP2C19, its moderate CYP2C9 inhibitors (it lists no strong one) and its strong CYP3A inhibitors, one row each for CYP3A4 and CYP3A5, as the FDA's table of clinical CYP inhibitors classes them, each row with its source. A ritonavir-boosted combination the FDA classes as one (`lopinavir and ritonavir`) has a row under ritonavir and one under its partner. For CYP2D6 the step applies CPIC's rule from its opioid guideline: "for patients on strong CYP2D6 inhibitors, the CYP2D6 activity score is adjusted to 0 and the predicted phenotype is a poor metabolizer", and for a moderate inhibitor "the activity score is multiplied by 0.5 and then converted to the predicted phenotype" (CPIC's bands: 0 poor, under 1.25 intermediate, 1.25 to 2.25 normal, above 2.25 ultrarapid). The activity score is the one PharmCAT gives the called diplotype. With several inhibitors of one gene, from several lines or from the drugs of one combination product, the strongest counts. The result is written as `Poor Metabolizer (phenoconverted by paroxetine)` beside the genotype phenotype, never in its place, with a `WARNING:` line in the section "Medications and Phenoconversion", so the summary carries it. Nothing is converted for a gene that is not called, ambiguous or unclassified, or for a moderate inhibitor when the report gives no activity score: the line says why. CPIC has no activity score for CYP2C19, and gives no rule that converts the phenotype of CYP2C9, CYP3A4 or CYP3A5, so an inhibitor of one of these genes gets a note and a `not computed: ...` value, with no converted phenotype. For a poor metabolizer of these genes (the most common CYP3A5 result) the note does not say the inhibitor can lower the activity further. A line none of whose drugs is in the table is listed as "Not in the inhibitor table", never dropped. A list that names no drug converts nothing and leaves the phenotype table as it is without a list. An unreadable list or table is reported in that section; the rest of the report is written as usual. The FDA's table lists no strong or moderate CYP2B6 inhibitor, so CYP2B6 has no rows. +8. With step 06's ClinVar hits, each hit classed `Pathogenic` or `Likely_pathogenic` (one part of `CLNSIG`; a conflicting classification or `risk_factor` alone does not count) in a gene of `assets/pgx_contraindications.tsv` adds a line in the section "Drug Contraindications From Disease Variants" for each drug the table names for the gene, with the label's note, the provisional CPIC level and the label; a drug whose name is on your medication list, as any drug of a line (the second drug of a combination product too), is marked `ON THE MEDICATION LIST`. The table's `Report_if` column says when the line is a `WARNING:`. For POLG, OTC and HPRT1 (`any`) one variant gives it. For the recessive urea cycle genes ASL, ASS1, CPS1 and NAGS (`biallelic`) it needs a genotype with both copies changed (`1/1`, a hemizygous `1`, or `1/.` and `./1`, which step 06 writes for the two alleles of a `1/2` site: the missing copy holds the other allele, not the reference) or two distinct variants in the gene, and the line says which. The hits are not phased, so two variants on the same copy still give the warning. One heterozygous variant in such a gene makes a carrier: it gives a `NOTE:` line instead, which says so, and only `WARNING:` lines reach the summary. The table holds the CPIC gene-drug pairs at level A, B or A/B (CPIC API, queried 2026-11-03) whose gene is a disease gene PharmCAT does not call, whose drug the variant makes risky and whose risk a drug label names: POLG with valproic acid (the valproate label contraindicates it in POLG mitochondrial disorders), ASL, ASS1, CPS1, NAGS and OTC with valproic acid (the label contraindicates it in urea cycle disorders), and HPRT1 with mycophenolic acid (the label says to avoid it in hereditary HGPRT deficiency). Every one of these pairs is provisional in CPIC: no CPIC guideline covers it, and the line says so. SCN1A with carbamazepine and phenytoin is left out: no label or guideline names the gene, and SCN1A also has gain-of-function disease variants, for which a warning against these drugs does not follow. Without the hits table the section lists the table's genes as not checked. 9. A report that cannot be read, or that yields no gene, writes a "PARSING FAILED" report and the step exits 1. It never writes an all-clear report from a format it could not read. A field of the report with an unexpected type (a number where a name belongs, say) reads as missing. 10. The `Date:` line of the recommendations is today's date in UTC. The output files are written under a temporary name and renamed into place, so a step that stops half way leaves no half-written table. diff --git a/docs/32-pypgx.md b/docs/32-pypgx.md index 6d187fdb3..12bc25056 100644 --- a/docs/32-pypgx.md +++ b/docs/32-pypgx.md @@ -165,7 +165,7 @@ All output is written to `${GENOME_DIR}/${SAMPLE}/pypgx/`. | Column | Description | |---|---| | Gene | Gene symbol | -| Diplotype | Star allele call (e.g., \*1/\*4); `Indeterminate` for CYP2D6 when the depth check failed (multi-mapped reads, or flanks under 10x); `FAILED` for a gene pypgx wrote no result for; `N/A` for a gene not callable on this reference (see Status) | +| Diplotype | Star allele call (e.g., \*1/\*4); `Indeterminate` for CYP2D6 when the depth check failed (multi-mapped reads, or flanks under 10x), or for a gene pypgx itself could not call; `FAILED` for a gene pypgx wrote no result for; `N/A` for a gene not callable on this reference (see Status) | | Phenotype | Metabolizer status (e.g., Intermediate Metabolizer). `no pypgx phenotype rule` for a gene pypgx has no phenotype rule for (no `PhenotypeMethod` in its own gene table; in 0.26.0 the 17 genes listed under [Weakly supported genes](#weakly-supported-genes)), where pypgx itself writes `Indeterminate` | | CNV_call | The copy-number call pypgx itself made from read depth (its `CNV` field, for example `Normal` or `WholeDel1`). BAM-based genes only; `N/A` for VCF-based genes, and `N/A (CYP2D6 depth check)` for CYP2D6 when the depth check failed. Earlier versions guessed a Yes/No flag from the allele names, which flagged any name containing `*5` and missed the GSTM1/GSTT1/CYP2A6 whole-gene deletions | | Source | BAM (SV genes) or VCF (variant-based genes) | @@ -176,6 +176,8 @@ All output is written to `${GENOME_DIR}/${SAMPLE}/pypgx/`. | Defining_filter | `PASS` when every defining variant is in the VCF with FILTER `PASS`; otherwise the ones that are not PASS (`RefCall`, `LowQual`..., or `no filter` for FILTER `.`) or not in the VCF, and any star allele pypgx's table lacks (`not in pypgx's table: *99`) | | Quality | `low-confidence: check the reads` when a defining variant has a GQ below the floor, `GQ not in the VCF: check the reads` when one has no GQ, `ok` otherwise, `N/A` without defining variants | +A Diplotype of `Indeterminate` is no call, in any case and also as one allele (`*1/Indeterminate`): step 36 counts it neither as agreeing nor as disagreeing with Cyrius, and step 27 gives it no comparison match and no warning. + ### Low-GQ star alleles A star allele can rest on a single heterozygous variant called with a low genotype quality. Such a call prints like any other diplotype, so the summary looks each defining variant up in the VCF, and a call with one below GQ 20 gets `low-confidence: check the reads` in its Quality column and in both reports, with the weakest variant and its AD. Set the floor with `PYPGX_GQ_FLOOR=15 ./scripts/32-pypgx.sh your_name`, or for the Nextflow module with `withName: 'PYPGX' { ext.gq_floor = 15 }` in a config passed with `-c`. Mapping the defining variants back to the VCF uses pypgx's allele table; a variant pypgx names differently from the VCF (a multi-base change written another way) reads as `not in the VCF`, so check such a call by hand. `og pgx pypgx-summary` (`rust/crates/og-steps/src/pypgx_summary.rs`) writes the summary and finds the allele table beside the pypgx on `PATH` (`--pypgx-data` names it). Without the tables, or without an index for the VCF, it still writes the summary and warns on stderr: Defining_variants says `not looked up` and the other four evidence columns say `N/A`, so a Quality of `N/A` there means no check ran. @@ -192,7 +194,7 @@ The pypgx section of both reports also lists the genes not assessable on this re | PharmCAT_diplotype | Diplotype from step 7 | | pypgx_diplotype | Diplotype from this step | | Match | Yes, No, pypgx only, or PharmCAT only. Allele names are compared after one rename: PharmCAT's `rs9923231 reference (C)` and pypgx's `Reference` are the same allele, and so are `rs9923231 variant (T)` and `rs9923231` | -| Called_by | `both` when both tools called the gene (Match is Yes or No), otherwise which tool did. PharmCAT's `Unknown/Unknown` and pypgx's `FAILED` and `N/A` count as no call; a gene neither tool called is left out | +| Called_by | `both` when both tools called the gene (Match is Yes or No), otherwise which tool did. PharmCAT's `Unknown/Unknown` and pypgx's `FAILED`, `N/A` and `Indeterminate` (in any case, or as one allele such as `*1/Indeterminate`) count as no call; a gene neither tool called is left out | | PharmCAT_phenotype | PharmCAT's phenotype for the gene, as step 27's phenotypes table shows it | | pypgx_phenotype | The Phenotype column of the summary above | diff --git a/docs/37-y-haplogroup.md b/docs/37-y-haplogroup.md index 392de3c2e..eeb4fdcaa 100644 --- a/docs/37-y-haplogroup.md +++ b/docs/37-y-haplogroup.md @@ -106,6 +106,102 @@ The header lines of the file record the gates, the mask, the sample's median chr The scan needs step 03's VCF (`${SAMPLE}/vcf/${SAMPLE}.vcf.gz`) and the current tree's tables and mask. Without them, the step says so in one line and writes the rest of its output as before. +## How well it places + +We measured both placements against Y haplogroups other people published for the same men. The men are 58 men of the 1000 Genomes Project. Poznik et al. 2016 (Nature Genetics 48:593) published their haplogroups in `1000Y.sample_level_haplogroup_calls.ver5.txt` on the 1000 Genomes FTP; we use its column 3, the most derived known SNP the authors saw (`R1b-DF27`). The reads are each man's chrY from his 30x CRAM of the New York Genome Center (NYGC), the same file the public-evidence run reads. + +How the men were chosen, fixed before any result was seen. We take the men of that file who have a 30x CRAM, 1,233 of its 1,244. A man counts only when his published SNP sits on one branch of the current tree. Where YFull gives one SNP name to two branches (a recurrent mutation: `M356` is on `C-K98` and on `T-Y31540`), the clade letter of the published name settles it (`C5-M356` is `C-K98`). For each of the file's high-level haplogroups we take its most common published branches: one branch, two for a haplogroup of 10 men or more, three for 50 or more. Branches with the same count are taken in the sort order of their published name. From each branch we take its first man by ID. That gives 59 men in 35 haplogroups, from A0 to T. One branch was left out because its SNP is on two R branches (`R2-L294`). One man was left out because EBI serves a truncated CRAM index for him (HG03846, 4,096 bytes), and his branch had no other man. + +Each man ran through step 37's own commands: the two pileups with the step's flags, then Yleaf 4.1.7 with `-rp` on its pileup and `og y tree-place` on the other. `og dev evidence y-bench` then places the published SNP and each tool's call on YFull YTree 14.06.0. It names the call by what it is against the published branch: + +| Result | The tool's branch is | +|---|---| +| exact | the published branch | +| below | below it: the same line, deeper | +| ancestor only | above it, on its line | +| sibling | on another line that splits off at the published branch's parent | +| wrong | on a line that splits off higher | +| no call, unknown name | no branch, or a name the tree cannot place | + +The published calls come from the ISOGG tree of 2014, which is much shallower than YFull's, so at 30x a correct call is nearly always "below". The benchmark checks the line down to the published branch. It cannot check the levels below it. + +| Tool | Men | Exact | Below | Ancestor only | Sibling | Wrong | No call | Unknown name | Published branch or below | +|---|---|---|---|---|---|---|---|---|---| +| Yleaf 4.1.7 | 58 | 1 | 57 | 0 | 0 | 0 | 0 | 0 | 58 of 58 (100.0%) | +| og y tree-place | 58 | 1 | 57 | 0 | 0 | 0 | 0 | 0 | 58 of 58 (100.0%) | + +Both tools put every man on his published line, and both go 7 levels below the published branch on average (og 7.4, Yleaf 7.1). The two named the same branch for 44 men. For 13 more both are on one line, and og's branch is deeper for 12 of them (one to three levels). For one man, HG00530, they split into two sister branches below the published one: og's `O-K481` against Yleaf's `O-Y174537`, and og names `O-Y174537` in `Conflict` with support 1 against 4 for its own branch. The published call does not say which of the two is right. + +The outvote rule fired for one man (HG01583: one ancestral SNP, `BY37185`, outvoted). That placement is on the published line, 9 levels below it. No man was placed off his published line by the rule or for lack of it, so the data give no reason to change it, and it stays as it is. `Conflict` named a branch for 4 men, each with support 1 against 4 to 31 for the terminal branch. For three R1a men it was the same branch, `R-Y40`. + +The public-evidence workflow's `y-benchmark` job runs the step itself again (`scripts/37-y-haplogroup.sh`, with `setup.sh --yleaf-data`) on 6 of these men by default (input `y_samples`), and runs `og dev evidence y-bench` on its calls. Its gate fails when a tool puts fewer than 95% of the men on the published branch or below it. Its red control gives each tool's calls to the man half the list away, and that table must fail the gate. Before it reads any CRAM, it refuses a `y_samples` list in which so many men share a line with that partner that the handed calls could still pass the gate. + +
+The 58 men + +| Man | Population | Published (Poznik et al. 2016), its branch on YFull 14.06.0 | Yleaf 4.1.7 | og y tree-place | +|---|---|---|---|---| +| HG01890 | ACB | `A0-L1038`, `A-L1035` | `A-L1035`, exact | `A-L1035`, exact | +| HG02613 | GWD | `A1a-M31`, `A1a` | `A-Y32399`, 3 levels below | `A-Y32399`, 3 levels below | +| HG02588 | GWD | `B-M181`, `B` | `B-Y311987*(xB-ZS382)`, 3 levels below | `B-ZS382`, 4 levels below | +| HG00553 | PUR | `B2-M109`, `B-Y10267` | `B-C137898`, 7 levels below | `B-C137898`, 7 levels below | +| NA18971 | JPT | `C1-P121`, `C-M8` | `C-Y161239`, 3 levels below | `C-Y161239`, 3 levels below | +| HG03917 | BEB | `C3-Z1300`, `C-Z1300` | `C-F3777*(xC-F9966)`, 3 levels below | `C-F3777`, 3 levels below | +| HG03850 | STU | `C5-P92`, `C-P92` | `C-Y33575*(xC-FT358545,C-FTB33293,C-Y82010,C-FTA79799)`, 5 levels below | `C-Y33575`, 5 levels below | +| HG03228 | PJL | `C5-M356`, `C-K98` | `C-FT327336`, 2 levels below | `C-FT327336`, 2 levels below | +| NA18952 | JPT | `D2-M179`, `D-M64.1` | `D-CTS2826`, 5 levels below | `D-CTS2826`, 5 levels below | +| NA18966 | JPT | `D2-M125`, `D-M125` | `D-Y439573`, 2 levels below | `D-Y439573`, 2 levels below | +| HG02461 | GWD | `E1a-L94`, `E-L94` | `E-Z5986*(xE-Z5987,E-Y478064,E-Z5989)`, 2 levels below | `E-FT333060`, 3 levels below | +| HG01094 | PUR | `E1a-CTS2808`, `E-M132` | `E-FT296631`, 6 levels below | `E-FT296631`, 6 levels below | +| HG01556 | CLM | `E1b-P252`, `E-U174` | `E-Z5968*(xE-FTA5825,E-Z5969,E-Z37526)`, 5 levels below | `E-Z5968`, 5 levels below | +| HG01204 | PUR | `E1b-P277`, `E-M4254` | `E-FT295369`, 14 levels below | `E-FT295369`, 14 levels below | +| HG02947 | ESN | `E1b-CTS8030`, `E-Z1704` | `E-Z37814`, 1 level below | `E-Z37814`, 1 level below | +| NA19035 | LWK | `E2-M200`, `E-M200` | `E-Y25288`, 3 levels below | `E-Y25288`, 3 levels below | +| HG02040 | KHV | `F-M213`, `F` | `F4`, 2 levels below | `F4`, 2 levels below | +| NA20858 | GIH | `G1-L833`, `G-M342` | `G-Y262772*(xG-Y99680)`, 9 levels below | `G-Y262772`, 9 levels below | +| NA19670 | MXL | `G2-CTS4803`, `G-CTS4803` | `G-S2784`, 5 levels below | `G-S2784`, 5 levels below | +| HG02224 | IBS | `G2-L30`, `G-L30` | `G-PF3345*(xG-CTS342,G-Y46211,G-FTA56155,G-L497,G-Y507487,G-BY149069,G-Y65645,G-Z39367,G-BY1433)`, 8 levels below | `G-PF3345`, 8 levels below | +| HG03809 | BEB | `H-M370`, `H-M69` | `H-Z34843*(xH-Y168231)`, 9 levels below | `H-Y166956`, 10 levels below | +| HG02684 | PJL | `FH-M2713`, `H` | `H-PH1063`, 5 levels below | `H-M4771`, 6 levels below | +| HG01583 | PJL | `H1-M52`, `H-M52` | `H-Z5885*(xH-Z4695,H-Z12600,H-FT282446,H-Y291749)`, 6 levels below | `H-FT167357`, 9 levels below | +| HG04198 | ITU | `H2-Apt`, `H-Z13973` | `H-Z13969*(xH-Y373420)`, 3 levels below | `H-C116648`, 4 levels below | +| HG00186 | FIN | `I1-L258`, `I-L258` | `I-Y131678`, 7 levels below | `I-Y131678`, 7 levels below | +| HG00117 | GBR | `I1-L118`, `I1` | `I-Y172395`, 7 levels below | `I-Y172395`, 7 levels below | +| HG01527 | IBS | `I2-CTS9183`, `I-CTS616` | `I-Y167082`, 9 levels below | `I-Y167082`, 9 levels below | +| HG01101 | PUR | `I2-L160`, `I-L160` | `I-Y2181`, 9 levels below | `I-Y2181`, 9 levels below | +| HG01069 | PUR | `J1-P58`, `J-P58` | `J-Z2289`, 18 levels below | `J-Z2289`, 18 levels below | +| HG01253 | CLM | `J1-L321`, `J1` | `J-M10096`, 7 levels below | `J-M10096`, 7 levels below | +| HG02236 | IBS | `J2-F4326`, `J-L26` | `J-Y33322*(xJ-Y87966)`, 18 levels below | `J-Y33322`, 18 levels below | +| HG02690 | PJL | `J2-M241`, `J-M241` | `J-FT19136`, 11 levels below | `J-Z8372`, 10 levels below | +| HG01412 | PUR | `J2-L397`, `J-L70` | `J-Z45235*(xJ-Y275293)`, 5 levels below | `J-Z45235`, 5 levels below | +| HG03742 | ITU | `KxLT-M526`, `K2` | `K-Y28301`, 6 levels below | `K-Y28301`, 6 levels below | +| HG02654 | PJL | `L1-M27`, `L-M27` | `L-Y28521*(xL-MF68713)`, 10 levels below | `L-FTA9535`, 12 levels below | +| HG03672 | STU | `L1-L1307`, `L-L1307` | `L-Y35083*(xL-M5513,L-Y106928)`, 6 levels below | `L-Y35083`, 6 levels below | +| HG02138 | KHV | `N-M2283`, `N-Z4762` | `N-Y24355*(xN-Y61250,N-Y65390,N-Y64234)`, 16 levels below | `N-Y24355`, 16 levels below | +| HG00185 | FIN | `N1-Z1940`, `N-Z1940` | `N-Y141790`, 4 levels below | `N-Y141790`, 4 levels below | +| HG00182 | FIN | `N1-CTS10035`, `N-Z1925` | `N-BY142665`, 9 levels below | `N-BY142665`, 9 levels below | +| HG00580 | CHS | `O1-F589`, `O-M119` | `O-FT323420*(xO-MF37402)`, 11 levels below | `O-FT323420`, 11 levels below | +| HG02396 | CDX | `O1-M119`, `O-M119` | `O-Z23482*(xO-Y252345)`, 18 levels below | `O-Y252345`, 19 levels below | +| HG00701 | CHS | `O2-M88`, `O-M111` | `O-CTS3745`, 9 levels below | `O-CTS3745`, 9 levels below | +| HG00442 | CHS | `O2-M95`, `O-M95` | `O-CTS10007`, 2 levels below | `O-CTS10007`, 2 levels below | +| HG00530 | CHS | `O2-F1942`, `O-F1658` | `O-Y174537*(xO-MF81782)`, 5 levels below | `O-K481`, 5 levels below | +| HG00622 | CHS | `O3-M188`, `O-M188` | `O-FT315752`, 5 levels below | `O-FT315752`, 5 levels below | +| HG00524 | CHS | `O3-M1561`, `O-P164` | `O-CTS1174`, 19 levels below | `O-CTS1174`, 19 levels below | +| HG00403 | CHS | `O3-F11`, `O-F11` | `O-ACT3882`, 3 levels below | `O-ACT3882`, 3 levels below | +| HG01124 | CLM | `Q1a-M3`, `Q-M3` | `Q-M848*(xQ-Y126607,Q-Z5906,Q-M925,Q-Z35841,Q-Y592325,Q-CTS2731,Q-MPB139,Q-Y166361,Q-Z19357,Q-Y27993,Q-Z19301,Q-MPB096,Q-Y202441,Q-Z35597,Q-Z19432)`, 1 level below | `Q-M848`, 1 level below | +| HG01944 | PEL | `Q1a-M120`, `Q-M120` | `Q-FT8312*(xQ-Y280718)`, 6 levels below | `Q-FT9713`, 7 levels below | +| HG03652 | PJL | `Q1b-F711`, `Q-L275` | `Q-Y478758`, 5 levels below | `Q-Y478758`, 5 levels below | +| HG02600 | PJL | `R1a-Y7`, `R-Y7` | `R-C10046`, 9 levels below | `R-C10046`, 9 levels below | +| HG02597 | PJL | `R1a-L657`, `R-L657` | `R-C131139`, 13 levels below | `R-Y672703`, 14 levels below | +| HG03636 | PJL | `R1a-Z2123`, `R-Z2123` | `R-YP6547*(xR-FT74091)`, 6 levels below | `R-YP6547`, 6 levels below | +| HG00107 | GBR | `R1b-DF27`, `R-DF27` | `R-A17453*(xR-Y235960,R-Y32852,R-A16155)`, 5 levels below | `R-A17453`, 5 levels below | +| HG00129 | GBR | `R1b-PF6570`, `R-U152` | `R-Y222042`, 3 levels below | `R-Y222042`, 3 levels below | +| HG00126 | GBR | `R1b-M167`, `R-Z201` | `R-BY75410`, 10 levels below | `R-BY75410`, 10 levels below | +| HG01586 | PJL | `R2-L266`, `R2` | `R-Y154917*(xR-Y555693,R-Y155742)`, 18 levels below | `R-FT302409`, 19 levels below | +| HG01133 | CLM | `T-Z709`, `T-Z709` | `T-CTS9882*(xT-Y169564,T-Y15648,T-Y4981,T-Y273415)`, 12 levels below | `T-CTS6901`, 13 levels below | + +
+ ## Runtime Not measured on a full genome; the end test runs the step on the fixture's chrY slices only. An estimate is a few minutes: Yleaf reads the pileup at about 183,000 Y marker positions, and the current tree's pileup is at about 369,000 more. The placement reads the tree and the tables in about a second. diff --git a/docs/42-traits.md b/docs/42-traits.md index 6bd27af64..c3d1ec3c1 100644 --- a/docs/42-traits.md +++ b/docs/42-traits.md @@ -2,7 +2,7 @@ ## What This Does -Reads your genotype at a small set of well-replicated variants and prints what each genotype means in plain words: lactase persistence (digesting milk as an adult), caffeine metabolism and coffee intake, alcohol flush, bitter taste, earwax type, the photic sneeze reflex, smelling asparagus in urine, muscle fibre type, eye, hair and skin pigmentation, red hair, hair thickness, an enzyme note on MTHFR C677T (not health advice), secretor status, vitamin D level, bilirubin (the Gilbert syndrome genotype), nicotine breakdown, two variants linked to a high Lp(a) level, CFTR R117H (read with the CFTR poly-T tract of [step 9](09-str-expansions.md)), and the APOE ε2/ε3/ε4 alleles. A rule table reads several sites together: the APOE pair, the TAS2R38 bitter-taste haplotypes (PAV, AVI), phased from the reads when that is needed, the APOL1 G1 kidney risk variant, the alpha-1 antitrypsin Pi type (SERPINA1 M, S, Z), the HFE haemochromatosis genotype (C282Y, H63D), the vitamin D-binding protein type (GC), the chromosome 9p21 heart disease haplotype and the 17q21.31 H1/H2 lineage. Each row names the primary paper it comes from. On by default. +Reads your genotype at a small set of well-replicated variants and prints what each genotype means in plain words: lactase persistence (digesting milk as an adult), caffeine metabolism and coffee intake, alcohol flush, bitter taste, earwax type, the photic sneeze reflex, smelling asparagus in urine, muscle fibre type, eye, hair and skin pigmentation, red hair, hair thickness, an enzyme note on MTHFR C677T (not health advice), secretor status, vitamin D level, bilirubin (a Gilbert syndrome marker), nicotine breakdown, two variants linked to a high Lp(a) level, CFTR R117H (read with the CFTR poly-T tract of [step 9](09-str-expansions.md)), and the APOE ε2/ε3/ε4 alleles (and the rare ε1, whose pairs are not read). A rule table reads several sites together: the APOE pair, the TAS2R38 bitter-taste haplotypes (PAV, AVI), phased from the reads when that is needed, the APOL1 G1 kidney risk variant, the alpha-1 antitrypsin Pi type (SERPINA1 M, S, Z), the HFE haemochromatosis genotype (C282Y, H63D), the vitamin D-binding protein type (GC), the chromosome 9p21 heart disease haplotype and the 17q21.31 H1/H2 lineage. Each row names the primary paper it comes from. On by default. ## Why @@ -50,7 +50,7 @@ Both reports have a Traits section: every row with its genotype, reading, note a | Trait | Variants | Papers (PubMed) | |---|---|---| -| Lactase persistence | rs4988235 (-13910), rs182549 (-22018), rs145946881 (-14010), rs41380347 (-13915), rs41525747 (-13907) | 11788828, 17159977 | +| Lactase persistence | rs4988235 (-13910), rs182549 (-22018), rs145946881 (-14010), rs41380347 (-13915), rs41525747 (-13907), rs869051967 (-14009) | 11788828, 17159977, 23993196 | | Alcohol flush, alcohol metabolism | rs671 (ALDH2*2), rs1229984 (ADH1B*2) | 2562960, 2014795 | | Caffeine and coffee | rs762551 (CYP1A2), rs2472297 (CYP1A1-CYP1A2), rs4410790 (AHR) | 10233211, 21357676, 21490707 | | Bitter taste (PTC, PROP) | rs713598, rs1726866, rs10246939 (TAS2R38) | 12595690, 15723792 | @@ -58,16 +58,16 @@ Both reports have a Traits section: every row with its genotype, reading, note a | Photic sneeze reflex, asparagus smell | rs10427255, rs4481887 | 20585627, 27965198 | | Muscle fibre type | rs1815739 (ACTN3 R577X) | 10192379, 12879365 | | Eye colour | rs12913832 (HERC2), rs1800407 (OCA2) | 18252222, 20457092 | -| Pigmentation, freckles, red hair | rs16891982 (SLC45A2), rs1426654 (SLC24A5), rs12203592 (IRF4), rs1805007, rs1805008, rs1805009 (MC1R) | 15714523, 16357253, 18483556, 24267888, 9302268, 7581459 | +| Pigmentation, freckles, red hair | rs16891982 (SLC45A2), rs1426654 (SLC24A5), rs12203592 (IRF4), rs1805007, rs1805008, rs1805009 (MC1R) | 15714523, 16357253, 18483556, 24267888, 9302268, 7581459, 11030758 | | Hair thickness | rs3827760 (EDAR) | 18065779 | | MTHFR enzyme note, not health advice ([why](not-reported.md#not-reported-on-purpose)) | rs1801133 (MTHFR C677T) | 7647779, 23288205 | | Secretor status | rs601338 (FUT2) | 7876235, 12692541 | | Vitamin D level | rs2282679 in GC, the vitamin D binding protein gene | 20541252 | -| Bilirubin, Gilbert syndrome genotype | rs887829 (UGT1A1) | 19414484, 37456363 | +| Bilirubin, a Gilbert syndrome marker | rs887829 (UGT1A1) | 19414484, 37456363 | | Nicotine breakdown | rs1801272 (CYP2A6*2) | 2322567, 9827545 | | Lp(a) level | rs10455872, rs3798220 (LPA); [step 46](46-lpa.md) adds the KIV-2 repeat count, and the report's Lp(a) section reads the three together | 20032323, 36036785, 29973585 | | CFTR R117H, read with the poly-T tract of [step 9](09-str-expansions.md) | rs78655421 (CFTR) | 32404922 | -| APOE ε2/ε3/ε4 | rs429358, rs7412 | 8346443, 9343467 | +| APOE ε2/ε3/ε4 (and the rare ε1) | rs429358, rs7412 | 8346443, 9343467 | Variants whose effect allele the papers we could read did not state were left out, among them SLC24A4 rs12896399 and TYR rs1393350 of the eye-colour panels. @@ -101,17 +101,17 @@ The rules today: | TAS2R38 | rs713598 (codon 49), rs1726866 (262), rs10246939 (296) | PAV (taster), AVI (non-taster) | 12595690, 15723792 | | APOL1 | rs73885319 (S342G), rs60910145 (I384M) | G0, G1 | 20647424 | | SERPINA1 | rs17580 (S, Glu264Val), rs28929474 (Z, Glu342Lys) | M, S, Z (PiMM to PiZZ) | 14522813 | -| HFE | rs1800562 (C282Y), rs1799945 (H63D) | wt, C282Y, H63D | 8696333 | -| GC | rs7041 (Asp432Glu), rs4588 (Thr436Lys) | Gc1F, Gc1S, Gc2 | 24256378 | +| HFE | rs1800562 (C282Y), rs1799945 (H63D) | wt, C282Y, H63D | 8696333, 18199861, 19554541, 11812557 | +| GC | rs7041 (Asp432Glu), rs4588 (Thr436Lys) | Gc1F, Gc1S, Gc2 | 24256378, 27250744 | | CAD9p21 | rs10757278, rs1333049 | nonrisk, risk | 17478679, 17554300, 17478681 | | MAPT | rs1052553, rs8070723 | H1, H2 | 15654335, 19912324, 21685912 | The six rules after TAS2R38 are read without phase. A double heterozygote of each rests on an assumption, and the reading says it (see `needs_phase no` above). SERPINA1 PiSZ, HFE C282Y/H63D and GC Gc1S/Gc2 assume the two changes sit on different copies. APOL1 G0/G1, CAD9p21 "one risk copy" and MAPT H1/H2 assume both sites sit on one copy: the two APOL1 G1 changes were in perfect linkage disequilibrium in Genovese et al. (2010), and the 9p21 and 17q21.31 tags each mark one haplotype. A copy the rule does not describe, such as a 9p21 copy with only one of the two risk alleles, is `other_haplotype`, not read. What each rule leaves out, said in its note: -- APOL1 G2 (rs71785313) is a 6-base deletion. The rule table reads single-base sites only, so the rule itself reads G1, and a G2 copy reads as G0 there. The HTML and text reports read G2 beside the rule (`og summary collect`, the `apol1` module of `rust/crates/og-summary/src/collect/group_c/traits.rs`) from the sample's gVCF, or its VCF when there is no gVCF, and show the G1 plus G2 result under "With G2": G1/G2, G2/G2 and G1/G1 are the high-risk genotype. Two G2 copies read G2/G2 even when the G1 sites were not read. A VCF without reference blocks cannot tell a reference base from a position with no reads, so there a result with no G2 copy is marked `(no G2 call)`. The G2 reading uses the same floor as the trait sites: a call over the deletion span with a GQ below 20, a G2 call whose FILTER is not `PASS` or `.`, or a reference call whose FILTER is not `PASS`, `.` or `RefCall`, leaves G2 not read, so two G2 copies at GQ 5 never read G2/G2. The Nextflow report gets the sample's gVCF too (DeepVariant's, or the samplesheet's `gvcf` column), so both runs read the gVCF when there is one. +- APOL1 G2 (rs71785313) is a 6-base deletion. The rule table reads single-base sites only, so the rule itself reads G1, and a G2 copy reads as G0 there. The HTML and text reports read G2 beside the rule (`og summary collect`, the `apol1` module of `rust/crates/og-summary/src/collect/group_c/traits.rs`) from the sample's gVCF, or its VCF when there is no gVCF, and show the G1 plus G2 result under "With G2": G1/G2, G2/G2 and G1/G1 are the high-risk genotype. Two G2 copies read G2/G2 even when the G1 sites were not read. A VCF without reference blocks cannot tell a reference base from a position with no reads, so there a result with no G2 copy is marked `(no G2 call)`. The G2 reading uses the same floor as the trait sites: a call over the deletion span with a GQ below 20, a G2 call whose FILTER is not `PASS` or `.`, or a reference call whose FILTER is not `PASS`, `.` or `RefCall`, leaves G2 not read, so two G2 copies at GQ 5 never read G2/G2. The Nextflow report gets the sample's gVCF too (DeepVariant's, or the samplesheet's `gvcf` column), so both runs read the gVCF when there is one. The `result` column of `traits/${SAMPLE}_genotype_rules.tsv` is the rule's own result and never shows G2, so a script that needs the APOL1 result with G2 reads `summary.json`: in `sections.traits.data.rules`, the line whose `rule` is `APOL1` has the field `g2`, whose `result` and `reading` are the G1 plus G2 answer the reports show. - SERPINA1: M is any copy with neither S nor Z; rarer deficiency and null alleles are not read. A serum level confirms a deficiency. -- HFE: the C282Y homozygote is also a secondary finding (`assets/acmg_sf.tsv`, [step 23](23-clinical-filter.md)); the rule gives the reading of every genotype. -- GC: the protein levels are those of Powe et al. (2013), whose monoclonal assay was questioned in later letters. A protein type, not a vitamin D deficiency. +- HFE: the C282Y homozygote is also a secondary finding (`assets/acmg_sf.tsv`, [step 23](23-clinical-filter.md)); the rule gives the reading of every genotype. Feder et al. (1996) found the two changes in patients. How many people with a risk genotype become ill comes from later cohorts, and the note gives their numbers: iron-overload disease in 28% of male and 1% of female C282Y homozygotes followed for 12 years in Melbourne (Allen et al. 2008), rare in C282Y/H63D compound heterozygotes of the same cohort (Gurrin et al. 2009), and frank haemochromatosis in under 1% of homozygotes in a US screening (Beutler et al. 2002). +- GC: the readings name the protein type and give no protein level. Powe et al. (2013) reported levels that differed widely by type, but its monoclonal assay reads the types differently: Denburg et al. (2016, PMID 27250744) found that genotype explained 85% of the variation in the protein level with that assay and at most 9% with mass spectrometry or a polyclonal assay. A protein type, not a vitamin D deficiency. Two more markers of the planned list are not rules here, because each sits in sequence the genome holds more than once and a gVCF genotype at one position cannot read it. They are read from the BAM by the step that counts the copies, as a fraction of copies rather than a genotype (`MARKERS` in `rust/crates/og-cn/src/markers.rs`, which pins each one's dbSNP b157 record and the GRCh38 sequence around every copy): @@ -122,29 +122,29 @@ Kim et al. (2003) found five TAS2R38 haplotypes worldwide. The rule reads the tw ### Checks on the rule table -- `bin/og traits check --table assets/traits.tsv --rules assets/genotype_rules.tsv` (also run by every step-42 run and by `tests/test_traits_report.py`): every haplotype allele is its site's ref or alt, the combination lines are exactly the pairs the genotypes can give (none missing, none that can never be read), and a site that is also a traits table row has the row's position and bases. +- `bin/og traits check --table assets/traits.tsv --rules assets/genotype_rules.tsv` (also run by every step-42 run and by `rust/crates/og-steps/tests/genome_traits.rs`): every haplotype allele is its site's ref or alt, the combination lines are exactly the pairs the genotypes can give (none missing, none that can never be read), and a site that is also a traits table row has the row's position and bases. - `scripts/ci/check-rule-alleles.sh` checks every site against dbSNP: the rsID at that GRCh38 position, ref as dbSNP's REF and alt among its ALT alleles. A pair written the wrong way round, or as the minus-strand bases (TAS2R38 lies on the minus strand, where the papers name the codon bases), fails with the reason. The dbSNP records it compares with sit in the script, taken from dbSNP's GRCh38 VCF of the build `CONSUMER_DBSNP_BUILD` pins in `versions.env` (b157); `--online` reads them again over HTTPS and fails when they differ, and the e2e case `genotype-rules-1-phase` runs it. A C/G or A/T site is its own complement, so there a strand mistake keeps valid alleles: the check prints a note for each such site (rs713598 is one), and its haplotype alleles rest on the codon: Pro49 has C at the first base of codon 49 on the coding strand (NM_176817.5 c.145C, where Ala49 has G), so the PAV base on GRCh38 is its complement, G. The other such sites: rs17580 (SERPINA1 S, minus strand: Glu264 GAA, Val264 GTA, so the S base on GRCh38 is the complement of T, A), rs1799945 (HFE H63D, plus strand: His63 CAT, Asp63 GAT, so D is G) and rs1333049 (no gene; the risk allele C is the plus-strand base of the WTCCC 2007 record). ### Adding a rule -Write the rule, site, haplotype and combination lines from the primary papers. Take each site's position and bases from dbSNP's GRCh38 record (plus strand), and turn each haplotype's codon bases into plus-strand bases for a gene on the minus strand. A new site needs its dbSNP record in `scripts/ci/check-rule-alleles.sh`: run it with `--print-table` and add each printed line at the end of the script after `#: `. Then run `OG=bin/og python3 tests/test_traits_report.py`. +Write the rule, site, haplotype and combination lines from the primary papers. Take each site's position and bases from dbSNP's GRCh38 record (plus strand), and turn each haplotype's codon bases into plus-strand bases for a gene on the minus strand. A new site needs its dbSNP record in `scripts/ci/check-rule-alleles.sh`: run it with `--print-table` and add each printed line at the end of the script after `#: `. Then run `cargo test -p og-steps --test genome_traits` in `rust/`. ## APOE Many people choose not to learn their APOE result. Decide before you open the Traits section; the reports keep the APOE lines last, and the HTML report folds them away. -The APOE line is the APOE rule of `assets/genotype_rules.tsv`, written into the traits table after the two APOE rows; it is byte for byte the line the earlier hand code wrote (tests/test_traits_report.py compares the sha256 of every genotype pair). It combines rs429358 (Cys112Arg) and rs7412 (Arg158Cys) into the ε2, ε3 and ε4 alleles: Arg112 with Arg158 is ε4, Cys112 with Cys158 is ε2, Cys112 with Arg158 is ε3. The odds ratios it quotes for Alzheimer's disease, against ε3/ε3, are those of a meta-analysis of people of European descent from clinic and autopsy studies (Farrer et al., JAMA 1997): ε2/ε3 0.6, ε3/ε4 3.2, ε2/ε4 2.6, ε4/ε4 14.9. The ε4 effect was weaker in African American and Hispanic people and stronger in Japanese people. ε4 is a risk factor, not a diagnosis. Without phase a heterozygote at both sites reads as ε2/ε4, which could also be the rare ε1/ε3; a pair with the rare ε1 allele is not read. +The APOE line is the APOE rule of `assets/genotype_rules.tsv`, written into the traits table after the two APOE rows; `rust/crates/og-steps/tests/genome_traits.rs` pins its sha256 for eleven genotype pairs, so any change to it shows. It combines rs429358 (Cys112Arg) and rs7412 (Arg158Cys) into the ε2, ε3 and ε4 alleles: Arg112 with Arg158 is ε4, Cys112 with Cys158 is ε2, Cys112 with Arg158 is ε3, and Arg112 with Cys158 is the rare ε1. The odds ratios it quotes for Alzheimer's disease, against ε3/ε3, are those of a meta-analysis of people of European descent from clinic and autopsy studies (Farrer et al., JAMA 1997), adjusted for age: ε2/ε3 0.6, ε3/ε4 3.2, ε2/ε4 2.6, ε4/ε4 14.9. They are odds ratios of those case-control studies, not a lifetime risk. The ε4 effect was seen at every age from 40 to 90 but was smaller after 70, and the risk of a genotype differed between men and women. The ε4 effect was weaker in African American and Hispanic people and stronger in Japanese people. ε4 is a risk factor, not a diagnosis. Without phase a heterozygote at both sites reads as ε2/ε4, which could also be the rare ε1/ε3; a pair with the rare ε1 allele is not read. [Step 25](25-prs.md) computes a polygenic score for late-onset Alzheimer's disease (PGS000334) whose two largest weights are the same two sites ([Before you look](interpreting-results.md#before-you-look-what-the-pipeline-can-tell-you)). [Step 6](06-clinvar-screen.md) does not list APOE: it screens the ClinVar records classified as pathogenic or likely pathogenic, and on 2026-10-15 ClinVar classified rs429358 (variation 17864) as "conflicting classifications of pathogenicity; other; risk factor". ## Runtime -Seconds: bcftools reads the gVCF at the table's positions through its index (36 rows and 17 rule sites), and WhatsHap, when a rule has two or more heterozygous sites to phase, reads the BAM over them (each rule's sites lie within about 7 kb). +Seconds: bcftools reads the gVCF at the table's positions through its index (37 rows and 17 rule sites), and WhatsHap, when a rule has two or more heterozygous sites to phase, reads the BAM over them (each rule's sites lie within about 7 kb). ## Notes - A reading is what the cited studies found on average in the people they studied. It is not a diagnosis, and for most traits it says little about one person. -- Rows marked `ancestry` read differently in other populations; their note says how. Lactase persistence is the clearest case: -13910T explains it in Europeans, and the three African alleles have their own rows. +- Rows marked `ancestry` read differently in other populations; their note says how. Lactase persistence is the clearest case: -13910T explains it in Europeans, and the three African alleles and the Ethiopian -14009G have their own rows. These are the main known persistence alleles, not every one, so a genotype with none of them does not prove lactose intolerance; the -22018 marker travels with -13910T, is not counted as a persistence allele, and is worth reading only when the -13910 row has no genotype. - Each traits table row is read alone. The genotype rules read several sites together: APOE, the TAS2R38 taster haplotype with read-backed phase, and the six rules of the table above, which use read-backed phase when it covers their heterozygous sites. Two different MC1R red-hair variants on the same or on different copies are still read one by one; their note says so. - SNVs only; the table has no indel and no chrX, chrY or chrM row. - Positions with no genotype (no reads, or a no-call) are listed as not read. On the e2e fixture, a small slice of the HG002 genome, only the table positions inside its slices are read, and the e2e case checks their genotypes against the GIAB truth. diff --git a/docs/delta-report.md b/docs/delta-report.md index d9dde62c8..f8ebb31ef 100644 --- a/docs/delta-report.md +++ b/docs/delta-report.md @@ -66,6 +66,6 @@ With a previous run on record, the delta heads its changes with "New since" and | PharmCAT | With each release | Steps 07 and 27 | | PGS Catalog | When a score you use gets a new version | Step 25 | -`bin/owngenome_recheck.py` finds these for you: it says which database your run used has a newer release, which steps that changes, and with `--rerun` runs the same `run-all.sh` command again ([Newer Databases Since a Run](recheck.md)). +`bin/og recheck` finds these for you: it says which database your run used has a newer release, which steps that changes, and with `--rerun` runs the same `run-all.sh` command again ([Newer Databases Since a Run](recheck.md)). `run-all.sh` runs Nextflow with `-resume`, so running the same command again redoes only the processes whose inputs or code changed. A new ClinVar file reruns the ClinVar screen and what reads it; the alignment and the variant calls come from the cache. One exception: when the sample started from FASTQ, the first call after that run ended with every task done starts from the BAM and VCF it published instead, so the steps after the variant calls run once more, without aligning or calling again; later calls hit the cache as usual. The delta then shows what the new release moved. diff --git a/docs/faq.md b/docs/faq.md index 5744ee9c8..92a8dce1c 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -5,7 +5,7 @@ | Problem | Cause | Fix | |---|---|---| | Container exits silently with code 137 | SIGKILL, usually a kill for memory | Give Docker more memory, or run fewer or smaller tasks. 137 alone does not prove a kill for memory: [troubleshooting](troubleshooting.md#container-exits-immediately-with-no-output) says how to confirm it. | -| "Permission denied" writing output | The container's user cannot write to the mounted folder | The step scripts run each container as you, except the few whose image cannot run unprivileged (Strelka2, Manta, CPSR and `benchmark-variants.sh` pass `--root`). A sample folder written by an older version holds root-owned files: take it back with `sudo chown -R "$(id -u):$(id -g)" "${GENOME_DIR:?Set GENOME_DIR to your data directory}/"`. See [troubleshooting](troubleshooting.md#permission-denied-writing-to-mounted-volumes). | +| "Permission denied" writing output | The container's user cannot write to the mounted folder | The step scripts run each container as you, and so do Nextflow's `docker` profile tasks. The exceptions run as root: CPSR and `benchmark-variants.sh` among the steps (`--root`), `CPSR` and `CNVPYTOR` among the Nextflow tasks. A sample folder written by an older version holds root-owned files: take it back with `sudo chown -R "$(id -u):$(id -g)" "${GENOME_DIR:?Set GENOME_DIR to your data directory}/"`. See [troubleshooting](troubleshooting.md#permission-denied-writing-to-mounted-volumes). | | VEP cache download fails/times out | 26 GB download over unreliable connection | Use `wget -c` (supports resume). See [docs/13-vep-annotation.md](13-vep-annotation.md) | | DeepVariant crashes on Mac | amd64 emulation + memory pressure | `THREADS=2 DV_MEM=12g ./scripts/03-deepvariant.sh `, the sex being `male` or `female`. Will be slow. See [troubleshooting](troubleshooting.md#step-3-deepvariant-crashes-on-mac-amd64-emulation). | | Wrong number of variants (too few) | Genome build mismatch | Ensure your BAM is aligned to GRCh38 (hg38), not hg19/GRCh37. Check the BAM header with the command under this table | diff --git a/docs/getting-started.md b/docs/getting-started.md index 2ea026818..f867fe45a 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -16,7 +16,7 @@ | Resource | Minimum | Recommended | Notes | |---|---|---|---| | **CPU** | 4 cores | 8-16 cores | `run-all.sh` runs DeepVariant with at most 8 shards, fewer when `--max_cpus` is lower; more cores run other steps beside it ([CPU](hardware-requirements.md#cpu-requirements)) | -| **RAM** | 16 GB | 32 GB | Every container has a hard memory limit; the 8-CPU steps ask for 32 GB, capped at the machine's RAM | +| **RAM** | 16 GB | 32 GB | Every container has a hard memory limit; the 8-CPU steps ask for 32 GB, capped at the machine's RAM. A 16 GB machine needs swap for step 25's pgsc_calc ([step 25, Memory](25-prs.md#memory)) | | **Disk** | 500 GB free from a BAM, about 700 GB from FASTQ | 1 TB+ | See [detailed breakdown](hardware-requirements.md#disk-space-breakdown) | | **Internet** | Broadband | 100+ Mbps | ~73-78 GB core downloads + ~175 GB optional annotation databases | | **OS** | Linux (amd64) | Ubuntu 22.04+ | macOS/ARM works but slower (see below) | @@ -28,9 +28,9 @@ | Software | Version | Install | |---|---|---| | Docker | 20.10+ | [docs.docker.com/get-docker](https://docs.docker.com/get-docker/) | -| bash | 4.4+ | Pre-installed on Linux; macOS ships 3.2, install a newer one with `brew install bash` | -| Java *(for a full run)* | 17+ | Any OpenJDK build, e.g. `apt install openjdk-17-jre-headless` or [Adoptium Temurin](https://adoptium.net/) | -| Nextflow *(for a full run)* | `NEXTFLOW_VERSION` in `versions.env` | The pinned install command in [Full run](#full-run) | +| bash | 4.4+ | Pre-installed on Linux; macOS ships 3.2, and the [Mac steps](#intel-mac) install a newer one with Homebrew | +| Java *(for a full run)* | 17+ | Any OpenJDK build, such as [Adoptium Temurin](https://adoptium.net/); [Install the requirements](#install-the-requirements) has the command for each platform | +| Nextflow *(for a full run)* | `NEXTFLOW_VERSION` in `versions.env` | The pinned install command in [Install the requirements](#install-the-requirements) | | wget or curl | Any | For downloading references | | python3 *(optional)* | 3.6+ | Used by long-read alignment (02b) for symlink resolution. Falls back to `readlink -f` on GNU/Linux if absent | @@ -67,6 +67,131 @@ Setup ends in one of three ways: The PGS Catalog scores, pgsc_calc and its Nextflow plugin are not on the list: step 25 fetches what is missing when it runs. +## Install the requirements + +Pick your platform and run its steps in order. Each section ends with the same check: four commands that each print a version. When all four do, go on to [Get the code](#get-the-code). The [platform notes](#platform-notes) below say how each platform runs the images. + +### Linux + +Run these in a terminal. The package names are Debian's and Ubuntu's; other distributions have the same packages under their own names. Debian 13 has no Java 17 package; there, install `openjdk-21-jre-headless` instead. bash 4.4 or later comes with every current distribution. + +```bash +# 1. Java 17, git and curl +sudo apt-get update +sudo apt-get install -y openjdk-17-jre-headless git curl +# 2. Docker Engine, and the right to run it without sudo +curl -fsSL https://get.docker.com | sudo sh +sudo usermod -aG docker "$USER" +# 3. Nextflow, the release versions.env pins +curl -s https://get.nextflow.io | NXF_VER=26.10.5 bash +sudo mv nextflow /usr/local/bin/ +``` + +Log out and back in, so that your user is in the `docker` group. Then check: + +```bash +docker info +bash --version +java -version +nextflow -version +``` + +`docker info` must show a `Server:` part with no error under it. Next: [Get the code](#get-the-code). + +### Intel Mac + +Run these in Terminal. Homebrew installs the rest, so it comes first; its installer asks for your password. + +```bash +# 1. Homebrew (https://brew.sh) +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +# 2. bash 4.4 or later (macOS ships 3.2), Java 17 and Docker Desktop +brew install bash +brew install --cask temurin@17 docker-desktop +open -a Docker +# 3. Nextflow, the release versions.env pins +curl -s https://get.nextflow.io | NXF_VER=26.10.5 bash +sudo mkdir -p /usr/local/bin +sudo mv nextflow /usr/local/bin/ +``` + +Docker Desktop asks you to accept its terms the first time it opens. Set its memory as the [Intel Mac notes](#macos-intel) say. Open a new Terminal window, so that the new bash comes first on your `PATH`. Then check: + +```bash +docker info +bash --version +java -version +nextflow -version +``` + +`docker info` must show a `Server:` part with no error under it, and `bash --version` must print 4.4 or later. Next: [Get the code](#get-the-code). + +### Apple Silicon Mac + +Run these in Terminal. Homebrew installs the rest, so it comes first; its installer asks for your password. On Apple Silicon it installs to `/opt/homebrew`, which is not on your `PATH` until step 2 puts it there. + +```bash +# 1. Homebrew (https://brew.sh) +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +# 2. Put Homebrew on your PATH, now and in every new Terminal window +echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> ~/.zprofile +eval "$(/opt/homebrew/bin/brew shellenv)" +# 3. Rosetta 2, which runs the amd64 images +softwareupdate --install-rosetta --agree-to-license +# 4. bash 4.4 or later (macOS ships 3.2), Java 17 and Docker Desktop +brew install bash +brew install --cask temurin@17 docker-desktop +open -a Docker +# 5. Nextflow, the release versions.env pins +curl -s https://get.nextflow.io | NXF_VER=26.10.5 bash +sudo mkdir -p /usr/local/bin +sudo mv nextflow /usr/local/bin/ +``` + +Docker Desktop asks you to accept its terms the first time it opens. Turn on its Rosetta setting as the [Apple Silicon notes](#macos-apple-silicon-m1-m4) say. Open a new Terminal window, so that the new bash comes first on your `PATH`. Then check: + +```bash +docker info +bash --version +java -version +nextflow -version +``` + +`docker info` must show a `Server:` part with no error under it, and `bash --version` must print 4.4 or later. If `java -version` crashes with `SIGBUS` in `CodeHeap::allocate`, run `csrutil status`: Java on Apple Silicon can crash at start this way when System Integrity Protection is turned off ([OpenJDK bug JDK-8326663](https://bugs.openjdk.org/browse/JDK-8326663)), and turning it back on is the fix. Next: [Get the code](#get-the-code). + +### Windows with WSL2 + +The pipeline runs inside Linux on WSL2, and Docker Desktop gives that Linux its Docker. + +1. Open PowerShell as Administrator and install WSL2 with Ubuntu. Restart Windows when it asks. + + ```powershell + wsl --install -d Ubuntu-24.04 + ``` + +2. Open **Ubuntu** from the Start menu and create your Linux user. +3. Install [Docker Desktop for Windows](https://docs.docker.com/desktop/setup/install/windows-install/) with the WSL 2 backend. In its Settings, under Resources > WSL integration, turn on Ubuntu-24.04. +4. Give WSL2 its memory: write `%UserProfile%\.wslconfig` as the [WSL2 notes](#windows-wsl2) say, then run `wsl --shutdown` in PowerShell and open Ubuntu again. +5. In the Ubuntu terminal, install Java 17, git and curl, then Nextflow, the release versions.env pins: + + ```bash + sudo apt-get update + sudo apt-get install -y openjdk-17-jre-headless git curl + curl -s https://get.nextflow.io | NXF_VER=26.10.5 bash + sudo mv nextflow /usr/local/bin/ + ``` + +6. Check, in the same Ubuntu terminal: + + ```bash + docker info + bash --version + java -version + nextflow -version + ``` + +`docker info` must show a `Server:` part with no error under it. Run every later command in the Ubuntu terminal, and keep the code and your data in your Linux home (`cd ~`), not under `/mnt/c`. Next: [Get the code](#get-the-code). + ## Platform Notes ### Linux (Recommended) @@ -75,7 +200,7 @@ Best performance. Docker runs natively. All pipeline images are linux/amd64. No ### macOS (Intel) Works fine. Docker Desktop runs a Linux VM, so there's a ~10-20% I/O overhead on file operations. Set Docker Desktop memory to at least 16 GB (Preferences > Resources). -### macOS (Apple Silicon / M1-M4) +### macOS (Apple Silicon M1-M4) Works but **slower**. All bioinformatics Docker images are amd64 and run under Rosetta 2 emulation (2-5x performance penalty). DeepVariant and BWA-MEM2 are the most affected. In Docker Desktop, turn on "Use Rosetta for x86_64/amd64 emulation on Apple Silicon" if your version offers it; it needs the Apple Virtualization framework as the virtual machine manager, not Docker VMM. Without it, Docker falls back to QEMU emulation, which is slower still. ### Windows (WSL2) @@ -96,10 +221,10 @@ Works for long-running analyses. Each step has a hard memory limit. On the `run- ```bash git clone https://github.com/GeiserX/OwnGenome.git cd OwnGenome -git tag --sort=-v:refname | head -n 1 # the newest release; `git checkout ` stays on it, main moves on +git checkout "$(git tag --list 'v[0-9]*' --sort=-v:refname | head -n 1)" # the newest release ``` -Every command on these pages runs from this folder, the repository root. +On a release tag, `setup.sh` downloads that release's `og` binaries. On `main` it builds `og` in a container instead, which takes longer; skip the checkout to follow `main`, which moves on. Every command on these pages runs from this folder, the repository root. ### Step 0: Quick Test (Optional) diff --git a/docs/hardware-requirements.md b/docs/hardware-requirements.md index 45685c047..98f46cfa7 100644 --- a/docs/hardware-requirements.md +++ b/docs/hardware-requirements.md @@ -246,20 +246,20 @@ Both ends of each measured figure are also in `assets/steps.tsv` (`hours_low` an ### The report step -`run-all.sh` writes both reports ([step 24](24-html-report.md)) after the pipeline, outside Nextflow's trace, so the end test above has no time for them. We measured them apart on the same three genomes in November 2026, on `main` at e7d5919 and on the same server, while it ran other work (load average 10-14). Each command ran three times per genome in the pinned Python image, in its own container capped at one CPU and under `nice -n 19`. The genome folder was mounted read-only and every output went to a scratch folder. Wall time and peak memory come from `getrusage` of the child process. +`run-all.sh` writes both reports ([step 24](24-html-report.md)) after the pipeline, outside Nextflow's trace, so the end test above has no time for them. We measured them apart on the same three genomes in November 2026, on `main` at e7d5919 and on the same server, while it ran other work (load average 10-14). Both scripts then ran the Python renderer `bin/render_report.py`. Step 24, the Nextflow `HTML_REPORT` module and `scripts/generate-report.sh` now run `og report render` instead, and we have not measured it on these genomes yet, so the numbers below are the Python renderer's. Each command ran three times per genome in the pinned Python image, in its own container capped at one CPU and under `nice -n 19`. The genome folder was mounted read-only and every output went to a scratch folder. Wall time and peak memory come from `getrusage` of the child process. | Command | Wall time | Peak memory | What it does | |---|---|---|---| -| `scripts/24-html-report.sh` (`render_report.py`, HTML, search index, charts) | 11.2-14.5 s | 403-424 MB | collects `summary.json` from every step's output, then renders | -| `scripts/generate-report.sh`, the text report | 11.6-12.7 s | 403-424 MB | collects `summary.json` again, then renders the text | +| `scripts/24-html-report.sh` with `render_report.py` (HTML, search index, charts) | 11.2-14.5 s | 403-424 MB | collects `summary.json` from every step's output, then renders | +| `scripts/generate-report.sh` with `render_report.py`, the text report | 11.6-12.7 s | 403-424 MB | collects `summary.json` again, then renders the text | | `render_report.py --summary` (render only, from a written `summary.json`) | 0.40-0.42 s | 62-73 MB | | | `run_delta.py` (what changed since the previous run) | 0.11-0.16 s | 19-21 MB | | So the reports cost about 25 s per sample, not 1-3 minutes, and almost all of it is collecting the summary, which each of the two scripts does once. Under `cProfile` (one HTML run), 82% of the collection was `sec_variants` reading every record of the sample's small-variant VCF in Python, and 9% the ROH section. Rendering is under half a second. The HTML report was 2.0-2.7 MiB and the search index 8.3-12.2 MiB. -Since then the Rust `og summary collect` writes `summary.json`, and `generate-report.sh` renders from the `summary.json` step 24 wrote unless a step output is newer than it, so a run collects each sample once. `run_delta.py` is gone: `og run delta` makes the comparison. +Since then the Rust `og summary collect` writes `summary.json`, `og report render` renders it, and `generate-report.sh` renders from the `summary.json` step 24 wrote unless a step output is newer than it, so a run collects each sample once. `run_delta.py` is gone: `og run delta` makes the comparison. -The command, per genome (`` holds the sample folder, `` is empty and writable, run from the repo root): +To time today's step the same way, per genome (`` holds the sample folder, `` is empty and writable, run from the repo root; `bin/og` runs the build `scripts/setup.sh --og` puts in `bin/.og/`): ```sh IMG=$(. ./versions.env; echo "$PYTHON_IMAGE") @@ -272,12 +272,12 @@ print(f"rc={rc} wall={time.perf_counter() - s:.2f}s maxrss_MB={r.ru_maxrss / 102 PY docker run --rm --cpus 1 --network none -v :/genome:ro -v "$PWD/bin":/pgp-bin:ro \ -v "$PWD/assets":/pgp-assets:ro -v :/out "$IMG" nice -n 19 python3 /out/tm.py \ - "python3 /pgp-bin/render_report.py --assets /pgp-assets --sample --sample-dir /genome/ \ + "/pgp-bin/og report render --assets /pgp-assets --sample --sample-dir /genome/ \ --json /out/summary.json -o /out/report.html --search-index /out/search_index.json \ --charts-json /out/charts.json >/dev/null" ``` -The text report is the same line with `-o /out/report.txt` and no `--search-index` or `--charts-json`; render only adds `--summary /out/summary.json`. +The text report is the same line with `-o /out/report.txt` and no `--search-index` or `--charts-json`; render only replaces `--assets` and `--json` with `--summary /out/summary.json`, as `generate-report.sh` does. ### The clinical filter, stage by stage diff --git a/docs/lessons-learned.md b/docs/lessons-learned.md index db179a266..d7c596a42 100644 --- a/docs/lessons-learned.md +++ b/docs/lessons-learned.md @@ -113,7 +113,7 @@ macOS is arm64; most Linux servers are amd64. Images built on Mac without `--pla Always use `--rm` for analysis containers to avoid accumulating stopped containers. Use `-d` (detached) for long-running jobs. ### Run containers as the caller, not as root -Most bioinformatics containers name a non-root user of their own, which may not be able to write to a bind-mounted host folder. The early scripts ran every container with `--user root`, which left root-owned files in the sample folder. The step scripts now start each container through `run_in` (`scripts/lib/common.sh`) as the caller (`--user` with the caller's UID and GID, `HOME=/tmp`), with only the sample folder writable, plus any folder the call names with `--rw` (a shared index, for example). A step whose image cannot run unprivileged passes `--root` at its call, with the reason in a comment. The Nextflow `docker` profile runs tasks as root with `docker.fixOwnership`, which gives the files back to the caller. +Most bioinformatics containers name a non-root user of their own, which may not be able to write to a bind-mounted host folder. The early scripts ran every container with `--user root`, which left root-owned files in the sample folder. The step scripts now start each container through `run_in` (`scripts/lib/common.sh`) as the caller (`--user` with the caller's UID and GID, `HOME=/tmp`), with only the sample folder writable, plus any folder the call names with `--rw` (a shared index, for example). A step whose image cannot run unprivileged passes `--root` at its call, with the reason in a comment. The Nextflow `docker` profile runs each task as the caller too (`-u ${UID}:${GROUPS}`, `HOME=/tmp`). `CPSR` and `CNVPYTOR` run as root, and Nextflow (`docker.fixOwnership`) gives their work files back to the caller; a file they write outside their work directory stays root's. ## CI / Workflow Issues diff --git a/docs/live-site.md b/docs/live-site.md index 782f22ae5..30361dd1a 100644 --- a/docs/live-site.md +++ b/docs/live-site.md @@ -30,6 +30,8 @@ OWNGENOME_LIVE_URL=http://127.0.0.1:8788 ./scripts/run-all.sh your_name male # The site runs in the foreground of its terminal. Press Ctrl+C there to stop it. A run that posts to it goes on without it. Under the systemd unit of [Deploy it on a server](#deploy-it-on-a-server), `systemctl stop owngenome-live` stops it. +`bin/og live serve --root "$GENOME_DIR"` serves the same pages from `og`, OwnGenome's one binary, on 127.0.0.1:8787. It is the Rust port of the site, and it has no login yet. It takes no password, so it refuses a `--bind` that is not loopback. It also refuses the options of a site with a password, TLS or a proxy (`--password-file`, `--allowed-host`, `--trust-proxy`, `--tls-cert`, `--tls-key`, `--cookie-secure`, a set `OWNGENOME_LIVE_PASSWORD_HASH`). Each refusal exits 2. `--plugin-dir` is refused too: the plugins are compiled into `og`. It starts no ingest listener. It reads the messages that the Python server stores in `GENOME_DIR/.owngenome-live/runs//events.jsonl`, so both can share one state folder. It does not serve the results yet: `results.json` holds only a note that says so, a finding page is 404 and the map answers 503. `scripts/owngenome-live.sh` still starts the Python server. + With `OWNGENOME_LIVE_URL` set, `run-all.sh` writes `/nextflow/live.config` and passes it to Nextflow with `-c`. That config loads the nf-weblog plugin (pinned as `NF_WEBLOG_PLUGIN` in `versions.env`; Nextflow downloads it on the first live run) and points it at `OWNGENOME_LIVE_URL/ingest/`, so every task's progress is posted. The numbered script steps that run after the pipeline (GRIDSS, imputation prep, the somatic caller, the extra callers) post when they start and how they end (the benchmark, `benchmark-variants.sh`, has no step number and posts nothing), and `run-all.sh` posts once more as it ends, with its exit code. A site that is down or slow costs one warning; the run never waits for it and never fails for it. `OWNGENOME_LIVE_URL` must be a plain `http://host:port` URL. Set `OWNGENOME_LIVE_PAGE_URL` to the address you open the site at (`http://127.0.0.1:8787`, or the name behind your proxy) and `run-all.sh` also prints the run's page, `OWNGENOME_LIVE_PAGE_URL/runs//`, as it starts; it prints it only together with `OWNGENOME_LIVE_URL`. A bare `nextflow run main.nf` can report too. Write a config like this one, with the sample id at the end of the URL (the page counts only the tasks tagged with it), and pass it with `-c`: @@ -214,4 +216,4 @@ The core files (`bin/owngenome_live.py`, `assets/live/index.html`, `home.html`, The page has a replay mode for a demo with no server: `` reads `frames.json` (`{"interval_ms": 1000, "frames": [{"t": ..., "state": }]}`) and plays it with a play button and a slider, through the same drawing code. A replay whose last frame is a finished run opens at that frame, on the finished report, until you press Play, which starts again from the first frame; with `"autoplay": true` it plays from the start at once. The bar shows each frame's moment as day, month and time in UTC ("2 Jan 10:05"), the full time on hover. -`tests/test_owngenome_live.py` checks the server, the login, the stored messages, the plugin interface and the page in a headless Chrome; `tests/test_live_results.py` checks `results.json` (every card's caveat word for word, no wording that reads as an all-clear, `results.json` behind the login); `tests/test_live_journal.py` checks the levels, the lead paragraph and the finding page's route; `tests/test_live_demo.py` builds the demo and checks every page, phone width included; `tests/test_live_post.sh` and `tests/fake-docker/run-all-live.sh` check what `run-all.sh` posts. +`tests/test_owngenome_live.py` checks the Python server's password, login, Host check, stored messages and plugin interface, and its login form in a headless Chrome; `cargo test -p og-live` (`tests/port_live_server*.rs`) checks `og live serve`: `state.json` and its headlines, the event stream, the policy on every answer, the run files, the family folder, the map, the re-check plugin and the run page in a headless Chrome; `tests/test_live_results.py` checks `results.json` (every card's caveat word for word, no wording that reads as an all-clear, `results.json` behind the login); `tests/test_live_journal.py` checks the levels, the lead paragraph and the finding page's route; `tests/test_live_demo.py` builds the demo and checks every page, phone width included; `tests/test_live_post.sh` and `tests/fake-docker/run-all-live.sh` check what `run-all.sh` posts. diff --git a/docs/nextflow.md b/docs/nextflow.md index a4538387e..cf19df46b 100644 --- a/docs/nextflow.md +++ b/docs/nextflow.md @@ -91,7 +91,7 @@ nextflow run main.nf \ # it names (two or more, each with a gVCF; docs/multi-sample.md) ``` -**Keep `--outdir` private.** The results hold genome data. Create the folder before the run with `mkdir -m 700 results`, or put it in a folder only you can open. The `og` tools write their analysis outputs owner-only (mode 0600; run as root in a container of the docker profile, they give each file to the user who started the run, named in Nextflow's `NXF_OWNER`, so you can read it), but Nextflow creates the folders, and the other tools write their files, with the modes your umask gives. +**Keep `--outdir` private.** The results hold genome data. Create the folder before the run with `mkdir -m 700 results`, or put it in a folder only you can open. The `og` tools write their analysis outputs owner-only (mode 0600). The docker profile runs them as you, so the files are yours; run as root, they give each file to the user who started the run, named in Nextflow's `NXF_OWNER`. Nextflow creates the folders, and the other tools write their files, with the modes your umask gives. **Family workflow (opt-in).** `--pedigree` (tab separated: sample, sex, father, mother, partner; [`assets/pedigree.example.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/pedigree.example.tsv)) runs `workflows/family.nf` over the samplesheet rows it names. `--pedigree` may be a families file (a leading family column, [`assets/families.example.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/assets/families.example.tsv)). `--family_name` names the family to run, which a file with more than one family needs, and the folder its results go to (`//` instead of `/family/`). The workflow does the same work as step 38: `FAMILY_PREPARE` checks the pedigree and that each member's gVCF names its own sample, `GLNEXUS` and `GLNEXUS_VCF` joint-call the gVCFs (`GLNEXUS` asks for 16 GB and gives GLnexus half of its memory as `--mem-gbytes`), `FAMILY_RELATE` and `FAMILY_CHECK` stop the run when somalier's relatedness of a pair disagrees with the pedigree (a sample swap), `WHATSHAP_PED` phases the children of complete trios by inheritance, and `FAMILY_REPORT` writes each child's allele origin, de novo candidates when both parents are sequenced, and compound heterozygotes among the clinical filter's candidates. With `--ancestry_fine_panel` (the panel folder of step 26b, `setup.sh --ancestry-fine-panel`), four more processes find the DNA each pair shares by descent, once `FAMILY_CHECK` has passed: `FAMILY_IBD_SITES` keeps the joint call's genotypes at the panel's SNVs on each autosome (a chromosome with fewer than 100 is left out), `FAMILY_IBD_PHASE` phases them with Beagle against the panel, `FAMILY_IBD_SEGMENTS` runs hap-ibd (its default 2 cM minimum) with the panel's genetic maps, and `FAMILY_IBD_REPORT` writes `ibd_pairs.tsv`, `ibd_segments.tsv` and `ibd_report.md`. They feed only the report, so a failure of one is logged and the run goes on. Without the panel none of them runs. Each member needs a gVCF: DEEPVARIANT's for a FASTQ or BAM row, the `gvcf` column for a VCF row (the run stops on a VCF row without one). Fewer than two named samples, or no `--somalier_sites`, stops the run at the start. When the pedigree marks two of those samples as each other's partner, `COUPLE_CARRIER` runs the couple carrier screen of step 39 on them ([Couple Carrier Screen](39-couple.md)): it reads each partner's ClinVar hits, so it needs `clinvar` in `--tools` (without it the run logs a warning and skips the screen), uses deletions over an exon only with `--gencode_exons` (the GENCODE exons BED `setup.sh` installs as `reference/gencode.v50.basic.exons.bed`), reads Delly's calls, ExpansionHunter's, and Parascopy's SMN1, interval and CYP21A2 tables when those steps ran, and, when `--annotsv_annotations` holds AnnotSV's benign loss set (`benign_Loss_SV_GRCh38.sorted.bed`, with gnomAD-SV's common deletions merged in by `setup.sh --annotsv-data`), sets aside a Delly deletion that matches a common one there. Without `--pedigree` none of these processes exists and a run is unchanged. What it checks and what the files hold: [Multi-sample comparison](multi-sample.md#family-analysis-step-38-opt-in). @@ -161,7 +161,7 @@ sample1,results/variant_calling/deepvariant/sample1/sample1.deepvariant.vcf.gz,r | Profile | Description | |---------|-------------| -| `docker` | Run with Docker containers (default for local). Every container runs with `--network none`. | +| `docker` | Run with Docker containers (default for local). Every container runs with `--network none`, and as you (`-u` with your UID and GID, `HOME=/tmp`), as the step scripts do. `CPSR` and `CNVPYTOR` run as root, with `HOME=/root`: PCGR has not been shown to run unprivileged, and CNVpytor copies its resources into its image. Nextflow (`docker.fixOwnership`) gives their work files back to you; a file they write outside their work directory stays root's. A task can write beside its inputs under `GENOME_DIR`: the step scripts mount `GENOME_DIR` read-only, but Nextflow mounts the folders of a task's inputs read-write. | | `singularity` | Singularity/Apptainer. **Untested.** One module writes inside its image and needs a writable container: `cnvpytor` (copies resources into its `site-packages`). This profile does not cut the network. | | `test` | Minimal test with reduced resources | | `test_full` | Full-size test with real WGS data | @@ -172,7 +172,7 @@ Combine profiles: `-profile docker,test` ## Resource Configuration -Each process asks for a fixed CPU count, a memory and a time limit, from its label or from its own block in `conf/base.config`. After an exit that usually means a kill for memory (137 and the others `conf/base.config` lists), the task is retried once with its memory doubled. Time doubles too, except DeepVariant's, which gets `--max_time` from its first attempt. On Nextflow before 26.10.0, a task that runs past its time limit can stop the whole run instead of being retried ([nextflow-io/nextflow#7569](https://github.com/nextflow-io/nextflow/issues/7569), fixed in 26.10.0); we read this in the source and have not reproduced it. These limits cap each request: +Each process asks for a fixed CPU count, a memory and a time limit, from its label or from its own block in `conf/base.config`. After an exit that usually means a kill for memory (137 and the others `conf/base.config` lists), the task is retried once with its memory doubled. Time doubles too, except DeepVariant's, which gets `--max_time` from its first attempt. On Nextflow before 26.10.0, a task that runs past its time limit can stop the whole run instead of being retried ([nextflow-io/nextflow#7569](https://github.com/nextflow-io/nextflow/issues/7569), fixed in 26.10.0); we read this in the source and have not reproduced it. A process that feeds only the report (the list in `nextflow.config`: TelomereHunter, ExpansionHunter, VNtyper, the CHIP and ancestry processes and the others there) does not stop the run when it fails: after a memory-kill exit it is retried once as above, and when it fails for another reason, or fails again, the run goes on without it. `HTML_REPORT` records the run's `--tools` list in the sample's summary, so the step's section is listed as "selected, no result" instead of not run (`selected_no_result` in `summary.json`, [step 24](24-html-report.md#results-from-an-earlier-run)). These limits cap each request: | Parameter | Default | Description | |-----------|---------|-------------| @@ -361,7 +361,7 @@ Where a module and its script differ on purpose: | Mito variants (20) | extracts the chrM reads with `samtools view`; annotates the calls with `clinvar/clinvar_chr.vcf.gz` and the gnomAD chrM sites under `gnomad_mito/` when they are installed | with GATK `PrintReads`. Mutect2 applies its own read filters to either, so the calls are expected to match; CI does not compare them. Both mark possible NuMTs at the median autosomal depth from mosdepth (step 16b, or `MOSDEPTH` when `mosdepth` is in `--tools`). `MITO_VARIANTS_ANNOTATE` runs the same annotation (`og mito annotate`) with the files of `--mito_clinvar` and `--gnomad_mito`; without them the table names each file as not given and leaves its columns `.`. `run-all.sh` passes `--mito_clinvar` when `clinvar/clinvar_chr.vcf.gz` and its `.tbi` are installed, and `--gnomad_mito` when the gnomAD chrM sites are | | AnnotSV (05) | annotates step 15's duphold-filtered calls; without them, or when Manta's calls are newer, Manta's calls, and says so | `ANNOTSV` always reads `DUPHOLD_FILTER`'s output: `annotsv` in `--tools` needs `duphold` and `manta` | | Y haplogroup (37) | reads the sex step 16 (indexcov) infers | runs on the rows whose samplesheet sex is male, which `INDEXCOV` has checked against the reads | -| HTML report (24) | renders every section from the summary `og summary collect` writes | `HTML_REPORT` runs the same code on the outputs of this run's steps, among them the cDNA read check of the ClinVar hits, the CYP2D6 depth check, the phased compound-het pairs, the RFC1 motif count, AnnotSV's class 4 and 5 table, the chrM annotation and step 35's HBA1/HBA2 count, CYP21A2 and gene-dosage tables. It also gets the sample's gVCF (DEEPVARIANT's or the `gvcf` column's, linked as `vcf/.g.vcf.gz`, where step 24 finds step 03's) and the release in nextflow.config's manifest, as a `run pipeline_version` row of the report's run manifest, and the regions of `--intervals`, when given, as a `run intervals` row (`bin/write_manifest.sh` writes the same two rows for `run-all.sh`, the second from `INTERVALS`), so the summary marks the run partial. It also gets the VEP VCF (each ClinVar hit's gnomAD AF and rsIDs), CPSR's classification table and the cDNA read check of its P/LP rows, `ALIGN_MINIMAP2`'s read count, and the clinical filter's and slivar's summary tables. The slivar card has no compound-het variant and gene counts: the Nextflow slivar module writes no `_compound_hets.tsv`. For every section, run `GENOME_DIR= scripts/24-html-report.sh ` on the Nextflow output | +| HTML report (24) | renders every section from the summary `og summary collect` writes | `HTML_REPORT` runs the same code on the outputs of this run's steps, among them the cDNA read check of the ClinVar hits, the CYP2D6 depth check, the phased compound-het pairs, the RFC1 motif count, AnnotSV's class 4 and 5 table, the chrM annotation and step 35's HBA1/HBA2 count, CYP21A2 and gene-dosage tables. It also gets the sample's gVCF (DEEPVARIANT's or the `gvcf` column's, linked as `vcf/.g.vcf.gz`, where step 24 finds step 03's) and the release in nextflow.config's manifest, as a `run pipeline_version` row of the report's run manifest, and the regions of `--intervals`, when given, as a `run intervals` row (`bin/write_manifest.sh` writes the same two rows for `run-all.sh`, the second from `INTERVALS`), so the summary marks the run partial. The report manifest also carries the effective `--tools` list as a `run tools` row. The bash runner omits that row, so only the Nextflow report uses `selected_no_result` and "(selected, no result)" in Steps Not Run. `HTML_REPORT` also gets the VEP VCF (each ClinVar hit's gnomAD AF and rsIDs), CPSR's classification table and the cDNA read check of its P/LP rows, `ALIGN_MINIMAP2`'s read count, and the clinical filter's and slivar's summary tables. The slivar card has no compound-het variant and gene counts: the Nextflow slivar module writes no `_compound_hets.tsv`. For every section, run `GENOME_DIR= scripts/24-html-report.sh ` on the Nextflow output | | CNVpytor (18) | mounts each resource file over the image's data folder | copies the files into the image's `site-packages`, so it needs a writable container | | Cyrius (21) | runs Cyrius from the install `setup.sh --cyrius` made | the same install, given as `--cyrius_install` (see [Cyrius, opt-in](#cyrius-opt-in)) | diff --git a/docs/output-contract.md b/docs/output-contract.md index 31ab738ba..7cc575900 100644 --- a/docs/output-contract.md +++ b/docs/output-contract.md @@ -11,6 +11,7 @@ The full list is one file, [`assets/output_contract.tsv`](https://github.com/Gei - **`run-all.sh`:** its two arguments (` `) and every switch in its `SWITCHES` list (`TOOLS`, `THREADS`, `SKIP_TRIM`, `INTERVALS`, `REF_FASTA`, ...). - **Result files:** every file the Nextflow pipeline publishes, which is also what `run-all.sh` writes, at its path under `GENOME_DIR//` (or `--outdir`). For each one the contract gives its step, its format and the fields a reader can count on: the columns of a table, the INFO and FORMAT fields of a VCF header, the top-level keys of a JSON file (`summary.json`'s keys are the ones its schema, `tests/schema/summary.schema.json`, requires). - **Single-step scripts:** the files listed with leg `bash` or `both`. A row is `both` when the step's script (`scripts/-*.sh`) writes the same path under `GENOME_DIR`. Some single steps write to other folders than the pipeline does (step 7 writes PharmCAT's reports to `vcf/`, the pipeline to `pharmcat/`; step 16b writes to `mosdepth/`, the pipeline to `coverage/`). Both names are kept. A single-step output that is not in the file is not part of the promise yet. +- **One table that is not the whole answer:** `traits/_genotype_rules.tsv` keeps its columns, but its APOL1 line's `result` never shows G2, the 6-base deletion the rule table cannot read. A script that needs the APOL1 result with G2 reads `summary.json`: in `sections.traits.data.rules`, the line whose `rule` is `APOL1` has the field `g2`, with the G1 plus G2 `result` and `reading` ([step 42](42-traits.md#genotype-rules)). - **The OwnHealth file:** `run-all.sh` also writes `facts/_genome_facts.json`, the file [OwnHealth](ownhealth.md) imports. Its path and top-level keys are covered here. Its format is OwnHealth's schema `ownhealth.genome-facts/v1`, which OwnHealth owns. ## What counts as a break diff --git a/docs/recheck.md b/docs/recheck.md index 3f5f93c52..e5df67db4 100644 --- a/docs/recheck.md +++ b/docs/recheck.md @@ -11,7 +11,7 @@ bin/og recheck --sample your_sample It is part of `og`, OwnGenome's one binary, which `./scripts/setup.sh --og` builds ([og](rust.md)). It asks the network through the host's `curl`. It needs the sample's `run_manifest.tsv`, which `run-all.sh` writes (`bin/write_manifest.sh`). -`bin/owngenome_recheck.py`, the old Python version, now only runs `bin/og recheck` with the same arguments and says on stderr that it is deprecated. It stays for crontabs until 1.1.0; point them at `bin/og recheck`. +The old Python version, `bin/owngenome_recheck.py`, is gone: point crontabs at `bin/og recheck`. ## What it checks @@ -170,4 +170,4 @@ launchd runs an interval missed during sleep when the Mac wakes; one missed whil ## Tests -`cargo test -p og-recheck` checks the classes, the steps each item names, a failed lookup (an offline answer and a server on 127.0.0.1), the files it writes, the refusals during a run and while a lock is held, the lock held while `setup.sh` refreshes ClinVar and let go before the replay, the comparison after a rerun, `--launch-dir`, the last line when lookups fail (connections refused), a recorded command read back word by word and refused when it is not a `run-all.sh` call, and the replay with typed arguments. `tests/test_owngenome_recheck.py` runs `og recheck` from the outside, the deprecated `bin/owngenome_recheck.py`, the live-site plugin and the ClinVar lock block of `run-all.sh` and step 06, each against a planted mistake too. `tests/fake-docker/run-all-recheck.sh` checks that `--rerun` replays the recorded command exactly, with a control where one dropped argument makes it differ. +`cargo test -p og-recheck` checks the classes, the steps each item names, a failed lookup (an offline answer and a server on 127.0.0.1), the files it writes, the refusals during a run and while a lock is held, the lock held while `setup.sh` refreshes ClinVar and let go before the replay, the comparison after a rerun, `--launch-dir`, the last line when lookups fail (connections refused), a recorded command read back word by word and refused when it is not a `run-all.sh` call, and the replay with typed arguments. `cargo test -p og-live --test port_live_server_recheck` checks the live-site plugin and the ClinVar lock block of `run-all.sh` and step 06, and that the block with its `flock` call taken out fails the check. `tests/fake-docker/run-all-recheck.sh` checks that `--rerun` replays the recorded command exactly, with a control where one dropped argument makes it differ. diff --git a/docs/releases/v1.0.0.md b/docs/releases/v1.0.0.md index af37293d5..1836fbebc 100644 --- a/docs/releases/v1.0.0.md +++ b/docs/releases/v1.0.0.md @@ -20,7 +20,7 @@ OwnGenome 1.0.0 is the first stable release. It is one Nextflow pipeline that ta - A VCF from a vendor is checked before any step runs: a GRCh37 file, a multi-sample file and a file without contig lengths are refused with the reason. - `run-all.sh` resumes, keeps per-step logs, starts from the BAM and VCF an earlier run produced, and runs the sample identity and contamination checks in a default run. - Polygenic score files are prepared at launch, beside alignment and variant calling, instead of after them, so a run from FASTQ or a BAM ends minutes sooner. -- `bin/owngenome_recheck.py` says which results a newer ClinVar or catalog would change, and `--rerun` reruns only those. +- `bin/og recheck` says which results a newer ClinVar or catalog would change, and `--rerun` reruns only those. ## Watching a run diff --git a/docs/rust.md b/docs/rust.md index 47e460a46..b030c0d3f 100644 --- a/docs/rust.md +++ b/docs/rust.md @@ -21,7 +21,7 @@ rust/ crates/og-qc/ og qc sex-loss: mosaic loss of Y or X for step 33 crates/og-setup/ what setup.sh runs once: the STRchive catalog, zips, the genetic map crates/og-steps/ the step helpers: constraint-join, prs format, step 40's consumer export; og family (step 38); og pgx outside-calls|hla-concordance|pypgx-summary (steps 08, 32, 36); og paraphase (step 35b) - crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; og summary collect, the whole summary.json with its post pass (render_report.py runs it for steps 24 and HTML_REPORT and generate-report.sh), or one group with --only a|b|c; og roh summary (step 11) + crates/og-summary/ the step tables of steps 20, 25 and 33: og mito, og prs table, og qc sample; og summary collect, the whole summary.json with its post pass (og report render runs it for steps 24 and HTML_REPORT and generate-report.sh; the Python live site runs it through bin/collect_summary.py), or one group with --only a|b|c; og roh summary (step 11) crates/og-pedigree/ the pedigree rules: families files, members, kinship, PARs crates/og-chip/ step 49, clonal haematopoiesis: og chip prepare, report, check-table crates/og-cn/ copy number from depth (steps 32, 35, 41, 46): og cn bed, call, intervals, check-bam, marker-regions, marker-read @@ -31,8 +31,8 @@ rust/ crates/og-lookup/ the genotype lookup of scripts/owngenome-lookup.sh: og lookup query, resolve, report crates/og-fresh/ og dev freshness: the monthly freshness issue's upstream lookups, typed answers crates/og-recheck/ og recheck: newer databases since a run, the rerun and the ClinVar lock - crates/og-render/ og report render|index|charts: the reports from summary.json (render: the frame, QC and variants so far; index: ported; neither has a caller yet) - crates/og-live/ og live serve|results|family|demo, og view serve: the live site (family and view serve ported; serve, results and demo not ported yet) + crates/og-render/ og report render|index|charts: the reports from summary.json (render: both reports, every section, called by steps 24, HTML_REPORT and generate-report.sh; a section that cannot be written says so in place and the rest is written) + crates/og-live/ og live serve|results|family|demo, og view serve: the live site (family, view serve and serve ported (serve without login or results); results and demo not ported yet) crates/og-couple/ og couple screen|genes: step 39's couple screen (not ported yet) crates/og-admix/ what the fine-ancestry engines share: panel, map, ref.geno, RNG, simulator; painting and projection (not ported yet) og/ the binary: og [args] @@ -40,13 +40,13 @@ bin/og the shim tasks call; runs bin/.og/og-linux- (og-da tests/rust/ the differential test against CPython, the image test ``` -Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). Step 39's couple screen applies the same rule. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (were the commands of `chip_report.py`). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py`. Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|freqs|sample-bed|chroms|report`. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`), and the archaic jobs of `public-evidence.yml` run `og archaic compare|genomes` (were the two commands of `archaic_compare.py`), and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`), and index the phased family VCF with `og vcf index` (was a `pysam.tabix_index` one-liner). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. Step 39 and the Nextflow `COUPLE_CARRIER` process run `og couple screen`, and `og couple genes` rebuilds the carrier list (were the two commands of `couple_report.py`); the screen computes every couple before it writes a file. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)); the file stays one release as a shim that runs it, for crontabs. `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. Any failed lookup makes it exit 1. With `--unreachable-warns`, a lookup whose upstream did not answer (no connection, a timeout, HTTP 5xx or 429 after the retries, or HTTP 403, which an upstream that refuses a whole network answers) is listed under its own heading as a warning and does not change the exit; `freshness.yml` passes it only in the pull-request dry run, so a third-party outage does not block a pull request, and the monthly report stays strict. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. +Each ported tool becomes a subcommand named after its file in `bin/` without `.py`, in kebab-case. Today `og` has `og qc sex-loss` (step 33), `og version`, `og --tree-hash`, `og map`, `og setup mask-fasta`, `og run state|tasks|eta|headroom|delta`, `og dev parity-compare`, `og pgx cyp2d6-bed|cyp2d6-measure|cyp2d6-check`, `og str motif|catalog`, `og cdna check|scan` (were `clip_check.py` and `cdna_scan.py`), `og comphet phase` (was `comphet_phase.py`), `og bam contigs`, `og setup unzip|genetic-map` and two hidden ones the tests drive: `og pycompat-probe` and `og bam-fetch-probe`. The filters of steps 04, 05 and 07 are grouped by step: `og sv svlen-fix` and `og sv inv-filter` (were `sv_svlen_fix.py` and `manta_inv_filter.py`), `og pgx refblock-fill` and `og pgx mito-merge` (were `pgx_refblock_fill.py` and `pgx_mito_merge.py`). Step 05 builds AnnotSV's input with `og sv add-callers` and checks its class 4 and 5 calls with `og sv postfilter` (were `cnvpytor_to_vcf.py` and `annotsv_postfilter.py`); both apply one read-depth rule for long Delly calls (`og-steps::sv`). Step 39's couple screen applies the same rule. `og str catalog` (was `strchive_convert.py`) builds the STRchive catalog of steps 09 and 9b; `og setup unzip` and `og setup genetic-map` replace two small Python programs inside `setup.sh`. Step 25 has `og prs format` (a port of `collect_summary.py prs-format`, named by what it works on) and `og prs table` (was `prs-table`). Step 33 has `og qc sample` (was `sample-qc`), and step 20 has `og mito annotate|deletions|copies` (were `mito-annotate`, `mito-deletions` and `mito-copies`). Steps 23 and 31 use `og vcf csq-rank` and `og table constraint-join` (ports of the awk programs `bin/csq_rank.awk` and `bin/constraint_join.awk`, named by what they work on). Step 27 uses `og pgx cpic-report` and `og pgx step36-outside` (were the two commands of `pgx_parse.py`). Step 49 runs `og chip prepare`, `og chip report` and `og chip check-table` (were the commands of `chip_report.py`). Steps 32, 35, 41 and 46 run `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions` and `og cn marker-read` ([Copy number from depth](cn-loci.md)); they replace `hba_copy_number.py`, `lpa_kiv2.py`, `copy_number.py` (the catalog tests materialise its pinned historical version as a differential oracle) and the marker reader of `traits_report.py`. Step 41 runs `og bloodtype report` (was `bloodtype_report.py`), which reads the marker sites through the VCF's index. Step 26b's six commands are `og ancestry-fine panel-map|check-map|freqs|sample-bed|chroms|report`. Step 43 runs `og archaic table|strict|genotypes|report` (were the four commands of `archaic_report.py`), and the archaic jobs of `public-evidence.yml` run `og archaic compare|genomes` (were the two commands of `archaic_compare.py`), and step 44 runs `og ancient select|lift|person-geno|align|report` (were the five commands of `ancient_report.py`); `og ancient lift` reads UCSC's chain file with `og-intervals::chain`. Step 40 runs `og consumer build-sites`, `og consumer export` and `og consumer check` (were the three commands of `export_consumer.py`). Step 38 and the Nextflow family workflow run `og family plan`, `prepare`, `check`, `report`, `segments` and `ibd` (were `family_report.py`, `family_segments.py` and `ibd_report.py`), and index the phased family VCF with `og vcf index` (was a `pysam.tabix_index` one-liner). `og family report` reads the joint VCF once, compresses each child's inheritance table on its own thread, and stops with exit 2 on a joint VCF that is not sorted. Step 39 and the Nextflow `COUPLE_CARRIER` process run `og couple screen`, and `og couple genes` rebuilds the carrier list (were the two commands of `couple_report.py`); the screen computes every couple before it writes a file. The opt-in steps read their tool's output with `og y str-bed|str-report` (45), `og viral report` (47), `og exomiser prepare|report` (48), `og mei report|annotsv-input` (50), `og mocha report` (51), `og vntyper report` (52) and `og locityper report` (53); they replaced `ystr_report.py`, `viral_report.py`, `exomiser_report.py`, `mei_report.py`, `mocha_report.py`, `vntyper_report.py` and `locityper_report.py`. Step 37 places the sample on the current YFull YTree with `og y tree-positions` and `og y tree-place` (were the two commands of `y_tree_place.py`), lists its private SNPs with `og y private-snps` (was `y_private_snps.py`) and writes Yleaf's marker positions with `og y yleaf-positions` (was `yleaf_run.py positions`; `yleaf_run.py predict` stays Python, as glue inside Yleaf's own code). Steps 08, 32 and 36 run `og pgx hla-concordance`, `og pgx pypgx-summary` and `og pgx outside-calls` (were the three commands of `pgx_outside_calls.py`), and step 35b runs `og paraphase check-bam|table` (were `paraphase_report.py`). Both write their two output files together or not at all. `scripts/owngenome-lookup.sh` runs `og lookup query` ([genotype lookup](lookup.md)), which answers every query of the command line in one process; `og lookup resolve` and `og lookup report` are its two halves, with the command lines of `genome_lookup.py`, which keeps only the query reading and the names rule that `results_index.py` imports. `og recheck` replaced `owngenome_recheck.py` ([Newer databases since a run](recheck.md)). `og dev freshness` replaced `scripts/ci/freshness.py`, the monthly freshness issue of `freshness.yml`. Any failed lookup makes it exit 1. With `--unreachable-warns`, a lookup whose upstream did not answer (no connection, a timeout, HTTP 5xx or 429 after the retries, or HTTP 403, which an upstream that refuses a whole network answers) is listed under its own heading as a warning and does not change the exit; `freshness.yml` passes it only in the pull-request dry run, so a third-party outage does not block a pull request, and the monthly report stays strict. `og dev build cn-intervals|callability|sv-rule-regions` rebuild the committed tables `bin/cn_loci.tsv` (its interval rows), `assets/callability_genes.tsv` and `assets/sv_rule_regions.tsv`; they replaced `scripts/ci/build-cn-intervals.py` and the Python inside `build-callability.sh` and `build-sv-rule-regions.sh`. Those scripts keep the downloads (`curl` through `scripts/ci/lib/fetch-pinned.sh`, each file checked against its sha256) and hand `og` the folder, so `og dev build` reads local files only. Both ask the network through the host's `curl` (the config on stdin, so no token reaches the process list), which keeps a TLS stack out of the static binary and its cross-builds. -Every verb the 1.0.0 release still has to port is already registered, each in its own module, as a stub that prints `og : not ported yet` on stderr and exits 2: `og live serve|results|demo`, `og vcf shift-back`, `og setup chrm-shift|revel|roh-af`, `og sv consensus-check`, `og ancestry-fine project|paint`, `og map svg`, `og dev demo-events|python-gate|docs-check|steps-check` and `og dev build roh-af|acmg-lof`. A port fills its module's body and does not edit `og/src/main.rs`, the workspace member list or a crate's module list. `rust/og/tests/port_scaffold.rs` lists these verbs and fails when one has no line in `og --help` (`og dev --help` for an `og dev` verb) or reaches a router's "unknown" error. `og report render` is part way: it writes the frame of both reports and the quality-control and variant sections, leaves the other sections out, and no step calls it yet; `bin/render_report.py` still writes the reports. `og report charts` writes the five chart specs and their inline SVG as `bin/render_report.py --charts-json` does; no step calls it yet either. +Every verb the 1.0.0 release still has to port is already registered, each in its own module, as a stub that prints `og : not ported yet` on stderr and exits 2: `og live results|demo`, `og vcf shift-back`, `og setup chrm-shift|revel|roh-af`, `og sv consensus-check`, `og ancestry-fine project|paint`, `og map svg`, `og dev demo-events|python-gate|docs-check|steps-check` and `og dev build roh-af|acmg-lof`. A port fills its module's body and does not edit `og/src/main.rs`, the workspace member list or a crate's module list. `rust/og/tests/port_scaffold.rs` lists these verbs and fails when one has no line in `og --help` (`og dev --help` for an `og dev` verb) or reaches a router's "unknown" error. `og report charts` writes the five chart specs and their inline SVG on their own; the steps get the same file from `og report render --charts-json`, so no step calls `og report charts`. `og ancestry-fine simulate` builds an admixed genome with known truth from held-out people of the fine-ancestry panel: an unphased VCF and a BED of each copy's ancestry. The engines that replace ADMIXTURE and RFMix test against it. It and the parts those engines share (the panel reader, the genetic map, `model/ref.geno` and a seeded xoshiro256** generator) live in `og-admix`. -A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/genome_lookup.py` is one: it keeps the query reading and the summary names rule that `results_index.py` imports. +A Python file whose tool moved to `og` but whose names other Python still imports stays as a *remnant*: a `# remnant:` header names its importers and the lane that deletes it, and `tests/parity/remnants.tsv` lists the names it may hold and the files that import it. `og dev remnants` (Guard, job `rust`) fails when the file defines a name the list does not allow, when an allowed name has no user left, when a listed importer no longer imports it, or when an unlisted file starts to. So a remnant only shrinks. `bin/genome_lookup.py` is one: it keeps the query reading and the summary names rule that `results_index.py` imports. `bin/render_report.py`, `bin/report_charts.py` and `bin/results_index.py` are no remnants but whole programs: no step calls them since `og report render` took over steps 24, HTML_REPORT and generate-report.sh, and they stay only for the Python live site (`bin/live_results.py` imports `render_report`) until its port deletes them with `genome_lookup.py` and `collect_summary.py`. The rules: no `unsafe` (the workspace forbids it), `cargo clippy -D warnings`, `cargo fmt`, and few, well-known crates (`indexmap`, `flate2`, `zlib-rs`, `memchr`, `smallvec`, `rustc-hash` and `rust-htslib` so far, declared in `rust/Cargo.toml` and locked in `Cargo.lock`). The musl builds also link `mimalloc` as their allocator: musl's own lets a long multi-threaded read hold several times its live memory. @@ -140,9 +140,11 @@ To add a primitive: write it in `pycompat` with unit tests, add an operation to `og dev evidence clashes` reads two gzip VCFs and gzip archaic tables. `og dev evidence hg002-gvcf` reads a gzip truth VCF, plain BED, plain FASTA -and FAI, and a plain sites table. `og dev renovate-check` reads a plain log -and plain repository sources. Text uses strict UTF-8 and universal newlines -(LF, CRLF and CR). Compression is explicit, even when a filename is misleading. +and FAI, and a plain sites table. `og dev evidence y-bench` reads the current +YFull tree as plain JSON and a plain calls table. `og dev renovate-check` +reads a plain log and plain repository sources. Text uses strict UTF-8 and +universal newlines (LF, CRLF and CR). Compression is explicit, even when a +filename is misleading. Evidence, `og dev fixture comphet`, `og dev fixture trim-pharmcat` and the Renovate summary stage output beside its resolved destination. A file symlink diff --git a/docs/testing.md b/docs/testing.md index edf9ecd8b..f5d53322f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -77,10 +77,11 @@ You can build it yourself on Linux with Docker, `bgzip`, `tabix` and about 10 GB In CI the cases run in parallel legs, each case at most once: a pull request runs only the cases it can affect ([Which cases a pull request runs](#which-cases-a-pull-request-runs)), main and the monthly run run them all. On 2026-11-02 the 130 cases took 230 minutes one after another on one runner (run 41390434697), against a 300-minute job limit. Now: - `e2e-plan` splits the cases with [`scripts/ci/e2e-shards.sh`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/e2e-shards.sh). The leg `e2e base` runs the cases of stage `base` in [`scripts/ci/e2e-case-times.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/e2e-case-times.tsv) (every numbered case: the fixture check, setup validation, alignment, DeepVariant, GATK, the single steps on HG002, the reports and the Nextflow run; and `bash-step-08-hla`), because most other cases read what those wrote. Every other picked case goes to one of the shard legs (`e2e shard 1`, `e2e shard 2`, ...). A full run uses all `E2E_SHARDS` (6) legs. For picked cases, `e2e-shards.sh count` sets how many: the cases' total seconds divided by the heaviest group's seconds (the critical path, since a group is never split) or by 900 seconds when that is larger, rounded down, plus one more leg when the rest is 900 seconds or more, and never more than `E2E_SHARDS` (6). A shard leg spends 18 to 20 minutes on top of its cases (pulling images, fetching and unpacking the base outputs, the parity checks and the uploads; run 41847167444), so a leg with less than 15 minutes of cases costs more runner time than it saves in wall time. With every case that rule gives 4 or 5 legs, and that is why a full run does not use it: the runner's disk, not time, is what a leg runs out of. With 5 legs, shards of 30 and 37 cases filled one runner's disk until the runner itself died (run 41995851301, shard 3), while 6 legs passed on the same main (run 41995719742). -- The table holds each case's median time on the passing runs of nine e2e runs of 2026-11-04 (41766267038, 41767270253, 41773233871, 41774860458, 41779067613, 41790595358, 41799741941, 41807346887 and 41821518376), and its `needs`: the non-base cases whose outputs it reads. The needs were written from what each case says it reads (its header and its checks) and from the cases that failed in a split run without them (run 41455829120). A case and every case it needs, followed through, form a group that stays in one shard, in name order. Groups go to the least loaded shard, heaviest first. Every case needs a row: `e2e-shards.sh check`, which `tests/test_e2e_shards.sh` runs on the real table in the unit tests, fails and names each case without one. The split itself still runs a case with no row, as a 300-second case that needs nothing, so add the row with the case's measured seconds from its first passing run. +- The table holds each case's median time on the passing runs of nine e2e runs of 2026-11-04 (41766267038, 41767270253, 41773233871, 41774860458, 41779067613, 41790595358, 41799741941, 41807346887 and 41821518376), and its `needs`: the non-base cases whose outputs it reads. The needs were written from what each case says it reads (its header and its checks) and from the cases that failed in a split run without them (run 41455829120). A case and every case it needs, followed through, form a group that stays in one shard, in name order. Groups go to the least loaded shard, heaviest first. Every case needs a row: `e2e-shards.sh check`, which `tests/test_e2e_shards.sh` runs on the real table in the unit tests, fails and names each case without one. The split itself still runs a case with no row, as a 300-second case that needs nothing, so add the row with the case's measured seconds from its first passing run. A case whose cost depends on the run gets its time for that run from `E2E_SECONDS` (`CASE=SECONDS ...`), which `e2e.yml` sets for every job of the run, so they all split alike: `prs-3-panel-measure` measures the 1000 Genomes panel only on a run dispatched on a branch other than main, where it took 2684 to 3452 seconds (median 2900), and takes 0 seconds elsewhere, as its row says. Before, a dispatched run gave its shard a full share of other cases on top of those 48 minutes, and that shard finished last in 12 of 18 runs. `e2e-shards.sh seconds` prints the time each case is weighed by, and the plan job's summary names any time `E2E_SECONDS` replaced. +- The job `og` builds `og` once, for the runners only (`OG_ARCHES=x86_64`, no macOS binaries; see `rust/og.sh`), and hands it to every leg as the artifact `og-e2e`. Guard's `rust` job and the Rust workflow's `images` job still build all four binaries. The base leg does not wait for `og` to start: it mounts the spare disk, frees disk, sets up Docker, Java and Nextflow, restores the image cache and pulls the images while `og` builds, and only then waits for `og-e2e` (step "Wait for og"). A shard leg finds the artifact at once. When the job `og` ends without it, the leg stops. - The base leg runs first. It packs every file its cases wrote or changed (at its absolute path, with owners, modes and times) and uploads it as the artifact `e2e-base`, kept for 3 days so that "Re-run failed jobs" the next day still finds it. The shard legs start only after it passed (`needs:`), so a shard never holds a runner while it waits: when many runs share the account's runners, shards that started early and waited took the runners the base legs needed (run 41506673662 had three shards running and its base leg still queued). Each shard fetches the artifact, tried up to five times, takes the base leg's work area, unpacks the archive there, deletes it (about 4 GB the cases need) and runs its cases. When the base leg fails, the shards are skipped and `e2e` is red. - The shard that holds `nextflow-from-fastq-2-nextflow` and the one that holds `nextflow-hardening-2-nf` run the parity checks; the plan fails when either case runs and the case before it in its pair is in a different shard. -- The required check is the job `e2e`. It merges every leg's results into one table of case, result, time and job, followed by the failed checks of each failed case, and it is red unless the plan and every leg succeeded, every picked case reported exactly one pass and each parity check passed once when its cases ran (and not at all when they did not). It is red too when `e2e-scope` did not succeed. A leg that was skipped, failed or cut by its time limit leaves cases with no result, and that is red too. When the run itself is cancelled, by a newer push to the pull request or by hand, `e2e` is cancelled with it and reads cancelled, not failed: no case ran, so none failed. A cancelled `e2e` blocks a merge as a red one does; re-run the workflow, or push again, to get a result. The full logs are in the `e2e-logs-` artifacts. +- The required check is the job `e2e`. It merges every leg's results into one table of case, result, time and job, followed by the failed checks of each failed case, and it is red unless the plan, the job `og` and every leg succeeded, every picked case reported exactly one pass and each parity check passed once when its cases ran (and not at all when they did not). It is red too when `e2e-scope` did not succeed. A leg that was skipped, failed or cut by its time limit leaves cases with no result, and that is red too. When the run itself is cancelled, by a newer push to the pull request or by hand, `e2e` is cancelled with it and reads cancelled, not failed: no case ran, so none failed. A cancelled `e2e` blocks a merge as a red one does; re-run the workflow, or push again, to get a result. The full logs are in the `e2e-logs-` artifacts. `tests/test_e2e_shards.sh` (unit tests) checks the split against planted faults: a case listed twice or missing, a stale or malformed table row, a need on a case that runs later, a pick that leaves out a need, and results missing, failed or reported twice; the leg count; and a case with no row, which `check` must name. `tests/test_e2e_scope.sh` checks the pick: a case with no paths row, a second row, a row for no case and a glob that matches no file must each fail `e2e-scope.sh check`, and a docs-only change, a change to `main.nf`, a change to one helper, to one module, to a case, to a table row and to `rust/` must each pick the cases listed below. On the real table, a change to `rust/` must pick what a change to `bin/og` picks. @@ -100,7 +101,7 @@ The base leg runs every base case whenever any case runs, because every shard st [`scripts/ci/e2e-case-paths.tsv`](https://github.com/GeiserX/OwnGenome/blob/main/scripts/ci/e2e-case-paths.tsv) holds, for each case, the files it runs: the step scripts, bin helpers, modules (`modules/local//*`) and assets it calls, and what those call in turn. The globs are matched as in bash `[[ PATH == GLOB ]]`, so `*` matches slashes too. A base case's row is `-`. The table was first written from the text of each case, followed through the scripts, helpers and modules it names (comments left out). `e2e-scope.sh check` runs first in `e2e-scope` and fails when a case has no row or two, a row names no case, a base row lists paths, a glob matches no tracked file, or a glob matches a file outside the e2e scope (a change to that file would start no run, so the case would never see it); then `e2e-scope` is red and so is `e2e`. A glob too many costs runner time; a glob too few lets a break through to main, where the full run after the merge finds it. -The images the cases use are cached in four GitHub Actions cache entries, one per group. `scripts/ci/e2e-run.sh --image-groups` names each image's group, taken from its repository name without the tag, so a version bump changes the key of one group only; a group whose key missed restores the newest older entry of that group and pulls only what changed. A group keeps its smallest images up to 1500 MB of zstd output, and the larger ones are pulled on every run. Four groups of 1500 MB and the six data entries below stay under the repository's 10 GB cache limit even while main replaces one group: 5.9 GB of images, 1.5 GB more for the group being replaced, and about 2.4 GB of data. Images pinned by digest are not cached, because `docker load` gives them back without the digest, and they are left out of the group's key, so a digest bump misses no entry. Only the base leg saves, and only on main (or in a run by hand with `save_images`, on any branch), after it pulled every image the cases use and got through its cases, and only the groups whose key missed: a cache saved by a pull request can be read by that pull request alone and would only push main's entries out. The job summary lists every image with its size and what the cache did with it: restored on an exact key hit, restored from an older entry, saved, or pulled. Every merge in the e2e scope starts a run on main, so the cache follows main's images. GitHub drops an entry no run has read for 7 days, so after a quiet week the next run on main fills the cache again. Until this layout the cache was one entry, and by October 2026 it had grown to about 11 GB, over the limit, so every run missed it. +The images the cases use are cached in four GitHub Actions cache entries, one per group. `scripts/ci/e2e-run.sh --image-groups` names each image's group, taken from its repository name without the tag, so a version bump changes the key of one group only; a group whose key missed restores the newest older entry of that group and pulls only what changed. A group keeps its smallest images up to 1500 MB of zstd output, and the larger ones are pulled on every run. Four groups of 1500 MB and the six data entries below stay under the repository's 10 GB cache limit even while main replaces one group: 5.9 GB of images, 1.5 GB more for the group being replaced, and about 2.4 GB of data. Images pinned by digest are not cached, because `docker load` gives them back without the digest, and they are left out of the group's key, so a digest bump misses no entry. Only the base leg saves, and only on main (or in a run by hand with `save_images`, on any branch), after it pulled every image the cases use and got through its cases, and only the groups whose key missed: a cache saved by a pull request can be read by that pull request alone and would only push main's entries out. The job summary lists every image with its size and what the cache did with it: restored on an exact key hit, restored from an older entry, saved, or pulled. Every merge in the e2e scope starts a run on main, so the cache follows main's images. A fork reads only its own caches: on a fork where pull request CI runs, start one run by hand on the fork's main, after it is synced, with `job: e2e` and `save_images: true`, or every leg there pulls every image and downloads every data file. The 10 GB is shared with the Rust caches, so only Guard's `rust` job keeps one, saved on main alone; the jobs that only build `og --release` have none, because it did not make that build faster, and their per-branch entries had filled the budget (10.69 GB in 55 entries on one fork, none of them from main). GitHub drops an entry no run has read for 7 days, so after a quiet week the next run on main fills the cache again. Until this layout the cache was one entry, and by October 2026 it had grown to about 11 GB, over the limit, so every run missed it. Six downloads are cached the same way. Two come from EBI's 1000 Genomes server, which is slow and at times stalls: MoChA's chr20 panel with its manifest (0.64 GB, case `mocha-1`), keyed on `MOCHA_PANEL_MANIFEST_SHA256` and the chromosome, and the 1000 Genomes phase 3 sites VCF (1.36 GB, case `archaic-ancestry-1`), keyed on `ARCHAIC_1KG_SITES_MD5`. On a miss, each of those two is tried first at `E2E_DATA_MIRROR`, the GitHub release `e2e-data-1` that holds copies of the pinned files, and then at EBI; the pin checks both. The release also holds the four PGS Catalog score files, the HAPNEST panel and the table builders' downloads below, so an EBI outage costs none of them on a cold cache. It holds the TRExplorer catalog of `build-sv-rule-regions.sh` too (the v2.1 file its `TRX_SHA256` pins), so a miss tries the mirror first and the catalog's own GitHub release after it. The release keeps each file unchanged, and its notes name the source and the terms of every file it holds. Two are small and failed whole runs on 2026-11-04, because their case is in the base leg or a shard that cannot start without it: `hla.dat` of the two IPD-IMGT/HLA releases `bash-step-08-hla` installs, from raw.githubusercontent.com (which served it under 1 KB/s), with GENCODE's gene and exon files from EBI's FTP server (curl exit 7 and 28), and the four real PGS Catalog score files of `prs-scores-and-run-options-1-strand-sex`, from the same server. `scripts/ci/e2e-run.sh --data-key` names their keys: the releases and a hash of the code in `scripts/lib/common.sh` that installs them, and a hash of the score ids and their server. `scripts/ci/e2e-run.sh` hands each case its files just before it runs, at the paths `setup.sh` and `install_data_file` read them from, so they download nothing; the score files the case copies from the cache itself. On a miss it downloads them first, checked against their pins or the checksum files the servers publish beside them (`MD5SUMS`, `md5checksum.txt`, the `.md5` of each score), trying up to 6 times 30 s apart. The four score files are pinned by md5 in `PGS_MD5` of `e2e-run.sh` (the md5 the PGS Catalog served beside each on 2026-11-06), tried on the mirror first and then at EBI; a restored or downloaded score must match its pin as well as its `.md5`. A file gets at most `E2E_MIRROR_BUDGET` seconds on the mirror (900, or half the fill's budget when that is less), so a mirror that trickles leaves EBI its turn. One fill may take at most `E2E_DATA_BUDGET` seconds (2400 in `e2e.yml`): a source that trickles above `fetch`'s stall limit would otherwise run on to the leg's 240-minute limit, as EBI's 1000 Genomes server did twice on 2026-11-05. A fill still running then is stopped and its case fails as `FAIL (data fill over 2400s)` without running, since its own download would be as slow. A restored entry is checked again before use (`--data-check`): the IPD-IMGT/HLA and GENCODE entry against the sha256 list written when it was filled, the score files against their `.md5`; one that fails is downloaded again. `bash-step-08-hla` then runs with every HTTP and HTTPS request sent to a closed port, after a control shows that a request under that block fails at the closed port, so a warm cache cannot reach either host. Only the leg whose cases include the case restores and saves its entry, and it saves only on main or in a run by hand with `save_images`, as with the images. The job summary says whether each key hit, and the case's log says where its files came from. The fifth is the PGS Catalog's synthetic HAPNEST panel (0.27 GB, cases `prs-2-ancestry` and `prs-scores-and-run-options-2-panel-chrx`) from EBI's FTP server, which failed cases with curl exit 7 on 2026-11-04. It is keyed on the sha256 `tests/e2e/lib.sh` pins (`HAPNEST_SHA256`). The cases handle it themselves (`setup_ancestry_panel` in `lib.sh`): with the panel in the cache, or on a miss from `E2E_DATA_MIRROR`, they check the sha256 and run `setup.sh --ancestry-panel` with every HTTP and HTTPS request sent to a closed port, so an EBI outage cannot fail them; when neither has it, `setup.sh` downloads the panel from EBI as a user's run does. The first case puts a panel that did not come from the cache in the cache for the save step. A red control in `prs-2-ancestry` runs `setup.sh` with the network blocked and no panel, which must fail. The sixth, entry `catalogs`, holds the pinned downloads of `scripts/ci/build-callability.sh`, `scripts/ci/build-cn-intervals.sh` and `scripts/ci/build-sv-rule-regions.sh`, whose cases `callability-1-catalog`, `cn-intervals-1-catalog` and `followups-ci-3-sv-rule-regions` rebuild `assets/callability_genes.tsv`, the interval rows of `bin/cn_loci.tsv` and `assets/sv_rule_regions.tsv` and compare them with the committed files. Each builder lists its downloads with `--list`; the key is a hash of those lists, so a new pin is a new key. GENCODE's GTF among them comes from EBI's FTP server, which refused every connection for minutes on 2026-11-06 and failed `callability-1-catalog` on main (run 42259432681). The runner fills the entry as it does the others, mirror first and every file checked against its sha256, and each case points its builder's cache at its folder there; a builder that still has to download tries for about five minutes, 30 s apart, and gives up at once on a 403 or 404 (`fetch_pinned` in `scripts/ci/lib/fetch-pinned.sh`, which the three builders share). The builders download with `curl` and hand the folder to `og dev build`, which reads local files only. Every download, there and in `setup.sh`, gives up on a transfer that stalls (`FETCH_STALL`, see `fetch` in `scripts/lib/common.sh`), and the next try resumes it. @@ -121,7 +122,7 @@ What it covers: - **Index times** (`run-all-sample-qc-and-stale-index-1-index-times.sh`): every index the direct `nextflow run` and the launcher published is not older than its data file, and samtools and bcftools print no stale-index warning on the BAM and the VCF. Then a short VCF-only run with three planted pairs checks `main.nf`'s completion handler: it sets the index times of the run's own files after the last copy, and leaves an index older than the run as it was. A copied pair with its index set an hour older is the negative control. - **The Nextflow report's tables** (`paralogs-report-section-1-nf-report-tables.sh`): `HTML_REPORT` runs for real on the CYP2D6 depth check, the cDNA read check and the phased compound-het pairs that earlier cases wrote, and its reports must show the depth-check line, the cDNA column with its flagged count and the pairs by phase; the same run without the three tables must show none of them. A stub run of every tool then checks that `main.nf` passes the cDNA table to each `HTML_REPORT` task that got ClinVar hits, against a copy of the pipeline with that join removed. -- **Locityper, step 53** (`ci-e2e-2-locityper.sh`): `setup.sh --locityper-data` builds the database for the HLA-DQB1 row of `assets/locityper_loci.bed` from a chr6 and chr20 reference cut from the fixture's, and `scripts/53-locityper.sh` genotypes the fixture's reads aligned to it, with chr20 as the background region. The fixture has no reads over FCGR or KIR, and HLA-DRB1 is left out for time: with it the case took 59 minutes, 35 of them jellyfish's k-mer count and 23 `locityper augment`. With HLA-DQB1 alone `augment` takes about a minute. Jellyfish does not depend on the loci: with its hash fixed at 3G entries it took 36 minutes (2,289 s for the case, fork run 42325306077), and with the hash sized to the reference it takes 36 seconds (142 s for the case, fork run 42421585308). The case keeps a limit of 90 minutes (`# case-timeout: 5400`; every other case keeps 60) as room for a slow runner. The locus must come out called, with two panel haplotypes, a quality, reads and the authors' filter, in the columns `og locityper report` writes. Red controls: the step stops without the database, without the BAM's index, and with a database whose `DONE` names another reference. +- **Locityper, step 53** (`ci-e2e-2-locityper.sh`): `setup.sh --locityper-data` builds the database for the HLA-DQB1 row of `assets/locityper_loci.bed` from a chr6 and chr20 reference cut from the fixture's, and `scripts/53-locityper.sh` genotypes the fixture's reads aligned to it, with chr20 as the background region. The fixture has no reads over FCGR or KIR, and HLA-DRB1 is left out for time: with it the case took 59 minutes, 35 of them jellyfish's k-mer count and 23 `locityper augment`. With HLA-DQB1 alone `augment` takes about a minute. Jellyfish does not depend on the loci: with its hash fixed at 3G entries it took 36 minutes (2,289 s for the case, fork run 42325306077), and with the hash sized to the reference it takes 36 seconds (142 s for the case, fork run 42421585308). The case keeps a limit of 90 minutes (`# case-timeout: 5400`) as room for a slow runner. So does `prs-3-panel-measure.sh`, whose wait for the panel download (up to 30 minutes when EBI is slow) and pgsc_calc's 35 minutes can pass 60. Every other case keeps 60. The locus must come out called, with two panel haplotypes, a quality, reads and the authors' filter, in the columns `og locityper report` writes. Red controls: the step stops without the database, without the BAM's index, and with a database whose `DONE` names another reference. - **The SV rule regions** (`followups-ci-3-sv-rule-regions.sh`): `scripts/ci/build-sv-rule-regions.sh --check` rebuilds `assets/sv_rule_regions.tsv` from its four pinned downloads and must find the committed file byte for byte, with its row counts (2,409 coding tandem repeats). Red control: `--check` on a copy with the MHC row's end moved by one must exit 1 and show that row. The downloads come from the `catalogs` data cache entry. @@ -158,7 +159,7 @@ A new check has to be seen failing once before it is trusted: run it against the ### What no e2e case runs -This is every step script that no e2e case runs, plus the full-size index build of step 02a. The first eight rows need data, hardware or a binary an e2e job does not have. The three long-read rows need long reads, and the e2e fixture has short reads only. The others could run, but no case runs them yet. The last column says what checks each one instead. An image's smoke test is a row of `tests/smoke/commands.tsv` that `container-test.yml` runs on the fixture when the image changes and once a month; it runs the tool, not the step's script. Its required check `container-ok` is green on a pull request that changes no image: then no smoke row ran, so that green says only that no image changed. The row that runs AnnotSV with its 5.3 GB of full data runs monthly and on dispatch, never on a pull request, so no image pull request waits on that download. +This is every step script that no e2e case runs, plus the full-size index build of step 02a. The first eight rows need data, hardware or a binary an e2e job does not have. The three long-read rows need long reads, and the e2e fixture has short reads only. The others could run, but no case runs them yet. The last column says what checks each one instead. An image's smoke test is a row of `tests/smoke/commands.tsv` that `container-test.yml` runs on the fixture when the image changes and once a month; it runs the tool, not the step's script. At most 8 rows run at once (`max-parallel`), so a change that picks every image (about 66 rows) leaves the account's other runners to the E2E legs; every row still runs. Its required check `container-ok` is green on a pull request that changes no image: then no smoke row ran, so that green says only that no image changed. The row that runs AnnotSV with its 5.3 GB of full data runs monthly and on dispatch, never on a pull request, so no image pull request waits on that download. | Step | Why no e2e case runs it | What checks it instead | |---|---|---| @@ -233,7 +234,7 @@ Look at the pictures, copy them into `docs/images/`, and update the alt text whe ## The Rust jobs -The `rust` job of `guard.yml` checks the Rust code on every pull request: format, clippy, the unit tests with a planted failure that must fail, `pycompat` against CPython at 100,000 cases per operation, both musl builds as `scripts/setup.sh --og` makes them, and the stale-binary check on the real binaries. The **Rust** workflow (`rust.yml`) runs `og` by bare name in the task images on amd64 and arm64, and every night runs the CPython comparison at 10,000,000 cases per operation. [The Rust Core](rust.md) says what each check covers. +The `rust` job of `guard.yml` checks the Rust code on every pull request: format, clippy, the unit tests with a planted failure that must fail, `pycompat` against CPython at 100,000 cases per operation, both musl builds as `scripts/setup.sh --og` makes them, and the stale-binary check on the real binaries. The **Rust** workflow (`rust.yml`) runs `og` by bare name in the task images on amd64 and arm64, and every night runs the CPython comparison at 10,000,000 cases per operation, with only the runner's own `og-linux-x86_64` built for it (`OG_ARCHES`, `rust/og.sh`). [The Rust Core](rust.md) says what each check covers. How long `scripts/setup.sh --og` takes to build both binaries: diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 96c04971c..d28af4d3e 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -119,9 +119,9 @@ docker run --rm -it \ **Symptom:** `OSError: [Errno 13] Permission denied: '/genome/sample/output'` or similar. -**Cause:** the container's user cannot write to the bind-mounted host folder. The step scripts run each container as you (`--user` with your UID and GID, `HOME=/tmp`), with only the sample folder writable. Older versions ran every container as root, so a sample folder they wrote holds root-owned files that a step running as you cannot overwrite. A few steps whose image cannot run unprivileged pass `--root` at their call (Strelka2, Manta, CPSR and `benchmark-variants.sh`). Most bioinformatics images name a non-root user of their own (e.g., UID 1000), so a container you start by hand without `--user` may not be able to write either. +**Cause:** the container's user cannot write to the bind-mounted host folder. The step scripts run each container as you (`--user` with your UID and GID, `HOME=/tmp`), with only the sample folder writable. Older versions ran every container as root, so a sample folder they wrote holds root-owned files that a step running as you cannot overwrite. CPSR and `benchmark-variants.sh` pass `--root` at their call: their images (PCGR, hap.py) have not been shown to run unprivileged. Strelka2 and Manta pass it only to delete a leftover folder that an older version wrote as root. Nextflow's `docker` profile runs each task as you too, with `HOME=/tmp`. `CPSR` and `CNVPYTOR` are the exceptions and run as root: CNVpytor copies files into its own image. Nextflow (`docker.fixOwnership`) gives their work files back to you; a file they write outside their work directory stays root's. Most bioinformatics images name a non-root user of their own (e.g., UID 1000), so a container you start by hand without `--user` may not be able to write either. -**Fix for a step script:** when the sample folder holds a path you do not own, the step prints the command that takes it back. Run it, then run the step again: +**Fix for a step script or a Nextflow run:** when the sample folder holds a path you do not own, a step script prints the command that takes it back. A Nextflow task stops with the error above. Run the command, then run the step or `run-all.sh` again: ```bash sudo chown -R "$(id -u):$(id -g)" "${GENOME_DIR:?Set GENOME_DIR to your data directory}/${SAMPLE:?Set SAMPLE to the sample name}" diff --git a/docs/viewer.md b/docs/viewer.md index 5fa27e2fd..4a30b0b13 100644 --- a/docs/viewer.md +++ b/docs/viewer.md @@ -43,6 +43,10 @@ Press Ctrl-C to stop it. Open it in a browser. To jump somewhere else, type a locus in igv.js's search box, or change the part after `#` in the address. Gene names do not work in the search box: looking a name up would need igv.org (see Privacy below). +The page is dark. Its heading names the sample. The line under it says what the tracks are, that the search box takes coordinates and not gene names, and that reads show at a few kilobases or less. igv.js draws its tracks on white, in a framed panel. On a narrow screen such as a phone, that panel scrolls sideways and the page keeps the width of the screen. If the viewer cannot start (the server stopped, or igv.js did not load), the page says what failed and what to check, and **Try again** loads it again. Screen readers announce that message. + +In a viewport at most 330 px tall, such as the map's reads panel, the viewer hides the service and orientation text to leave room for the tracks. + | Variable | Default | Meaning | |---|---|---| | `OWNGENOME_VIEW_PORT` | `8765` | The port. The report's links use 8765, so keep it unless the port is taken. | @@ -115,6 +119,10 @@ The server starts at once. How fast a view loads depends on the disk and the rea - it answers byte ranges of the BAM exactly, and refuses unknown paths, a foreign `Host` and a forwarded request; - in headless Chrome, igv.js starts on the config, the policy blocks nothing it needs, and it reads the BAM, its index, the reference and the VCF. A control with an empty igv.js must not reach that state. +`rust/crates/og-browser/tests/rv_viewer_shell.rs`, in the same job, checks the page itself in headless Chrome, at 1440x900 and on a 390x844 phone, with igv.js started on a synthetic reference and VCF. The page is dark, one `h1` names the sample, one `main` holds the error box and igv.js, and the page says what search takes. On the phone the page is no wider than the screen. When `config.json` answers 500, the error box has `role="alert"`, says what failed and what to check, and offers Try again, which starts the viewer once the server answers. A stub serves the page with the policy of `og view serve`, and the test keeps that policy equal to og-live's. + +The browser checks also measure the contrast of the igv.js search input and variant track label. At 600x260 and 900x260, the variant track must fit without vertical scrolling. + `tests/test_owngenome_view.sh` checks the script with `og`: igv.js is the release `versions.env` pins, with its sha256, and the script sets no pin of its own; it prints the address, picks Manta's SV VCF, refuses a foreign `Host`, a forwarded request and a reversed locus, and never calls `python3`. `tests/test_viewer.py` checks that every findings table of the report ends with a Viewer cell that links each row to its position, with the note under it; a control with the links taken out must fail that check for every table. diff --git a/main.nf b/main.nf index debd9bdf7..fb7f1d2ea 100644 --- a/main.nf +++ b/main.nf @@ -1170,10 +1170,11 @@ workflow { // for every selected step it shows, the SV consensus merge included. // The ACMG SF rules, the carrier list, the gene callability table, the // confirmed mtDNA variants and the two paralog catalogs (step 35's loci, - // step 35b's regions) the report applies (bin/ is the only repository + // step 35b's regions) the report applies, and the step catalogue, which + // maps each --tools name to its sections (bin/ is the only repository // folder a task sees). def report_assets = ["acmg_sf.tsv", "carrier_genes.tsv", "callability_genes.tsv", "mito_confirmed.tsv", - "paralog_loci.tsv", "paraphase_genes.tsv"].collect { n -> file("${projectDir}/assets/${n}", checkIfExists: true) } + "paralog_loci.tsv", "paraphase_genes.tsv", "steps.tsv"].collect { n -> file("${projectDir}/assets/${n}", checkIfExists: true) } ch_report_inputs = ch_vcf .map { meta, vcf, idx -> [meta.id, meta, vcf] } .join(PGX.out.clinvar_dir.map { meta, f -> [meta.id, f] }, remainder: true) @@ -1272,11 +1273,15 @@ workflow { // gvcf columns) go in the list too, with their names in meta.gvcf: // HTML_REPORT links them as vcf/.g.vcf.gz, where the bash report // finds step 03's gVCF. A sample without a gVCF gets no meta.gvcf. + // meta.tools is the run's tool list (--tools, with exomiser when + // --hpo added it): HTML_REPORT records it, so a selected step that + // wrote nothing reads "selected, no result" instead of not run. .map { meta, vcf, files -> [meta.id, meta, vcf, files] } .join(ch_gvcf.map { meta, gvcf, idx -> [meta.id, [gvcf, idx]] }, remainder: true) .filter { items -> items[1] != null } .map { id, meta, vcf, files, gvcf -> - gvcf ? [meta + [gvcf: gvcf.collect { f -> f.name }], vcf, files + gvcf] : [meta, vcf, files] + def report_meta = meta + [tools: tools_list] + gvcf ? [report_meta + [gvcf: gvcf.collect { f -> f.name }], vcf, files + gvcf] : [report_meta, vcf, files] } // QC files for MultiQC (mosdepth summaries) diff --git a/modules/local/html_report/main.nf b/modules/local/html_report/main.nf index 85a2a56da..7d08e48d5 100644 --- a/modules/local/html_report/main.nf +++ b/modules/local/html_report/main.nf @@ -1,12 +1,12 @@ /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - HTML_REPORT — The sample's HTML report, rendered by bin/render_report.py + HTML_REPORT — The sample's HTML report, rendered by og report render ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The same code scripts/24-html-report.sh runs: og summary collect - (rust/crates/og-summary, run by render_report.py through bin/og) reads - the step outputs into summary.json (the bash step's name for it), - and render_report.py writes the report from it, so a number on the - Nextflow report is the number the bash report shows for the same file. + The same code scripts/24-html-report.sh runs: og report render + (rust/crates/og-render) collects the step outputs into summary.json + (og summary collect, the bash step's name for it) and writes the report + from it, so a number on the Nextflow report is the number the bash + report shows for the same file. The inputs are the outputs of the steps that ran for this sample, as one list (main.nf joins them, so the report waits for each). They are linked @@ -23,8 +23,9 @@ and assets/carrier_genes.tsv (the carrier list) and assets/callability_genes.tsv (the hard-to-read genes) come in the same list and go to assets/, which --assets names, as do assets/mito_confirmed.tsv - (the confirmed mtDNA variants step 20's section names) and the paralog - catalogs assets/paralog_loci.tsv and assets/paraphase_genes.tsv. Every + (the confirmed mtDNA variants step 20's section names), the paralog + catalogs assets/paralog_loci.tsv and assets/paraphase_genes.tsv, and + assets/steps.tsv (the step catalogue). Every __copy_number.tsv of step 35 goes to paralogs/, and so do step 35's _cn_intervals.tsv, _cn_dosage.tsv, _cyp21a2_psv.tsv and _smn1_marker.tsv; step 46's _lpa_kiv2.tsv @@ -35,10 +36,14 @@ OwnGenome release (nextflow.config's manifest.version), so the report names it as the bash report does, and the regions of --intervals when the run was limited to them, so summary.json marks the run partial as - the bash report does; it is not published, because + the bash report does, and the run's tool list (meta.tools, main.nf's + --tools list), so og summary collect marks each section of a selected + step that wrote nothing "selected, no result" instead of not run (it + maps each tool to its sections through assets/steps.tsv, which comes in + the list with the other assets); it is not published, because run-all.sh writes the full manifest there. Every gene and rsID the results mention also goes to ${id}_search_index.json beside it - (bin/results_index.py), for the live site's search, and the report's + (og report index), for the live site's search, and the report's charts to ${id}_charts.json, for the live page (docs/24-html-report.md). Equivalent to: scripts/24-html-report.sh @@ -73,6 +78,9 @@ process HTML_REPORT { def release = workflow.manifest.version ?: '' // --intervals as given, single-quoted below like gvcf; the task script puts it on one line def intervals = params.intervals ? params.intervals.toString().replace("'", "'\\''") : '' + // The run's --tools names (main.nf checks each against its known list), + // single-quoted below like gvcf + def tools = (meta.tools ?: []).join(' ').replace("'", "'\\''") """ # Where og summary collect looks for each output (rust/crates/og-summary) S=layout/${id} @@ -93,6 +101,11 @@ process HTML_REPORT { intervals='${intervals}' read -r -d '' -a regions <<< "\$intervals" || true [ "\${#regions[@]}" -eq 0 ] || printf 'run\\tintervals\\t%s\\n' "\${regions[*]}" >> "\$S/run_manifest.tsv" + # The steps this run selected (the bash runner records none), so og + # summary collect lists a selected step that wrote nothing as + # "selected, no result" + tools='${tools}' + [ -z "\$tools" ] || printf 'run\\ttools\\t%s\\n' "\$tools" >> "\$S/run_manifest.tsv" for f in input/*; do n=\$(basename "\$f") if [ -n "\$gvcf" ] && { [ "\$n" = "\$gvcf" ] || [ "\$n" = "\$gvcf_index" ]; }; then continue; fi @@ -148,13 +161,13 @@ process HTML_REPORT { ${id}_mei.tsv|${id}_mei_sites.tsv) place mei "\$f" ;; ${id}_mocha.tsv|${id}_mocha_summary.tsv) place mosaic "\$f" ;; ${id}_couple_carrier.tsv) place couple "\$f" ;; - acmg_sf.tsv|carrier_genes.tsv|callability_genes.tsv|mito_confirmed.tsv|paralog_loci.tsv|paraphase_genes.tsv) mkdir -p assets && ln -s "\$(readlink -f "\$f")" assets/ ;; + acmg_sf.tsv|carrier_genes.tsv|callability_genes.tsv|mito_confirmed.tsv|paralog_loci.tsv|paraphase_genes.tsv|steps.tsv) mkdir -p assets && ln -s "\$(readlink -f "\$f")" assets/ ;; *) echo "ERROR: HTML_REPORT got an input it has no place for: \$(basename "\$f")" >&2; exit 1 ;; esac done find layout | sort - render_report.py \\ + og report render \\ --assets assets \\ --sample ${id} \\ --sample-dir "\$S" \\ diff --git a/modules/local/prs/main.nf b/modules/local/prs/main.nf index 4b3f5237e..cae8b5e9b 100644 --- a/modules/local/prs/main.nf +++ b/modules/local/prs/main.nf @@ -185,8 +185,17 @@ process PRS { // process.containerOptions of docs/hardware-requirements.md's shared-host // config). The task script drops their user and network flags. Quoted for // the shell. + // >>> user flag (tests/test_prs_runopts.sh applies this pattern) + // Each user flag goes here first, with its whole value: -u VALUE, -uVALUE, + // -u=VALUE, --user VALUE or --user=VALUE, where VALUE is a run of plain + // characters, quoted strings, and $(...) or `...` substitutions, which + // can hold spaces. The task script splits the options into words, so it + // would drop only the first word of such a value: a config's + // -u $(id -u):$(id -g) handed pgsc_calc a stray -g). + def user_flag = '(? o }.join(' ').replace("'", "'\\''") + .findAll { o -> o }.join(' ').replaceAll(user_flag, ' ').replace("'", "'\\''") // The sex for pgsc_calc's plink2 (--update-sex: the VCF's sample name, 1 // male or 2 female), added to the ext.args its conf/modules.config gives // PLINK2_VCF at this release; the folder is mounted into its containers. diff --git a/nextflow.config b/nextflow.config index d6c7df852..022eb0cc6 100644 --- a/nextflow.config +++ b/nextflow.config @@ -203,7 +203,18 @@ process { profiles { docker { docker.enabled = true - docker.runOptions = '--user root:root' + // Tasks run as you, as the bash steps' run_in does (scripts/lib/common.sh): + // what a task writes, in its work directory or beside an input under + // GENOME_DIR, is yours, not root's. HOME=/tmp gives a tool a writable + // home. ${UID}:${GROUPS} is bash's own `$(id -u):$(id -g)`, expanded by + // the task's .command.run, which is always bash. It is written without a + // space, so a reader that splits these options into words sees the ids + // as one word. (The PRS module, which hands them on to pgsc_calc, also + // drops the spaced form whole.) + docker.runOptions = '-u ${UID}:${GROUPS} -e HOME=/tmp' + // CPSR and CNVPYTOR still run as root (below). Nextflow gives their work + // directory back to you at the end of the task; a file they write + // outside it stays root's. docker.fixOwnership = true singularity.enabled = false conda.enabled = false @@ -212,12 +223,20 @@ profiles { // downloading something unpinned. No exception: Cyrius runs from the // install setup.sh --cyrius made. process.containerOptions = '--network none' - // VNtyper's image starts an entrypoint that runs only its own command - // words (vntyper, celery, health): Nextflow's bash would be refused. process { + // VNtyper's image starts an entrypoint that runs only its own command + // words (vntyper, celery, health): Nextflow's bash would be refused. withName: 'VNTYPER' { containerOptions = '--network none --entrypoint ""' } + // Root, as before, for two images not shown to run as another user. + // CPSR: step 17 runs PCGR as root too (run_in --root). CNVPYTOR: the + // module copies its resources into the image's site-packages. A + // process's options come after docker.runOptions, so its --user + // wins, and so does its HOME: /root, the home root had before. + withName: 'CPSR|CNVPYTOR' { + containerOptions = '--network none --user 0:0 -e HOME=/root' + } } } singularity { diff --git a/rust/Cargo.lock b/rust/Cargo.lock index b56baa14b..307997f19 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -854,6 +854,7 @@ dependencies = [ "og-pedigree", "og-render", "og-runlog", + "og-summary", "pycompat", "regex", ] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 4abfbedcd..303adea33 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -70,5 +70,9 @@ all = { level = "warn", priority = -1 } [profile.release] lto = true codegen-units = 1 -panic = "abort" +# unwind, not abort: og report render catches a panic inside one section, so +# that section says it could not be written and the rest of the report is +# still written. A panic anywhere else still aborts og at once: og's main +# installs og_render::chapters::abort_outside_guarded. +panic = "unwind" strip = true diff --git a/rust/README.md b/rust/README.md index d18b07ee1..5c81c5834 100644 --- a/rust/README.md +++ b/rust/README.md @@ -10,7 +10,7 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-bam` | BAM reading as pysam does it (fetch, `reference_end`, `cigarstring`, `get_blocks`), on htslib linked statically, and the tools that read BAMs: the reads at a variant (`reads`: one CIGAR walk per read, SA pieces, filters per tool), `og cdna check`, `og cdna scan`, `og comphet phase`, `og bam contigs`, the CYP2D6 depth check and the RFC1 motif count | | `crates/og-pedigree` | the pedigree rules of step 38: families files, members, kinship, declared relationships, PARs | | `crates/og-qc` | `og qc sex-loss`: step 33's mosaic loss of Y or X, with CPython's medians, `fmean` and exact `variance` | -| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC), `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar) and `--only c` (the remaining summary sections), and without `--only` the whole summary.json with its post pass (the readings that join sections, the CYP2D6 block, `not_run`, `not_assessed`, `recheck`, `new_since`), which steps 24 and HTML_REPORT and `generate-report.sh` write through `bin/render_report.py`; `og roh summary` (step 11) | +| `crates/og-summary` | the step tables: `og qc sample` (step 33), `og prs table` (step 25), `og mito annotate`, `og mito deletions`, `og mito copies` (step 20); the sections of summary.json, `og summary collect --only a` (QC), `--only b` (ClinVar, carrier list, PGx, CPSR, the clinical filter with ACMG SF and two hits, slivar) and `--only c` (the remaining summary sections), and without `--only` the whole summary.json with its post pass (the readings that join sections, the CYP2D6 block, `not_run`, `not_assessed`, `recheck`, `new_since`), which steps 24 and HTML_REPORT and `generate-report.sh` write through `og report render`; `og roh summary` (step 11) | | `crates/og-chip` | step 49, clonal haematopoiesis: `og chip prepare`, `og chip report`, `og chip check-table` | | `crates/og-cn` | copy number from depth, steps 32, 35, 41 and 46: `og cn bed`, `og cn call`, `og cn intervals`, `og cn check-bam`, `og cn marker-regions`, `og cn marker-read` | | `crates/og-bloodtype` | step 41, blood groups: `og bloodtype report`, `og bloodtype check-markers`, `og bloodtype systems` | @@ -22,8 +22,8 @@ OwnGenome's own tools, ported from `bin/*.py` one at a time into one static bina | `crates/og-steps` | step helpers, among them `og pgx outside-calls`, `og pgx hla-concordance` (steps 08, 36), `og pgx pypgx-summary` (step 32) and `og paraphase check-bam`, `og paraphase table` (step 35b) | | `crates/og-fresh` | `og dev freshness`: the upstream lookups of the monthly freshness issue, each answer typed, HTTP through the host's `curl` | | `crates/og-recheck` | `og recheck`: which of a finished sample's results a newer ClinVar or catalog would change, `--rerun`, `--affected`, `--launch-dir` | -| `crates/og-render` | `og report render`, `og report index`, `og report charts`: the reports from summary.json (render: the frame, QC and variants so far, no caller yet; charts: ported, no caller yet; index not ported yet) | -| `crates/og-live` | `og live serve`, `og live results`, `og live family`, `og live demo`, `og view serve`: the live site (`og live family` and `og view serve` ported; serve, results and demo not ported yet) | +| `crates/og-render` | `og report render`, `og report index`, `og report charts`: the reports from summary.json (render: both reports, every section, called by steps 24, HTML_REPORT and `generate-report.sh`; charts and index: ported, and the steps get the charts from `og report render --charts-json`) | +| `crates/og-live` | `og live serve`, `og live results`, `og live family`, `og live demo`, `og view serve`: the live site (`og live family`, `og view serve` and `og live serve` ported (`og live serve` without login or results); results and demo not ported yet) | | `crates/og-couple` | `og couple screen`, `og couple genes`: step 39's couple screen and its carrier list | | `crates/og-admix` | what the fine-ancestry engines share: the panel's haplotypes, the genetic map, `model/ref.geno`, a seeded RNG and the admixed-genome simulator of `og ancestry-fine simulate`; our own painting and projection (not ported yet) | | `og` | the binary: `og [args]`, `og version`, `og --tree-hash` | diff --git a/rust/crates/og-ancestry/src/sample_bed.rs b/rust/crates/og-ancestry/src/sample_bed.rs index f59495ed2..e2280ef2a 100644 --- a/rust/crates/og-ancestry/src/sample_bed.rs +++ b/rust/crates/og-ancestry/src/sample_bed.rs @@ -143,13 +143,10 @@ fn gt_alleles(gt: &str) -> Option<[&str; 2]> { /// A1, 2 heterozygous, 3 homozygous A2; `None` for no call (an index that /// is no allele, or an allele the `.bim` does not have). pub fn bed_code(alleles: &[&str], idx: [&str; 2], a1: &str, a2: &str) -> Option { - let n = alleles.len() as i64; let (mut n1, mut n2) = (0, 0); for i in idx { let i = crate::int(i)?; - // A negative index counts from the end, as Python's list indexing. - let at = if i < 0 { n + i } else { i }; - let called = *alleles.get(usize::try_from(at).ok()?)?; + let called = *alleles.get(usize::try_from(i).ok()?)?; n1 += usize::from(called == a1); n2 += usize::from(called == a2); } @@ -392,8 +389,9 @@ mod tests { } #[test] - fn python_list_indexing_of_alleles() { - assert_eq!(bed_code(&["A", "G"], ["0", "-1"], "G", "A"), Some(2)); + fn genotype_indexes_must_name_alleles() { + assert_eq!(bed_code(&["A", "G"], ["0", "1"], "G", "A"), Some(2)); + assert_eq!(bed_code(&["A", "G"], ["0", "-1"], "G", "A"), None); assert_eq!(bed_code(&["A"], ["0", "2"], "G", "A"), None); assert_eq!(bed_code(&["A", "G"], ["x", "1"], "G", "A"), None); assert_eq!(gt_alleles("0/1/1"), None); diff --git a/rust/crates/og-browser/tests/rv_live_load_errors.rs b/rust/crates/og-browser/tests/rv_live_load_errors.rs new file mode 100644 index 000000000..6f87e5424 --- /dev/null +++ b/rust/crates/og-browser/tests/rv_live_load_errors.rs @@ -0,0 +1,580 @@ +//! A live page that cannot load its data says so, in a headless Chrome. A small demo site built from +//! assets/live (the home page, the adults DEMO-001 and DEMO-002, the child DEMO-003 and the family DEMO-FAMILY) +//! is served by a stub +//! that can answer one path with an error, with no answer at all, with a given body or after a delay, with +//! the live site's Content-Security-Policy. +//! +//! - the person page whose results.json answers 500 says the results could not be loaded, with Try again; +//! once the stub recovers, Try again draws them. An OK answer draws as before. An answer whose chapters +//! are not a list says it could not load them too. +//! - the home page whose runs.json answers 500, or no answer, says the list of people could not be loaded, +//! with Try again, and keeps the server's list of runs; Try again draws the page. +//! - the family page whose family.json answers `{}` says the family could not be loaded; Try again draws it. +//! - DEMO-001 with frames.json and results.json at 500: the closed page says "Results not loaded" and never +//! says child. So does DEMO-001 whose frames.json holds a state with no plugins. The control: DEMO-003, a +//! real child with good files, still reads "Results about a child". +//! - Each Try again check needs the click to find the button and the page to draw within 5 s, so main's own +//! 30 s retry cannot pass it. +//! - frames.json held 15 s: the wire, run health and re-check panels say they wait for the run's data, +//! never that there is nothing; once the frames arrive the replay shows its steps and the dated re-check. +//! DEMO-002, whose state has no re-check and no numbers, then reads "No re-check yet" and "No numbers yet.". +//! +//! Each case was red on main before the fix (the PR says where); the DEMO-002 check is red when the re-check +//! panel keeps its waiting line after a state with no re-check. Without Chrome the test says SKIP, and +//! fails in CI. + +use og_browser::{Browser, Scratch, Value, chrome_or_skip, js_str, repo_root}; +use pycompat::json::{self, Dump}; +use std::collections::HashMap; +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::net::{Shutdown, TcpListener, TcpStream}; +use std::path::{Component, Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +/// The live site's policy (`CSP` of bin/owngenome_live.py; live_replay.rs keeps its copy equal to it). +const CSP: &str = "default-src 'none'; script-src 'self'; connect-src 'self'; style-src 'self'; \ + img-src 'self' data:; form-action 'self'; base-uri 'none'; frame-src 'self'; frame-ancestors 'none'"; +const UNREAD: &str = "This page could not read this run's state, so it keeps its results closed."; +const WAITING: &str = "Waiting for the run's data."; +const NO_RECHECK: &str = "No re-check yet: bin/og recheck writes one (docs/recheck.md)."; +const BANNER: &str = "Database re-check of 2026-01-02T11:05:40Z: ClinVar 2026-01 is newer than the one this run used."; + +/// How the stub answers one path. +#[derive(Clone)] +enum Answer { + /// The file under the site's folder (404 when there is none). + File, + /// This status, with a plain-text body. + Status(u16), + /// No answer: the connection closes, so fetch rejects (the browser is offline for that file). + Abort, + /// 200 with this JSON body. + Json(&'static str), + /// The file, after this delay. + Hold(Duration), +} + +/// A site on 127.0.0.1 whose answer per path the test sets with [`Stub::set`]. Threads still holding an +/// answer when it is dropped end on their own. +struct Stub { + port: u16, + rules: Arc>>, + stop: Arc, +} + +impl Stub { + fn start(dir: &Path) -> Stub { + let root = Arc::new(dir.canonicalize().unwrap()); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let rules: Arc>> = Arc::default(); + let stop = Arc::new(AtomicBool::new(false)); + let (r, s) = (Arc::clone(&rules), Arc::clone(&stop)); + thread::spawn(move || { + for conn in listener.incoming() { + if s.load(Ordering::SeqCst) { + break; + } + if let Ok(c) = conn { + let (root, rules) = (Arc::clone(&root), Arc::clone(&r)); + thread::spawn(move || answer(c, &root, &rules)); + } + } + }); + Stub { port, rules, stop } + } + + /// Answer `path` (from the root, like "/runs.json") this way from now on. + fn set(&self, path: &str, a: Answer) { + self.rules.lock().unwrap().insert(path.to_string(), a); + } + + fn url(&self, path: &str) -> String { + format!("http://127.0.0.1:{}{path}", self.port) + } +} + +impl Drop for Stub { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + let _ = TcpStream::connect(("127.0.0.1", self.port)); + } +} + +fn answer(mut c: TcpStream, root: &Path, rules: &Mutex>) { + let _ = c.set_read_timeout(Some(Duration::from_secs(10))); + let mut r = BufReader::new(&c); + let mut first = String::new(); + if r.read_line(&mut first).is_err() { + return; + } + let mut line = String::new(); + while r.read_line(&mut line).is_ok_and(|n| n > 2) { + line.clear(); + } + let target = first.split_whitespace().nth(1).unwrap_or("/"); + let path = target.split(['?', '#']).next().unwrap_or("/").to_string(); + let rule = rules + .lock() + .unwrap() + .get(&path) + .cloned() + .unwrap_or(Answer::File); + let file = |p: &str| -> Option<(Vec, &'static str)> { + let mut rel = p.trim_start_matches('/').to_string(); + if rel.is_empty() || rel.ends_with('/') { + rel.push_str("index.html"); + } + let rel = PathBuf::from(rel); + if !rel.components().all(|c| matches!(c, Component::Normal(_))) { + return None; + } + let ctype = match rel.extension().and_then(|e| e.to_str()) { + Some("html") => "text/html; charset=utf-8", + Some("js") => "text/javascript; charset=utf-8", + Some("css") => "text/css; charset=utf-8", + Some("json") => "application/json", + _ => "application/octet-stream", + }; + fs::read(root.join(rel)).ok().map(|b| (b, ctype)) + }; + let (status, body, ctype) = match rule { + Answer::Abort => { + let _ = c.shutdown(Shutdown::Both); + return; + } + Answer::Status(n) => (format!("{n} Stub"), b"stub error\n".to_vec(), "text/plain"), + Answer::Json(j) => ("200 OK".into(), j.as_bytes().to_vec(), "application/json"), + Answer::Hold(d) => { + thread::sleep(d); + served(file(&path)) + } + Answer::File => served(file(&path)), + }; + let head = format!( + "HTTP/1.1 {status}\r\nContent-Type: {ctype}\r\nContent-Length: {}\r\nCache-Control: no-store\r\n\ + Content-Security-Policy: {CSP}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = c + .write_all(head.as_bytes()) + .and_then(|()| c.write_all(&body)); + let _ = c.shutdown(Shutdown::Write); +} + +fn served(f: Option<(Vec, &'static str)>) -> (String, Vec, &'static str) { + match f { + Some((b, t)) => ("200 OK".into(), b, t), + None => ( + "404 Not Found".into(), + b"not found\n".to_vec(), + "text/plain", + ), + } +} + +/// The frames of tests/fixtures/live/frames.json with `plugins` set on every frame's state. +fn frames_with(plugins: &str) -> String { + let text = fs::read_to_string(repo_root().join("tests/fixtures/live/frames.json")).unwrap(); + let mut fr = json::loads(&text).unwrap(); + let p = json::loads(plugins).unwrap(); + if let Value::Object(m) = &mut fr + && let Some(Value::Array(a)) = m.get_mut("frames") + { + for f in a.iter_mut() { + if let Value::Object(f) = f + && let Some(Value::Object(s)) = f.get_mut("state") + { + s.insert("plugins".into(), p.clone()); + } + } + } + json::dumps(&fr, &Dump::default()).unwrap() +} + +/// A person page of the demo, as tests/demo/build_live_demo.py run_page fills it, with the ops and +/// re-check plugins. +fn run_page(dir: &Path, name: &str, template: &str, frames: &str, results: Option<&str>) { + let tags = ["ops.js", "recheck.js"] + .map(|f| format!("")) + .join("\n"); + let page = fs::read_to_string(repo_root().join("assets/live").join(template)) + .unwrap() + .replace("{{assets}}", "../../assets") + .replace("{{run}}", name) + .replace("{{mode}}", "replay") + .replace( + "{{plugins}}", + if template == "index.html" { &tags } else { "" }, + ) + .replace("href=\"/\"", "href=\"../../\""); + let d = dir.join("runs").join(name); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("index.html"), page).unwrap(); + fs::write(d.join("frames.json"), frames).unwrap(); + if let Some(r) = results { + fs::write(d.join("results.json"), r).unwrap(); + } +} + +/// The demo site: assets/, the home page with runs.json, DEMO-001 (an adult), DEMO-003 (a child with a +/// result to act on) and the family page of DEMO-FAMILY. +fn demo_site(dir: &Path) { + let assets = repo_root().join("assets/live"); + fs::create_dir_all(dir.join("assets/plugins")).unwrap(); + for f in [ + "core.js", + "live.js", + "map.js", + "person.js", + "home.js", + "family-page.js", + "live.css", + "map.css", + ] { + fs::copy(assets.join(f), dir.join("assets").join(f)).unwrap(); + } + for f in ["ops.js", "recheck.js", "family.js"] { + fs::copy( + assets.join("plugins").join(f), + dir.join("assets/plugins").join(f), + ) + .unwrap(); + } + let home = fs::read_to_string(assets.join("home.html")) + .unwrap() + .replace("{{assets}}", "assets") + .replace("{{data}}", "runs.json") + .replace( + "{{runs}}", + "
  • DEMO-001
  • ", + ); + fs::write(dir.join("index.html"), home).unwrap(); + fs::write( + dir.join("runs.json"), + r#"{"runs": [{"name": "DEMO-001", "kind": "person", "url": "runs/DEMO-001/", "state": "finished", + "started_utc": "2026-01-02T09:00:00Z", "finished_utc": "2026-01-02T10:05:40Z", "planned": 2, "done": 2, + "running": 0, "failed": 0, "top": []}], "families": []}"#, + ) + .unwrap(); + let recheck = format!( + r#"{{"recheck": {{"file": "recheck/state.json", "mtime": 1, "checked_utc": "2026-01-02T11:05:40Z", + "banner": {}, "steps": {{"03": ["newer ClinVar available"]}}}}}}"#, + js_str(BANNER) + ); + let results = r#"{"schema_version": 1, "sample": "DEMO-001", "person": {"sample": "DEMO-001"}, "chapters": [], + "findings": [], "levels": {}, "note": null, "lead": [], + "journal": [{"n": 8, "key": "assessed", "title": "Assessed, no finding", "question": "What did the steps look at?", + "cards": [], "findings": []}]}"#; + run_page( + dir, + "DEMO-001", + "index.html", + &frames_with(&recheck), + Some(results), + ); + run_page( + dir, + "DEMO-002", + "index.html", + &frames_with("{}"), + Some(results), + ); + let child = r#"{"family": {"kind": "member", "child": true, "families": [{"name": "DEMO-FAMILY", "url": "../DEMO-FAMILY/"}]}}"#; + let acts = r#"{"schema_version": 1, "sample": "DEMO-003", "chapters": [], "note": null, + "findings": [{"id": "act-1", "level": "act", "card": "clinical", "title": "A result to act on"}]}"#; + run_page( + dir, + "DEMO-003", + "index.html", + &frames_with(child), + Some(acts), + ); + run_page(dir, "DEMO-FAMILY", "family.html", &frames_with("{}"), None); + fs::write( + dir.join("runs/DEMO-FAMILY/family.json"), + r#"{"state": {"kind": "family", "name": "DEMO-FAMILY", "members": [], "children": [], "couples": [], + "plan": [], "reports": []}, "runs": {}}"#, + ) + .unwrap(); +} + +/// Open `url` from a blank page and wait until it is the document shown, so no check reads the page before it +/// (`Browser::go` returns once the navigation starts). +fn visit(b: &mut Browser, url: &str) { + b.go("about:blank").unwrap(); + assert!( + b.wait("location.href === 'about:blank'", Duration::from_secs(10)) + .is_some(), + "about:blank never opened" + ); + b.go(url).unwrap(); + let at = format!("location.href === {}", js_str(url)); + assert!( + b.wait(&at, Duration::from_secs(20)).is_some(), + "{url} never opened" + ); +} + +fn text(v: Option) -> String { + match v { + Some(Value::Str(s)) => s, + Some(v) => format!("{v:?}"), + None => "None".into(), + } +} + +/// Run a script that clicks a Try again button and returns whether it found one (`!!t`). +fn click(b: &mut Browser, script: &str) -> bool { + matches!(b.eval(script), Ok(Value::Bool(true))) +} + +#[test] +fn a_page_that_cannot_load_its_data_says_so() { + let Some(chrome) = chrome_or_skip("load the live demo pages") else { + return; + }; + let scratch = Scratch::new(Path::new(env!("CARGO_TARGET_TMPDIR")), "load-errors"); + let site = scratch.path().join("site"); + demo_site(&site); + let stub = Stub::start(&site); + let mut b = Browser::launch(&chrome, &scratch.path().join("chrome")).unwrap(); + let mut fails = Vec::new(); + let mut check = |desc: &str, ok: bool, detail: String| { + println!( + "[{}] {desc}{}", + if ok { "PASS" } else { "FAIL" }, + if ok { + String::new() + } else { + format!(" -- {detail}") + } + ); + if !ok { + fails.push(desc.to_string()); + } + }; + let secs = Duration::from_secs; + let failed_box = |id: &str| { + format!( + "(function () {{ var e = document.getElementById({}); return e ? e.innerText.replace(/\\s+/g, ' ').trim() : ''; }}())", + js_str(id) + ) + }; + + // The person page: results.json at 500, then an OK answer through Try again. + stub.set("/runs/DEMO-001/results.json", Answer::Status(500)); + visit(&mut b, &stub.url("/runs/DEMO-001/")); + let said = text(b.wait(&failed_box("results-failed"), secs(20))); + check( + "person: results.json at 500 says 'The results could not be loaded.' with Try again", + said == "The results could not be loaded. Try again", + said.clone(), + ); + stub.set("/runs/DEMO-001/results.json", Answer::File); + let clicked = click( + &mut b, + "var t = document.getElementById('results-retry'); if (t) { t.click(); } !!t", + ); + let drawn = text(b.wait( + "!document.getElementById('results-failed') && document.querySelector('#results section.sec h2') && \ + document.querySelector('#results section.sec h2').textContent", + secs(5), + )); + check( + "person: once results.json answers, Try again draws the results within 5 s", + clicked && drawn == "Assessed, no finding", + format!("clicked={clicked} {drawn}"), + ); + // An OK answer from the start draws as before. + visit(&mut b, &stub.url("/runs/DEMO-001/?ok")); + let ok = text(b.wait( + "document.querySelector('#results section.sec h2') && [document.querySelector('#results section.sec h2').textContent, \ + !!document.getElementById('results-failed')].join('|')", + secs(20), + )); + check( + "person: an OK results.json draws as before, with no failure line", + ok == "Assessed, no finding|false", + ok, + ); + // An answer whose chapters are not a list is not results: the page says so instead of a broken report. + stub.set( + "/runs/DEMO-001/results.json", + Answer::Json(r#"{"chapters": {}}"#), + ); + visit(&mut b, &stub.url("/runs/DEMO-001/?shape")); + let said = text(b.wait(&failed_box("results-failed"), secs(20))); + check( + "person: results.json whose chapters are not a list says 'The results could not be loaded.' with Try again", + said == "The results could not be loaded. Try again", + said, + ); + stub.set("/runs/DEMO-001/results.json", Answer::File); + + // The home page: runs.json at 500, then no answer at all (offline), then Try again once it answers. + for (how, a) in [ + ("at 500", Answer::Status(500)), + ("aborted (offline)", Answer::Abort), + ] { + stub.set("/runs.json", a); + visit(&mut b, &stub.url("/")); + let said = text(b.wait(&failed_box("home-failed"), secs(20))); + let kept = text( + b.eval("String(document.querySelectorAll('#home ul.runs li a').length)") + .ok(), + ); + check( + &format!( + "home: runs.json {how} says 'The list of people could not be loaded.' with Try again, above the server's list" + ), + said == "The list of people could not be loaded. Try again" && kept == "1", + format!("{said:?} links {kept}"), + ); + } + stub.set("/runs.json", Answer::File); + let clicked = click( + &mut b, + "var t = document.querySelector('#home-failed button'); if (t) { t.click(); } !!t", + ); + let people = text(b.wait( + "!document.getElementById('home-failed') && document.documentElement.dataset.pgpStatus === 'ready' && \ + document.documentElement.dataset.pgpPeople", + secs(5), + )); + check( + "home: once runs.json answers, Try again draws the people within 5 s", + clicked && people == "1", + format!("clicked={clicked} {people}"), + ); + + // The family page: family.json answers 200 {}, then the family through Try again. + stub.set("/runs/DEMO-FAMILY/family.json", Answer::Json("{}")); + visit(&mut b, &stub.url("/runs/DEMO-FAMILY/")); + let said = text(b.wait( + "(function () { var m = document.getElementById('family'); var t = m.innerText.replace(/\\s+/g, ' ').trim(); \ + return m.querySelector('button') ? t : ''; }())", + secs(20), + )); + check( + "family: family.json answering {} says 'The family could not be loaded.' with Try again", + said == "The family could not be loaded. Try again", + said, + ); + stub.set("/runs/DEMO-FAMILY/family.json", Answer::File); + let clicked = click( + &mut b, + "var t = document.querySelector('#family button'); if (t) { t.click(); } !!t", + ); + let h1 = text(b.wait( + "document.querySelector('#family h1') && document.querySelector('#family h1').textContent", + secs(5), + )); + check( + "family: once family.json answers, Try again draws the family within 5 s", + clicked && h1 == "Family DEMO-FAMILY", + format!("clicked={clicked} {h1}"), + ); + + // A run whose state and results cannot be read: the gate stays closed and names no role. + stub.set("/runs/DEMO-001/frames.json", Answer::Status(500)); + stub.set("/runs/DEMO-001/results.json", Answer::Status(500)); + visit(&mut b, &stub.url("/runs/DEMO-001/?unread")); + let page = text(b.wait( + "document.getElementById('report-gate') && JSON.stringify([document.querySelector('#report-gate h1').textContent, \ + document.querySelector('#report-gate .lede').textContent, /child/i.test(document.body.innerText), \ + document.querySelectorAll('[data-finding], article.result, #results').length])", + secs(20), + )); + let want = format!("[\"Results not loaded\",{},false,0]", js_str(UNREAD)); + check( + "gate: DEMO-001 with frames.json and results.json at 500 reads 'Results not loaded', says no 'child' and draws no result", + page == want, + page, + ); + // A state that parses but has no plugins is read no better: the same closed page, with no role named. + stub.set( + "/runs/DEMO-001/frames.json", + Answer::Json(r#"{"frames": [{"state": {}}]}"#), + ); + visit(&mut b, &stub.url("/runs/DEMO-001/?noplugins")); + let page = text(b.wait( + "document.getElementById('report-gate') && JSON.stringify([document.querySelector('#report-gate h1').textContent, \ + document.querySelector('#report-gate .lede').textContent, /child/i.test(document.body.innerText), \ + document.querySelectorAll('[data-finding], article.result, #results').length])", + secs(20), + )); + check( + "gate: DEMO-001 whose state has no plugins, with results.json at 500, reads 'Results not loaded' and says no 'child'", + page == want, + page, + ); + stub.set("/runs/DEMO-001/frames.json", Answer::File); + stub.set("/runs/DEMO-001/results.json", Answer::File); + visit(&mut b, &stub.url("/runs/DEMO-003/")); + let child = text(b.wait( + "document.getElementById('report-gate') && document.querySelector('#report-gate h1').textContent", + secs(20), + )); + check( + "control: DEMO-003, a real child with good files, still reads 'Results about a child'", + child == "Results about a child", + child, + ); + + // frames.json held 15 s: the panels wait, then the finished replay draws its steps and the re-check. + stub.set("/runs/DEMO-001/frames.json", Answer::Hold(secs(15))); + let t0 = Instant::now(); + visit(&mut b, &stub.url("/runs/DEMO-001/?held")); + let panels = "JSON.stringify([document.getElementById('found').innerText.trim(), \ + (document.querySelector('[data-plugin=\"ops\"] p') || {}).textContent || '', \ + (document.querySelector('[data-plugin=\"recheck\"] p') || {}).textContent || ''])"; + let waiting = text(b.wait( + &format!("document.querySelector('[data-plugin=\"recheck\"] p') && document.querySelector('[data-plugin=\"ops\"] p') && {panels}"), + secs(10), + )); + let absent = text( + b.eval( + "['Nothing finished yet.', 'No numbers yet.', 'No re-check yet'].filter(function (s) { \ + return document.body.innerText.indexOf(s) >= 0; }).join('|')", + ) + .ok(), + ); + let held = t0.elapsed() < secs(14); + let want = format!("[{w},{w},{w}]", w = js_str(WAITING)); + check( + "held: before frames.json answers, the wire, run health and re-check panels read 'Waiting for the run's data.'", + held && waiting == want && absent.is_empty(), + format!("{waiting} absent={absent:?} in time={held}"), + ); + let arrived = text(b.wait( + "document.documentElement.dataset.pgpFrame && JSON.stringify([document.querySelectorAll('#map g.box').length, \ + (document.querySelector('[data-plugin=\"recheck\"] p') || {}).textContent])", + secs(30), + )); + check( + "held: once the frames arrive, the finished replay shows its steps and the dated re-check", + arrived == format!("[2,{}]", js_str(BANNER)), + arrived, + ); + + // An adult whose first state has no re-check and no run numbers: the panels then say so. + visit(&mut b, &stub.url("/runs/DEMO-002/")); + let none = text(b.wait( + "document.documentElement.dataset.pgpFrame && JSON.stringify([(document.querySelector('[data-plugin=\"ops\"] p') || {}).textContent, \ + (document.querySelector('[data-plugin=\"recheck\"] p') || {}).textContent])", + secs(20), + )); + check( + "open: a first state with no re-check and no numbers reads 'No numbers yet.' and 'No re-check yet' instead of waiting", + none == format!("[\"No numbers yet.\",{}]", js_str(NO_RECHECK)), + none, + ); + + drop((b, stub)); + assert!(fails.is_empty(), "failed: {fails:?}"); +} diff --git a/rust/crates/og-browser/tests/rv_viewer_shell.rs b/rust/crates/og-browser/tests/rv_viewer_shell.rs new file mode 100644 index 000000000..373deafe9 --- /dev/null +++ b/rust/crates/og-browser/tests/rv_viewer_shell.rs @@ -0,0 +1,657 @@ +//! The genome viewer's page (assets/viewer: index.html and viewer.js) in a headless Chrome, on a synthetic +//! sample: a random 4 kb reference with its .fai and a VCF with one variant. The page, its script, the pinned +//! igv.js release, config.json and the sample's files are served by a stub that answers the paths `og view +//! serve` answers, with that server's Content-Security-Policy, and can make config.json answer an error. +//! og-browser does not depend on og-live, so the stub keeps a copy of the policy, and +//! `the_policy_is_og_view_serves` keeps the two equal. +//! +//! - At 1440x900 and at 390x844 the page is dark: the root's colour scheme is dark and the body has an opaque +//! dark background (main's page had none, so the browser's white showed under the header). +//! - The igv.js search input and variant track label have readable text on their backgrounds. +//! - At the live map's short iframe height, the variant track fits without vertical scrolling. +//! - On the 390 px phone, with igv.js started, the page is no wider than the screen: igv.js's own bar +//! scrolls inside its panel (on main it widened the page past 390 px). +//! - The page has one h1, which names the sample, one main landmark holding the error box and igv.js, and a +//! line that says search takes coordinates, not gene names. +//! - When config.json answers 500, #error has role alert, names the failure, says what to check and offers +//! Try again; once the server answers again, Try again starts the viewer. +//! +//! The PR records the failing baseline for each check. Without Chrome the test says SKIP, and fails +//! in CI. Set `OG_BROWSER_SHOTS` to a folder to save a screenshot of each view there. + +use og_browser::{Browser, Scratch, Value, chrome_or_skip, js_str, repo_root}; +use pycompat::json; +use std::collections::HashMap; +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::net::{Shutdown, TcpListener, TcpStream}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, OnceLock}; +use std::thread; +use std::time::Duration; + +/// `CSP` of rust/crates/og-live/src/viewer.rs, the policy `og view serve` sends. +const CSP: &str = "default-src 'none'; script-src 'self'; connect-src 'self'; img-src 'self' data: blob:; \ + style-src 'self' 'unsafe-inline'; font-src 'self' data:; worker-src 'self' blob:; \ + base-uri 'none'; form-action 'none'; frame-ancestors 'none'"; +const PHONE: (u32, u32) = (390, 844); +const DESKTOP: (u32, u32) = (1440, 900); +const STATUS: &str = "document.documentElement.dataset.pgpStatus || ''"; +const SETTLED: &str = "(function(){var s=document.documentElement.dataset.pgpStatus||'';return s==='ready'||s.indexOf('error')===0;})()"; + +/// How the stub answers config.json. +#[derive(Clone)] +enum Config { + Json(String), + Status(u16), +} + +/// The paths `og view serve` answers, on a free port of 127.0.0.1: `/` and `/index.html`, `/viewer.js`, +/// `/igv.min.js`, `/config.json` and `/data//` (byte ranges too). +struct Stub { + port: u16, + config: Arc>, + stop: Arc, +} + +impl Stub { + fn start(files: HashMap, config: Config) -> Stub { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let config = Arc::new(Mutex::new(config)); + let stop = Arc::new(AtomicBool::new(false)); + let (files, c, s) = (Arc::new(files), Arc::clone(&config), Arc::clone(&stop)); + thread::spawn(move || { + for conn in listener.incoming() { + if s.load(Ordering::SeqCst) { + break; + } + if let Ok(conn) = conn { + let (files, c) = (Arc::clone(&files), Arc::clone(&c)); + thread::spawn(move || answer(conn, &files, &c)); + } + } + }); + Stub { port, config, stop } + } + + fn set_config(&self, c: Config) { + *self.config.lock().unwrap() = c; + } + + fn url(&self, path: &str) -> String { + format!("http://127.0.0.1:{}{path}", self.port) + } +} + +impl Drop for Stub { + fn drop(&mut self) { + self.stop.store(true, Ordering::SeqCst); + let _ = TcpStream::connect(("127.0.0.1", self.port)); + } +} + +fn answer(mut c: TcpStream, files: &HashMap, config: &Mutex) { + let _ = c.set_read_timeout(Some(Duration::from_secs(10))); + let mut r = BufReader::new(&c); + let mut first = String::new(); + if r.read_line(&mut first).is_err() { + return; + } + let (mut line, mut range) = (String::new(), None); + while r.read_line(&mut line).is_ok_and(|n| n > 2) { + if let Some((k, v)) = line.split_once(':') + && k.eq_ignore_ascii_case("range") + { + range = Some(v.trim().to_owned()); + } + line.clear(); + } + let target = first.split_whitespace().nth(1).unwrap_or("/"); + let path = target.split(['?', '#']).next().unwrap_or("/"); + let ctype = |p: &Path| match p.extension().and_then(|e| e.to_str()) { + Some("html") => "text/html; charset=utf-8", + Some("js") => "text/javascript; charset=utf-8", + _ => "application/octet-stream", + }; + let (status, ctype, body, extra) = if path == "/config.json" { + match config.lock().unwrap().clone() { + Config::Json(j) => ( + "200 OK".to_owned(), + "application/json", + j.into_bytes(), + String::new(), + ), + Config::Status(n) => ( + format!("{n} Stub"), + "text/plain", + b"stub error\n".to_vec(), + String::new(), + ), + } + } else if let Some(body) = files.get(path).and_then(|f| fs::read(f).ok()) { + let size = body.len(); + match range.as_deref().and_then(|r| byte_range(r, size)) { + Some((a, b)) => ( + "206 Partial Content".to_owned(), + ctype(&files[path]), + body[a..=b].to_vec(), + format!("Content-Range: bytes {a}-{b}/{size}\r\n"), + ), + None => ( + "200 OK".to_owned(), + ctype(&files[path]), + body, + String::new(), + ), + } + } else { + ( + "404 Not Found".to_owned(), + "text/plain", + b"not found\n".to_vec(), + String::new(), + ) + }; + let head = format!( + "HTTP/1.1 {status}\r\nContent-Type: {ctype}\r\nContent-Length: {}\r\nCache-Control: no-store\r\n\ + Accept-Ranges: bytes\r\nContent-Security-Policy: {CSP}\r\n{extra}Connection: close\r\n\r\n", + body.len() + ); + let _ = c.write_all(head.as_bytes()); + let _ = c.write_all(&body); + let _ = c.shutdown(Shutdown::Both); +} + +/// The first and last byte of `bytes=a-b` (or `a-`) within `size`, else `None` (the whole file). +fn byte_range(header: &str, size: usize) -> Option<(usize, usize)> { + let (a, b) = header.strip_prefix("bytes=")?.split_once('-')?; + let a: usize = a.trim().parse().ok()?; + let b = match b.trim() { + "" => size.checked_sub(1)?, + b => b.parse::().ok()?.min(size.checked_sub(1)?), + }; + (a <= b).then_some((a, b)) +} + +/// A random sequence of `n` bases from `seed`. +fn sequence(seed: &mut u64, n: usize) -> String { + (0..n) + .map(|_| { + *seed = seed + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + b"ACGT"[(*seed >> 62) as usize] as char + }) + .collect() +} + +/// The sample in `dir` (reference/synthetic.fasta with its .fai, S.vcf) and the stub's file routes. +fn make_sample(dir: &Path) -> HashMap { + let mut seed = 7; + let contigs = [ + ("chr1", sequence(&mut seed, 3000)), + ("chr2", sequence(&mut seed, 1000)), + ]; + let (mut fasta, mut fai, mut off) = (String::new(), String::new(), 0); + for (n, s) in &contigs { + off += n.len() + 2; + fai.push_str(&format!("{n}\t{}\t{off}\t60\t61\n", s.len())); + let lines: Vec<&str> = s + .as_bytes() + .chunks(60) + .map(|c| std::str::from_utf8(c).unwrap()) + .collect(); + fasta.push_str(&format!(">{n}\n{}\n", lines.join("\n"))); + off += s.len() + lines.len(); + } + let r = &contigs[0].1[450..451]; + let alt = "ACGT".replace(r, "")[..1].to_owned(); + let vcf = format!( + "##fileformat=VCFv4.2\n##contig=\n##contig=\n\ + ##FORMAT=\n\ + #CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\nchr1\t451\t.\t{r}\t{alt}\t50\tPASS\t.\tGT\t0/1\n" + ); + fs::write(dir.join("synthetic.fasta"), fasta).unwrap(); + fs::write(dir.join("synthetic.fasta.fai"), fai).unwrap(); + fs::write(dir.join("S.vcf"), vcf).unwrap(); + let assets = repo_root().join("assets/viewer"); + HashMap::from([ + ("/".to_owned(), assets.join("index.html")), + ("/index.html".to_owned(), assets.join("index.html")), + ("/viewer.js".to_owned(), assets.join("viewer.js")), + ("/igv.min.js".to_owned(), pinned_igv().clone()), + ( + "/data/0/synthetic.fasta".to_owned(), + dir.join("synthetic.fasta"), + ), + ( + "/data/1/synthetic.fasta.fai".to_owned(), + dir.join("synthetic.fasta.fai"), + ), + ("/data/2/S.vcf".to_owned(), dir.join("S.vcf")), + ]) +} + +/// config.json as `og view serve` writes it for this sample (a VCF without an index is read whole). +const CONFIG: &str = r#"{"sample": "S", "locus": "chr1:451", + "reference": {"id": "reference", "name": "synthetic.fasta", "fastaURL": "data/0/synthetic.fasta", + "indexURL": "data/1/synthetic.fasta.fai"}, + "tracks": [{"name": "S small variants", "type": "variant", "format": "vcf", "url": "data/2/S.vcf", + "indexed": false}]}"#; + +/// The igv.js release versions.env pins, downloaded once per test run into the target's tmp folder and +/// checked by sha256. +fn pinned_igv() -> &'static PathBuf { + static IGV: OnceLock = OnceLock::new(); + IGV.get_or_init(|| { + let env = fs::read_to_string(repo_root().join("versions.env")).unwrap(); + let pin = |k: &str| { + env.lines() + .find_map(|l| l.strip_prefix(&format!("{k}=\""))) + .and_then(|v| v.strip_suffix('"')) + .unwrap_or_else(|| panic!("versions.env sets no {k}")) + .to_owned() + }; + let (version, sha) = (pin("IGV_JS_VERSION"), pin("IGV_JS_SHA256")); + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")); + let path = dir.join(format!("igv-{version}.min.js")); + let sum = |p: &Path| { + let out = Command::new("shasum") + .args(["-a", "256"]) + .arg(p) + .output() + .or_else(|_| Command::new("sha256sum").arg(p).output()) + .expect("shasum or sha256sum"); + String::from_utf8_lossy(&out.stdout) + .split_whitespace() + .next() + .unwrap_or_default() + .to_owned() + }; + if !path.is_file() || sum(&path) != sha { + let part = dir.join(format!("igv-{version}.min.js.{}", std::process::id())); + let url = format!("https://cdn.jsdelivr.net/npm/igv@{version}/dist/igv.min.js"); + let ok = Command::new("curl") + .args(["-fsSL", "--retry", "3", "-o"]) + .arg(&part) + .arg(&url) + .status() + .is_ok_and(|s| s.success()); + assert!(ok, "cannot download {url}"); + let got = sum(&part); + if got != sha { + let _ = fs::remove_file(&part); + panic!("igv.min.js has sha256 {got}, versions.env pins {sha}"); + } + fs::rename(&part, &path).unwrap(); + } + path + }) +} + +/// A Chrome whose page is `w` x `h` CSS pixels; a phone (`mobile`) honours the page's viewport tag. +fn browser(chrome: &Path, profile: &Path, (w, h): (u32, u32), mobile: bool) -> Browser { + let mut b = Browser::launch(chrome, profile).unwrap(); + let params = + format!(r#"{{"width": {w}, "height": {h}, "deviceScaleFactor": 1, "mobile": {mobile}}}"#); + b.call( + "Emulation.setDeviceMetricsOverride", + json::loads(¶ms).unwrap(), + ) + .unwrap(); + b.call( + "Emulation.setEmulatedMedia", + json::loads(r#"{"features":[{"name":"prefers-color-scheme","value":"dark"}]}"#).unwrap(), + ) + .unwrap(); + b +} + +fn text(v: Result) -> String { + match v { + Ok(Value::Str(s)) => s, + Ok(v) => format!("{v:?}"), + Err(e) => format!("error: {e}"), + } +} + +/// The value at `key` of the JSON object the expression `expr` returns as a string. +fn probe(b: &mut Browser, expr: &str) -> Value { + let raw = text(b.eval(expr)); + json::loads(&raw).unwrap_or_else(|e| panic!("{expr}: {raw}: {e:?}")) +} + +fn get<'a>(v: &'a Value, key: &str) -> &'a Value { + match v { + Value::Object(m) => m.get(key).unwrap_or(&Value::Null), + _ => &Value::Null, + } +} + +/// The colour scheme of the root, and whether `sel`'s background is opaque and dark (luminance under 60 of +/// 255), as a JSON string. +fn dark(sel: &str) -> String { + format!( + "(function(){{var c=getComputedStyle(document.querySelector({})).backgroundColor;\ + var m=(c.match(/[0-9.]+/g)||[]).map(Number);var a=m.length>3?m[3]:1;\ + return JSON.stringify({{color:c,scheme:getComputedStyle(document.documentElement).colorScheme,\ + dark:m.length>=3&&a>=0.99&&0.2126*m[0]+0.7152*m[1]+0.0722*m[2]<60}});}})()", + js_str(sel) + ) +} + +/// Saves a screenshot as `/viewer-.png` when that variable names a folder. +fn shot(b: &mut Browser, name: &str) { + let Some(dir) = std::env::var_os("OG_BROWSER_SHOTS").filter(|d| !d.is_empty()) else { + return; + }; + let r = b + .call( + "Page.captureScreenshot", + json::loads(r#"{"format": "png"}"#).unwrap(), + ) + .unwrap(); + let Value::Str(data) = get(&r, "data") else { + panic!("no screenshot data") + }; + fs::write( + Path::new(&dir).join(format!("viewer-{name}.png")), + base64(data), + ) + .unwrap(); +} + +fn base64(text: &str) -> Vec { + let val = |c: u8| match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a' + 26, + b'0'..=b'9' => c - b'0' + 52, + b'+' => 62, + _ => 63, + }; + let bytes: Vec = text.bytes().filter(|c| *c != b'=').map(val).collect(); + bytes + .chunks(4) + .flat_map(|q| { + let n = q + .iter() + .enumerate() + .fold(0u32, |n, (i, v)| n | u32::from(*v) << (18 - 6 * i)); + n.to_be_bytes()[1..q.len()].to_vec() + }) + .collect() +} + +/// Opens the viewer at `size` on a fresh stub and waits until it reports ready or an error. +fn open( + chrome: &Path, + scratch: &Scratch, + name: &str, + size: (u32, u32), + mobile: bool, +) -> (Stub, Browser) { + let site = scratch.path().join(format!("site-{name}")); + fs::create_dir_all(&site).unwrap(); + let stub = Stub::start(make_sample(&site), Config::Json(CONFIG.to_owned())); + let mut b = browser( + chrome, + &scratch.path().join(format!("chrome-{name}")), + size, + mobile, + ); + b.go(&stub.url("/#chr1:400-500")).unwrap(); + b.wait(SETTLED, Duration::from_secs(60)); + (stub, b) +} + +#[test] +fn the_policy_is_og_view_serves() { + let src = fs::read_to_string(repo_root().join("rust/crates/og-live/src/viewer.rs")).unwrap(); + let start = src + .find("pub const CSP: &str = \"") + .expect("og-live's viewer.rs sets CSP") + + 23; + let raw = &src[start..start + src[start..].find("\";").unwrap()]; + // A Rust string continuation: a backslash, the line break and the next line's leading blanks are dropped. + let joined: String = raw + .split("\\\n") + .enumerate() + .map(|(i, p)| if i == 0 { p } else { p.trim_start() }) + .collect(); + assert_eq!(joined, CSP, "the stub must send og view serve's policy"); +} + +#[test] +fn the_page_is_dark_with_one_heading_and_a_main_landmark() { + let Some(chrome) = chrome_or_skip("load the viewer page") else { + return; + }; + let scratch = Scratch::new(Path::new(env!("CARGO_TARGET_TMPDIR")), "rv-viewer-desktop"); + let (_stub, mut b) = open(&chrome, &scratch, "desktop", DESKTOP, false); + assert_eq!( + text(b.eval(STATUS)), + "ready", + "igv.js did not start on the synthetic sample" + ); + assert_eq!( + text(b.eval("document.documentElement.dataset.pgpBlocked || ''")), + "", + "the policy blocked what the page needs" + ); + let d = probe(&mut b, &dark("body")); + assert!( + get(&d, "dark") == &Value::Bool(true) && get(&d, "scheme") == &Value::Str("dark".into()), + "desktop: the page must be dark: {d:?}" + ); + let outline = probe( + &mut b, + "JSON.stringify({h1:document.querySelectorAll('h1').length,main:document.querySelectorAll('main').length,\ + title:(document.querySelector('h1')||{}).textContent||'',\ + inside:!!document.querySelector('main #igv')&&!!document.querySelector('main #error'),\ + about:(document.querySelector('main')||document.body).textContent.replace(/\\s+/g,' ')})", + ); + assert!( + get(&outline, "h1") == &Value::Int(1) && get(&outline, "main") == &Value::Int(1), + "one h1 and one main: {outline:?}" + ); + assert_eq!( + get(&outline, "title"), + &Value::Str("Reads of S".into()), + "the h1 names the sample" + ); + assert_eq!( + get(&outline, "inside"), + &Value::Bool(true), + "main holds the error box and igv.js" + ); + let about = text(Ok(get(&outline, "about").clone())); + assert!( + about.contains("coordinates") + && about.contains("gene names do not work") + && about.contains("a few kb"), + "the page says what search takes and when reads show: {about}" + ); + shot(&mut b, "desktop"); +} + +#[test] +fn igv_search_and_track_label_have_readable_text() { + let Some(chrome) = chrome_or_skip("check the igv.js text contrast") else { + return; + }; + let scratch = Scratch::new(Path::new(env!("CARGO_TARGET_TMPDIR")), "rv-viewer-contrast"); + let (_stub, mut b) = open(&chrome, &scratch, "contrast", DESKTOP, false); + assert_eq!(text(b.eval(STATUS)), "ready"); + + let contrast = probe( + &mut b, + r#"JSON.stringify((() => { + const igv = document.querySelector('#igv').shadowRoot; + const search = igv.querySelector('.igv-search-input'); + const track = Array.from(igv.querySelectorAll('.igv-track-label')) + .find(label => label.textContent === 'S small variants'); + // The pinned igv.js styles both surfaces with opaque RGB backgrounds. + const luminance = rgb => { + const channels = rgb.match(/[\d.]+/g).slice(0, 3).map(channel => { + const value = Number(channel) / 255; + return value <= 0.04045 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4; + }); + return 0.2126 * channels[0] + 0.7152 * channels[1] + 0.0722 * channels[2]; + }; + const colors = element => { + const { color, backgroundColor } = getComputedStyle(element); + const fg = luminance(color); + const bg = luminance(backgroundColor); + return { + color, backgroundColor, + readable: (Math.max(fg, bg) + 0.05) / (Math.min(fg, bg) + 0.05) >= 4.5 + }; + }; + return { search: colors(search), track: colors(track) }; + })())"#, + ); + assert!( + ["search", "track"] + .iter() + .all(|key| get(get(&contrast, key), "readable") == &Value::Bool(true)), + "the search input and variant label must each have 4.5:1 text contrast: {contrast:?}" + ); +} + +#[test] +fn the_tracks_fit_at_the_live_map_frame_height() { + let Some(chrome) = chrome_or_skip("check the viewer at the map iframe height") else { + return; + }; + let scratch = Scratch::new(Path::new(env!("CARGO_TARGET_TMPDIR")), "rv-viewer-short"); + let layouts = [600, 900].map(|width| { + let (_stub, mut b) = open(&chrome, &scratch, &width.to_string(), (width, 260), false); + assert_eq!(text(b.eval(STATUS)), "ready"); + probe( + &mut b, + r#"JSON.stringify((() => { + const igv = document.querySelector('#igv'); + const track = igv.shadowRoot.querySelector('.igv-viewport[data-track-type=variant]'); + const rect = track.getBoundingClientRect(); + return { + width: innerWidth, height: innerHeight, + panelTop: Math.ceil(igv.getBoundingClientRect().top), + trackBottom: Math.ceil(rect.bottom), + fits: rect.height > 0 && rect.bottom <= innerHeight + }; + })())"#, + ) + }); + assert!( + layouts.iter().all(|layout| { + matches!(get(layout, "panelTop"), Value::Int(0..=60)) + && get(layout, "fits") == &Value::Bool(true) + }), + "at the map iframe height, igv.js must start within 60 px and show the variant track: {layouts:?}" + ); +} + +#[test] +fn on_a_phone_the_page_is_dark_and_no_wider_than_the_screen() { + let Some(chrome) = chrome_or_skip("load the viewer page on a phone") else { + return; + }; + let scratch = Scratch::new(Path::new(env!("CARGO_TARGET_TMPDIR")), "rv-viewer-phone"); + let (_stub, mut b) = open(&chrome, &scratch, "phone", PHONE, true); + assert_eq!( + text(b.eval(STATUS)), + "ready", + "igv.js did not start on the synthetic sample" + ); + // igv.js has drawn its bar and tracks. + assert!( + b.wait( + "document.getElementById('igv').scrollHeight > 150", + Duration::from_secs(20) + ) + .is_some(), + "igv.js drew nothing" + ); + thread::sleep(Duration::from_millis(500)); + let d = probe(&mut b, &dark("body")); + assert_eq!( + get(&d, "dark"), + &Value::Bool(true), + "phone: the page must be dark: {d:?}" + ); + let w = probe( + &mut b, + "JSON.stringify({scroll:document.scrollingElement.scrollWidth,client:document.documentElement.clientWidth})", + ); + let wide = |k: &str| match get(&w, k) { + Value::Int(n) => *n, + v => panic!("{k}: {v:?}"), + }; + assert!( + wide("client") == i128::from(PHONE.0) && wide("scroll") <= i128::from(PHONE.0), + "on a {} px phone the page must not be wider than the screen: {w:?}", + PHONE.0 + ); + shot(&mut b, "phone"); +} + +#[test] +fn a_config_error_says_what_to_check_and_try_again_starts_the_viewer() { + let Some(chrome) = chrome_or_skip("load the viewer page with a failing config") else { + return; + }; + let scratch = Scratch::new(Path::new(env!("CARGO_TARGET_TMPDIR")), "rv-viewer-error"); + let site = scratch.path().join("site"); + fs::create_dir_all(&site).unwrap(); + let stub = Stub::start(make_sample(&site), Config::Status(500)); + let mut b = browser(&chrome, &scratch.path().join("chrome"), PHONE, true); + b.go(&stub.url("/#chr1:400-500")).unwrap(); + b.wait(SETTLED, Duration::from_secs(60)); + assert_eq!(text(b.eval(STATUS)), "error: config.json answered 500"); + let e = probe( + &mut b, + "(function(){var e=document.getElementById('error');var r=e.getBoundingClientRect();\ + var btn=e.querySelector('button');return JSON.stringify({role:e.getAttribute('role'),\ + shown:!e.hidden&&r.height>0,text:e.textContent,button:btn?btn.textContent:''});})()", + ); + assert_eq!( + get(&e, "role"), + &Value::Str("alert".into()), + "#error is announced: {e:?}" + ); + assert_eq!(get(&e, "shown"), &Value::Bool(true), "#error shows: {e:?}"); + let said = text(Ok(get(&e, "text").clone())); + assert!( + said.contains("The viewer could not start: config.json answered 500") + && said.contains("is still running, then try again."), + "#error names the failure and says what to check: {said}" + ); + assert_eq!( + get(&e, "button"), + &Value::Str("Try again".into()), + "#error offers Try again: {e:?}" + ); + let d = probe(&mut b, &dark("#error")); + assert_eq!( + get(&d, "dark"), + &Value::Bool(true), + "the error box is dark: {d:?}" + ); + shot(&mut b, "error"); + // The server answers again: Try again starts the viewer. + stub.set_config(Config::Json(CONFIG.to_owned())); + b.eval("document.querySelector('#error button').click()") + .unwrap(); + let ready = b.wait( + "document.documentElement.dataset.pgpStatus === 'ready' && document.getElementById('error').hidden", + Duration::from_secs(30), + ); + assert!( + ready.is_some(), + "Try again did not start the viewer: {}", + text(b.eval(STATUS)) + ); +} diff --git a/rust/crates/og-cn/src/smn.rs b/rust/crates/og-cn/src/smn.rs index 9cf353689..9df0e5b8a 100644 --- a/rust/crates/og-cn/src/smn.rs +++ b/rust/crates/og-cn/src/smn.rs @@ -139,11 +139,14 @@ pub fn read(path: Option<&str>, chrom: &str, marker_copies: &[i64]) -> Result Table { - let dir = - std::env::temp_dir().join(format!("og-cn-smn-{}-{}", std::process::id(), rows.len())); + static NEXT: AtomicUsize = AtomicUsize::new(0); + let n = NEXT.fetch_add(1, Ordering::Relaxed); + let dir = std::env::temp_dir().join(format!("og-cn-smn-{}-{n}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); - let p = dir.join(format!("t{}.tsv", rows.join("").len())); + let p = dir.join("table.tsv"); std::fs::write(&p, format!("{HEAD}{}", rows.join(""))).unwrap(); Table(p.to_string_lossy().into_owned()) } const SITES: [i64; 2] = [70_952_074, 70_076_654]; + #[test] + fn two_tables_of_the_same_size_never_share_a_folder() { + let row = + "chr5\t70925000\t70955000\tSMN1\tPASS\t4\t30\tPASS\t2\t40\tchr5:70049000-70080000\n"; + let a = table(&[row]); + let b = table(&[row]); + let parent = |t: &Table| std::path::Path::new(&t.0).parent().unwrap().to_path_buf(); + assert_ne!(parent(&a), parent(&b)); + // Dropping one table leaves the other readable. + drop(a); + let r = read(Some(&b.0), "chr5", &SITES).unwrap(); + assert_eq!(r.pscn, Some(2)); + } + #[test] fn the_first_row_over_exon_7_wins() { // Two rows over exon 7: the first is unresolved, the second says 2. diff --git a/rust/crates/og-dev/src/demo/traits.rs b/rust/crates/og-dev/src/demo/traits.rs index 0bb19f41f..7ecee6d6f 100644 --- a/rust/crates/og-dev/src/demo/traits.rs +++ b/rust/crates/og-dev/src/demo/traits.rs @@ -217,6 +217,7 @@ const TRAIT_ALT_COPIES: &[(&str, usize)] = &[ ("rs145946881", 0), ("rs41380347", 0), ("rs41525747", 0), + ("rs869051967", 0), ("rs671", 0), ("rs1229984", 2), ("rs762551", 2), diff --git a/rust/crates/og-dev/src/evidence.rs b/rust/crates/og-dev/src/evidence.rs index a3e2f12db..291e0de26 100644 --- a/rust/crates/og-dev/src/evidence.rs +++ b/rust/crates/og-dev/src/evidence.rs @@ -12,13 +12,27 @@ //! panel site inside the benchmark regions that no GIAB record covers, //! with the reference base from the FASTA (through its `.fai`). The //! output is unsorted: the workflow sorts it with bcftools. +//! - `og dev evidence y-bench TREE.json CALLS.tsv OUT.tsv MIN`: the Y +//! benchmark. CALLS has the columns `sample`, `published`, `tool` and +//! `call`: a man's published haplogroup, written `-` as +//! Poznik et al. 2016 give it, and the branch a tool placed him on. Both +//! are placed on the current YFull tree (TREE, `current_tree.json`): the +//! published call by its SNP (a SNP on two branches by the clade letter +//! too), the tool's by its branch id, else by the SNP after the dash. +//! Each call is `exact`, `below` (deeper on the same +//! line), `ancestor` (above it only), `sibling` (splits off at the +//! published branch's parent), `wrong` (splits off higher), `no_call` or +//! `unknown_name`. OUT has one row per call; one table row per tool goes +//! to stdout. It exits 1 when a tool puts fewer than MIN of its men on +//! the published branch or below it (MIN is a share from 0 to 1): the +//! gate of the workflow's y-benchmark job. //! -//! Both stream their VCFs. hg002-gvcf keeps, per chromosome, the merged +//! The first two stream their VCFs. hg002-gvcf keeps, per chromosome, the merged //! spans the GIAB records cover instead of every covered position, and //! reads one reference base per site with a positioned read. //! -//! Exit 0 when written, 1 when an input cannot be read or parsed, 2 on a -//! usage error. +//! Exit 0 when written, 1 when an input cannot be read or parsed (or the +//! y-bench gate fails, after OUT is written), 2 on a usage error. use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fs::{self, File}; @@ -28,7 +42,7 @@ use std::path::Path; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; -const USAGE: &str = "usage: og dev evidence clashes RAW.vcf.gz SPLIT.vcf.gz DATA_DIR OUT.tsv\n og dev evidence hg002-gvcf TRUTH.vcf.gz TRUTH.bed REF.fasta SITES.tsv OUT.vcf\n"; +const USAGE: &str = "usage: og dev evidence clashes RAW.vcf.gz SPLIT.vcf.gz DATA_DIR OUT.tsv\n og dev evidence hg002-gvcf TRUTH.vcf.gz TRUTH.bed REF.fasta SITES.tsv OUT.vcf\n og dev evidence y-bench TREE.json CALLS.tsv OUT.tsv MIN\n"; /// Runs `og dev evidence ARGS`; returns the exit code. pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { @@ -47,7 +61,10 @@ pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { ); }) } - Some("clashes" | "hg002-gvcf") => return usage(err, "wrong number of arguments"), + Some("y-bench") if rest.len() == 4 => y_bench(&rest[0], &rest[1], &rest[2], &rest[3], out), + Some("clashes" | "hg002-gvcf" | "y-bench") => { + return usage(err, "wrong number of arguments"); + } Some("-h" | "--help") => { let _ = out.write_all(USAGE.as_bytes()); return 0; @@ -607,6 +624,284 @@ fn hg002_gvcf( Ok((records, blocks)) } +/// The current YFull tree as `y-bench` reads it: each branch's parent and +/// depth, its id, and the branches every id and every SNP name sits on. +struct YTree { + /// The parent's index; the root's is `usize::MAX`. + parent: Vec, + depth: Vec, + ids: Vec, + by_id: HashMap>, + by_snp: HashMap>, +} + +fn read_ytree(path: &str) -> Result { + use pycompat::json::{self, Value}; + let text = read_text(Path::new(path))?; + let root = json::loads(&text).map_err(|e| format!("{path}: {e}"))?; + let mut t = YTree { + parent: Vec::new(), + depth: Vec::new(), + ids: Vec::new(), + by_id: HashMap::new(), + by_snp: HashMap::new(), + }; + let mut stack: Vec<(&Value, usize, u32)> = vec![(&root, usize::MAX, 0)]; + while let Some((v, parent, depth)) = stack.pop() { + let Value::Object(m) = v else { + return Err(format!("{path}: a branch is not a JSON object")); + }; + let text_of = |key: &str| match m.get(key) { + Some(Value::Str(s)) => Ok(s.trim()), + None | Some(Value::Null) => Ok(""), + Some(_) => Err(format!("{path}: a branch's '{key}' is not text")), + }; + let i = t.parent.len(); + let id = text_of("id")?.to_owned(); + for name in text_of("snps")?.split([',', '/']).map(str::trim) { + if !name.is_empty() { + let at = t.by_snp.entry(name.to_owned()).or_default(); + if at.last() != Some(&i) { + at.push(i); + } + } + } + if !id.is_empty() { + t.by_id.entry(id.clone()).or_default().push(i); + } + t.parent.push(parent); + t.depth.push(depth); + t.ids.push(id); + match m.get("children") { + Some(Value::Array(kids)) => { + for k in kids.iter().rev() { + stack.push((k, i, depth + 1)); + } + } + None | Some(Value::Null) => {} + Some(_) => return Err(format!("{path}: a branch's 'children' is not a list")), + } + } + if t.parent.len() < 2 { + return Err(format!("{path}: the tree has no branch below its root")); + } + Ok(t) +} + +/// A branch name without Yleaf's marks: `R-M269*(xR-L23)` is `R-M269`. +fn bare(name: &str) -> &str { + name.split(['*', '(']).next().unwrap_or("").trim() +} + +/// The SNP of a name written `-` (`R1b-DF27` is `DF27`), or the +/// whole name when it has no dash. +fn snp_of(name: &str) -> &str { + name.split_once('-').map_or(name, |(_, snp)| snp) +} + +impl YTree { + fn one(v: Option<&Vec>) -> Option { + v.filter(|v| v.len() == 1).map(|v| v[0]) + } + + /// The branch a published call names: the one branch that carries its + /// SNP. A SNP name YFull gives to two branches (a recurrent mutation, + /// `M356` on `C-K98` and `T-Y31540`) is settled by the clade letter of + /// the published name (`C5-M356` is in C), never by anything else. A + /// published call is the truth, so a SNP the tree does not have, or one + /// the letter does not settle, is an error, never a guess. + fn published(&self, name: &str) -> Result { + let n = bare(name); + let snp = snp_of(n); + let Some(at) = self.by_snp.get(snp) else { + return Err(format!( + "published call '{name}': SNP {snp} is on no branch of the tree" + )); + }; + if let [one] = at[..] { + return Ok(one); + } + let letter = n.chars().next(); + let same: Vec = at + .iter() + .copied() + .filter(|&i| self.ids[i].chars().next() == letter) + .collect(); + match same[..] { + [one] => Ok(one), + _ => Err(format!( + "published call '{name}': SNP {snp} is on {} branches of the tree, {} of them in clade {}", + at.len(), + same.len(), + letter.unwrap_or('?') + )), + } + } + + /// The branch a tool's call names: its id on the tree, else the one + /// branch carrying the SNP after the dash (a branch renamed since the + /// tool's tree). `Ok(None)` is no call; `Err` a name the tree cannot place. + fn call(&self, name: &str) -> Result, ()> { + let n = bare(name); + if n.is_empty() || n == "NA" || n == "." { + return Ok(None); + } + Self::one(self.by_id.get(n)) + .or_else(|| Self::one(self.by_snp.get(snp_of(n)))) + .map(Some) + .ok_or(()) + } + + fn is_ancestor(&self, a: usize, mut b: usize) -> bool { + while b != usize::MAX && self.depth[b] > self.depth[a] { + b = self.parent[b]; + } + b == a + } + + fn common(&self, mut a: usize, mut b: usize) -> usize { + while a != b { + if self.depth[a] >= self.depth[b] { + a = self.parent[a]; + } else { + b = self.parent[b]; + } + } + a + } +} + +/// What a tool's call is, against the published branch. +const RESULTS: [&str; 7] = [ + "exact", + "below", + "ancestor", + "sibling", + "wrong", + "no_call", + "unknown_name", +]; + +/// One of [`RESULTS`]: the published branch itself, a branch below it +/// (the same line, placed deeper than the published call went), one above +/// it only, a branch that splits off at the published branch's parent, one +/// that splits off higher, no call, or a name the tree cannot place. +fn classify(t: &YTree, truth: usize, call: Result, ()>) -> (usize, Option) { + let c = match call { + Err(()) => return (6, None), + Ok(None) => return (5, None), + Ok(Some(c)) => c, + }; + let r = if c == truth { + 0 + } else if t.is_ancestor(truth, c) { + 1 + } else if t.is_ancestor(c, truth) { + 2 + } else if t.common(truth, c) == t.parent[truth] { + 3 + } else { + 4 + }; + (r, Some(c)) +} + +/// `og dev evidence y-bench TREE.json CALLS.tsv OUT.tsv MIN`: places every +/// published and every tool call of CALLS on the tree, writes one row per +/// call to OUT and one line per tool to `out`, and fails when a tool puts +/// fewer than MIN of its men on the published branch or below it. +fn y_bench(tree: &str, calls: &str, out: &str, min: &str, w: &mut dyn Write) -> Result<(), String> { + let min: f64 = min + .parse() + .ok() + .filter(|m: &f64| (0.0..=1.0).contains(m)) + .ok_or_else(|| format!("MIN '{min}' is not a share from 0 to 1"))?; + let t = read_ytree(tree)?; + let mut rows: Vec<[String; 4]> = Vec::new(); + let mut first = true; + lines(calls, |line| { + let line = std::str::from_utf8(line).map_err(|e| format!("{calls}: {e}"))?; + let f: Vec<&str> = line.split('\t').collect(); + if std::mem::take(&mut first) { + return if f == ["sample", "published", "tool", "call"] { + Ok(()) + } else { + Err(format!( + "{calls}: the header is not sample, published, tool, call" + )) + }; + } + let [s, p, tool, c] = f[..] else { + return Err(format!("{calls}: a row has {} columns, not 4", f.len())); + }; + if s.is_empty() || p.is_empty() || tool.is_empty() { + return Err(format!( + "{calls}: a row has no sample, published call or tool" + )); + } + rows.push([s, p, tool, c].map(str::to_owned)); + Ok(()) + })?; + if rows.is_empty() { + return Err(format!("{calls}: no calls")); + } + let mut seen = BTreeSet::new(); + let mut tools: Vec = Vec::new(); + let mut counts: HashMap = HashMap::new(); + let mut text = String::from( + "sample\tpublished\ttool\tcall\tpublished_branch\tcall_branch\tresult\tlevels\n", + ); + for [s, p, tool, c] in &rows { + if !seen.insert((s.clone(), tool.clone())) { + return Err(format!("{calls}: two calls of {tool} for {s}")); + } + let truth = t.published(p)?; + let (r, at) = classify(&t, truth, t.call(c)); + if !counts.contains_key(tool) { + tools.push(tool.clone()); + } + counts.entry(tool.clone()).or_default()[r] += 1; + let (branch, levels) = at.map_or((".".to_owned(), ".".to_owned()), |a| { + let d = i64::from(t.depth[a]) - i64::from(t.depth[truth]); + (t.ids[a].clone(), format!("{d:+}")) + }); + text += &format!( + "{s}\t{p}\t{tool}\t{c}\t{}\t{branch}\t{}\t{levels}\n", + t.ids[truth], RESULTS[r] + ); + } + publish(Path::new(out), text.as_bytes())?; + let _ = writeln!( + w, + "| Tool | Men | Exact | Below | Ancestor only | Sibling | Wrong | No call | Unknown name | Published branch or below |\n|---|---|---|---|---|---|---|---|---|---|" + ); + let mut short = Vec::new(); + for tool in &tools { + let n = counts[tool]; + let men: u64 = n.iter().sum(); + let share = (n[0] + n[1]) as f64 / men as f64; + let cells: Vec = n.iter().map(u64::to_string).collect(); + let _ = writeln!( + w, + "| {tool} | {men} | {} | {} of {men} ({:.1}%) |", + cells.join(" | "), + n[0] + n[1], + share * 100.0 + ); + if share < min { + short.push(format!("{tool} {:.3}", share)); + } + } + if short.is_empty() { + Ok(()) + } else { + Err(format!( + "y-bench gate: under {min} on the published branch or below: {}", + short.join(", ") + )) + } +} + #[cfg(test)] mod tests { use super::*; @@ -1137,4 +1432,191 @@ mod tests { inside(&m, 5) && !inside(&m, 6) && inside(&m, 15) && !inside(&m, 16) && !inside(&m, 4) ); } + + const YTREE: &str = r#"{"id": "", "snps": "", "children": [ + {"id": "A-a1", "snps": "a1, a2/a2b", "children": [ + {"id": "A-a11", "snps": "a11, dup2"}, + {"id": "A-a21", "snps": "a21", "children": [{"id": "A-a2x", "snps": "a2x, dup2"}]}]}, + {"id": "B-b1", "snps": "b1", "children": [ + {"id": "B-b11", "snps": "b11"}, + {"id": "B-b21", "snps": "b21, dup"}]}, + {"id": "C-c1", "snps": "dup"}]}"#; + + struct YCase(PathBuf); + impl YCase { + fn new(name: &str, calls: &str) -> Self { + let d = std::env::temp_dir().join(format!("og-ybench-{}-{name}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("tree.json"), YTREE).unwrap(); + fs::write(d.join("calls.tsv"), calls).unwrap(); + YCase(d) + } + fn run(&self, min: &str) -> (u8, String, String) { + let p = |n: &str| self.0.join(n).to_str().unwrap().to_owned(); + let args: Vec = vec![ + "y-bench".into(), + p("tree.json"), + p("calls.tsv"), + p("out.tsv"), + min.into(), + ]; + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&args, &mut out, &mut err); + ( + code, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ) + } + fn out(&self) -> String { + fs::read_to_string(self.0.join("out.tsv")).unwrap_or_default() + } + } + impl Drop for YCase { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + const YHEAD: &str = "sample\tpublished\ttool\tcall\n"; + + #[test] + fn y_bench_classifies_every_result() { + let calls = format!( + "{YHEAD}m1\tA2-a21\tt\tA-a21\nm2\tA2-a21\tt\tA-a2x\nm3\tA2-a21\tt\tA-a1\nm4\tA2-a21\tt\tA-a11\n\ + m5\tA2-a21\tt\tB-b11\nm6\tA2-a21\tt\tNA\nm7\tA2-a21\tt\tZ-zz\nm8\tA2-a21\tt\tB-dup\n" + ); + let c = YCase::new("classes", &calls); + let (code, table, err) = c.run("0"); + assert_eq!(code, 0, "{err}"); + let got: Vec = c + .out() + .lines() + .skip(1) + .map(|l| { + let f: Vec<&str> = l.split('\t').collect(); + format!("{}:{}:{}:{}", f[0], f[5], f[6], f[7]) + }) + .collect(); + assert_eq!( + got, + [ + "m1:A-a21:exact:+0", + "m2:A-a2x:below:+1", + "m3:A-a1:ancestor:-1", + "m4:A-a11:sibling:+0", + "m5:B-b11:wrong:+0", + "m6:.:no_call:.", + "m7:.:unknown_name:.", + "m8:.:unknown_name:.", + ] + ); + assert!( + table.contains("| t | 8 | 1 | 1 | 1 | 1 | 1 | 1 | 2 | 2 of 8 (25.0%) |"), + "{table}" + ); + } + + #[test] + fn y_bench_reads_names_as_the_tools_and_the_paper_write_them() { + // Yleaf's marks, a published clade name that is not the tree's id + // (found by its SNP), an alias after a slash, and a branch renamed + // since the tool's tree (found by the SNP after the dash). + let calls = format!( + "{YHEAD}m1\tA1-a2b\tyleaf\tA-a1*(xA-a11)\nm2\tA2b-a21\tog\tA-a21\nm3\tB2-b21\tog\tOld-b21\n" + ); + let c = YCase::new("names", &calls); + let (code, table, err) = c.run("1"); + assert_eq!(code, 0, "{err}{table}"); + assert!( + c.out() + .contains("m1\tA1-a2b\tyleaf\tA-a1*(xA-a11)\tA-a1\tA-a1\texact\t+0\n") + ); + assert!( + c.out() + .contains("m3\tB2-b21\tog\tOld-b21\tB-b21\tB-b21\texact\t+0\n") + ); + // A SNP on two branches is settled by the clade letter alone. + let c = YCase::new("letter", &format!("{YHEAD}m1\tC1-dup\tog\tC-c1\n")); + assert_eq!(c.run("1").0, 0); + assert!(c.out().contains("\tC-c1\tC-c1\texact\t")); + // A published SNP on no branch, or on two the letter does not + // settle, is refused: it is the truth. + for bad in ["X-nope", "D-dup", "A2-dup2"] { + let c = YCase::new("truth", &format!("{YHEAD}m1\t{bad}\tog\tA-a1\n")); + let (code, _, err) = c.run("0"); + assert_eq!(code, 1); + assert!(err.contains("published call"), "{err}"); + assert!(c.out().is_empty()); + } + } + + #[test] + fn y_bench_gate_and_its_red_control() { + let men = [ + ("m1", "A2-a21"), + ("m2", "A1-a11"), + ("m3", "B1-b11"), + ("m4", "B2-b21"), + ]; + let calls = ["A-a21", "A-a11", "B-b11", "B-b21"]; + let table = |rotate: usize| { + let mut t = YHEAD.to_owned(); + for (i, (m, p)) in men.iter().enumerate() { + t += &format!("{m}\t{p}\tog\t{}\n", calls[(i + rotate) % calls.len()]); + } + t + }; + let c = YCase::new("gate", &table(0)); + let (code, out, err) = c.run("1"); + assert_eq!(code, 0, "{err}{out}"); + assert!(out.contains("4 of 4 (100.0%)"), "{out}"); + // Red control: the same calls given to the wrong men. + let c = YCase::new("rotated", &table(2)); + let (code, out, err) = c.run("0.5"); + assert_eq!(code, 1, "{out}"); + assert!( + err.contains("y-bench gate: under 0.5") && err.contains("og 0.000"), + "{err}" + ); + assert_eq!( + c.out().matches("\twrong\t").count(), + 4, + "the rows are written before the gate fails" + ); + } + + #[test] + fn y_bench_refuses_bad_input() { + for (name, calls, min, want) in [ + ("header", "sample\tpublished\ttool\n", "0", "the header"), + ( + "columns", + &*format!("{YHEAD}m1\tA-a1\tog\n"), + "0", + "columns, not 4", + ), + ( + "twice", + &*format!("{YHEAD}m1\tA-a1\tog\tA-a1\nm1\tA-a1\tog\tA-a1\n"), + "0", + "two calls", + ), + ("empty", YHEAD, "0", "no calls"), + ( + "min", + &*format!("{YHEAD}m1\tA-a1\tog\tA-a1\n"), + "1.5", + "not a share", + ), + ] { + let c = YCase::new(name, calls); + let (code, _, err) = c.run(min); + assert_eq!(code, 1, "{name}"); + assert!(err.contains(want), "{name}: {err}"); + } + let (mut out, mut err) = (Vec::new(), Vec::new()); + assert_eq!(cli(&["y-bench".into(), "x".into()], &mut out, &mut err), 2); + } } diff --git a/rust/crates/og-dev/src/lib.rs b/rust/crates/og-dev/src/lib.rs index 6d7b81396..d98596234 100644 --- a/rust/crates/og-dev/src/lib.rs +++ b/rust/crates/og-dev/src/lib.rs @@ -28,7 +28,8 @@ //! - `og dev fixture` ([`fixture`]): writers of test fixtures (the //! compound-het reads, a trimmed PharmCAT report). //! - `og dev evidence` ([`evidence`]): the measurements of the public -//! evidence workflow on GIAB's HG002. +//! evidence workflow on GIAB's HG002, and its Y placement +//! benchmark on published 1000 Genomes men. //! - `og dev renovate-check LOG` ([`renovate`]): what a Renovate dry run //! found, against every pin in the repository. //! - `og dev build cn-intervals|callability|sv-rule-regions` ([`build`]): @@ -88,7 +89,7 @@ mod zip; use std::io::Write; -const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev fixture comphet|trim-pharmcat [args]\n og dev evidence clashes|hg002-gvcf [args]\n og dev renovate-check LOG [--pins FILE] [--repo DIR]\n og dev helpers --list|--static|--images|--orad-calls [--root DIR]\n og dev helpers --orad --help-file FILE [--orad-version V] [--root DIR]\n og dev build cn-intervals|callability|sv-rule-regions|roh-af|acmg-lof [args]\n og dev scan personal-data|images [args]\n og dev versions check-run|expand|coupled [args]\n og dev pgs trait|label-match|metric-match [args]\n\n og dev pharmcat diplotypes REPORT [--called|--called-lines]\n og dev nf-trace requests|attempts TRACE [options]\n og dev smoke mini-ref|long-reads|eh-catalog|csv-cell|check [args]\n og dev demo-sample [args]\n og dev demo-events [args]\n og dev params-doc [args]\n og dev python-gate [args]\n og dev everything-check [args]\n og dev licences [args]\n og dev docs-check [args]\n og dev stub-http [args]\n og dev steps-check [args]\n og dev site-check SITE_DIR [ZENSICAL_TOML]\n og dev traits-sources [--table T] [--fasta F [--min-fasta-rows N]] [--online]\n og dev pictures take|live|check|same-sample [args]"; +const USAGE: &str = "usage: og dev parity-compare KIND ORACLE NEW\n og dev parity-compare --self-test\n og dev contract --static [--repo DIR] [--contract FILE]\n og dev contract --outputs DIR --leg bash|nextflow --sample S [--observe]\n og dev schema SCHEMA.json DOC.json\n og dev report-only [--repo DIR]\n og dev remnants [--list FILE] [--root DIR]\n og dev fixture comphet|trim-pharmcat [args]\n og dev evidence clashes|hg002-gvcf|y-bench [args]\n og dev renovate-check LOG [--pins FILE] [--repo DIR]\n og dev helpers --list|--static|--images|--orad-calls [--root DIR]\n og dev helpers --orad --help-file FILE [--orad-version V] [--root DIR]\n og dev build cn-intervals|callability|sv-rule-regions|roh-af|acmg-lof [args]\n og dev scan personal-data|images [args]\n og dev versions check-run|expand|coupled [args]\n og dev pgs trait|label-match|metric-match [args]\n\n og dev pharmcat diplotypes REPORT [--called|--called-lines]\n og dev nf-trace requests|attempts TRACE [options]\n og dev smoke mini-ref|long-reads|eh-catalog|csv-cell|check [args]\n og dev demo-sample [args]\n og dev demo-events [args]\n og dev params-doc [args]\n og dev python-gate [args]\n og dev everything-check [args]\n og dev licences [args]\n og dev docs-check [args]\n og dev stub-http [args]\n og dev steps-check [args]\n og dev site-check SITE_DIR [ZENSICAL_TOML]\n og dev traits-sources [--table T] [--fasta F [--min-fasta-rows N]] [--online]\n og dev pictures take|live|check|same-sample [args]"; /// Python's `str(OSError)`: `[Errno 2] No such file or directory: 'path'`. pub(crate) fn py_os_error(e: &std::io::Error, path: &str) -> String { diff --git a/rust/crates/og-dev/src/licences.rs b/rust/crates/og-dev/src/licences.rs index e3d4fe1ed..d04d552d3 100644 --- a/rust/crates/og-dev/src/licences.rs +++ b/rust/crates/og-dev/src/licences.rs @@ -1,11 +1,24 @@ -//! `og dev licences`: the notices of the third-party code in the og binary. +//! `og dev licences`: the notices of the third-party code in the og binary, +//! and the register of the data files the repository ships. //! //! ```text //! og dev licences notices [--manifest-path FILE] [--package NAME] //! (--target TRIPLE | --target-no-default TRIPLE)... [--extra NAME=FILE]... //! [--out FILE] +//! og dev licences check [--root DIR] //! ``` //! +//! `check` reads `assets/SOURCES.tsv` (columns path, source, version, +//! licence, header) and fails, exit 1, naming each fault: a file under +//! `assets/` that git tracks or would add with no row (a row ending in `/` +//! covers a folder); a file in two rows; a row that matches no file; a row +//! with no source or licence; a header phrase (split on `|`, `-` for none) +//! missing as whole words from the file's opening `#` lines, read as one line +//! so a phrase may wrap; and a third-party row (a source that does not start +//! with "OwnGenome") without an entry for its full path in `NOTICE`. +//! Lint runs it on every push and pull request. The rest of this page is +//! about `notices`. +//! //! The og binaries link about seventy crates statically, and all four also //! link htslib and htscodecs (C, built by `hts-sys`). Their licences //! (MIT, BSD-3-Clause, Unicode-3.0 and others) ask that a binary carry their @@ -87,8 +100,13 @@ SOFTWARE. pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { match args.first().map(String::as_str) { Some("notices") => notices_cli(&args[1..], out, err), + Some("check") => check_cli(&args[1..], out, err), _ => { - let _ = writeln!(err, "{USAGE}"); + let _ = writeln!( + err, + "{USAGE}\n {}", + CHECK_USAGE.trim_start_matches("usage: ") + ); 2 } } @@ -643,6 +661,370 @@ impl<'a> Parser<'a> { } } +// ---- og dev licences check: the register of bundled data ---- + +/// The register of bundled data, relative to the repository root. +pub const REGISTER: &str = "assets/SOURCES.tsv"; +/// The notice file that names every third-party data file. +pub const NOTICE: &str = "NOTICE"; +/// The folder whose every file must have a row. +pub const SCOPE: &str = "assets/"; +/// The register's columns, in order. +pub const COLUMNS: [&str; 5] = ["path", "source", "version", "licence", "header"]; + +const CHECK_USAGE: &str = "usage: og dev licences check [--root DIR]"; + +/// One row of the register. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Row { + /// Line number in the register, for messages. + pub line: usize, + /// A file, or a folder when it ends with `/` (every file below it). + pub path: String, + pub source: String, + pub version: String, + pub licence: String, + /// Phrases the file's opening `#` lines must carry, split on `|`; + /// `-` when the file carries no header. + pub header: String, +} + +impl Row { + /// Our own work: the source starts with "OwnGenome". Any other row is + /// a third party's, and NOTICE must name its path. + pub fn own(&self) -> bool { + self.source.starts_with("OwnGenome") + } + + fn matches(&self, file: &str) -> bool { + if self.path.ends_with('/') { + file.starts_with(&self.path) + } else { + file == self.path + } + } + + fn phrases(&self) -> Vec { + if self.header == "-" { + return Vec::new(); + } + self.header + .split('|') + .map(words) + .filter(|p| !p.is_empty()) + .collect() + } +} + +/// Runs of blanks folded to one space, ends trimmed. +fn words(s: &str) -> String { + s.split_whitespace().collect::>().join(" ") +} + +/// A value that names nothing. +fn placeholder(v: &str) -> bool { + ["-", "?", "unknown", "todo", "tbd", "none", "n/a"].contains(&v.to_ascii_lowercase().as_str()) +} + +/// The register's rows, or every problem in it. `#` lines and blank lines +/// are skipped; the first other line names the columns. +pub fn parse_register(text: &str) -> Result, Vec> { + let mut problems = Vec::new(); + let mut rows: Vec = Vec::new(); + let mut header_seen = false; + for (i, raw) in text.lines().enumerate() { + let line = i + 1; + let l = raw.trim_end_matches('\r'); + if l.starts_with('#') || l.trim().is_empty() { + continue; + } + let cells: Vec<&str> = l.split('\t').collect(); + if !header_seen { + header_seen = true; + if cells != COLUMNS { + problems.push(format!( + "{REGISTER}:{line}: the column line must be {:?}, got {:?}", + COLUMNS.join("\t"), + l + )); + return Err(problems); + } + continue; + } + if cells.len() != COLUMNS.len() { + problems.push(format!( + "{REGISTER}:{line}: {} tab-separated fields, want {}", + cells.len(), + COLUMNS.len() + )); + continue; + } + let cell: Vec = cells.iter().map(|c| c.trim().to_string()).collect(); + let row = Row { + line, + path: cell[0].clone(), + source: cell[1].clone(), + version: cell[2].clone(), + licence: cell[3].clone(), + header: cell[4].clone(), + }; + let p = &row.path; + if p.is_empty() + || p.starts_with('/') + || p.contains('\\') + || p.split('/').any(|s| s == ".." || s == ".") + { + problems.push(format!( + "{REGISTER}:{line}: path {p:?} must be relative to the repository root" + )); + } + for (name, v) in [ + ("source", &row.source), + ("licence", &row.licence), + ("version", &row.version), + ("header", &row.header), + ] { + if v.is_empty() { + problems.push(format!( + "{REGISTER}:{line}: {p}: the {name} is empty (write - for none)" + )); + } + } + for (name, v) in [("source", &row.source), ("licence", &row.licence)] { + if !v.is_empty() && placeholder(v) { + problems.push(format!( + "{REGISTER}:{line}: {p}: the {name} is {v:?}; name the real one" + )); + } + } + if row.header != "-" && row.phrases().is_empty() { + problems.push(format!( + "{REGISTER}:{line}: {p}: the header names no phrase" + )); + } + if let Some(first) = rows.iter().find(|r| r.path == row.path) { + problems.push(format!( + "{REGISTER}:{line}: {p} has a row already, on line {}", + first.line + )); + } + rows.push(row); + } + if !header_seen { + problems.push(format!("{REGISTER}: no column line and no rows")); + } + if problems.is_empty() { + Ok(rows) + } else { + Err(problems) + } +} + +/// The opening comment of a file: its leading `#` lines, the `#` and the +/// blanks dropped, joined into one line of words. A phrase can so wrap +/// across lines; a line after the first non-`#` line is never header. +pub fn header_text(file: &str) -> String { + let lines: Vec<&str> = file + .lines() + .take_while(|l| l.starts_with('#')) + .map(|l| l.trim_start_matches('#')) + .collect(); + words(&lines.join(" ")) +} + +/// Match a normalized phrase with word boundaries, keeping its punctuation. +fn contains_header_phrase(header: &str, phrase: &str) -> bool { + let word_char = |c: char| c.is_alphanumeric() || c == '_'; + header.char_indices().any(|(start, _)| { + !header[..start].ends_with(word_char) + && header[start..] + .strip_prefix(phrase) + .is_some_and(|rest| !rest.starts_with(word_char)) + }) +} + +/// Every problem with the register against the files of the tree. +/// `files`: every file of the tree, relative to its root; `read`: a file's +/// text; `notice`: a path on its own line or followed by `: ` and a description. +pub fn audit( + rows: &[Row], + files: &[String], + read: &dyn Fn(&str) -> Result, + notice: &str, +) -> Vec { + let mut problems = Vec::new(); + let mut used = vec![false; rows.len()]; + for f in files { + let hits: Vec = (0..rows.len()).filter(|&i| rows[i].matches(f)).collect(); + for &i in &hits { + used[i] = true; + } + if f.starts_with(SCOPE) && f != REGISTER && hits.is_empty() { + problems.push(format!( + "{f} has no row in {REGISTER}: add its source, version, licence and header" + )); + } + if hits.len() > 1 { + let lines: Vec = hits.iter().map(|&i| rows[i].line.to_string()).collect(); + problems.push(format!( + "{f} is in {} rows of {REGISTER} (lines {}); give it one", + hits.len(), + lines.join(", ") + )); + } + let Some(&i) = hits.first() else { continue }; + let phrases = rows[i].phrases(); + if phrases.is_empty() || hits.len() > 1 { + continue; + } + match read(f) { + Ok(text) => { + let head = header_text(&text); + for p in phrases { + if !contains_header_phrase(&head, &p) { + problems.push(format!( + "{f}: its opening # lines lack {p:?}, which {REGISTER}:{} says it carries", + rows[i].line + )); + } + } + } + Err(e) => problems.push(format!("{f}: {e}")), + } + } + let named_in_notice = |path: &str| { + notice.lines().map(str::trim).any(|line| { + line == path + || line + .strip_prefix(path) + .is_some_and(|description| description.starts_with(": ")) + }) + }; + for (r, u) in rows.iter().zip(&used) { + if !u { + let hint = if r.path.ends_with('/') { + "" + } else { + " (a folder row ends with /)" + }; + problems.push(format!( + "{REGISTER}:{}: {} matches no file{hint}", + r.line, r.path + )); + } + if !r.own() && !named_in_notice(&r.path) { + problems.push(format!( + "{NOTICE} does not name {}, which comes from a third party ({REGISTER}:{})", + r.path, r.line + )); + } + } + problems +} + +/// The files of the tree at `root` that git tracks or would add: tracked +/// ones, and untracked ones that .gitignore does not exclude, so a file +/// not yet added is checked too. +pub fn tree_files(root: &Path) -> Result, String> { + let run = Command::new("git") + .arg("-C") + .arg(root) + .args([ + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + ]) + .output() + .map_err(|e| format!("git did not start: {e}"))?; + if !run.status.success() { + return Err(format!( + "git ls-files in {} failed: {}", + root.display(), + String::from_utf8_lossy(&run.stderr).trim() + )); + } + let mut files: Vec = String::from_utf8_lossy(&run.stdout) + .split('\0') + .filter(|s| !s.is_empty()) + .map(str::to_owned) + .collect(); + files.sort(); + files.dedup(); + Ok(files) +} + +/// `og dev licences check` on the tree at `root`: Ok(summary) or every +/// problem. +pub fn check(root: &Path) -> Result> { + let reg = root.join(REGISTER); + let text = + std::fs::read_to_string(®).map_err(|e| vec![format!("cannot read {REGISTER}: {e}")])?; + let rows = parse_register(&text)?; + let notice = std::fs::read_to_string(root.join(NOTICE)) + .map_err(|e| vec![format!("cannot read {NOTICE}: {e}")])?; + let files = tree_files(root).map_err(|e| vec![e])?; + let read = |f: &str| -> Result { + std::fs::read(root.join(f)) + .map(|b| String::from_utf8_lossy(&b).into_owned()) + .map_err(|e| format!("cannot read it: {e}")) + }; + let problems = audit(&rows, &files, &read, ¬ice); + if !problems.is_empty() { + return Err(problems); + } + let in_scope = files + .iter() + .filter(|f| f.starts_with(SCOPE) && f.as_str() != REGISTER) + .count(); + let third = rows.iter().filter(|r| !r.own()).count(); + Ok(format!( + "{in_scope} files under {SCOPE} and {} rows in {REGISTER}: every file has its source and licence, \ +every header phrase is in its file, {NOTICE} names all {third} third-party rows", + rows.len() + )) +} + +fn check_cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + let mut root = PathBuf::from("."); + let mut it = args.iter(); + while let Some(a) = it.next() { + match (a.as_str(), it.next()) { + ("--root", Some(v)) => root = PathBuf::from(v), + _ => { + let _ = writeln!( + err, + "og dev licences check: unknown or incomplete argument {a:?}\n{CHECK_USAGE}" + ); + return 2; + } + } + } + match check(&root) { + Ok(summary) => { + let _ = writeln!(out, "og dev licences check: {summary}"); + 0 + } + Err(problems) => { + for p in &problems { + let _ = writeln!(err, "og dev licences check: {p}"); + } + let _ = writeln!( + err, + "og dev licences check: {} {}; {REGISTER} says how each column is filled", + problems.len(), + if problems.len() == 1 { + "problem" + } else { + "problems" + } + ); + 1 + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -693,4 +1075,252 @@ mod tests { assert!(allowed(e).is_err(), "{e:?} should not parse"); } } + + // ---- og dev licences check ---- + + const REG: &str = "# the register\npath\tsource\tversion\tlicence\theader\n\ +assets/third.tsv\tSomeone, Some data\t1.0\tCC-BY-4.0\tSource: Someone, Some data | licensed CC BY 4.0\n\ +assets/own.tsv\tOwnGenome\t-\tGPL-3.0-or-later\t-\n\ +assets/stub/\tOwnGenome, synthetic\t-\tGPL-3.0-or-later\t-\n"; + + const THIRD: &str = "# third.tsv: a table.\n# Source: Someone, Some data, https://example.org,\n\ +# licensed CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/).\na\tb\n"; + + fn files(list: &[&str]) -> Vec { + list.iter().map(|s| s.to_string()).collect() + } + + /// The audit of REG-like rows against a tree whose third.tsv is `third`. + fn run(reg: &str, tree: &[&str], third: &str, notice: &str) -> Vec { + let rows = parse_register(reg).expect("register parses"); + let third = third.to_string(); + let read = move |f: &str| -> Result { + match f { + "assets/third.tsv" => Ok(third.clone()), + _ => Ok(String::new()), + } + }; + audit(&rows, &files(tree), &read, notice) + } + + const TREE: &[&str] = &[ + "README.md", + "assets/SOURCES.tsv", + "assets/own.tsv", + "assets/stub/a.bam", + "assets/stub/db/.keep", + "assets/third.tsv", + ]; + const NOTICE_OK: &str = "assets/third.tsv: Some data, CC BY 4.0\n"; + + #[test] + fn a_registered_tree_passes() { + assert_eq!(run(REG, TREE, THIRD, NOTICE_OK), Vec::::new()); + } + + #[test] + fn an_unregistered_asset_fails() { + let mut tree = TREE.to_vec(); + tree.push("assets/planted.tsv"); + let p = run(REG, &tree, THIRD, NOTICE_OK); + assert_eq!(p.len(), 1, "{p:?}"); + assert!(p[0].starts_with("assets/planted.tsv has no row"), "{p:?}"); + } + + #[test] + fn a_file_outside_assets_needs_no_row() { + let mut tree = TREE.to_vec(); + tree.push("docs/x.md"); + assert!(run(REG, &tree, THIRD, NOTICE_OK).is_empty()); + } + + #[test] + fn a_stripped_source_line_fails() { + let stripped: String = THIRD + .lines() + .filter(|l| !l.contains("Source:")) + .map(|l| format!("{l}\n")) + .collect(); + let p = run(REG, TREE, &stripped, NOTICE_OK); + assert_eq!(p.len(), 1, "{p:?}"); + assert!( + p[0].contains( + "assets/third.tsv: its opening # lines lack \"Source: Someone, Some data\"" + ), + "{p:?}" + ); + } + + #[test] + fn a_phrase_may_wrap_and_is_read_only_from_the_opening_lines() { + // "licensed CC BY 4.0" wraps here: still found. + let wrapped = "# Source: Someone, Some data, licensed\n# CC BY 4.0\nrow\n"; + assert!(run(REG, TREE, wrapped, NOTICE_OK).is_empty()); + // The same words after the first data line are no header. + let late = "# a table\nrow\n# Source: Someone, Some data, licensed CC BY 4.0\n"; + assert_eq!(run(REG, TREE, late, NOTICE_OK).len(), 2); + } + + #[test] + fn a_header_phrase_must_match_whole_words() { + let reg = REG.replace("Source: Someone, Some data", "Source: GENCODE release 50"); + let header = "# Source: GENCODE release 500, unlicensed CC BY 4.0\nrow\n"; + let problems = run(®, TREE, header, NOTICE_OK); + assert_eq!(problems.len(), 2, "{problems:?}"); + assert!(problems[0].contains("lack \"Source: GENCODE release 50\"")); + assert!(problems[1].contains("lack \"licensed CC BY 4.0\"")); + + let correct = "# Source: GENCODE release 50, licensed\n# CC BY 4.0\nrow\n"; + assert!(run(®, TREE, correct, NOTICE_OK).is_empty()); + } + + #[test] + fn a_third_party_row_missing_from_notice_fails() { + let p = run(REG, TREE, THIRD, "nothing here\n"); + assert_eq!(p.len(), 1, "{p:?}"); + assert!( + p[0].starts_with("NOTICE does not name assets/third.tsv"), + "{p:?}" + ); + } + + #[test] + fn a_notice_must_name_the_complete_path() { + for notice in [ + "assets/third.tsv.gz: Some data, CC BY 4.0\n", + "other/assets/third.tsv: Some data, CC BY 4.0\n", + ] { + let problems = run(REG, TREE, THIRD, notice); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!(problems[0].starts_with("NOTICE does not name assets/third.tsv")); + } + + assert!(run(REG, TREE, THIRD, "assets/third.tsv\n").is_empty()); + } + + #[test] + fn a_row_with_no_file_fails() { + let tree: Vec<&str> = TREE + .iter() + .copied() + .filter(|f| *f != "assets/own.tsv") + .collect(); + let p = run(REG, &tree, THIRD, NOTICE_OK); + assert_eq!(p.len(), 1, "{p:?}"); + assert!(p[0].contains("assets/own.tsv matches no file"), "{p:?}"); + } + + #[test] + fn a_file_in_two_rows_fails() { + let reg = format!("{REG}assets/stub/a.bam\tOwnGenome\t-\tGPL-3.0-or-later\t-\n"); + let p = run(®, TREE, THIRD, NOTICE_OK); + assert_eq!(p.len(), 1, "{p:?}"); + assert!(p[0].contains("assets/stub/a.bam is in 2 rows"), "{p:?}"); + } + + #[test] + fn a_broken_register_fails_to_parse() { + let bad = [ + ("path\tsource\tlicence\n", "the column line"), + ( + "path\tsource\tversion\tlicence\theader\nassets/a\tX\t1\tMIT\n", + "4 tab-separated fields", + ), + ( + "path\tsource\tversion\tlicence\theader\nassets/a\tX\t1\t\t-\n", + "the licence is empty", + ), + ( + "path\tsource\tversion\tlicence\theader\nassets/a\tX\t1\tunknown\t-\n", + "the licence is \"unknown\"", + ), + ( + "path\tsource\tversion\tlicence\theader\nassets/a\t-\t1\tMIT\t-\n", + "the source is \"-\"", + ), + ( + "path\tsource\tversion\tlicence\theader\n../a\tX\t1\tMIT\t-\n", + "relative to the repository root", + ), + ( + "path\tsource\tversion\tlicence\theader\nassets/a\tX\t1\tMIT\t | \n", + "names no phrase", + ), + ( + "path\tsource\tversion\tlicence\theader\nassets/a\tX\t1\tMIT\t-\nassets/a\tY\t1\tMIT\t-\n", + "has a row already, on line 2", + ), + ("# only comments\n", "no column line"), + ]; + for (text, want) in bad { + let e = parse_register(text).expect_err(text); + assert!(e.iter().any(|p| p.contains(want)), "{text:?}: {e:?}"); + } + } + + #[test] + fn check_reads_the_tree_git_would_commit() { + let root = std::env::temp_dir().join(format!("og-licences-check-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("assets")).unwrap(); + let w = |rel: &str, text: &str| std::fs::write(root.join(rel), text).unwrap(); + w( + "assets/SOURCES.tsv", + "path\tsource\tversion\tlicence\theader\nassets/third.tsv\tSomeone, Some data\t1.0\tCC-BY-4.0\tSource: Someone, Some data\n", + ); + w("assets/third.tsv", THIRD); + w("NOTICE", NOTICE_OK); + w(".gitignore", "*.bed\n"); + let git = |args: &[&str]| { + let s = Command::new("git") + .arg("-C") + .arg(&root) + .args(args) + .output() + .unwrap(); + assert!( + s.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&s.stderr) + ); + }; + git(&["init", "-q"]); + git(&["add", "assets/SOURCES.tsv", "assets/third.tsv"]); + // An ignored file is not checked; an untracked one is. + w("assets/ignored.bed", "x\n"); + let ok = check(&root); + assert!(ok.is_ok(), "{ok:?}"); + w("assets/planted.tsv", "x\n"); + let e = check(&root).expect_err("an untracked asset with no row must fail"); + assert!( + e.iter() + .any(|p| p.starts_with("assets/planted.tsv has no row")), + "{e:?}" + ); + let mut out = Vec::new(); + let mut err = Vec::new(); + let root_s = root.to_str().unwrap().to_string(); + assert_eq!( + cli( + &["check".into(), "--root".into(), root_s.clone()], + &mut out, + &mut err + ), + 1 + ); + std::fs::remove_file(root.join("assets/planted.tsv")).unwrap(); + assert_eq!( + cli( + &["check".into(), "--root".into(), root_s], + &mut out, + &mut err + ), + 0 + ); + assert_eq!( + cli(&["check".into(), "--bogus".into()], &mut out, &mut err), + 2 + ); + let _ = std::fs::remove_dir_all(&root); + } } diff --git a/rust/crates/og-dev/src/traits_sources.rs b/rust/crates/og-dev/src/traits_sources.rs index 9f3e99df7..ebda1c7f6 100644 --- a/rust/crates/og-dev/src/traits_sources.rs +++ b/rust/crates/og-dev/src/traits_sources.rs @@ -449,26 +449,66 @@ mod tests { d } - /// A FASTA whose chr2, 60 bases a line, holds the lactase row's base at - /// its position and A everywhere else. + /// The test FASTA's chr2 length, and where the lactase row's base sits + /// in it: line 3, column 5, so the check must use the .fai's line width. + const LEN: u64 = 150; + const POS: u64 = 125; + + /// A 150-base chr2 (60 bases a line, as its .fai says), N everywhere but + /// the lactase row's base at POS, and the lactase row moved to POS. No + /// test writes a chromosome; the e2e case traits-panel-1 checks the real + /// coordinates against the real reference. fn lactase_fasta(d: &Path) -> (PathBuf, Row) { let lct = shipped() .into_iter() .find(|r| r.rsid == "rs4988235") .unwrap(); - let n = usize::try_from(lct.pos).unwrap(); - let mut seq = vec![b'A'; n]; - seq[n - 1] = lct.ref_.as_bytes()[0]; + let lct = Row { pos: POS, ..lct }; + let mut seq = vec![b'N'; usize::try_from(LEN).unwrap()]; + seq[usize::try_from(POS - 1).unwrap()] = lct.ref_.as_bytes()[0]; let body: Vec = seq .chunks(60) .flat_map(|c| c.iter().copied().chain([b'\n'])) .collect(); let fa = d.join("ref.fasta"); std::fs::write(&fa, [b">chr2\n".as_slice(), &body].concat()).unwrap(); - std::fs::write(d.join("ref.fasta.fai"), format!("chr2\t{n}\t6\t60\t61\n")).unwrap(); + std::fs::write(d.join("ref.fasta.fai"), format!("chr2\t{LEN}\t6\t60\t61\n")).unwrap(); (fa, lct) } + #[test] + fn the_test_fasta_is_small() { + let d = scratch("small"); + let (fa, _) = lactase_fasta(&d); + let size = std::fs::metadata(&fa).unwrap().len(); + assert!(size < 4096, "{size} bytes"); + let _ = std::fs::remove_dir_all(&d); + } + + #[test] + fn fasta_check_reads_exactly_the_row_position_and_stops_at_the_contig_end() { + let d = scratch("exact"); + let (fa, lct) = lactase_fasta(&d); + let at = |pos: u64| Row { pos, ..lct.clone() }; + let want = |pos: u64, got: &str| { + vec![format!( + "rs4988235 chr2:{pos}: table ref {}, FASTA {got}", + lct.ref_ + )] + }; + // One base either side holds N: an offset off by one fails. + for pos in [POS - 1, POS + 1] { + assert_eq!(fasta_errors(&fa, &[at(pos)]).unwrap(), (want(pos, "N"), 1)); + } + // The last base reads; one past the declared contig does not. + assert_eq!(fasta_errors(&fa, &[at(LEN)]).unwrap(), (want(LEN, "N"), 1)); + assert_eq!( + fasta_errors(&fa, &[at(LEN + 1)]).unwrap(), + (want(LEN + 1, "past the end of the contig"), 1) + ); + let _ = std::fs::remove_dir_all(&d); + } + #[test] fn the_shipped_table_reads() { let rows = shipped(); @@ -523,15 +563,12 @@ mod tests { #[test] fn the_command_line_fails_on_a_wrong_base_and_too_few_rows() { let d = scratch("cli"); - let (fa, _) = lactase_fasta(&d); + let (fa, lct) = lactase_fasta(&d); let fa = fa.to_string_lossy().into_owned(); - let lct = shipped() - .into_iter() - .find(|r| r.rsid == "rs4988235") - .unwrap(); // The shipped table without its other chr2 rows: one row on the - // FASTA's contig, every other row on a contig it lacks. With - // `wrong`, the lactase row's ref is that base. + // FASTA's contig, moved to the test FASTA's POS, every other row on a + // contig it lacks. With `wrong`, the lactase row's ref is that base. + let pos = POS.to_string(); let text = std::fs::read_to_string(format!("{REPO}/{TABLE}")).unwrap(); let write = |name: &str, wrong: Option<&str>| { let kept: String = text @@ -542,6 +579,9 @@ mod tests { if c.first() == Some(&"chr2") && !lactase { return None; } + if lactase { + c[1] = &pos; + } if let (true, Some(w)) = (lactase, wrong) { c[4] = w; } diff --git a/rust/crates/og-live/Cargo.toml b/rust/crates/og-live/Cargo.toml index f8b7f8ac5..779ae27d6 100644 --- a/rust/crates/og-live/Cargo.toml +++ b/rust/crates/og-live/Cargo.toml @@ -22,6 +22,9 @@ og-io.workspace = true og-pedigree.workspace = true og-render.workspace = true og-bloodtype.workspace = true +# og live serve: the summary the step headlines read while summary.json is +# older than the steps that are done (og summary collect, in-process). +og-summary.workspace = true [dev-dependencies] regex.workspace = true diff --git a/rust/crates/og-live/src/charts.rs b/rust/crates/og-live/src/charts.rs index 63070ae95..959cfa5f6 100644 --- a/rust/crates/og-live/src/charts.rs +++ b/rust/crates/og-live/src/charts.rs @@ -1 +1,16 @@ -//! The live site's charts. Empty until its unit of the 1.0.0 plan lands. +//! The live page's charts. +//! +//! Step 24 writes `_charts.json` beside the report. The browser may +//! fetch it (`/runs//_charts.json`, +//! [`crate::plugins::RUN_GLOBS`]), and `state.json` says under +//! `plugins.charts` whether it is there and when it was written; +//! `assets/live/plugins/charts.js` draws it. + +use pycompat::json::Value; +use std::path::Path; + +/// `plugins.charts` of `state.json`: `{file, mtime}` (mtime in ns) of the +/// run's `_charts.json`, or `{}` while there is none. +pub fn state(run_dir: &Path) -> Value { + crate::search::file_state(run_dir, "_charts.json") +} diff --git a/rust/crates/og-live/src/csp.rs b/rust/crates/og-live/src/csp.rs index a3a75eb2d..2f32881ed 100644 --- a/rust/crates/og-live/src/csp.rs +++ b/rust/crates/og-live/src/csp.rs @@ -1,2 +1,61 @@ -//! The live site's Content-Security-Policy. Empty until its unit of the -//! 1.0.0 plan lands. +//! The live site's Content-Security-Policy and the headers that go with it +//! on every answer of `og live serve`. +//! +//! [`CSP`] is the policy of the site's own pages: scripts, styles, data and +//! frames from this site only, nothing inline, no outside URL, and no page +//! of another site may frame it. A run's own HTML (the report, CPSR, +//! PharmCAT, MultiQC) carries inline styles and, for third-party pages, +//! scripts and outside links; it is sent under [`RUN_HTML_CSP`], sandboxed: +//! no script runs, nothing is fetched from outside, and it keeps its +//! styles. `allow-same-origin` lets its links reach the run's other files; +//! without `allow-scripts` that grants the page nothing else. + +/// The policy of every page, script, style and data answer of the site. +pub const CSP: &str = "default-src 'none'; script-src 'self'; connect-src 'self'; style-src 'self'; \ + img-src 'self' data:; form-action 'self'; base-uri 'none'; frame-src 'self'; frame-ancestors 'none'"; + +/// The policy of a run's own HTML and SVG files. +pub const RUN_HTML_CSP: &str = "sandbox allow-same-origin allow-popups allow-popups-to-escape-sandbox; \ + default-src 'none'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; form-action 'none'; \ + base-uri 'none'; frame-ancestors 'none'"; + +/// `X-Frame-Options` of every answer but the genome viewer's. +pub const DENY: &str = "DENY"; + +/// `X-Frame-Options` of the genome viewer, which the map's reads panel +/// frames. +pub const SAME_ORIGIN: &str = "SAMEORIGIN"; + +/// The headers every answer carries besides its type and length: the policy +/// `csp`, no type sniffing, no referrer, no cache, and the frame option. +pub fn common(csp: &str, frame_options: &str) -> [(&'static str, String); 5] { + [ + ("Content-Security-Policy", csp.to_owned()), + ("X-Content-Type-Options", "nosniff".to_owned()), + ("Referrer-Policy", "no-referrer".to_owned()), + ("Cache-Control", "no-store".to_owned()), + ("X-Frame-Options", frame_options.to_owned()), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_site_policy_allows_nothing_from_outside_and_no_framing() { + for part in [ + "default-src 'none'", + "script-src 'self'", + "frame-ancestors 'none'", + "base-uri 'none'", + ] { + assert!(CSP.contains(part), "{part}"); + } + assert!(!CSP.contains("unsafe")); + assert!(RUN_HTML_CSP.starts_with("sandbox ") && !RUN_HTML_CSP.contains("allow-scripts")); + let h = common(CSP, DENY); + assert_eq!(h[0], ("Content-Security-Policy", CSP.to_owned())); + assert_eq!(h[4].1, "DENY"); + } +} diff --git a/rust/crates/og-live/src/map.rs b/rust/crates/og-live/src/map.rs index 288f8a36f..e3bbc3aef 100644 --- a/rust/crates/og-live/src/map.rs +++ b/rust/crates/og-live/src/map.rs @@ -1,2 +1,320 @@ -//! The genome map on the live site. Empty until its unit of the 1.0.0 plan -//! lands. +//! The genome map's files on the live site. +//! +//! ```text +//! GET /runs//map.json `og map build` of a person's run: stdin is {"results": the +//! body /runs//results.json sends}, the answer is og's +//! stdout (gzip) +//! GET /map/genes/.json `og map genes --chrom `: one chromosome's genes (gzip) +//! GET /map/genes/names.json `og map genes --names`: every gene's name and place (gzip) +//! ``` +//! +//! The site decides nothing about the map: og builds each file and caches it +//! under `STATE_DIR/map`, and the site sends og's bytes unchanged, with its +//! own headers. og writes a file only once it is complete and its exit code +//! sets the status, so the site waits for og to end: 0 is 200; 3 (no map for +//! this run, no gene files) is 404 with og's reason; 127 (og cannot run on +//! this host) is 503; anything else is 500. On 127 and 500, og's first stderr +//! line goes to the server's stderr. Each request writes one access-log row +//! `map`, with the run (or `genes` as the section for a gene file) and never +//! a finding, a gene or a chromosome. +//! +//! A run's map is built from its results, which hold the veil of the results +//! a child's page keeps closed. While the site has no results +//! ([`crate::plugins::ResultsFns`]), the map is not built at all: 503, and og +//! never runs. + +use crate::server::Req; +use crate::state::dumps; +use pycompat::json::Value; +use std::io::{self, Read, Write}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +/// How long og may take. +pub const OG_SECONDS: u64 = 300; +pub const GZIP: &str = "application/gzip"; +const TEXT: &str = "text/plain; charset=utf-8"; +/// The answer when og cannot run on this host. +pub const NO_OG: &str = "The map needs og on this host: run scripts/setup.sh --og\n"; +/// The answer when og failed. +pub const FAILED: &str = "The map could not be built: the server's log says why.\n"; +/// The answer while the site has no results to build a map from. +pub const NO_RESULTS: &str = + "The map needs the results, which og live serve does not serve yet: the report still works.\n"; + +/// How og ended: its exit code, or why it has none. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Ended { + Code(i32), + /// Not found, or not runnable here. + Missing, + Timeout, + /// Killed by a signal, or another error starting it. + Other(String), +} + +/// Runs `og ARGS` with `stdin`, up to [`OG_SECONDS`]: how it ended, its +/// stdout and its first stderr line. +pub fn run_og( + og: &Path, + args: &[String], + stdin: &[u8], + limit: Duration, +) -> (Ended, Vec, String) { + let child = Command::new(og) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn(); + let mut child = match child { + Ok(c) => c, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + return (Ended::Missing, Vec::new(), String::new()); + } + Err(e) => { + return ( + Ended::Other(e.kind().to_string()), + Vec::new(), + e.to_string(), + ); + } + }; + let mut pipe_in = child.stdin.take(); + let input = stdin.to_vec(); + let writer = thread::spawn(move || { + if let Some(p) = pipe_in.as_mut() { + let _ = p.write_all(&input); + } + }); + let mut so = child.stdout.take(); + let mut se = child.stderr.take(); + let out_t = thread::spawn(move || { + let mut b = Vec::new(); + if let Some(p) = so.as_mut() { + let _ = p.read_to_end(&mut b); + } + b + }); + let err_t = thread::spawn(move || { + let mut b = Vec::new(); + if let Some(p) = se.as_mut() { + let _ = p.read_to_end(&mut b); + } + b + }); + let end = Instant::now() + limit; + let status = loop { + match child.try_wait() { + Ok(Some(s)) => break Some(s), + Ok(None) if Instant::now() >= end => { + let _ = child.kill(); + let _ = child.wait(); + break None; + } + Ok(None) => thread::sleep(Duration::from_millis(20)), + Err(_) => break None, + } + }; + let _ = writer.join(); + let out = out_t.join().unwrap_or_default(); + let err = err_t.join().unwrap_or_default(); + let first = String::from_utf8_lossy(&err) + .trim() + .lines() + .next() + .unwrap_or("") + .to_owned(); + let Some(status) = status else { + return ( + Ended::Timeout, + Vec::new(), + format!("og did not end within {} s", limit.as_secs()), + ); + }; + let ended = match status.code() { + Some(c) => Ended::Code(c), + None => { + use std::os::unix::process::ExitStatusExt; + Ended::Other(format!("-{}", status.signal().unwrap_or(0))) + } + }; + (ended, out, first) +} + +/// Runs og with `args` and `stdin` and sends its answer; one access-log row. +pub fn relay( + req: &mut Req<'_>, + args: &[String], + stdin: &[u8], + run: &str, + section: &str, +) -> io::Result<()> { + let og = req.live.cfg.og.clone(); + let (ended, out, err) = run_og(&og, args, stdin, Duration::from_secs(OG_SECONDS)); + let verb = args.get(1).map_or("", String::as_str); + match ended { + Ended::Code(0) => { + req.log("map", run, section, 200); + req.send_body(200, GZIP, out) + } + Ended::Code(3) => { + req.log("map", run, section, 404); + req.send_body(404, TEXT, format!("{err}\n")) + } + Ended::Code(127) | Ended::Missing => { + if !err.is_empty() { + eprintln!("og live serve: og map {verb} unavailable: {err}"); + } + req.log("map", run, section, 503); + req.send_body(503, TEXT, NO_OG) + } + other => { + let code = match other { + Ended::Code(c) => c.to_string(), + Ended::Timeout => "timeout".into(), + Ended::Other(w) => w, + Ended::Missing => unreachable!("answered above"), + }; + eprintln!("og live serve: og map {verb} failed ({code}): {err}"); + req.log("map", run, section, 500); + req.send_body(500, TEXT, FAILED) + } + } +} + +/// Where og keeps the map files it built: `STATE_DIR/map`. +fn cache_dir(req: &Req<'_>) -> String { + req.live + .cfg + .state_dir + .join("map") + .to_string_lossy() + .into_owned() +} + +/// `GET /runs//map.json`: og map build of a person's run. +pub fn build(req: &mut Req<'_>, name: &str) -> io::Result<()> { + let live = req.live; + let Some(run) = live.run(name).filter(|r| !r.is_family()) else { + req.log("map", name, "-", 404); + return req.send_body(404, TEXT, "not found\n"); + }; + let Some(fns) = live.cfg.results else { + req.log("map", &run.name, "-", 503); + return req.send_body(503, TEXT, NO_RESULTS); + }; + let body = match crate::plugins::results_body(live, &run, &fns) { + Ok(b) => b, + Err(e) => { + eprintln!( + "og live serve: results of {} not built for the map: {e}", + run.name + ); + req.log("map", &run.name, "-", 500); + return req.send_body(500, TEXT, FAILED); + } + }; + let mut o = indexmap::IndexMap::new(); + o.insert("results".to_owned(), body); + let stdin = dumps(&Value::Object(o)); + let args: Vec = [ + "map", + "build", + "--root", + &live.root.to_string_lossy(), + "--run", + &run.name, + "--assets", + &live.cfg.repo_assets.to_string_lossy(), + "--cache-dir", + &cache_dir(req), + ] + .iter() + .map(|s| (*s).to_owned()) + .collect(); + relay(req, &args, stdin.as_bytes(), &run.name, "-") +} + +/// Whether `which` is a gene file the map has: `chr` and one or two letters +/// or digits, or `names`. +pub fn valid_genes(which: &str) -> bool { + which == "names" + || which.strip_prefix("chr").is_some_and(|c| { + (1..=2).contains(&c.len()) && c.bytes().all(|b| b.is_ascii_alphanumeric()) + }) +} + +/// `GET /map/genes/.json` and `/map/genes/names.json`: og map genes. +pub fn genes(req: &mut Req<'_>, which: &str) -> io::Result<()> { + let mut args: Vec = vec![ + "map".into(), + "genes".into(), + "--root".into(), + req.live.root.to_string_lossy().into_owned(), + "--cache-dir".into(), + cache_dir(req), + ]; + if which == "names" { + args.push("--names".into()); + } else { + args.push("--chrom".into()); + args.push(which.into()); + } + relay(req, &args, b"", "-", "genes") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gene_files() { + assert!( + valid_genes("names") + && valid_genes("chr1") + && valid_genes("chrX") + && valid_genes("chr22") + ); + assert!( + !valid_genes("chr") + && !valid_genes("chr123") + && !valid_genes("chr1.x") + && !valid_genes("x1") + ); + } + + #[test] + fn og_missing_or_failing() { + let (e, _, _) = run_og( + Path::new("/nonexistent/og"), + &[], + b"", + Duration::from_secs(5), + ); + assert_eq!(e, Ended::Missing); + let (e, out, err) = run_og( + Path::new("/bin/sh"), + &[ + "-c".into(), + "cat; echo why >&2; echo more >&2; exit 3".into(), + ], + b"in", + Duration::from_secs(5), + ); + assert_eq!( + (e, out.as_slice(), err.as_str()), + (Ended::Code(3), &b"in"[..], "why") + ); + let (e, _, err) = run_og( + Path::new("/bin/sh"), + &["-c".into(), "sleep 5".into()], + b"", + Duration::from_millis(200), + ); + assert_eq!(e, Ended::Timeout); + assert!(err.contains("did not end")); + } +} diff --git a/rust/crates/og-live/src/plugins.rs b/rust/crates/og-live/src/plugins.rs index e6939bce0..eac6b131b 100644 --- a/rust/crates/og-live/src/plugins.rs +++ b/rust/crates/og-live/src/plugins.rs @@ -1 +1,882 @@ -//! The live site's plugins. Empty until its unit of the 1.0.0 plan lands. +//! The live site's plugins, compiled into og: each adds a part of +//! `state.json` (`plugins.`), routes, files of a run folder the +//! browser may fetch, or a change to the served report. Their page scripts +//! are `assets/live/plugins/.js`, which call +//! `window.PGP.register({name, onState, onEvent, panel})`. +//! +//! | plugin | state.json | routes | run files | report | +//! |---------|------------|--------|-----------|--------| +//! | charts | [`crate::charts`] | | `*_charts.json` | | +//! | family | [`crate::family`] | `/family//`, `/runs//family.json` | | | +//! | map | | [`crate::map`] | | | +//! | ops | [`crate::ops`] | `/metrics` | | | +//! | recheck | [`crate::recheck`] | | `recheck/state.json` | | +//! | results | | `/runs//results.json`, `/runs//finding//` | | | +//! | search | [`crate::search`] | | `*_search_index.json` | | +//! | viewer | | `/runs//viewer/...` ([`crate::viewer`]) | | its links point here | +//! +//! The results are a seam ([`ResultsFns`]): `og live results` fills it when +//! it lands. Until then `results.json` carries a note and no card, a +//! finding's page is 404 and the map is not built. + +use crate::family::{self, inside, realpath}; +use crate::ops::{self, Asker, Ctx}; +use crate::runs::{self, Run}; +use crate::server::{Req, quote}; +use crate::state::{Live, get, now, obj, s}; +use crate::static_files as sf; +use crate::viewer::{self, LIVE_CSP, unquote}; +use indexmap::IndexMap; +use og_runlog::host::{Host, System}; +use og_runlog::state::State; +use pycompat::json::{Dump, Value, dumps}; +use std::collections::BTreeSet; +use std::fs::{self, File}; +use std::io::{self, Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +/// The files of a run folder the plugins let the browser fetch, as +/// `fnmatch` patterns. +pub const RUN_GLOBS: [&str; 3] = ["*_charts.json", "recheck/state.json", "*_search_index.json"]; + +/// `results.json` of run `name` from its summary, given the files the +/// run's report links (`None`: any). +pub type BuildFn = fn(&str, &Value, Option<&BTreeSet>) -> Result; + +/// Each person's results: what `og live results` gives the site. +#[derive(Clone, Copy)] +pub struct ResultsFns { + /// `results.json` of run `name` from its summary: the chapters, the + /// cards and the findings (`[{id, level, ...}]`). The files the run's + /// report links (`None`: any). + pub build: BuildFn, + /// The home page's overview of a summary: `attention`, `levels`, `top`, + /// `lead`, `lead_closed`, `screened`. + pub overview: fn(&Value) -> Result, +} + +/// The note of `results.json` while the site has no results. +pub const NO_RESULTS_NOTE: &str = + "The results are not on og live serve yet: the report and the Progress tab still work."; +/// The note of `results.json` while no step has finished. +pub const NO_STEP_NOTE: &str = + "No step has finished yet: the results appear here as the steps end."; + +const TEXT: &str = "text/plain; charset=utf-8"; + +/// A finding's id: a lower-case letter or digit, then up to 79 of them or +/// `-`. +pub fn valid_finding_id(id: &str) -> bool { + let b = id.as_bytes(); + let ok = |c: &u8| c.is_ascii_lowercase() || c.is_ascii_digit(); + !b.is_empty() && b.len() <= 80 && ok(&b[0]) && b[1..].iter().all(|c| ok(c) || *c == b'-') +} + +// --- state.json --------------------------------------------------------------------------- + +/// `plugins` of `state.json`: each plugin's part, in the order of the +/// plugins' names. +pub fn state_hooks( + live: &Live, + run: &Run, + d: &Path, + st: &State<'_>, + _v: &Value, + events: &[Value], +) -> Value { + let mut o = IndexMap::new(); + o.insert("charts".into(), crate::charts::state(d)); + o.insert("family".into(), family_hook(live, d)); + o.insert("ops".into(), ops_hook(live, d, st, events)); + o.insert("recheck".into(), crate::recheck::state(d)); + o.insert("search".into(), crate::search::state(d)); + let _ = run; + Value::Object(o) +} + +/// `plugins.ops`: the run health card, or the line it shows instead. +pub fn ops_hook(live: &Live, d: &Path, st: &State<'_>, events: &[Value]) -> Value { + let host = System; + let ctx = Ctx { + now: now(), + steps: &live.steps, + host: &host, + tz: &live.tz, + }; + ops::hook(Some(d), st, events, &live.cfg.ops, &ctx).to_json() +} + +fn folder_name(d: &Path) -> String { + d.components() + .next_back() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// `plugins.family`: the family's state on a family's page; on a person's +/// page the families that list it and whether it is a child +/// ([`member_state`]). +pub fn family_hook(live: &Live, d: &Path) -> Value { + let name = folder_name(d); + if family::is_family(d) && inside(d, &live.root) { + return match family::family_state(&name, d) { + Ok(v) => v, + Err(e) => { + eprintln!( + "og live serve: the family state of {name} could not be read: {}", + e.0 + ); + obj([("error", s("the family's files cannot be read"))]) + } + }; + } + member_state(&live.root, &name) +} + +/// On the page of sample `sample`: the families that list it, and whether +/// its pages treat it as a child. With no readable family listing it while +/// a family cannot be read, the answer is unknown, never "in no family". +pub fn member_state(root: &Path, sample: &str) -> Value { + let fams = family::families(root); + let child = runs::child_of(&fams, sample); + let mine: Vec = fams + .iter() + .filter(|f| match get(f, "members") { + Some(Value::Array(ms)) => ms + .iter() + .any(|m| matches!(get(m, "sample"), Some(Value::Str(x)) if x == sample)), + _ => false, + }) + .map(|f| { + let n = get(f, "name").cloned().unwrap_or(Value::Null); + let url = Value::Str(format!( + "/runs/{}/", + match &n { + Value::Str(x) => x.clone(), + _ => String::new(), + } + )); + obj([("name", n), ("url", url)]) + }) + .collect(); + if !mine.is_empty() { + return obj([ + ("kind", s("member")), + ("families", Value::Array(mine)), + ("child", Value::Bool(child)), + ]); + } + if child { + obj([ + ("kind", s("member")), + ("families", Value::Array(Vec::new())), + ("unknown", Value::Bool(true)), + ]) + } else { + Value::Null + } +} + +// --- the report --------------------------------------------------------------------------- + +/// The served report: its viewer links and note pointed at this site's +/// genome viewer of the run, when the viewer is set up. +pub fn report_filter(live: &Live, run: &str, html: String) -> String { + if viewer_ready(live).is_some() { + return html; + } + let here = format!("/runs/{}/viewer/", quote(run)); + html.replace( + &format!("href=\"{}#", og_render::html::VIEWER_URL), + &format!("href=\"{here}#"), + ) + .replace(og_render::notes::VIEWER_NOTE, LIVE_NOTE) +} + +/// The note the served report carries for the site's viewer. +pub const LIVE_NOTE: &str = "

    The Viewer column opens the reads at each row in this site's genome viewer \ + (docs/live-site.md).

    "; + +// --- the routes --------------------------------------------------------------------------- + +/// `/runs//` with a valid run name: `(name, tail)`. +fn run_route<'p>(path: &'p str, tail: &str) -> Option<&'p str> { + let rest = path.strip_prefix("/runs/")?; + let name = rest.strip_suffix(tail)?; + runs::valid_run(name).then_some(name) +} + +/// Answers `path` when a plugin has a route for it: `Ok(true)` once +/// answered, `Ok(false)` for a path no plugin takes (or a method its route +/// does not). +pub fn route(req: &mut Req<'_>, path: &str) -> io::Result { + let get_ = req.method == "GET"; + // family + if let Some(rest) = path.strip_prefix("/family/") + && let Some((name, f)) = rest.split_once('/') + && runs::valid_run(name) + && runs::valid_run(f) + { + if !get_ { + return Ok(false); + } + family_file(req, name, f)?; + return Ok(true); + } + if let Some(name) = run_route(path, "/family.json") { + if !get_ { + return Ok(false); + } + family_json(req, name)?; + return Ok(true); + } + // map + if let Some(name) = run_route(path, "/map.json") { + if !get_ { + return Ok(false); + } + crate::map::build(req, name)?; + return Ok(true); + } + if let Some(which) = path + .strip_prefix("/map/genes/") + .and_then(|w| w.strip_suffix(".json")) + .filter(|w| crate::map::valid_genes(w)) + { + if !get_ { + return Ok(false); + } + crate::map::genes(req, which)?; + return Ok(true); + } + // ops + if path == "/metrics" { + if !get_ { + return Ok(false); + } + metrics(req)?; + return Ok(true); + } + // results + if let Some(name) = run_route(path, "/results.json") { + if !get_ { + return Ok(false); + } + results(req, name)?; + return Ok(true); + } + if let Some(rest) = path.strip_prefix("/runs/") + && let Some((name, tail)) = rest.split_once("/finding/") + && runs::valid_run(name) + && let Some(id) = tail.strip_suffix('/') + && !id.is_empty() + && id.chars().count() <= 200 + && !id.contains('/') + { + if !get_ { + return Ok(false); + } + finding(req, name, id)?; + return Ok(true); + } + // viewer + if let Some(rest) = path.strip_prefix("/runs/") { + if let Some(run) = rest + .strip_suffix("/viewer") + .filter(|r| !r.is_empty() && !r.contains('/')) + { + if !get_ { + return Ok(false); + } + viewer_slash(req, run)?; + return Ok(true); + } + if let Some((run, tail)) = rest.split_once('/') + && !run.is_empty() + && let Some(file) = tail.strip_prefix("viewer/") + { + if !get_ { + return Ok(false); + } + viewer_route(req, run, file)?; + return Ok(true); + } + } + Ok(false) +} + +// --- family ------------------------------------------------------------------------------- + +/// The family folder of `name`, when it is one inside the root. +fn family_dir(req: &Req<'_>, name: &str) -> Option { + req.run_dir(name) + .filter(|d| family::is_family(d) && inside(d, &req.live.root)) +} + +/// `GET /family//`: the pedigree drawn now, or one of the +/// family's files listed in `state.json`; any other name is 404. +fn family_file(req: &mut Req<'_>, name: &str, f: &str) -> io::Result<()> { + let Some(d) = family_dir(req, name) else { + req.log("family", name, f, 404); + return req.send_body(404, TEXT, "not found\n"); + }; + if f == "pedigree.svg" { + let svg = family::members_of(&d).and_then(|p| og_pedigree::layout::draw(&p, name)); + return match svg { + Ok(svg) => { + req.log("family", name, f, 200); + req.send_body(200, "image/svg+xml", svg) + } + Err(_) => { + req.log("family", name, f, 404); + req.send_body(404, TEXT, "not found\n") + } + }; + } + let files = family::members_of(&d) + .map(|p| family::served_files(&d, &p)) + .unwrap_or_default(); + let p = d.join(f); + let body = (files.iter().any(|x| x == f) && inside(&p, &d)) + .then(|| fs::read(&p).ok()) + .flatten(); + let Some(body) = body else { + req.log("family", name, f, 404); + return req.send_body(404, TEXT, "not found\n"); + }; + let ctype = match sf::ext(f) { + ".svg" => "image/svg+xml", + _ => TEXT, + }; + req.log("family", name, f, 200); + req.send_body(200, ctype, body) +} + +/// `GET /runs//family.json`: the family page's data, with what +/// only the site knows: each member run's progress, attention count and +/// whether its pages treat it as a child. +fn family_json(req: &mut Req<'_>, name: &str) -> io::Result<()> { + let Some(d) = family_dir(req, name) else { + req.log("family", name, "family.json", 404); + return req.send_body(404, TEXT, "not found\n"); + }; + match family_page(req.live, name, &d) { + Ok(v) => { + req.log("family", name, "family.json", 200); + req.send_json(&v) + } + Err(kind) => { + eprintln!("og live serve: family.json of {name} not built: {kind}"); + req.log("family", name, "family.json", 500); + req.send_body(500, TEXT, "family not readable\n") + } + } +} + +fn family_page(live: &Live, name: &str, d: &Path) -> Result { + let st = family::family_state(name, d).map_err(|e| e.0)?; + let listed = live.runs(); + let (mut runs_o, mut candidates, mut summaries) = + (IndexMap::new(), IndexMap::new(), IndexMap::new()); + let members = match get(&st, "members") { + Some(Value::Array(m)) => m.clone(), + _ => Vec::new(), + }; + for p in &members { + let Some(Value::Str(sample)) = get(p, "sample") else { + continue; + }; + let Some(r) = listed.get(sample) else { + continue; + }; + let rd = r.folder(&live.root); + let rd_s = Value::Str(rd.to_string_lossy().into_owned()); + summaries.insert(sample.clone(), rd_s.clone()); + if r.is_family() { + continue; + } + candidates.insert(sample.clone(), rd_s); + let (bst, events) = live.base_state(r); + let v = live.base_json(&bst, events.len()); + let planned: Vec<_> = bst + .steps + .iter() + .filter(|x| x.state.as_str() != "not planned") + .collect(); + let done = planned + .iter() + .filter(|x| x.state.as_str() == "done") + .count(); + let (mut n, mtime) = live.attention(&rd); + if live.gated(sample, &rd, &v, Some(&listed)) { + n = None; + } + runs_o.insert( + sample.clone(), + obj([ + ("url", Value::Str(format!("/runs/{sample}/"))), + ("state", s(bst.run.as_str())), + ("planned", Value::Int(planned.len() as i128)), + ("done", Value::Int(done as i128)), + ("attention", n.unwrap_or(Value::Null)), + ("results", Value::Bool(mtime.is_some())), + ("child", Value::Bool(live.child(sample, Some(&listed)))), + ]), + ); + } + let input = obj([ + ("runs", Value::Object(runs_o)), + ("candidates", Value::Object(candidates)), + ("summaries", Value::Object(summaries)), + ]); + family::site_page(name, d, &live.root, &input).map_err(|e| e.0) +} + +// --- ops ---------------------------------------------------------------------------------- + +/// `GET /metrics`: every run's health in Prometheus' text format. Without a +/// password the site is loopback only and `/metrics` is open, like every +/// page. +fn metrics(req: &mut Req<'_>) -> io::Result<()> { + let who = Asker { + login_required: false, + logged_in: false, + authorization: req.header("authorization"), + }; + let token = req.live.cfg.metrics_token_file.clone(); + match ops::metrics_allowed(&who, || ops::read_token(token.as_deref())) { + Err(e) => { + eprintln!( + "og live serve: --metrics-token-file cannot be read: {}", + e.kind() + ); + req.log("metrics", "-", "-", 503); + return req.send_body(503, TEXT, "the metrics token file cannot be read\n"); + } + Ok(false) => { + req.log("metrics", "-", "-", 401); + return req.send_body(401, TEXT, "login or bearer token required\n"); + } + Ok(true) => {} + } + let live = req.live; + let mut cards = Vec::new(); + let mut errors = 0; + for (name, r) in live.runs() { + let (st, events) = live.base_state(&r); + match ops::hook( + Some(&r.folder(&live.root)), + &st, + &events, + &live.cfg.ops, + &Ctx { + now: now(), + steps: &live.steps, + host: &System, + tz: &live.tz, + }, + ) { + ops::OpsState::Ok(o) => cards.push((name, o)), + ops::OpsState::Error(_) => { + errors += 1; + eprintln!("og live serve: {name}: run health not read"); + } + } + } + let refs: Vec<(&str, &ops::RunOps)> = cards + .iter() + .map(|(n, o)| (n.as_str(), o.as_ref())) + .collect(); + let text = ops::metrics_text( + &refs, + ops::disk_warn_ratio(&live.cfg.ops), + System.physical_memory(), + errors, + ); + req.log("metrics", "-", "-", 200); + req.send_body(200, ops::CONTENT_TYPE, text) +} + +// --- results ------------------------------------------------------------------------------ + +/// What `GET /runs//results.json` sends, built the same way for the +/// map: the results of the run's summary, or a note while no step is done. +pub fn results_body(live: &Live, run: &Run, fns: &ResultsFns) -> Result { + let d = run.folder(&live.root); + let (st, events) = live.base_state(run); + let v = live.base_json(&st, events.len()); + match live.summary(&run.name, &d, &v) { + Some(sm) => { + let files = if run.dir.is_some() { + runs::run_files(&run.name, run.dir.as_deref()) + } else { + BTreeSet::new() + }; + (fns.build)(&run.name, &sm, Some(&files)) + } + None => Ok(note(&run.name, NO_STEP_NOTE)), + } +} + +fn note(name: &str, text: &str) -> Value { + obj([ + ("schema_version", Value::Int(1)), + ("sample", s(name)), + ("chapters", Value::Array(Vec::new())), + ("note", s(text)), + ]) +} + +/// `GET /runs//results.json`: the person's results page data. A reader +/// error gives no cards and a note, never a 500. +fn results(req: &mut Req<'_>, name: &str) -> io::Result<()> { + let live = req.live; + let Some(run) = live.run(name).filter(|r| !r.is_family()) else { + req.log("results", name, "-", 404); + return req.send_body(404, TEXT, "not found\n"); + }; + let out = match live.cfg.results { + None => note(&run.name, NO_RESULTS_NOTE), + Some(fns) => { + let d = run.folder(&live.root); + let (st, events) = live.base_state(&run); + let v = live.base_json(&st, events.len()); + match results_body(live, &run, &fns) { + Ok(mut out) => { + // A child whose report the site keeps closed: the + // flagged-card count of summary.json, which the family + // page shows once every result to act on is opened. + if live.summary(&run.name, &d, &v).is_some() + && live.gated(&run.name, &d, &v, None) + && let Value::Object(o) = &mut out + { + o.insert( + "saved_attention".into(), + live.attention(&d).0.unwrap_or(Value::Null), + ); + } + out + } + Err(_) => { + eprintln!("og live serve: results of {} not built", run.name); + note( + &run.name, + "The results could not be read; the report and the Progress tab still work.", + ) + } + } + } + }; + req.log("results", &run.name, "-", 200); + req.send_json(&out) +} + +/// `GET /runs//finding//`: one finding's page, 404 unless `id` is +/// a finding of the run's results now; for a child with a result to act on, +/// the page for any well-formed id (a 404 would tell a guessed id apart +/// from a result the child has). +fn finding(req: &mut Req<'_>, name: &str, id: &str) -> io::Result<()> { + let live = req.live; + let run = live.run(name).filter(|r| !r.is_family()); + let mut ok = false; + if let (Some(run), Some(fns)) = (&run, live.cfg.results) + && valid_finding_id(id) + { + let d = run.folder(&live.root); + let (st, events) = live.base_state(run); + let v = live.base_json(&st, events.len()); + let found = match live.summary(&run.name, &d, &v) { + Some(sm) => (fns.build)(&run.name, &sm, None).map(|b| match get(&b, "findings") { + Some(Value::Array(f)) => f.clone(), + _ => Vec::new(), + }), + None => Ok(Vec::new()), + }; + match found { + Ok(found) => { + let is = |f: &Value, k: &str, want: &str| matches!(get(f, k), Some(Value::Str(x)) if x == want); + ok = found.iter().any(|f| is(f, "id", id)) + || (found.iter().any(|f| is(f, "level", "act")) && live.child(&run.name, None)); + } + Err(e) => eprintln!("og live serve: finding of {} not checked: {e}", run.name), + } + } + let Some(run) = run.filter(|_| ok) else { + req.log("finding", name, "-", 404); + return req.send_body(404, TEXT, "not found\n"); + }; + match sf::finding_page(&live.cfg.assets, &run.name, id) { + Ok(page) => { + req.log("finding", &run.name, "-", 200); + req.send_body(200, "text/html; charset=utf-8", page) + } + Err(_) => req.deny(500, None), + } +} + +// --- viewer ------------------------------------------------------------------------------- + +const LOCAL_TEXT: &str = "served from this computer only."; +const LIVE_TEXT: &str = "served by the run site, behind its login."; + +/// Why the viewer cannot run, or `None` when it can. +pub fn viewer_ready(live: &Live) -> Option<&'static str> { + let (Some(fasta), Some(igv)) = (&live.cfg.viewer_fasta, &live.cfg.viewer_igv_js) else { + return Some( + "the genome viewer is not set up: start the site with --viewer-fasta and --viewer-igv-js \ + (scripts/owngenome-live.sh passes both)", + ); + }; + let mut fai = fasta.clone().into_os_string(); + fai.push(".fai"); + if !fasta.is_file() || !Path::new(&fai).is_file() { + return Some("the genome viewer's reference or its .fai is missing"); + } + if !igv.is_file() { + return Some("igv.min.js is missing"); + } + None +} + +/// The viewer's headers: same-origin resources, its own policy, framing by +/// this site only. +fn viewer_head( + req: &mut Req<'_>, + code: u16, + ctype: &str, + len: u64, + extra: &[(&str, String)], +) -> io::Result<()> { + let mut all = vec![("Cross-Origin-Resource-Policy", "same-origin".to_owned())]; + all.extend(extra.iter().cloned()); + req.head( + code, + ctype, + Some(len), + &all, + LIVE_CSP, + crate::csp::SAME_ORIGIN, + ) +} + +fn viewer_send(req: &mut Req<'_>, code: u16, ctype: &str, body: &[u8]) -> io::Result<()> { + viewer_head(req, code, ctype, body.len() as u64, &[])?; + req.body(body) +} + +/// `src`, or the one byte range the request asks for (206, or 416 past the +/// end), under the viewer's headers. +fn viewer_file(req: &mut Req<'_>, ctype: &str, src: &Path) -> io::Result<()> { + let mut f = File::open(src)?; + let size = f.metadata()?.len(); + let rng = viewer::parse_range(req.header("range"), size); + if rng == viewer::Range::Bad { + return viewer_head( + req, + 416, + TEXT, + 0, + &[("Content-Range", format!("bytes */{size}"))], + ); + } + let (start, end) = match rng { + viewer::Range::Part(a, b) => (a, b), + _ => (0, size.saturating_sub(1)), + }; + let length = if size == 0 { 0 } else { end - start + 1 }; + let mut extra = vec![("Accept-Ranges", "bytes".to_owned())]; + if rng != viewer::Range::Whole { + extra.push(("Content-Range", format!("bytes {start}-{end}/{size}"))); + } + let code = if rng == viewer::Range::Whole { + 200 + } else { + 206 + }; + viewer_head(req, code, ctype, length, &extra)?; + if req.head_only || length == 0 { + return Ok(()); + } + f.seek(SeekFrom::Start(start))?; + let mut left = length; + let mut buf = vec![0u8; (1usize << 20).min(length as usize)]; + while left > 0 { + let want = buf.len().min(left as usize); + let n = f.read(&mut buf[..want])?; + if n == 0 { + break; + } + req.out().write_all(&buf[..n])?; + left -= n as u64; + } + req.out().flush() +} + +/// `(config, files)` of one run's viewer: the run folder's reads, VCF and SV +/// VCF that lie inside the run and the root, each track with its index +/// there too. Files 0 and 1 are the reference and its `.fai`, from any +/// folder. +fn run_config(live: &Live, run: &str, d: &Path) -> Result<(Value, Vec>), String> { + let top = realpath(d); + let root = live.root.clone(); + let ok = |p: &Path| { + let real = realpath(p); + [&top, &root].iter().all(|t| { + let mut pre = t.to_string_lossy().trim_end_matches('/').to_owned(); + pre.push('/'); + real.to_string_lossy().starts_with(&pre) + }) + }; + let sfiles = viewer::sample_files(d, run); + let keep = |p: Option| p.filter(|p| ok(p)); + let fasta = live.cfg.viewer_fasta.clone().unwrap_or_default(); + let cfg = viewer::build_config( + run, + &fasta, + keep(sfiles.reads).as_deref(), + keep(sfiles.vcf).as_deref(), + keep(sfiles.sv_vcf).as_deref(), + None, + )?; + let mut files: Vec> = cfg + .files + .iter() + .enumerate() + .map(|(n, f)| (n < 2 || ok(f)).then(|| f.clone())) + .collect(); + let mut json = cfg.json; + let nums = |t: &Value| -> Vec { + ["url", "indexURL"] + .iter() + .filter_map(|k| match get(t, k) { + Some(Value::Str(u)) => u.split('/').nth(1).and_then(|n| n.parse().ok()), + _ => None, + }) + .collect() + }; + if let Value::Object(o) = &mut json + && let Some(Value::Array(tracks)) = o.get_mut("tracks") + { + for t in tracks.iter() { + let ns = nums(t); + if !ns + .iter() + .all(|&n| files.get(n).is_some_and(Option::is_some)) + { + for n in ns { + if let Some(f) = files.get_mut(n) { + *f = None; + } + } + } + } + tracks.retain(|t| { + nums(t) + .iter() + .all(|&n| files.get(n).is_some_and(Option::is_some)) + }); + } + Ok((json, files)) +} + +/// `GET /runs//viewer`: to the page. +fn viewer_slash(req: &mut Req<'_>, raw: &str) -> io::Result<()> { + let run = unquote(raw); + if req.run_dir(&run).is_none() { + return req.deny(404, None); + } + let to = format!("/runs/{}/viewer/", quote(&run)); + req.redirect(301, &to) +} + +/// `GET /runs//viewer/`: the page, its script, igv.js, the +/// config and the config's files by byte range. +fn viewer_route(req: &mut Req<'_>, raw: &str, rest: &str) -> io::Result<()> { + let run = unquote(raw); + let live = req.live; + let page = rest.is_empty() || rest == "index.html"; + let Some(d) = req.run_dir(&run) else { + req.log("viewer", "-", "-", 404); + return req.deny(404, None); + }; + if let Some(why) = viewer_ready(live) { + req.log("viewer", &run, "-", 404); + return viewer_send(req, 404, TEXT, why.as_bytes()); + } + let dir = live.cfg.repo_assets.join("viewer"); + if page { + let Ok(html) = fs::read_to_string(dir.join("index.html")) else { + return req.deny(404, None); + }; + req.log("viewer", &run, "-", 200); + return viewer_send( + req, + 200, + "text/html; charset=utf-8", + html.replace(LOCAL_TEXT, LIVE_TEXT).as_bytes(), + ); + } + if rest == "viewer.js" { + let Ok(js) = fs::read(dir.join("viewer.js")) else { + return req.deny(404, None); + }; + return viewer_send(req, 200, "text/javascript; charset=utf-8", &js); + } + if rest == "igv.min.js" { + let igv = live.cfg.viewer_igv_js.clone().unwrap_or_default(); + return viewer_file(req, "text/javascript; charset=utf-8", &igv); + } + let (cfg, files) = match run_config(live, &run, &d) { + Ok(x) => x, + Err(e) => { + req.log("viewer", &run, "-", 409); + return req.deny( + 409, + Some(&format!("the genome viewer cannot open this run: {e}")), + ); + } + }; + if rest == "config.json" { + let text = dumps( + &cfg, + &Dump { + indent: Some(1), + ..Dump::default() + }, + ) + .unwrap_or_default(); + return viewer_send(req, 200, "application/json", text.as_bytes()); + } + if let Some(dm) = rest.strip_prefix("data/") + && let Some((n, fname)) = dm.split_once('/') + && !n.is_empty() + && n.bytes().all(|b| b.is_ascii_digit()) + && !fname.is_empty() + && let Ok(n) = n.parse::() + && let Some(Some(f)) = files.get(n) + && f.file_name().map(|x| x.to_string_lossy().into_owned()) == Some(unquote(fname)) + { + let f = f.clone(); + return viewer_file(req, "application/octet-stream", &f); + } + req.deny(404, None) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn finding_ids() { + assert!(valid_finding_id("clinvar-brca1-1") && valid_finding_id("0")); + assert!(!valid_finding_id("") && !valid_finding_id("-a") && !valid_finding_id("A")); + assert!(!valid_finding_id(&"a".repeat(81)) && valid_finding_id(&"a".repeat(80))); + } + + #[test] + fn run_routes() { + assert_eq!(run_route("/runs/S1/map.json", "/map.json"), Some("S1")); + assert_eq!(run_route("/runs/a/b/map.json", "/map.json"), None); + assert_eq!(run_route("/runs/.x/map.json", "/map.json"), None); + } +} diff --git a/rust/crates/og-live/src/recheck.rs b/rust/crates/og-live/src/recheck.rs index 9c9f1bd55..8cd3e1d44 100644 --- a/rust/crates/og-live/src/recheck.rs +++ b/rust/crates/og-live/src/recheck.rs @@ -1,2 +1,222 @@ -//! The recheck view of the live site. Empty until its unit of the 1.0.0 -//! plan lands. +//! The database re-check on the live page. +//! +//! `og recheck` writes `/recheck/state.json`: which databases the run +//! used have a newer release, which steps that would change, and the last +//! rerun. The browser may fetch that file (`/runs//recheck/state.json`, +//! [`crate::plugins::RUN_GLOBS`]), and `state.json` carries a short form of +//! it under `plugins.recheck` ([`state`]): a banner line and, per step, the +//! badges the step map shows ("newer ClinVar available", "redone +//! 2026-10-05"). `assets/live/plugins/recheck.js` draws them. Release names +//! and step numbers only: the check holds no result. + +use indexmap::IndexMap; +use pycompat::json::{self, Value}; +use pycompat::value::{py_str, truthy}; +use std::fs; +use std::path::Path; + +/// What each class of item says on the page. +fn act(class: &str, name: &str) -> Option { + match class { + "rerun-ready" => Some(format!("a rerun picks up the newer {name}")), + "refresh-available" => Some(format!("newer {name} available")), + _ => None, + } +} + +fn text(v: &Value) -> String { + match v { + Value::Str(s) => s.clone(), + other => py_str(other).unwrap_or_default(), + } +} + +fn get<'a>(v: &'a Value, key: &str) -> Option<&'a Value> { + match v { + Value::Object(o) => o.get(key), + _ => None, + } +} + +/// `v or ""` as text. +fn or_blank(v: Option<&Value>) -> String { + v.filter(|v| truthy(v)).map(text).unwrap_or_default() +} + +/// `'ClinVar (step 06's file)'` as `'ClinVar'`: a last part in brackets +/// cut, `database` for nothing left. +pub fn short(label: &str) -> String { + let body = label.strip_suffix('\n').unwrap_or(label); + let cut = label + .char_indices() + .filter(|(_, c)| *c == '(') + .map(|(i, _)| i) + .find(|&i| { + let rest = &body.get(i..).unwrap_or(""); + rest.ends_with(')') && !rest.contains('\n') + }) + .map_or(label, |i| &label[..i]); + let t = cut.trim(); + if t.is_empty() { + "database".into() + } else { + t.into() + } +} + +/// `'2026-09-02 12:00:00 UTC'` or `'2026-09-02T12:00:00Z'` as +/// `'20260902120000'`: the first 14 digits; `""` without any. +pub fn utc14(text: &str) -> String { + text.chars().filter(char::is_ascii_digit).take(14).collect() +} + +/// The `written_utc` row of `/run_manifest.tsv`, or `""`. +pub fn manifest_written(run_dir: &Path) -> String { + let Ok(b) = fs::read(run_dir.join("run_manifest.tsv")) else { + return String::new(); + }; + let t = String::from_utf8_lossy(&b) + .replace("\r\n", "\n") + .replace('\r', "\n"); + for line in t.split('\n') { + let c: Vec<&str> = line.split('\t').collect(); + if c.len() >= 3 && c[0] == "run" && c[1] == "written_utc" { + return c[2].to_owned(); + } + } + String::new() +} + +/// A step number as a JSON key: text as it is, a number as Python writes +/// a key. +fn key(v: &Value) -> String { + match v { + Value::Bool(b) => (if *b { "true" } else { "false" }).into(), + Value::Null => "null".into(), + other => text(other), + } +} + +/// `plugins.recheck` of `state.json`: `{file, mtime (ns), checked_utc, +/// banner, steps: {step: [badge]}}`, or `{}` while there is no check, or +/// while the check is older than the run's `run_manifest.tsv` (then it +/// describes an earlier run, as the reports read it: `og run delta`). +pub fn state(run_dir: &Path) -> Value { + let empty = Value::Object(IndexMap::new()); + let p = run_dir.join("recheck").join("state.json"); + let Ok(mtime) = pycompat::mtime::mtime_ns(&p) else { + return empty; + }; + let Some(st) = fs::read(&p) + .ok() + .and_then(|b| json::loads(&String::from_utf8_lossy(&b)).ok()) + else { + return empty; + }; + if !matches!(st, Value::Object(_)) { + return empty; + } + let items: Vec<&Value> = match get(&st, "items") { + Some(Value::Array(a)) => a.iter().filter(|i| matches!(i, Value::Object(_))).collect(), + _ => Vec::new(), + }; + let checked = get(&st, "checked_utc"); + if utc14(&manifest_written(run_dir)) > utc14(&or_blank(checked)) { + return empty; + } + let mut steps: IndexMap> = IndexMap::new(); + let mut newer = Vec::new(); + let label = |i: &Value| { + short(&or_blank( + get(i, "label").filter(|v| truthy(v)).or(get(i, "item")), + )) + }; + for i in items { + let class = get(i, "class").map(text).unwrap_or_default(); + let is_str = matches!(get(i, "class"), Some(Value::Str(_))); + if let Some(t) = act(&class, &label(i)).filter(|_| is_str) { + newer.push(t.clone()); + if let Some(Value::Array(ns)) = get(i, "steps").filter(|v| truthy(v)) { + for n in ns { + steps.entry(key(n)).or_default().push(Value::Str(t.clone())); + } + } + } else if is_str + && ["in-pipeline-release", "upstream-only", "error"].contains(&class.as_str()) + { + newer.push(format!("{}: {class}", label(i))); + } + } + let rr = match get(&st, "rerun") { + Some(v @ Value::Object(_)) => Some(v), + _ => None, + }; + let done: String = or_blank(rr.and_then(|r| get(r, "finished_utc"))) + .chars() + .take(10) + .collect(); + if !done.is_empty() + && let Some(Value::Array(ns)) = rr + .and_then(|r| get(r, "predicted_vs_redone")) + .and_then(|p| get(p, "redone")) + .filter(|v| truthy(v)) + { + for n in ns { + steps + .entry(key(n)) + .or_default() + .push(Value::Str(format!("redone {done}"))); + } + } + let when = checked + .filter(|v| truthy(v)) + .map_or_else(|| "unknown date".to_owned(), text); + let banner = format!( + "Database re-check of {when}: {}", + if newer.is_empty() { + "nothing newer than this run".to_owned() + } else { + newer.join("; ") + } + ); + let mut o = IndexMap::new(); + o.insert("file".into(), Value::Str("recheck/state.json".into())); + o.insert("mtime".into(), Value::Int(mtime)); + o.insert( + "checked_utc".into(), + checked.cloned().unwrap_or(Value::Null), + ); + o.insert("banner".into(), Value::Str(banner)); + o.insert( + "steps".into(), + Value::Object( + steps + .into_iter() + .map(|(k, v)| (k, Value::Array(v))) + .collect(), + ), + ); + Value::Object(o) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn short_names() { + assert_eq!(short("ClinVar (step 06's file)"), "ClinVar"); + assert_eq!(short("gnomAD"), "gnomAD"); + assert_eq!(short(""), "database"); + assert_eq!(short("(x)"), "database"); + assert_eq!(short("A (b) c"), "A (b) c"); + assert_eq!(short("A (b) (c)"), "A"); + } + + #[test] + fn utc_digits() { + assert_eq!(utc14("2026-09-02 12:00:00 UTC"), "20260902120000"); + assert_eq!(utc14("2026-09-02T12:00:00.123Z"), "20260902120000"); + assert_eq!(utc14(""), ""); + } +} diff --git a/rust/crates/og-live/src/runs.rs b/rust/crates/og-live/src/runs.rs index 2b7502103..6d123451c 100644 --- a/rust/crates/og-live/src/runs.rs +++ b/rust/crates/og-live/src/runs.rs @@ -1,2 +1,335 @@ -//! The runs the live site knows. Empty until its unit of the 1.0.0 plan -//! lands. +//! The runs the live site knows, and the files of a run folder a browser +//! may fetch. +//! +//! A run is a folder under the root that holds `logs/run_status.tsv`, +//! `nextflow/` or `run_manifest.tsv` (a person's run), a run known only +//! from its stored messages, or a family folder (`og live family`). Who is a +//! child ([`child_of`]) is computed from the families' files on each +//! request, failing closed while a family cannot be read. +//! +//! A run folder serves only what its report links to by a relative href, +//! the files a plugin names ([`crate::plugins::RUN_GLOBS`]), and the one +//! HTML file of a folder the report links (CPSR's report in `cpsr/`). +//! Nothing that resolves out of the run folder or the root is served. + +use crate::family::{inside, realpath}; +use crate::static_files::{normpath, unescape}; +use indexmap::IndexMap; +use pycompat::json::Value; +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; + +/// A run name, as `validate_sample` in `scripts/lib/common.sh` takes it: a +/// letter or digit, then letters, digits, `.`, `_` and `-`. +pub fn valid_run(name: &str) -> bool { + crate::family::valid_name(name) +} + +/// One run the site knows. +#[derive(Debug, Clone, PartialEq)] +pub struct Run { + pub name: String, + /// Its folder under the root; `None` for a run known from its messages + /// alone. + pub dir: Option, + /// `sample` or `family`. + pub kind: String, + /// A family's members (`[{sample, father, mother, ...}]`), as + /// `og live family families` lists them. + pub members: Vec, + /// A family whose page cannot be built from its folder. + pub unreadable: bool, +} + +impl Run { + /// The folder the run's files are read from: its own, else the one it + /// would have under the root. + pub fn folder(&self, root: &Path) -> PathBuf { + self.dir.clone().unwrap_or_else(|| root.join(&self.name)) + } + + pub fn is_family(&self) -> bool { + self.kind == "family" + } + + /// The run as `/runs.json` and `child_of` read a family: + /// `{name, members, unreadable}`. + pub fn family_value(&self) -> Value { + let mut o = IndexMap::new(); + o.insert("name".into(), Value::Str(self.name.clone())); + o.insert("members".into(), Value::Array(self.members.clone())); + o.insert("unreadable".into(), Value::Bool(self.unreadable)); + Value::Object(o) + } +} + +fn field<'a>(v: &'a Value, key: &str) -> Option<&'a Value> { + match v { + Value::Object(o) => o.get(key), + _ => None, + } +} + +fn truthy(v: Option<&Value>) -> bool { + v.is_some_and(pycompat::value::truthy) +} + +/// Whether run `name`'s pages treat it as a child: a run that any readable +/// family lists with a father or a mother is a child, whatever the other +/// families say. While a family cannot be read, a run it might list is +/// treated as a child too (fail closed): every run while its members cannot +/// be read (an empty member list), and the members it lists while another +/// of its files cannot be read (`unreadable`). +pub fn child_of(families: &[Value], name: &str) -> bool { + let listed = |f: &Value| -> Vec { + match field(f, "members") { + Some(Value::Array(ms)) => ms + .iter() + .filter(|m| matches!(field(m, "sample"), Some(Value::Str(s)) if s == name)) + .cloned() + .collect(), + _ => Vec::new(), + } + }; + let members_empty = |f: &Value| !truthy(field(f, "members")); + if families.iter().any(|f| { + listed(f) + .iter() + .any(|m| truthy(field(m, "father")) || truthy(field(m, "mother"))) + }) { + return true; + } + families + .iter() + .any(|f| members_empty(f) || (truthy(field(f, "unreadable")) && !listed(f).is_empty())) +} + +/// The runs under `root`, by name: the person runs' folders, then the runs +/// known from `known` (their stored messages), then the family folders. +pub fn list(root: &Path, known: &[String]) -> BTreeMap { + let mut out = BTreeMap::new(); + let mut names: Vec = fs::read_dir(root) + .map(|rd| { + rd.filter_map(|e| e.ok()?.file_name().into_string().ok()) + .collect() + }) + .unwrap_or_default(); + names.sort(); + for n in names { + let d = root.join(&n); + if n.starts_with('.') || !valid_run(&n) || !inside(&d, root) || !d.is_dir() { + continue; + } + if ["logs/run_status.tsv", "nextflow", "run_manifest.tsv"] + .iter() + .any(|x| d.join(x).exists()) + { + out.insert( + n.clone(), + Run { + name: n, + dir: Some(d), + kind: "sample".into(), + members: Vec::new(), + unreadable: false, + }, + ); + } + } + for n in known { + out.entry(n.clone()).or_insert_with(|| Run { + name: n.clone(), + dir: None, + kind: "sample".into(), + members: Vec::new(), + unreadable: false, + }); + } + for f in crate::family::families(root) { + let Some(Value::Str(name)) = field(&f, "name") else { + continue; + }; + if !valid_run(name) || out.contains_key(name) { + continue; + } + let dir = match field(&f, "dir") { + Some(Value::Str(d)) => Some(PathBuf::from(d)), + _ => None, + }; + let members = match field(&f, "members") { + Some(Value::Array(m)) => m.clone(), + _ => Vec::new(), + }; + out.insert( + name.clone(), + Run { + name: name.clone(), + dir, + kind: match field(&f, "kind") { + Some(Value::Str(k)) => k.clone(), + _ => "other".into(), + }, + members, + unreadable: truthy(field(&f, "unreadable")), + }, + ); + } + out +} + +/// The families among `runs`, as [`child_of`] reads them. +pub fn families_of(runs: &BTreeMap) -> Vec { + runs.values() + .filter(|r| r.is_family()) + .map(Run::family_value) + .collect() +} + +/// The relative links of `_report.html`: every `href="..."` up to a +/// `#` or `?`, unescaped and unquoted, that names no scheme, is not +/// absolute and does not leave the run folder. +pub fn report_links(html: &str) -> BTreeSet { + let mut out = BTreeSet::new(); + let mut rest = html; + while let Some(i) = rest.find("href=\"") { + rest = &rest[i + 6..]; + let end = rest.find(['"', '#', '?']).unwrap_or(rest.len()); + if end == 0 { + continue; + } + let h = unescape(&rest[..end]); + rest = &rest[end..]; + if has_scheme(&h) || h.starts_with('/') { + continue; + } + let rel = normpath(&crate::viewer::unquote(&h)); + if !rel.starts_with("..") && rel != "." { + out.insert(rel); + } + } + out +} + +/// `^[a-z][a-z0-9+.-]*:`, any case. +fn has_scheme(h: &str) -> bool { + let b = h.as_bytes(); + if !b.first().is_some_and(u8::is_ascii_alphabetic) { + return false; + } + for &c in &b[1..] { + if c == b':' { + return true; + } + if !(c.is_ascii_alphanumeric() || matches!(c, b'+' | b'.' | b'-')) { + return false; + } + } + false +} + +/// The relative paths of run `name`'s folder `d` that its report links to. +pub fn run_files(name: &str, d: Option<&Path>) -> BTreeSet { + let Some(d) = d else { + return BTreeSet::new(); + }; + let rep = d.join(format!("{name}_report.html")); + match fs::read(&rep) { + Ok(b) if rep.is_file() => report_links(&String::from_utf8_lossy(&b)), + _ => BTreeSet::new(), + } +} + +/// Whether the browser may fetch `rel` of the run: a report link, a file a +/// plugin names, or the one HTML file of a folder the report links. +pub fn file_allowed(run: &Run, root: &Path, rel: &str) -> bool { + let files = run_files(&run.name, run.dir.as_deref()); + if files.contains(rel) + || crate::plugins::RUN_GLOBS + .iter() + .any(|g| pycompat::mtime::fnmatch(rel, g)) + { + return true; + } + let (folder, base) = match rel.rsplit_once('/') { + Some((f, b)) => (f, b), + None => ("", rel), + }; + !folder.is_empty() && sole_html(run, root, folder, &files).as_deref() == Some(base) +} + +/// The name of the only HTML file directly in a run folder the report +/// links, inside the run and the root; `None` when the folder is not +/// linked or holds none or several. +pub fn sole_html(run: &Run, root: &Path, folder: &str, files: &BTreeSet) -> Option { + let d = run.dir.as_deref()?; + if !files.contains(folder) { + return None; + } + let p = d.join(folder); + if !p.is_dir() { + return None; + } + let top = realpath(d); + let pages: Vec = fs::read_dir(&p) + .ok()? + .filter_map(|e| e.ok()?.file_name().into_string().ok()) + .filter(|f| { + let fp = p.join(f); + f.to_lowercase().ends_with(".html") + && fp.is_file() + && inside(&fp, &top) + && inside(&fp, root) + }) + .collect(); + match pages.as_slice() { + [one] => Some(one.clone()), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fam(members: &[(&str, &str, &str)], unreadable: bool) -> Value { + let ms = members + .iter() + .map(|(s, f, m)| { + let mut o = IndexMap::new(); + o.insert("sample".into(), Value::Str((*s).into())); + o.insert("father".into(), Value::Str((*f).into())); + o.insert("mother".into(), Value::Str((*m).into())); + Value::Object(o) + }) + .collect(); + let mut o = IndexMap::new(); + o.insert("members".into(), Value::Array(ms)); + o.insert("unreadable".into(), Value::Bool(unreadable)); + Value::Object(o) + } + + #[test] + fn who_is_a_child() { + let a = fam(&[("kid", "dad", "mum"), ("dad", "", "")], false); + let b = fam(&[("kid", "", "")], false); + assert!(child_of(&[a.clone(), b.clone()], "kid")); + assert!(child_of(&[b.clone(), a.clone()], "kid")); + assert!(!child_of(std::slice::from_ref(&a), "dad")); + assert!(!child_of(std::slice::from_ref(&a), "loner")); + // A family whose members cannot be read closes everyone. + assert!(child_of(&[a.clone(), fam(&[], false)], "loner")); + // One that lists the run but cannot be read closes it alone. + let c = fam(&[("dad", "", "")], true); + assert!(child_of(std::slice::from_ref(&c), "dad") && !child_of(&[c], "loner")); + } + + #[test] + fn report_links_relative_only() { + let html = r##"xto +acqu +emd"##; + let got: Vec = report_links(html).into_iter().collect(); + assert_eq!(got, ["a b.txt", "clinvar/hits.tsv", "cpsr", "x&y.tsv"]); + } +} diff --git a/rust/crates/og-live/src/search.rs b/rust/crates/og-live/src/search.rs index a4e302f07..103e4fcd3 100644 --- a/rust/crates/og-live/src/search.rs +++ b/rust/crates/og-live/src/search.rs @@ -1 +1,57 @@ -//! The live site's search. Empty until its unit of the 1.0.0 plan lands. +//! The live page's search box. +//! +//! Step 24 writes `_search_index.json` beside the report: every gene +//! symbol and rsID the results mention, with the sections that name them. +//! The browser may fetch that file (`/runs//_search_index.json`, +//! [`crate::plugins::RUN_GLOBS`]), and `state.json` says under +//! `plugins.search` whether it is there and when it was written. +//! `assets/live/plugins/search.js` filters it in the browser, so a query is +//! in no request and no log. + +use pycompat::json::Value; +use std::path::Path; + +/// `{file, mtime}` (mtime in ns) of the file `` in +/// `run_dir`, or `{}` while there is none. +pub fn file_state(run_dir: &Path, suffix: &str) -> Value { + let base = run_dir + .components() + .next_back() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .unwrap_or_default(); + let name = format!("{base}{suffix}"); + let p = run_dir.join(&name); + let mut o = indexmap::IndexMap::new(); + if p.is_file() + && let Ok(ns) = pycompat::mtime::mtime_ns(&p) + { + o.insert("file".to_owned(), Value::Str(name)); + o.insert("mtime".to_owned(), Value::Int(ns)); + } + Value::Object(o) +} + +/// `plugins.search` of `state.json`: `{file, mtime}` of the run's +/// `_search_index.json`, or `{}` while there is none. +pub fn state(run_dir: &Path) -> Value { + file_state(run_dir, "_search_index.json") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_index_file_or_nothing() { + let d = std::env::temp_dir().join(format!("og-live-search-{}", std::process::id())); + let run = d.join("S1"); + std::fs::create_dir_all(&run).unwrap(); + assert_eq!(state(&run), Value::Object(indexmap::IndexMap::new())); + std::fs::write(run.join("S1_search_index.json"), "{}").unwrap(); + let v = state(&run); + let Value::Object(o) = &v else { panic!() }; + assert_eq!(o["file"], Value::Str("S1_search_index.json".into())); + assert!(matches!(o["mtime"], Value::Int(n) if n > 0)); + std::fs::remove_dir_all(&d).unwrap(); + } +} diff --git a/rust/crates/og-live/src/server.rs b/rust/crates/og-live/src/server.rs index b15cb7cb8..5d65e5f58 100644 --- a/rust/crates/og-live/src/server.rs +++ b/rust/crates/og-live/src/server.rs @@ -1,13 +1,1275 @@ -//! `og live serve`: the live site's server, which `bin/owngenome_live.py` -//! runs today. +//! `og live serve`: the live site's server. It shows each run under the +//! genome folder while it goes: the step map, the event stream, the report +//! and its files, the family pages, the genome viewer, the map and the +//! Prometheus metrics. //! -//! Not ported yet: until its unit of the 1.0.0 plan lands, the verb says so -//! on stderr and exits 2, before reading any argument. +//! ```text +//! GET / the people under --root, by family (assets/live/home.html) +//! GET /runs.json each run's progress and each family's members +//! GET /assets/, /assets/plugins/ +//! GET /runs// a person's page (index.html), or a family's (family.html) +//! GET /runs//state.json the run's state, the headline of each finished +//! section and what the plugins add +//! GET /runs//events the stored messages as Server-Sent Events +//! (Last-Event-ID resumes) +//! GET /runs//report.html the run's _report.html +//! GET /runs// a file that report links to by a relative href, or +//! one a plugin names; nothing else from disk +//! ``` +//! +//! and the plugins' routes ([`crate::plugins`]). The plugins are compiled +//! in: there is no plugin folder. +//! +//! Not here yet: the password, the sessions, TLS, the proxy options and the +//! ingest listener. Until they land, `og live serve` listens on loopback +//! only, takes no password, and refuses the flags that would make a site +//! less safe if they were ignored (exit 2). It reads the messages another +//! process stores under `STATE_DIR/runs//events.jsonl` (the Python +//! server's ingest listener today), so both can run beside each other on one +//! state folder. The access log (`STATE_DIR/access.log`, TSV) records who +//! opened what, never a query string, a request body or a result. + +use crate::csp::{self, CSP, DENY, RUN_HTML_CSP}; +use crate::family::{inside, realpath}; +use crate::ops::OpsConfig; +use crate::plugins::{self, ResultsFns}; +use crate::runs::{self, Run}; +use crate::state::{self, Live, dumps, lock}; +use crate::static_files as sf; +use crate::viewer::unquote; +use std::collections::BTreeMap; +use std::fs::{self, OpenOptions}; +use std::io::{self, BufRead, BufReader, Read, Write}; +use std::net::{IpAddr, Shutdown, SocketAddr, TcpListener, TcpStream, ToSocketAddrs}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::Ordering; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +/// How long a client may take to send its request (an event stream writes +/// every [`HEARTBEAT`]). +pub const REQUEST_SECONDS: u64 = 60; +/// How often an open event stream writes when nothing happens. +pub const HEARTBEAT: Duration = Duration::from_secs(15); +/// The largest request head read. +const MAX_HEAD: usize = 64 * 1024; +/// The cookie the login sets (`og live serve` sets none yet; logout clears +/// it). +pub const COOKIE: &str = "owngenome_live"; + +const USAGE: &str = + "usage: og live serve --root GENOME_DIR [--state-dir DIR] [--bind ADDR] [--port N] + [--max-streams N] [--access-log FILE] [--access-log-max-mb N] + [--config FILE] [--quiet] + [--viewer-fasta FILE --viewer-igv-js FILE] + [--disk-warn-percent N] [--mem-warn-percent N] [--stall-minutes N] + [--metrics-token-file FILE] [--eta-box NAME] [--eta-cores N] +"; + +/// What the site is started with. +#[derive(Clone)] +pub struct Config { + /// GENOME_DIR: the folder holding one folder per run. + pub root: PathBuf, + /// Where the messages are read and the access log and map cache are + /// written (`ROOT/.owngenome-live`). + pub state_dir: PathBuf, + pub bind: String, + pub port: u16, + /// `assets/live`. + pub assets: PathBuf, + /// `assets/`: the viewer's page, the map's and the summary's tables. + pub repo_assets: PathBuf, + /// `steps.tsv`. + pub steps: PathBuf, + /// Event streams open at once. + pub max_streams: usize, + /// `STATE_DIR/access.log`. + pub access_log: PathBuf, + /// Then it moves to `access.log.1`; 0 never. + pub access_log_max_mb: f64, + /// No request line on stderr. + pub quiet: bool, + pub ops: OpsConfig, + /// The bearer token Prometheus sends to `/metrics` (with a password). + pub metrics_token_file: Option, + /// The genome viewer's reference FASTA, with its `.fai` beside it. + pub viewer_fasta: Option, + /// The checked igv.min.js. + pub viewer_igv_js: Option, + /// The og the map runs (`og map build`, `og map genes`): this binary. + pub og: PathBuf, + /// Each person's results (`results.json`); `None` until `og live + /// results` lands, and then the results, the findings and the map say + /// they are not here yet. + pub results: Option, + /// How often a quiet event stream writes. + pub heartbeat: Duration, +} + +/// The first ancestor of this binary that holds `rel`. +fn beside_exe(rel: &str) -> Option { + let exe = std::env::current_exe().ok()?; + exe.ancestors() + .skip(1) + .find(|d| d.join(rel).exists()) + .map(Path::to_path_buf) +} + +impl Config { + /// The defaults for genome folder `root`: assets and steps.tsv of the + /// checkout this binary sits in. + pub fn new(root: impl Into) -> Config { + let root = root.into(); + let repo = beside_exe("assets/live/index.html").unwrap_or_default(); + let state_dir = root.join(".owngenome-live"); + Config { + access_log: state_dir.join("access.log"), + state_dir, + root, + bind: "127.0.0.1".into(), + port: 8787, + assets: repo.join("assets").join("live"), + repo_assets: repo.join("assets"), + steps: og_runlog::steps::default_path().unwrap_or_else(|| repo.join("steps.tsv")), + max_streams: 32, + access_log_max_mb: 10.0, + quiet: false, + ops: OpsConfig::default(), + metrics_token_file: None, + viewer_fasta: None, + viewer_igv_js: None, + og: std::env::current_exe().unwrap_or_else(|_| PathBuf::from("og")), + results: None, + heartbeat: HEARTBEAT, + } + } +} + +// --- the access log ----------------------------------------------------------------------- + +/// `STATE_DIR/access.log`: one TSV row per request that opens something. +pub struct AccessLog { + path: PathBuf, + max_bytes: u64, + lock: Mutex<()>, +} + +/// The UTC time now as `2026-11-20T10:00:00Z`. +pub fn utc_now() -> String { + jiff::Timestamp::now() + .strftime("%Y-%m-%dT%H:%M:%SZ") + .to_string() +} + +impl AccessLog { + pub const HEADER: &str = "utc\tclient\tsession8\taction\trun\tsection\tstatus\n"; + + pub fn new(path: &Path, max_mb: f64) -> io::Result { + if let Some(d) = std::path::absolute(path)?.parent() { + fs::create_dir_all(d)?; + } + Ok(AccessLog { + path: path.to_owned(), + max_bytes: (max_mb * 1024.0 * 1024.0) as u64, + lock: Mutex::new(()), + }) + } + + /// One row: the time, then each field with tabs and line breaks as + /// spaces, cut at 200 characters, `-` for none. + pub fn write(&self, fields: &[&str]) { + let mut row = utc_now(); + for f in fields { + let f = if f.is_empty() { "-" } else { f }; + let clean: String = f + .chars() + .map(|c| { + if matches!(c, '\t' | '\r' | '\n') { + ' ' + } else { + c + } + }) + .take(200) + .collect(); + row.push('\t'); + row.push_str(&clean); + } + row.push('\n'); + let _g = lock(&self.lock); + if self.max_bytes > 0 && fs::metadata(&self.path).is_ok_and(|m| m.len() > self.max_bytes) { + let mut old = self.path.clone().into_os_string(); + old.push(".1"); + let _ = fs::rename(&self.path, old); + } + let new = !self.path.exists(); + if let Ok(mut f) = OpenOptions::new() + .create(true) + .append(true) + .open(&self.path) + { + let text = if new { + format!("{}{row}", Self::HEADER) + } else { + row + }; + let _ = f.write_all(text.as_bytes()); + } + } +} + +// --- one request -------------------------------------------------------------------------- + +/// A request and the connection its answer goes to. +pub struct Req<'a> { + pub live: &'a Live, + /// `GET` (also for a HEAD, then [`Req::head_only`]) or `POST`. + pub method: String, + pub head_only: bool, + /// The path, still quoted, without its query. + pub path: String, + pub query: String, + headers: Vec<(String, String)>, + pub client: String, + out: &'a mut dyn Write, +} + +/// The status line's reason. +fn reason(code: u16) -> &'static str { + match code { + 200 => "OK", + 206 => "Partial Content", + 301 => "Moved Permanently", + 302 => "Found", + 303 => "See Other", + 400 => "Bad Request", + 401 => "Unauthorized", + 403 => "Forbidden", + 404 => "Not Found", + 409 => "Conflict", + 413 => "Request Entity Too Large", + 416 => "Requested Range Not Satisfiable", + 500 => "Internal Server Error", + 501 => "Not Implemented", + 503 => "Service Unavailable", + _ => "", + } +} + +/// The text of a refusal without its own. +fn deny_text(code: u16) -> &'static str { + match code { + 401 => "login required", + 403 => "forbidden", + 404 => "not found", + 429 => "too many failed logins", + _ => "error", + } +} + +/// `urllib.parse.quote(text)`: letters, digits and `_.-~/` as they are. +pub fn quote(text: &str) -> String { + crate::viewer::quote(text) +} + +impl Req<'_> { + /// Header `name` (any case), the first one. + pub fn header(&self, name: &str) -> Option<&str> { + let n = name.to_ascii_lowercase(); + self.headers + .iter() + .find(|(k, _)| *k == n) + .map(|(_, v)| v.as_str()) + } + + /// The status line and headers: the type and length, `extra`, then the + /// policy `csp`, no sniffing, no referrer, no cache and frame option + /// `frame`. + pub fn head( + &mut self, + code: u16, + ctype: &str, + length: Option, + extra: &[(&str, String)], + csp: &str, + frame: &str, + ) -> io::Result<()> { + let date = jiff::Timestamp::now().strftime("%a, %d %b %Y %H:%M:%S GMT"); + let mut h = format!( + "HTTP/1.1 {code} {}\r\nServer: owngenome-live\r\nDate: {date}\r\nContent-Type: {ctype}\r\n", + reason(code) + ); + if let Some(n) = length { + h.push_str(&format!("Content-Length: {n}\r\n")); + } + for (k, v) in extra { + h.push_str(&format!("{k}: {v}\r\n")); + } + for (k, v) in csp::common(csp, frame) { + h.push_str(&format!("{k}: {v}\r\n")); + } + h.push_str("Connection: close\r\n\r\n"); + self.out.write_all(h.as_bytes()) + } + + /// The bytes after the head, unless the request is a HEAD. + pub fn body(&mut self, body: &[u8]) -> io::Result<()> { + if !self.head_only { + self.out.write_all(body)?; + } + self.out.flush() + } + + /// A whole answer under policy `csp`. + pub fn send( + &mut self, + code: u16, + ctype: &str, + body: &[u8], + extra: &[(&str, String)], + csp: &str, + ) -> io::Result<()> { + self.head(code, ctype, Some(body.len() as u64), extra, csp, DENY)?; + self.body(body) + } + + /// A whole answer under the site's policy. + pub fn send_body(&mut self, code: u16, ctype: &str, body: impl AsRef<[u8]>) -> io::Result<()> { + self.send(code, ctype, body.as_ref(), &[], CSP) + } + + /// An empty redirect. + pub fn redirect(&mut self, code: u16, location: &str) -> io::Result<()> { + self.send( + code, + "text/plain", + b"", + &[("Location", location.to_owned())], + CSP, + ) + } + + /// `json.dumps(v)`. + pub fn send_json(&mut self, v: &pycompat::json::Value) -> io::Result<()> { + self.send_body(200, "application/json", dumps(v)) + } + + /// A refusal: `text`, else the code's own line. + pub fn deny(&mut self, code: u16, text: Option<&str>) -> io::Result<()> { + let t = format!("{}\n", text.unwrap_or_else(|| deny_text(code))); + self.send_body(code, "text/plain; charset=utf-8", t) + } + + /// One access-log row for this request. + pub fn log(&self, action: &str, run: &str, section: &str, status: u16) { + self.live + .access + .write(&[&self.client, "-", action, run, section, &status.to_string()]); + } + + /// The folder of run `name`, when the site knows it and it has one. + pub fn run_dir(&self, name: &str) -> Option { + self.live.run(name).and_then(|r| r.dir) + } + + /// The writer for an answer streamed by hand (the event stream, a file + /// by ranges). + pub fn out(&mut self) -> &mut dyn Write { + &mut *self.out + } +} + +// --- the dispatch ------------------------------------------------------------------------- + +/// Whether the Host header names a host the site answers to: loopback or +/// the bind address, with any port. +fn host_ok(req: &Req<'_>) -> bool { + let h = req.header("host").unwrap_or("").trim().to_lowercase(); + let name = if let Some(r) = h.strip_prefix('[') { + match r.split_once(']') { + Some((inner, rest)) if port_ok(rest) => inner.to_owned(), + _ => return false, + } + } else { + let (n, rest) = match h.find(':') { + Some(i) => (&h[..i], &h[i..]), + None => (h.as_str(), ""), + }; + if n.is_empty() || !port_ok(rest) { + return false; + } + n.to_owned() + }; + req.live + .allowed + .iter() + .any(|a| a.trim_matches(|c| c == '[' || c == ']') == name) +} + +/// `""` or `:` and digits. +fn port_ok(rest: &str) -> bool { + rest.is_empty() + || rest + .strip_prefix(':') + .is_some_and(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit())) +} + +/// A POST from another site is refused. Browsers say where a request comes +/// from in Sec-Fetch-Site; with Referrer-Policy no-referrer they send +/// `Origin: null` for the site's own forms. +fn origin_ok(req: &Req<'_>) -> bool { + if let Some(site) = req.header("sec-fetch-site").filter(|s| !s.is_empty()) { + return site == "same-origin" || site == "none"; + } + let o = req.header("origin").unwrap_or(""); + if o.is_empty() || o == "null" { + return true; + } + netloc(o).to_lowercase() == req.header("host").unwrap_or("").to_lowercase() +} + +/// The network location of a URL (`urlsplit(url).netloc`). +fn netloc(url: &str) -> &str { + let Some(i) = url.find("//") else { return "" }; + let scheme_ok = url[..i] + .strip_suffix(':') + .is_some_and(|s| s.is_empty() || s.chars().next().is_some_and(|c| c.is_ascii_alphabetic())) + || i == 0; + if !scheme_ok { + return ""; + } + let rest = &url[i + 2..]; + &rest[..rest.find(['/', '?', '#']).unwrap_or(rest.len())] +} + +/// Answers one request. +pub fn handle(req: &mut Req<'_>) -> io::Result<()> { + if !host_ok(req) { + req.log("host-refused", "-", "-", 403); + return req.deny(403, None); + } + if req.method == "POST" && !origin_ok(req) { + req.log("origin-refused", "-", "-", 403); + return req.deny(403, None); + } + let path = req.path.clone(); + if path == "/login" { + return login(req); + } + if path == "/logout" && req.method == "POST" { + req.log("logout", "-", "-", 303); + let clear = format!("{COOKIE}=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0"); + return req.send( + 303, + "text/plain", + b"", + &[("Location", "/login".into()), ("Set-Cookie", clear)], + CSP, + ); + } + if req.method == "GET" + && let Some(name) = path.strip_prefix("/assets/") + { + return asset(req, name); + } + if plugins::route(req, &path)? { + return Ok(()); + } + if req.method != "GET" { + return req.deny(404, None); + } + if path == "/" { + return index(req); + } + if path == "/runs.json" { + req.log("runs", "-", "-", 200); + let v = req.live.runs_index(); + return req.send_json(&v); + } + let Some(tail) = path.strip_prefix("/runs/") else { + return req.deny(404, None); + }; + let (seg, rest) = match tail.find('/') { + Some(i) => (&tail[..i], Some(&tail[i + 1..])), + None => (tail, None), + }; + if seg.is_empty() { + return req.deny(404, None); + } + let Some(run) = req.live.run(&unquote(seg)) else { + req.log("run", "-", "-", 404); + return req.deny(404, None); + }; + match rest { + None => { + let to = format!("/runs/{}/", run.name); + req.redirect(301, &to) + } + Some(rest) => run_path(req, &run, rest), + } +} + +/// `/login`: the login page; a POST goes home (no password is set). +fn login(req: &mut Req<'_>) -> io::Result<()> { + if req.method == "POST" { + return req.redirect(303, "/"); + } + let next = sf::query_value(&req.query, "next").unwrap_or_else(|| "/".into()); + match sf::login_page(&req.live.cfg.assets, "", &next) { + Ok(page) => req.send_body(200, "text/html; charset=utf-8", page), + Err(_) => req.deny(500, None), + } +} + +fn asset(req: &mut Req<'_>, name: &str) -> io::Result<()> { + let top = realpath(&req.live.cfg.assets); + match sf::asset(&top, name) { + Some((ctype, body)) => req.send_body(200, ctype, body), + None => req.deny(404, None), + } +} + +/// The home page: the people grouped by family, drawn by home.js from +/// `/runs.json`; the plain list in it is what a browser without scripts +/// shows. +fn index(req: &mut Req<'_>) -> io::Result<()> { + let runs: Vec<(String, String)> = req + .live + .runs() + .into_values() + .map(|r| (r.name, r.kind)) + .collect(); + req.log("index", "-", "-", 200); + match sf::home_page(&req.live.cfg.assets, &runs) { + Ok(page) => req.send_body(200, "text/html; charset=utf-8", page), + Err(_) => req.deny(500, None), + } +} + +fn run_path(req: &mut Req<'_>, run: &Run, rest: &str) -> io::Result<()> { + let name = run.name.as_str(); + match rest { + "" => { + let page = sf::run_page(&req.live.cfg.assets, name, run.is_family()); + req.log("run", name, "-", 200); + return match page { + Ok(p) => req.send_body(200, "text/html; charset=utf-8", p), + Err(_) => req.deny(500, None), + }; + } + "state.json" => { + req.log("state", name, "-", 200); + let v = req.live.state(run); + return req.send_json(&v); + } + "events" => return events(req, run), + _ => {} + } + let root = req.live.root.clone(); + let d = run.dir.as_deref(); + if rest == "report.html" { + let p = d.map(|d| d.join(format!("{name}_report.html"))); + let ok = matches!((&p, d), (Some(p), Some(d)) if p.is_file() && inside(p, &realpath(d))); + let page = p.filter(|_| ok).and_then(|p| fs::read(p).ok()); + let Some(page) = page else { + req.log("report", name, "-", 404); + return req.deny(404, None); + }; + let page = + plugins::report_filter(req.live, name, String::from_utf8_lossy(&page).into_owned()); + req.log("report", name, "-", 200); + return req.send( + 200, + "text/html; charset=utf-8", + page.as_bytes(), + &[], + RUN_HTML_CSP, + ); + } + let rel = sf::normpath(&unquote(rest)); + let Some(d) = d.filter(|_| { + !rel.starts_with("..") && !rel.starts_with('/') && runs::file_allowed(run, &root, &rel) + }) else { + req.log("file", name, &rel, 404); + return req.deny(404, None); + }; + let p = d.join(&rel); + let top = realpath(d); + if p.is_dir() + && inside(&p, &top) + && inside(&p, &root) + && let Some(page) = runs::sole_html(run, &root, &rel, &runs::run_files(name, Some(d))) + { + req.log("file", name, &rel, 302); + let to = format!("/runs/{}/{}", quote(name), quote(&format!("{rel}/{page}"))); + return req.redirect(302, &to); + } + if !p.is_file() || !inside(&p, &top) || !inside(&p, &root) { + req.log("file", name, &rel, 404); + return req.deny(404, None); + } + let Ok(body) = fs::read(&p) else { + req.log("file", name, &rel, 404); + return req.deny(404, None); + }; + let ext = sf::ext(&rel).to_lowercase(); + req.log("file", name, &rel, 200); + let csp = if [".html", ".htm", ".svg"].contains(&ext.as_str()) { + RUN_HTML_CSP + } else { + CSP + }; + req.send(200, sf::type_or_bytes(&ext), &body, &[], csp) +} + +/// Gives back an event stream's place when it ends. +struct StreamSlot<'a>(&'a Live); + +impl Drop for StreamSlot<'_> { + fn drop(&mut self) { + self.0.streams.fetch_sub(1, Ordering::SeqCst); + } +} + +/// `/runs//events`: the run's stored messages as Server-Sent Events, +/// from the one after `Last-Event-ID`, then each new one as it is stored; a +/// comment line every heartbeat while nothing comes. +fn events(req: &mut Req<'_>, run: &Run) -> io::Result<()> { + let live = req.live; + let name = run.name.as_str(); + let max = live.max_streams.load(Ordering::SeqCst); + if live + .streams + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| { + (n < max).then_some(n + 1) + }) + .is_err() + { + req.log("events", name, "-", 503); + return req.deny(503, Some("too many open event streams")); + } + let _slot = StreamSlot(live); + let after: usize = req + .header("last-event-id") + .and_then(|v| v.trim().parse::().ok()) + .map_or(0, |n| n.max(0) as usize); + req.log("events", name, "-", 200); + req.head( + 200, + "text/event-stream", + None, + &[("X-Accel-Buffering", "no".into())], + CSP, + DENY, + )?; + if req.head_only { + return req.out().flush(); + } + let out = req.out(); + out.write_all(b"retry: 3000\n\n")?; + out.flush()?; + let mut sent = after; + while !live.stopping.load(Ordering::SeqCst) { + let new = live + .store + .wait(name, sent, live.cfg.heartbeat, &live.stopping); + if new.is_empty() { + out.write_all(b": heartbeat\n\n")?; + } else { + let mut text = String::new(); + for (i, payload) in state::messages(&new, &live.steps, sent) { + text.push_str(&format!( + "id: {i}\ndata: {}\n\n", + state::dumps_compact(&payload) + )); + } + out.write_all(text.as_bytes())?; + sent += new.len(); + } + out.flush()?; + } + Ok(()) +} + +// --- the connection ----------------------------------------------------------------------- + +/// A request's headers: (name lowered, value), in order. +type Headers = Vec<(String, String)>; + +/// The head of one request: method, target and headers (names lowered); +/// `None` for one that cannot be read. +fn read_head(r: &mut impl BufRead) -> Option<(String, String, Headers)> { + let mut head = Vec::new(); + loop { + let mut line = Vec::new(); + let n = r + .by_ref() + .take((MAX_HEAD - head.len()) as u64 + 1) + .read_until(b'\n', &mut line) + .ok()?; + if n == 0 { + if head.is_empty() { + return None; + } + break; + } + head.extend_from_slice(&line); + if head.len() > MAX_HEAD { + return None; + } + if line == b"\r\n" || line == b"\n" { + break; + } + } + let text = String::from_utf8_lossy(&head).into_owned(); + let mut lines = text.split('\n').map(|l| l.trim_end_matches('\r')); + let first = lines.next()?; + let mut parts = first.split(' '); + let (method, target, version) = (parts.next()?, parts.next()?, parts.next()?); + if parts.next().is_some() || !version.starts_with("HTTP/") { + return None; + } + let mut headers = Vec::new(); + for l in lines { + if l.is_empty() { + break; + } + let (k, v) = l.split_once(':')?; + headers.push((k.trim().to_ascii_lowercase(), v.trim().to_owned())); + } + Some((method.to_owned(), target.to_owned(), headers)) +} + +fn serve_conn(live: &Live, stream: TcpStream) { + let timeout = Some(Duration::from_secs(REQUEST_SECONDS)); + let _ = stream.set_read_timeout(timeout); + let _ = stream.set_write_timeout(timeout); + let client = stream + .peer_addr() + .map(|a| a.ip().to_string()) + .unwrap_or_else(|_| "-".into()); + let Ok(read_half) = stream.try_clone() else { + return; + }; + let mut reader = BufReader::new(read_half); + let mut out = io::BufWriter::new(&stream); + let Some((method, target, headers)) = read_head(&mut reader) else { + let _ = out.write_all( + b"HTTP/1.1 400 Bad Request\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: 12\r\n\ + Connection: close\r\n\r\nbad request\n", + ); + return; + }; + let (path, query) = match target.split_once('?') { + Some((p, q)) => (p, q), + None => (target.as_str(), ""), + }; + let path = path.split('#').next().unwrap_or("").to_owned(); + if !live.cfg.quiet { + eprintln!("og live serve: {method} {path}"); + } + let (m, head_only) = match method.as_str() { + "GET" => ("GET", false), + "HEAD" => ("GET", true), + "POST" => ("POST", false), + _ => { + let _ = out.write_all( + b"HTTP/1.1 501 Not Implemented\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: 20\r\n\ + Connection: close\r\n\r\nunsupported method\r\n", + ); + return; + } + }; + let mut req = Req { + live, + method: m.into(), + head_only, + path, + query: query.split('#').next().unwrap_or("").to_owned(), + headers, + client, + out: &mut out, + }; + let _ = handle(&mut req); + let _ = out.flush(); + drop(out); + let _ = stream.shutdown(Shutdown::Write); + // No route reads a body: what the client still sends is read and + // dropped once the answer is out, so closing does not reset the + // connection before the client has read it. + let _ = stream.set_read_timeout(Some(Duration::from_secs(1))); + let _ = io::copy(&mut reader.take(MAX_HEAD as u64), &mut io::sink()); +} + +/// A running site: [`Server::start`] binds and serves on threads, dropping +/// it stops it. +pub struct Server { + live: Arc, + addr: SocketAddr, + accept: Option>, +} + +impl Server { + /// Reads steps.tsv, opens the access log, binds `cfg.bind:cfg.port` (0: + /// any free port) and serves. + pub fn start(cfg: Config) -> Result { + let steps = og_runlog::steps::read_steps(&cfg.steps).map_err(|e| format!("steps: {e}"))?; + fs::create_dir_all(&cfg.state_dir) + .map_err(|e| format!("--state-dir {}: {e}", cfg.state_dir.display()))?; + let access = AccessLog::new(&cfg.access_log, cfg.access_log_max_mb) + .map_err(|e| format!("--access-log {}: {e}", cfg.access_log.display()))?; + let host = cfg.bind.trim_matches(|c| c == '[' || c == ']').to_owned(); + let listener = TcpListener::bind((host.as_str(), cfg.port)) + .map_err(|e| format!("cannot listen on {}:{}: {e}", cfg.bind, cfg.port))?; + let addr = listener.local_addr().map_err(|e| e.to_string())?; + let live = Arc::new(Live::new(cfg, steps, access)); + let l2 = Arc::clone(&live); + let accept = thread::spawn(move || { + for conn in listener.incoming() { + if l2.stopping.load(Ordering::SeqCst) { + break; + } + let Ok(stream) = conn else { continue }; + let l3 = Arc::clone(&l2); + thread::spawn(move || serve_conn(&l3, stream)); + } + }); + Ok(Server { + live, + addr, + accept: Some(accept), + }) + } + + /// The port it listens on. + pub fn port(&self) -> u16 { + self.addr.port() + } + + /// The state it serves. + pub fn live(&self) -> &Live { + &self.live + } + + /// Serves until the process ends. + pub fn wait(mut self) { + if let Some(h) = self.accept.take() { + let _ = h.join(); + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.live.stopping.store(true, Ordering::SeqCst); + // Wakes the accept loop so it sees the flag. + let _ = TcpStream::connect_timeout(&self.addr, Duration::from_secs(1)); + if let Some(h) = self.accept.take() { + let _ = h.join(); + } + } +} + +// --- the command line --------------------------------------------------------------------- + +/// Whether `addr` is a loopback address, or a name that resolves only to +/// loopback. +pub fn is_loopback(addr: &str) -> bool { + if addr == "localhost" { + return true; + } + let bare = addr.trim_matches(|c| c == '[' || c == ']'); + if let Ok(ip) = bare.parse::() { + return ip.is_loopback(); + } + match (bare, 0).to_socket_addrs() { + Ok(it) => { + let all: Vec = it.collect(); + !all.is_empty() && all.iter().all(|a| a.ip().is_loopback()) + } + Err(_) => false, + } +} + +/// The flags that take no value. +const SWITCHES: [&str; 2] = ["quiet", "cookie_secure"]; + +/// The flags of a site with a password, TLS or a proxy: refused until they +/// are ported, because a site that ignored them would be less safe than +/// its owner asked for. +const NOT_YET: [&str; 6] = [ + "password_file", + "allowed_host", + "trust_proxy", + "tls_cert", + "tls_key", + "cookie_secure", +]; + +/// Flags read and not used yet: the ingest listener's and the sessions'. +/// `og live serve` starts neither, so they change nothing. +const UNUSED: [&str; 6] = [ + "ingest_bind", + "ingest_port", + "max_body_kb", + "session_hours", + "session_idle_minutes", + "login_max_failures", +]; + +/// Every flag, by its name with `_`. +const FLAGS: [&str; 26] = [ + "root", + "state_dir", + "bind", + "port", + "ingest_bind", + "ingest_port", + "password_file", + "allowed_host", + "trust_proxy", + "tls_cert", + "tls_key", + "cookie_secure", + "session_hours", + "session_idle_minutes", + "max_streams", + "max_body_kb", + "login_max_failures", + "access_log", + "access_log_max_mb", + "assets", + "steps", + "quiet", + "viewer_fasta", + "viewer_igv_js", + "metrics_token_file", + "plugin_dir", +]; + +/// The ops plugin's flags. +const OPS_FLAGS: [&str; 5] = [ + "disk_warn_percent", + "mem_warn_percent", + "stall_minutes", + "eta_box", + "eta_cores", +]; + +fn known(k: &str) -> bool { + FLAGS.contains(&k) || OPS_FLAGS.contains(&k) +} + +/// The flags as given: each name (with `_`) and its values, in order. +type Flags = BTreeMap>; + +/// `KEY=VALUE` lines of `--config`: any flag, by its name with `_` or `-`. +fn read_config(path: &str) -> Result { + let text = fs::read(path).map_err(|e| format!("{path}: {e}"))?; + let text = String::from_utf8_lossy(&text); + let mut out = Flags::new(); + for (n, line) in text.lines().enumerate() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let bad = || format!("{path}:{}: not a KEY=VALUE line of a known flag", n + 1); + let (k, v) = line.split_once('=').ok_or_else(bad)?; + let k = k.trim().replace('-', "_"); + if !known(&k) || k == "config" { + return Err(bad()); + } + let v = v.trim().to_owned(); + let v = if SWITCHES.contains(&k.as_str()) { + if ["1", "true", "yes"].contains(&v.to_lowercase().as_str()) { + "1".into() + } else { + String::new() + } + } else { + v + }; + if k == "allowed_host" || k == "trust_proxy" { + out.entry(k).or_default().push(v); + } else { + out.insert(k, vec![v]); + } + } + Ok(out) +} + +/// The command line: (the flags of `--config`, then those given, which win) +/// or the error to print. +fn parse_args(args: &[String]) -> Result { + let mut given = Flags::new(); + let mut config = None; + let mut i = 0; + while i < args.len() { + let a = &args[i]; + i += 1; + let Some(flag) = a.strip_prefix("--") else { + return Err(format!("unrecognized arguments: {a}")); + }; + let (name, inline) = match flag.split_once('=') { + Some((n, v)) => (n, Some(v.to_owned())), + None => (flag, None), + }; + let k = name.replace('-', "_"); + if k != "config" && !known(&k) { + return Err(format!("unrecognized arguments: {a}")); + } + if SWITCHES.contains(&k.as_str()) { + if inline.is_some() { + return Err(format!("argument --{name}: ignored explicit argument")); + } + given.insert(k, vec!["1".into()]); + continue; + } + let v = match inline { + Some(v) => v, + None if i < args.len() && !args[i].starts_with("--") => { + i += 1; + args[i - 1].clone() + } + None => return Err(format!("argument --{name}: expected one argument")), + }; + if k == "config" { + config = Some(v); + } else if k == "allowed_host" || k == "trust_proxy" { + given.entry(k).or_default().push(v); + } else { + given.insert(k, vec![v]); + } + } + let mut out = match config { + Some(p) => read_config(&p)?, + None => Flags::new(), + }; + for (k, v) in given { + if k == "allowed_host" || k == "trust_proxy" { + out.entry(k).or_default().extend(v); + } else { + out.insert(k, v); + } + } + Ok(out) +} + +fn one<'a>(f: &'a Flags, k: &str) -> Option<&'a str> { + f.get(k).and_then(|v| v.last()).map(String::as_str) +} + +fn number(f: &Flags, k: &str, kind: &str) -> Result, String> { + match one(f, k) { + None => Ok(None), + Some(v) => v.trim().parse().map(Some).map_err(|_| { + format!( + "argument --{}: invalid {kind} value: '{v}'", + k.replace('_', "-") + ) + }), + } +} + +/// The site's [`Config`] from the command line, or (exit code, message). +pub fn configure(args: &[String]) -> Result { + let usage = |m: String| (2u8, format!("{USAGE}og live serve: error: {m}")); + let f = parse_args(args).map_err(usage)?; + for k in NOT_YET { + if f.get(k).is_some_and(|v| v.iter().any(|x| !x.is_empty())) { + return Err(( + 2, + format!( + "og live serve: --{} is not ported yet: the password, TLS and proxy options \ + land with the login (bin/owngenome_live.py has them until then)", + k.replace('_', "-") + ), + )); + } + } + if f.contains_key("plugin_dir") { + return Err(( + 2, + "og live serve: --plugin-dir: the plugins are compiled into og".into(), + )); + } + if std::env::var("OWNGENOME_LIVE_PASSWORD_HASH").is_ok_and(|h| !h.trim().is_empty()) { + return Err(( + 2, + "og live serve: OWNGENOME_LIVE_PASSWORD_HASH is set, and the password is not ported \ + yet (bin/owngenome_live.py has it until then)" + .into(), + )); + } + let Some(root) = one(&f, "root").filter(|r| !r.is_empty()) else { + return Err(usage( + "--root is required (on the command line or in --config)".into(), + )); + }; + let mut cfg = Config::new(root); + if let Some(s) = one(&f, "state_dir").filter(|s| !s.is_empty()) { + cfg.state_dir = PathBuf::from(s); + } + cfg.access_log = one(&f, "access_log") + .filter(|s| !s.is_empty()) + .map_or_else(|| cfg.state_dir.join("access.log"), PathBuf::from); + if let Some(b) = one(&f, "bind") { + cfg.bind = b.to_owned(); + } + if let Some(p) = number::(&f, "port", "int").map_err(usage)? { + cfg.port = p; + } + if let Some(n) = number::(&f, "max_streams", "int").map_err(usage)? { + cfg.max_streams = n; + } + if let Some(n) = number::(&f, "access_log_max_mb", "float").map_err(usage)? { + cfg.access_log_max_mb = n; + } + for k in UNUSED { + if k.ends_with("hours") || k.ends_with("minutes") { + number::(&f, k, "float").map_err(usage)?; + } else if k != "ingest_bind" { + number::(&f, k, "int").map_err(usage)?; + } + } + if let Some(a) = one(&f, "assets") { + cfg.assets = PathBuf::from(a); + if let Some(p) = cfg.assets.parent() { + cfg.repo_assets = p.to_path_buf(); + } + } + if let Some(s) = one(&f, "steps") { + cfg.steps = PathBuf::from(s); + } + cfg.quiet = one(&f, "quiet").is_some_and(|v| !v.is_empty()); + let path = |k: &str| one(&f, k).filter(|v| !v.is_empty()).map(PathBuf::from); + cfg.viewer_fasta = path("viewer_fasta"); + cfg.viewer_igv_js = path("viewer_igv_js"); + cfg.metrics_token_file = path("metrics_token_file"); + cfg.ops = OpsConfig { + root: Some(cfg.root.clone()), + eta_box: one(&f, "eta_box").map(str::to_owned), + eta_cores: number::(&f, "eta_cores", "int").map_err(usage)?, + disk_warn_percent: one(&f, "disk_warn_percent").map(str::to_owned), + mem_warn_percent: one(&f, "mem_warn_percent").map(str::to_owned), + stall_minutes: number::(&f, "stall_minutes", "float").map_err(usage)?, + }; + if !is_loopback(&cfg.bind) { + return Err(( + 2, + format!( + "ERROR: --bind {} is not loopback, and og live serve takes no password yet: \ + it listens on loopback only until the login is ported", + cfg.bind + ), + )); + } + if !cfg.root.is_dir() { + return Err(( + 1, + format!("ERROR: --root {} is not a folder", cfg.root.display()), + )); + } + Ok(cfg) +} + +/// `og live serve ARGS`: serves until it is stopped. Exit 2 for flags it +/// cannot take, 1 when it cannot start. +pub fn cli(args: &[String], out: &mut dyn Write, err: &mut dyn Write) -> u8 { + if args.iter().any(|a| a == "-h" || a == "--help") { + let _ = write!(out, "{USAGE}"); + return 0; + } + let cfg = match configure(args) { + Ok(c) => c, + Err((code, m)) => { + let _ = writeln!(err, "{m}"); + return code; + } + }; + let bind = cfg.bind.clone(); + let srv = match Server::start(cfg) { + Ok(s) => s, + Err(e) => { + let _ = writeln!(err, "og live serve: {e}"); + return 1; + } + }; + let _ = writeln!( + err, + "og live serve: no password hash set: anyone who can reach the loopback port sees the runs" + ); + let _ = writeln!(err, "og live serve: browser http://{bind}:{}/", srv.port()); + let _ = err.flush(); + srv.wait(); + 0 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn args(a: &[&str]) -> Vec { + a.iter().map(|x| (*x).to_owned()).collect() + } + + #[test] + fn hosts_with_and_without_ports() { + assert!(port_ok("") && port_ok(":8787") && !port_ok(":") && !port_ok(":x")); + assert_eq!(netloc("http://127.0.0.1:8787/x"), "127.0.0.1:8787"); + assert_eq!(netloc("null"), ""); + } + + #[test] + fn flags_and_config() { + let dir = std::env::temp_dir().join(format!("og-live-cfg-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let c = dir.join("live.conf"); + fs::write( + &c, + format!( + "# x\nroot={}\nport=9000\nquiet=yes\nviewer-fasta=/r.fa\n", + dir.display() + ), + ) + .unwrap(); + let cfg = configure(&args(&["--config", c.to_str().unwrap(), "--port=9001"])).unwrap(); + assert_eq!(cfg.port, 9001); + assert!(cfg.quiet); + assert_eq!(cfg.viewer_fasta.as_deref(), Some(Path::new("/r.fa"))); + assert_eq!(cfg.state_dir, dir.join(".owngenome-live")); + let refused = |a: &[&str]| configure(&args(a)).err().map(|e| e.0); + let root = dir.to_str().unwrap(); + assert_eq!(refused(&["--root", root, "--password-file", "/x"]), Some(2)); + assert_eq!(refused(&["--root", root, "--bind", "0.0.0.0"]), Some(2)); + assert_eq!(refused(&["--root", root, "--tls-cert", "a"]), Some(2)); + assert_eq!(refused(&["--root", root, "--port", "x"]), Some(2)); + assert_eq!(refused(&["--root", root, "--nope"]), Some(2)); + assert_eq!(refused(&["--port", "1"]), Some(2)); + assert_eq!(refused(&["--root", root, "--ingest-port", "8788"]), None); + fs::write(&c, "nope=1\n").unwrap(); + assert_eq!(refused(&["--config", c.to_str().unwrap()]), Some(2)); + fs::remove_dir_all(&dir).unwrap(); + } -use std::io::Write; + #[test] + fn loopback() { + assert!(is_loopback("127.0.0.1") && is_loopback("::1") && is_loopback("[::1]")); + assert!(is_loopback("localhost") && !is_loopback("0.0.0.0") && !is_loopback("192.0.2.1")); + } -/// `og live serve ARGS`: not ported yet, so one line on stderr and exit 2. -pub fn cli(_args: &[String], _out: &mut dyn Write, err: &mut dyn Write) -> u8 { - let _ = writeln!(err, "og live serve: not ported yet"); - 2 + #[test] + fn access_rows() { + let dir = std::env::temp_dir().join(format!("og-live-log-{}", std::process::id())); + let log = AccessLog::new(&dir.join("a.log"), 0.0).unwrap(); + log.write(&["127.0.0.1", "", "file", "S1", "a\tb\nc", "200"]); + let text = fs::read_to_string(dir.join("a.log")).unwrap(); + let mut lines = text.lines(); + assert_eq!(lines.next(), Some(AccessLog::HEADER.trim_end())); + let row: Vec<&str> = lines.next().unwrap().split('\t').collect(); + assert_eq!(&row[1..], ["127.0.0.1", "-", "file", "S1", "a b c", "200"]); + fs::remove_dir_all(&dir).unwrap(); + } } diff --git a/rust/crates/og-live/src/state.rs b/rust/crates/og-live/src/state.rs index abe3b071f..ac2f54208 100644 --- a/rust/crates/og-live/src/state.rs +++ b/rust/crates/og-live/src/state.rs @@ -1 +1,1154 @@ -//! The live site's run state. Empty until its unit of the 1.0.0 plan lands. +//! The live site's run state: what `state.json`, `/runs.json` and the +//! event stream of `og live serve` say about each run. +//! +//! - [`Store`]: the stored messages of each run, `STATE_DIR/runs// +//! events.jsonl`, read again whenever the file changes, so a message +//! another process appends reaches every open stream. +//! - [`messages`], [`current_run`], [`run_finished`]: what a browser is sent +//! of each message, and which messages are the newest run's. +//! - [`headline`]: one line per finished section, from og-render's own +//! helpers, so it reads as the report does. +//! - [`Live`]: the state every request shares: the runs, each run's state +//! ([`Live::base_state`], [`Live::state`]), the summary the headlines read +//! and the home page's list ([`Live::runs_index`]). +//! +//! No path, task tag or parameter of a message reaches a page. + +use crate::family::{inside, realpath}; +use crate::runs::{self, Run}; +use crate::server::{AccessLog, Config}; +use indexmap::IndexMap; +use jiff::tz::TimeZone; +use og_render::model::{Malformed, Summary}; +use og_render::sections::clinvar::D; +use og_runlog::files; +use og_runlog::state::{Inputs, RunState, State, trace_files}; +use og_runlog::steps::Step; +use og_runlog::tasks; +use pycompat::json::{self, Dump, Value}; +use pycompat::value::{py_str, truthy}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +pub type Obj = IndexMap; + +/// The lock's data, also after another request panicked holding it. +pub(crate) fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +pub(crate) fn s(text: &str) -> Value { + Value::Str(text.to_owned()) +} + +pub(crate) fn obj(pairs: [(&str, Value); N]) -> Value { + Value::Object(pairs.into_iter().map(|(k, v)| (k.to_owned(), v)).collect()) +} + +pub(crate) fn get<'a>(v: &'a Value, key: &str) -> Option<&'a Value> { + match v { + Value::Object(o) => o.get(key), + _ => None, + } +} + +fn get_str<'a>(v: &'a Value, key: &str) -> Option<&'a str> { + match get(v, key) { + Some(Value::Str(x)) => Some(x), + _ => None, + } +} + +/// `json.dumps(value)` with Python's default separators. +pub fn dumps(value: &Value) -> String { + json::dumps(value, &Dump::default()).unwrap_or_else(|_| "null".into()) +} + +/// `json.dumps(value, separators=(",", ":"))`. +pub fn dumps_compact(value: &Value) -> String { + json::dumps( + value, + &Dump { + separators: Some((",", ":")), + ..Dump::default() + }, + ) + .unwrap_or_else(|_| "null".into()) +} + +/// Seconds since the epoch. +pub fn now() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0.0, |d| d.as_secs_f64()) +} + +/// A file's (mtime in ns, size); `None` when it cannot be read. +pub fn file_stamp(p: &Path) -> Option<(i128, u64)> { + let m = fs::metadata(p).ok()?; + Some((pycompat::mtime::mtime_ns(p).ok()?, m.len())) +} + +// --- the stored messages ------------------------------------------------------------------ + +struct Cached { + stamp: Option<(i128, u64)>, + events: Arc>, +} + +/// `STATE_DIR/runs//events.jsonl` of every run, kept in memory and read +/// again when the file's time or size changes. +pub struct Store { + dir: PathBuf, + runs: Mutex>, +} + +/// How often an open stream looks at its run's file. +const POLL: Duration = Duration::from_millis(200); + +impl Store { + pub fn new(state_dir: &Path) -> Store { + Store { + dir: state_dir.join("runs"), + runs: Mutex::new(HashMap::new()), + } + } + + /// The events file of `run`. + pub fn path(&self, run: &str) -> PathBuf { + self.dir.join(run).join("events.jsonl") + } + + /// The runs with an events file, by name. + pub fn known(&self) -> Vec { + let mut out: Vec = fs::read_dir(&self.dir) + .map(|rd| { + rd.filter_map(|e| e.ok()?.file_name().into_string().ok()) + .filter(|r| runs::valid_run(r) && self.path(r).is_file()) + .collect() + }) + .unwrap_or_default(); + out.sort(); + out + } + + /// The stored messages of `run`: each line that is a JSON object. + pub fn events(&self, run: &str) -> Arc> { + let p = self.path(run); + let stamp = file_stamp(&p); + { + let runs = lock(&self.runs); + if let Some(c) = runs.get(run) + && c.stamp == stamp + { + return Arc::clone(&c.events); + } + } + let events = Arc::new(files::read_events(Some(&p)).unwrap_or_default()); + lock(&self.runs).insert( + run.to_owned(), + Cached { + stamp, + events: Arc::clone(&events), + }, + ); + events + } + + /// The messages of `run` after the first `after`, once there are any or + /// `timeout` has passed (then none), or `stop` is set. + pub fn wait( + &self, + run: &str, + after: usize, + timeout: Duration, + stop: &AtomicBool, + ) -> Vec { + let end = Instant::now() + timeout; + loop { + let evs = self.events(run); + if evs.len() > after { + return evs[after..].to_vec(); + } + let left = end.saturating_duration_since(Instant::now()); + if left.is_zero() || stop.load(Ordering::SeqCst) { + return Vec::new(); + } + std::thread::sleep(left.min(POLL)); + } + } +} + +// --- what a message says ------------------------------------------------------------------ + +fn event_is(ev: &Value, name: &str) -> bool { + get_str(ev, "event") == Some(name) +} + +/// Whether the run is over: run-all.sh posts `pgp_run_finished` as its last +/// message, so a run is over only while that is the newest message (a +/// message after it is a later run of the same sample). +pub fn run_finished(events: &[Value]) -> bool { + events + .last() + .is_some_and(|e| event_is(e, "pgp_run_finished")) +} + +/// The messages of the newest run of a sample: those after the last +/// `pgp_run_finished` that is not the newest message, and from the last +/// weblog `started` when that comes later (a bare nextflow run posts no +/// `pgp_run_finished`). +pub fn current_run(events: &[Value]) -> &[Value] { + let mut start = 0; + if events.len() > 1 { + for (i, ev) in events[..events.len() - 1].iter().enumerate() { + if event_is(ev, "pgp_run_finished") { + start = i + 1; + } + } + } + if let Some(i) = (start..events.len()) + .rev() + .find(|&i| event_is(&events[i], "started")) + { + start = i; + } + &events[start..] +} + +/// `'SV:MANTA (s1)'` as `(MANTA, s1)`: the process without its workflow +/// prefix and the tag (og-nf's reading); `None` for a name of two words. +pub fn task_name(name: &str) -> (Option, Option) { + match og_nf::name::task_name(name) { + Some((p, tag)) => (Some(p.to_owned()), tag.map(str::to_owned)), + None => (None, None), + } +} + +/// Which step runs each process. +pub fn step_of_process(steps: &[Step]) -> HashMap<&str, &str> { + let mut out = HashMap::new(); + for s in steps { + for p in &s.processes { + out.insert(p.as_str(), s.step.as_str()); + } + } + out +} + +fn text_of(v: &Value) -> String { + match v { + Value::Str(x) => x.clone(), + other => py_str(other).unwrap_or_default(), + } +} + +/// What the browser is sent for each stored message, in order: `(id, +/// payload)`. The id is the message's place in the run's messages (the +/// first of `events` is `start + 1`). No path, parameter or tag leaves here. +pub fn messages(events: &[Value], steps: &[Step], start: usize) -> Vec<(usize, Value)> { + let owner = step_of_process(steps); + events + .iter() + .enumerate() + .map(|(i, ev)| { + let mut p = Obj::new(); + p.insert( + "event".into(), + get(ev, "event").cloned().unwrap_or(Value::Null), + ); + p.insert( + "utc".into(), + get(ev, "utcTime").cloned().unwrap_or(Value::Null), + ); + if let Some(tr @ Value::Object(_)) = get(ev, "trace") { + let name = [get(tr, "name"), get(tr, "process")] + .into_iter() + .flatten() + .find(|v| truthy(v)) + .map(text_of) + .unwrap_or_default(); + let (proc, _) = task_name(&name); + let step = proc + .as_deref() + .and_then(|p| owner.get(p)) + .map_or(Value::Null, |st| s(st)); + p.insert("process".into(), proc.map_or(Value::Null, Value::Str)); + p.insert("step".into(), step); + p.insert( + "status".into(), + get(tr, "status").cloned().unwrap_or(Value::Null), + ); + p.insert( + "attempt".into(), + get(tr, "attempt").cloned().unwrap_or(Value::Null), + ); + } + if event_is(ev, "pgp_step") { + p.insert( + "step".into(), + get(ev, "step").cloned().unwrap_or(Value::Null), + ); + p.insert( + "state".into(), + get(ev, "state").cloned().unwrap_or(Value::Null), + ); + } + if event_is(ev, "pgp_run_finished") { + p.insert( + "exit".into(), + get(ev, "exit").cloned().unwrap_or(Value::Null), + ); + } + p.retain(|_, v| *v != Value::Null); + (start + i + 1, Value::Object(p)) + }) + .collect() +} + +// --- headlines ---------------------------------------------------------------------------- + +/// The first line, and the section's caveat when the helper ends with it +/// (never re-worded), as two sentences the page can cut after the first. +fn with_note(lines: &[String], note: &str) -> Option { + let first = lines.first()?; + if lines.last().map(String::as_str) != Some(note) { + return Some(first.clone()); + } + let dot = if first.trim_end().ends_with(['.', '!', '?']) { + "" + } else { + "." + }; + Some(format!("{first}{dot} {note}")) +} + +fn first(lines: Vec) -> Result { + lines + .into_iter() + .next() + .ok_or_else(|| Malformed("no line".into())) +} + +/// A number Python compares with `> 0`; an error for any other value. +fn positive(v: &Value) -> Result { + match v { + Value::Int(n) => Ok(*n > 0), + Value::Float(f) => Ok(*f > 0.0), + Value::Bool(b) => Ok(*b), + _ => Err(Malformed("not a number".into())), + } +} + +fn or_zero(v: Option<&Value>) -> Value { + v.filter(|v| truthy(v)).cloned().unwrap_or(Value::Int(0)) +} + +/// The keys [`headline`] has a line of its own for. +pub const HEADLINE_KEYS: [&str; 11] = [ + "clinvar", "carrier", "couple", "roh", "pypgx", "annotsv", "viral", "archaic", "lpa", "cpsr", + "mito", +]; + +/// `(text, attention)` of section `key` from its helper, `None` for a key +/// without one. An error is a summary the helper cannot read. +fn line_of(key: &str, sm: &Summary) -> Result, bool)>, Malformed> { + use og_render::notes::{COUPLE_NOTE, ROH_NOTE, VIRAL_NOTE}; + use og_render::sections as sec; + let x = sm.section(key); + let d = D::of(&x); + let data = x.data; + let flag = |k: &str| get(data, k).is_some_and(truthy); + Ok(Some(match key { + "clinvar" => { + let n = or_zero(get(data, "count")); + ( + Some(format!( + "Pathogenic/Likely Pathogenic hits: {}. {}", + py_str(&n).map_err(|e| Malformed(e.to_string()))?, + og_render::notes::CLINVAR_FREQ_NOTE + )), + positive(&n)?, + ) + } + "carrier" => (Some(first(sec::carrier::lines(&d)?)?), flag("carriers")), + "couple" => ( + with_note(&sec::couple::lines(&d)?, COUPLE_NOTE), + flag("shared") || flag("x_linked"), + ), + "roh" => { + let lines = sec::roh::lines(&x)?; + let f = lines + .iter() + .find(|l| l.starts_with("**")) + .cloned() + .map_or_else(|| first(lines.clone()), Ok)?; + let text = if f == ROH_NOTE { + Some(f) + } else { + with_note(&[f, ROH_NOTE.to_owned()], ROH_NOTE) + }; + (text, flag("upd_flag")) + } + "pypgx" => ( + sec::pgx::pypgx_lines(&x)?.into_iter().next(), + flag("low_gq") || flag("not_callable"), + ), + "annotsv" => (Some(first(sec::annotsv::lines(&d)?)?), flag("count")), + "viral" => ( + with_note(&sec::viral::lines(&x)?, VIRAL_NOTE), + flag("found"), + ), + "archaic" => (Some(first(sec::archaic::lines(data)?)?), false), + "lpa" => (Some(first(sec::lpa::lines(&x)?)?), false), + "cpsr" => { + let empty = Value::Object(IndexMap::new()); + let dbs = sm.get("databases").filter(|v| truthy(v)).unwrap_or(&empty); + let lines = sec::cpsr::lines(&d, &D::new(dbs, "databases"))?; + let Some(l0) = lines.into_iter().next() else { + return Ok(Some((None, false))); + }; + let plp = get(data, "plp").filter(|v| truthy(v)); + let n = match plp { + None => 0, + Some(Value::Array(a)) => a.len(), + Some(Value::Object(o)) => o.len(), + Some(Value::Str(t)) => t.chars().count(), + Some(_) => return Err(Malformed("plp has no length".into())), + }; + ( + Some(format!("Pathogenic/Likely Pathogenic rows: {n}. {l0}")), + n > 0, + ) + } + "mito" => { + let lines = sec::mito::mito_lines(&x)?; + let f = lines + .iter() + .find(|l| l.starts_with("ClinVar pathogenic")) + .or(lines.first()) + .cloned(); + let a = get(data, "annotation").filter(|v| truthy(v)); + let plp = or_zero(a.and_then(|a| get(a, "clinvar_plp"))); + (f, positive(&plp)?) + } + _ => return Ok(None), + })) +} + +/// `(text, tone)` of one summary section, tone `info` or `attention`; +/// `None` while it has no file. The text comes from og-render's helpers, so +/// it reads as the report does; no tone says a section is fine (a ClinVar +/// screen with no hit is `info`, never an all-clear). +pub fn headline( + key: &str, + sec: Option<&Value>, + summary: &Summary, +) -> Option<(String, &'static str)> { + let sec = sec?; + let state = get(sec, "state"); + let state_s = match state { + None | Some(Value::Null) => return None, + Some(Value::Str(x)) if x == "missing" => return None, + Some(v) => text_of(v), + }; + let title = get(sec, "title") + .filter(|v| truthy(v)) + .map_or_else(|| key.to_owned(), text_of); + if state_s == "failed" || state_s == "unreadable" { + let note = get(sec, "note") + .filter(|v| truthy(v)) + .map_or_else(|| state_s.clone(), text_of); + return Some((format!("{title}: {note}"), "attention")); + } + if state_s == "stale" { + let note = og_render::notes::stale_note(&summary.section(key)).unwrap_or_default(); + return Some((format!("{title}: {note}"), "attention")); + } + let (text, attention) = line_of(key, summary) + .ok() + .flatten() + .unwrap_or((None, false)); + let text = text + .unwrap_or_else(|| format!("{title}: finished, see the report")) + .replace("**", ""); + Some((text, if attention { "attention" } else { "info" })) +} + +/// The report's anchor of a summary key: the id of the section that lists +/// it among its keys. +pub fn anchor_of(key: &str) -> Option<&'static str> { + og_render::chapters::CHAPTERS + .iter() + .flat_map(|c| c.sections.iter()) + .find(|s| s.keys.contains(&key)) + .map(|s| s.id) +} + +// --- the summary the headlines read -------------------------------------------------------- + +/// `YYYY-MM-DDTHH:MM:SS` (the first 19 characters) as UTC epoch seconds. +fn utc_seconds(text: &str) -> Option { + let t: String = text.chars().take(19).collect(); + let (date, time) = t.split_once('T')?; + let mut d = date.split('-'); + let mut h = time.split(':'); + let num = |x: Option<&str>, max: usize| -> Option { + let x = x?; + (!x.is_empty() && x.len() <= max && x.bytes().all(|b| b.is_ascii_digit())) + .then(|| x.parse().ok()) + .flatten() + }; + let (y, mo, dd) = (num(d.next(), 4)?, num(d.next(), 2)?, num(d.next(), 2)?); + let (hh, mi, ss) = (num(h.next(), 2)?, num(h.next(), 2)?, num(h.next(), 2)?); + if d.next().is_some() || h.next().is_some() || y < 1 { + return None; + } + let dt = jiff::civil::DateTime::new( + i16::try_from(y).ok()?, + i8::try_from(mo).ok()?, + i8::try_from(dd).ok()?, + i8::try_from(hh).ok()?, + i8::try_from(mi).ok()?, + i8::try_from(ss).ok()?, + 0, + ) + .ok()?; + let z = dt.to_zoned(TimeZone::UTC).ok()?; + Some(z.timestamp().as_second() as f64) +} + +/// The done steps of a state's JSON, by number. +fn done_steps(st: &Value) -> Vec<&Value> { + match get(st, "steps") { + Some(Value::Array(a)) => a + .iter() + .filter(|s| get_str(s, "state") == Some("done")) + .collect(), + _ => Vec::new(), + } +} + +/// `/summary.json` when it is newer than every done step's end, else +/// `None`. A summary.json that links out of the run folder is never read. +pub fn summary_file(d: &Path, st: &Value) -> Option { + let p = d.join("summary.json"); + if !inside(&p, &realpath(d)) { + return None; + } + let mtime = pycompat::mtime::mtime(&p).ok()?; + for s in done_steps(st) { + let e = get_str(s, "ended_utc").filter(|e| !e.is_empty())?; + if utc_seconds(e)? > mtime { + return None; + } + } + let v = json::loads(&String::from_utf8_lossy(&fs::read(&p).ok()?)).ok()?; + matches!(get(&v, "sections"), Some(Value::Object(_))).then_some(v) +} + +/// The cache key of a run's summary: the done steps, the message count and +/// summary.json's stamp. +#[derive(Debug, Clone, PartialEq)] +pub struct SummaryKey { + pub done: Vec, + pub events: Option, + pub stamp: Option<(i128, u64)>, +} + +struct SummaryEntry { + key: SummaryKey, + headlines: Option, + summary: Option>, +} + +/// A summary.json's stamp and the overview read from it. +type Stamped = ((i128, u64), Value); + +/// The overview of a summary for the home page (`levels`, `attention`, +/// `top`, `lead`, ...), and the results built from one: what +/// [`crate::plugins::ResultsFns`] gives when the site has them. +pub type Overview = Option; + +// --- the shared state --------------------------------------------------------------------- + +/// The state both the requests and the event streams share. +pub struct Live { + pub cfg: Config, + /// The root, links followed. + pub root: PathBuf, + pub store: Store, + pub steps: Vec, + pub tz: TimeZone, + pub access: AccessLog, + /// The Host names a request may carry. + pub allowed: BTreeSet, + /// Event streams open now, and how many may be. + pub streams: AtomicUsize, + pub max_streams: AtomicUsize, + pub stopping: AtomicBool, + summaries: Mutex>, + /// summary.json path -> (its stamp, its overview). + attention_cache: Mutex>, + /// run -> (the summary results.json reads now, its overview). + summary_overviews: Mutex, Overview)>>, + /// run -> (the summary key, the overview of that summary). + live_overview: Mutex>, +} + +impl Live { + pub fn new(cfg: Config, steps: Vec, access: AccessLog) -> Live { + let root = realpath(&cfg.root); + let mut allowed: BTreeSet = ["127.0.0.1", "localhost", "::1"] + .iter() + .map(|h| (*h).to_owned()) + .collect(); + if cfg.bind != "0.0.0.0" && cfg.bind != "::" { + allowed.insert( + cfg.bind + .trim_matches(|c| c == '[' || c == ']') + .to_lowercase(), + ); + } + Live { + store: Store::new(&cfg.state_dir), + max_streams: AtomicUsize::new(cfg.max_streams), + root, + steps, + tz: TimeZone::system(), + access, + allowed, + streams: AtomicUsize::new(0), + stopping: AtomicBool::new(false), + summaries: Mutex::new(HashMap::new()), + attention_cache: Mutex::new(HashMap::new()), + summary_overviews: Mutex::new(HashMap::new()), + live_overview: Mutex::new(HashMap::new()), + cfg, + } + } + + /// Every run the site knows now. + pub fn runs(&self) -> BTreeMap { + runs::list(&self.root, &self.store.known()) + } + + /// Run `name`, when it is a valid name of a run the site knows. + pub fn run(&self, name: &str) -> Option { + if !runs::valid_run(name) { + return None; + } + self.runs().remove(name) + } + + /// The state of `run`'s newest run, without headlines or plugins (the + /// home page's view), and its messages. + pub fn base_state(&self, run: &Run) -> (State<'_>, Arc>) { + let d = run.folder(&self.root); + let events = self.store.events(&run.name); + let st = self.base_state_of(&run.name, &d, &events); + (st, events) + } + + /// [`Live::base_state`] of the messages `events`. + pub fn base_state_of(&self, name: &str, d: &Path, events: &[Value]) -> State<'_> { + let cur = current_run(events); + let status = files::read_run_status(&d.join("logs").join("run_status.tsv")) + .ok() + .flatten(); + let started = status.as_ref().and_then(|s| s.started()); + let pt_path = d.join("nextflow").join("process_status.tsv"); + let mut pt = files::read_process_status(&pt_path).ok().flatten(); + if pt.is_some() + && let Some(st) = started + && pycompat::mtime::mtime(&pt_path).is_ok_and(|m| m < st) + { + pt = None; // an earlier run's table + } + let events_path = self.store.path(name); + let inp = Inputs { + status: status.as_ref(), + process_status: pt.as_ref(), + trace_tasks: tasks::trace_tasks(&trace_files(d, started), name, &self.tz), + event_tasks: tasks::event_tasks(cur, name), + events: cur, + sources: files::summary_sources(d), + heard: og_runlog::state::last_heard(d, &[events_path.as_path()]), + }; + let mut st = og_runlog::state::build(name, &self.steps, &inp, now(), &self.tz); + if run_finished(events) && matches!(st.run, RunState::Running | RunState::Stopped) { + st.run = RunState::Finished; + } + st + } + + /// [`Live::base_state`] as JSON, with `events`: the message count the + /// page paints a message ahead of `state.json` past. + pub fn base_json(&self, st: &State<'_>, events: usize) -> Value { + let mut v = st.to_json(); + if let Value::Object(o) = &mut v { + o.insert("events".into(), Value::Int(events as i128)); + } + v + } + + /// `state.json` of `run`: the base state, each section's report anchor, + /// the headlines, the summary's stamp and what the plugins add. + pub fn state(&self, run: &Run) -> Value { + let d = run.folder(&self.root); + let (st, events) = self.base_state(run); + let mut v = self.base_json(&st, events.len()); + let mut anchors = Obj::new(); + for k in st.steps.iter().flat_map(|s| s.step.sections.iter()) { + if let Some(a) = anchor_of(k) { + anchors.insert(k.clone(), s(a)); + } + } + let headlines = self.headlines(&run.name, &d, &v); + let stamp = file_stamp(&d.join("summary.json")) + .map_or(Value::Null, |(m, n)| Value::Str(format!("{m}-{n}"))); + let plugins = crate::plugins::state_hooks(self, run, &d, &st, &v, &events); + if let Value::Object(o) = &mut v { + o.insert("anchors".into(), Value::Object(anchors)); + o.insert("headlines".into(), headlines); + o.insert("results_stamp".into(), stamp); + o.insert("plugins".into(), plugins); + } + v + } + + fn summary_key(&self, d: Option<&Path>, st: &Value) -> SummaryKey { + SummaryKey { + done: done_steps(st) + .iter() + .filter_map(|s| get_str(s, "step").map(str::to_owned)) + .collect(), + events: get(st, "events").cloned(), + stamp: d.and_then(|d| file_stamp(&d.join("summary.json"))), + } + } + + /// The run's summary for the steps that are done: its summary.json when + /// that is newer than the end of every done step, else collected now. + /// Read again only when the done steps, the message count or + /// summary.json change; `None` while nothing is done. + pub fn summary(&self, sample: &str, d: &Path, st: &Value) -> Option> { + let key = self.summary_key(Some(d), st); + if let Some(e) = lock(&self.summaries).get(sample) + && e.key == key + { + return e.summary.clone(); + } + let summary = if !key.done.is_empty() && d.is_dir() { + summary_file(d, st) + .or_else(|| self.collect(sample, d)) + .map(Arc::new) + } else { + None + }; + lock(&self.summaries).insert( + sample.to_owned(), + SummaryEntry { + key, + headlines: None, + summary: summary.clone(), + }, + ); + summary + } + + /// `og summary collect` of the folder, in this process; `None` (and a + /// line on stderr) when it cannot be read. + fn collect(&self, sample: &str, d: &Path) -> Option { + use og_summary::collect::{Group, Input, group_c, post}; + let input = Input { + sample: sample.to_owned(), + sample_dir: d.to_string_lossy().into_owned(), + declared_sex: None, + assets: self.cfg.repo_assets.to_string_lossy().into_owned(), + group: Group::All, + tables: bin_tables(group_c::util::default_tables(), &self.cfg.repo_assets), + }; + let got = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| post::collect(&input))); + match got { + Ok(Ok(v)) => Some(v), + Ok(Err(_)) | Err(_) => { + eprintln!("og live serve: summary of {sample} not read"); + None + } + } + } + + /// `{section: {text, tone, step}}` for the sections of the done steps. + pub fn headlines(&self, sample: &str, d: &Path, st: &Value) -> Value { + let summary = self.summary(sample, d, st); + let key = self.summary_key(Some(d), st); + if let Some(e) = lock(&self.summaries).get(sample) + && e.key == key + && let Some(h) = &e.headlines + { + return h.clone(); + } + let mut out = Obj::new(); + let parsed = summary + .as_ref() + .and_then(|sm| Summary::parse(&dumps(sm)).ok()); + if let (Some(sm), Some(parsed)) = (&summary, &parsed) { + let sections = get(sm, "sections"); + for step in done_steps(st) { + let Some(Value::Array(keys)) = get(step, "sections") else { + continue; + }; + for k in keys.iter().filter_map(|k| match k { + Value::Str(x) => Some(x.as_str()), + _ => None, + }) { + let sec = sections.and_then(|x| get(x, k)); + if let Some((text, tone)) = headline(k, sec, parsed) { + out.insert( + k.to_owned(), + obj([ + ("text", Value::Str(text)), + ("tone", s(tone)), + ("step", get(step, "step").cloned().unwrap_or(Value::Null)), + ]), + ); + } + } + } + } + let out = Value::Object(out); + if let Some(e) = lock(&self.summaries).get_mut(sample) + && e.key == key + && e.summary.as_ref().map(Arc::as_ptr) == summary.as_ref().map(Arc::as_ptr) + { + e.headlines = Some(out.clone()); + } + out + } + + /// The overview of `/summary.json` and its time, cached by its + /// stamp; `(None, None)` without one. A summary.json that links out of + /// its run is never read. Without the results the overview is `None`. + pub fn overview(&self, d: &Path) -> (Overview, Option) { + let p = d.join("summary.json"); + if !inside(&p, &realpath(d)) || !inside(&p, &self.root) { + return (None, None); + } + let Ok(mtime) = pycompat::mtime::mtime(&p) else { + return (None, None); + }; + let Some(fns) = &self.cfg.results else { + return (None, Some(mtime)); + }; + let stamp = file_stamp(&p); + if let Some(st) = stamp + && let Some((h, v)) = lock(&self.attention_cache).get(&p) + && *h == st + { + return (Some(v.clone()), Some(mtime)); + } + let n = fs::read(&p) + .ok() + .and_then(|b| json::loads(&String::from_utf8_lossy(&b)).ok()) + .ok_or_else(|| "not JSON".to_owned()) + .and_then(|v| (fns.overview)(&v)); + match n { + Ok(v) => { + if let Some(st) = stamp { + lock(&self.attention_cache).insert(p, (st, v.clone())); + } + (Some(v), Some(mtime)) + } + Err(_) => { + let run = d.file_name().map(|n| n.to_string_lossy().into_owned()); + eprintln!( + "og live serve: {}/summary.json not read", + run.unwrap_or_default() + ); + (None, Some(mtime)) + } + } + } + + /// The count of cards worth a look, from `/summary.json` alone. + pub fn attention(&self, d: &Path) -> (Option, Option) { + let (o, mtime) = self.overview(d); + (o.and_then(|o| get(&o, "attention").cloned()), mtime) + } + + /// Whether `name`'s pages treat it as a child, over the families the + /// site lists now (or those of `runs`). + pub fn child(&self, name: &str, runs: Option<&BTreeMap>) -> bool { + let fams = match runs { + Some(r) => runs::families_of(r), + None => runs::families_of(&self.runs()), + }; + runs::child_of(&fams, name) + } + + /// The overview of the summary results.json reads now, cached with that + /// summary; `None` while there is none. + pub fn summary_overview(&self, name: &str, d: &Path, st: &Value) -> Overview { + let sm = self.summary(name, d, st)?; + let fns = self.cfg.results.as_ref()?; + if let Some((s0, o)) = lock(&self.summary_overviews).get(name) + && Arc::ptr_eq(s0, &sm) + { + return o.clone(); + } + let o = match (fns.overview)(&sm) { + Ok(v) => Some(v), + Err(_) => { + eprintln!("og live serve: overview of {name} not built"); + None + } + }; + lock(&self.summary_overviews).insert(name.to_owned(), (sm, o.clone())); + o + } + + /// Whether `name`'s own page keeps its report closed: a child whose + /// results hold a result to act on, counted from the summary its + /// results.json reads (summary.json while that has none). + pub fn gated( + &self, + name: &str, + d: &Path, + st: &Value, + runs: Option<&BTreeMap>, + ) -> bool { + if !self.child(name, runs) { + return false; + } + let o = self + .summary_overview(name, d, st) + .or_else(|| self.overview(d).0); + has_act(o.as_ref()) + } + + /// `/runs.json`: each run's progress, and the families with their + /// members, for the home page. No result beyond the attention count, + /// the counts by level, the titles of the first findings and the lead; + /// no path. + pub fn runs_index(&self) -> Value { + let runs = self.runs(); + lock(&self.summaries).retain(|k, _| runs.contains_key(k)); + lock(&self.live_overview).retain(|k, _| runs.contains_key(k)); + lock(&self.summary_overviews).retain(|k, _| runs.contains_key(k)); + let keep: BTreeSet = runs + .values() + .map(|r| r.folder(&self.root).join("summary.json")) + .collect(); + lock(&self.attention_cache).retain(|k, _| keep.contains(k)); + let mut people = Vec::new(); + let mut fams = Vec::new(); + for (name, r) in &runs { + let url = format!("/runs/{name}/"); + if r.is_family() { + fams.push(obj([ + ("name", s(name)), + ("url", Value::Str(url)), + ("members", Value::Array(r.members.clone())), + ("unreadable", Value::Bool(r.unreadable)), + ])); + continue; + } + let d = r.folder(&self.root); + let (st, events) = self.base_state(r); + let v = self.base_json(&st, events.len()); + let planned: Vec<_> = st + .steps + .iter() + .filter(|x| x.state.as_str() != "not planned") + .collect(); + let count = |w: &str| planned.iter().filter(|x| x.state.as_str() == w).count() as i128; + let run_v = get(&v, "run").cloned().unwrap_or(Value::Null); + let field = |k: &str| get(&run_v, k).cloned().unwrap_or(Value::Null); + let mut item = Obj::new(); + item.insert("name".into(), s(name)); + item.insert("kind".into(), s(&r.kind)); + item.insert("url".into(), Value::Str(url)); + item.insert("state".into(), field("state")); + item.insert("started_utc".into(), field("started_utc")); + item.insert("last_heard_utc".into(), field("last_heard_utc")); + item.insert("finished_utc".into(), field("finished_utc")); + item.insert("declared_sex".into(), field("declared_sex")); + item.insert("planned".into(), Value::Int(planned.len() as i128)); + item.insert("done".into(), Value::Int(count("done"))); + item.insert("running".into(), Value::Int(count("running"))); + item.insert("failed".into(), Value::Int(count("failed"))); + let (mut o, mtime) = self.overview(&d); + // While a run goes, the summary its page last read (results.json's + // own) is newer than summary.json: count from it while it is + // current, so the home never says less than the person page. + if st.run == RunState::Running && self.cfg.results.is_some() { + let key = self.summary_key(Some(&d), &v); + let hit = lock(&self.summaries) + .get(name) + .filter(|e| e.key.done == key.done && e.key.stamp == key.stamp) + .map(|e| (e.key.clone(), e.summary.clone())); + if let Some((hk, Some(sm))) = hit { + let cached = lock(&self.live_overview) + .get(name) + .filter(|(k, _)| *k == hk) + .map(|(_, o)| o.clone()); + let lo = cached.unwrap_or_else(|| { + let fns = self.cfg.results.as_ref().map(|f| f.overview); + let lo = fns.and_then(|f| match f(&sm) { + Ok(v) => Some(v), + Err(_) => { + eprintln!("og live serve: overview of {name} not built"); + None + } + }); + lock(&self.live_overview).insert(name.clone(), (hk, lo.clone())); + lo + }); + if lo.is_some() { + o = lo; + } + } + } + // A child's results to act on decide whether its own page is + // closed: when the summary its results.json reads, or the + // overview above, has one, that summary's overview replaces it, + // so the home page closes the child exactly when its page does. + if self.child(name, Some(&runs)) { + let so = self.summary_overview(name, &d, &v); + if so.is_some() && (has_act(so.as_ref()) || has_act(o.as_ref())) { + o = so; + } + } + item.insert( + "attention".into(), + o.as_ref() + .and_then(|x| get(x, "attention").cloned()) + .unwrap_or(Value::Null), + ); + item.insert("results".into(), Value::Bool(mtime.is_some())); + if let Some(o) = &o { + for k in ["levels", "top", "lead", "lead_closed"] { + item.insert(k.into(), get(o, k).cloned().unwrap_or(Value::Null)); + } + item.insert( + "screened".into(), + get(o, "screened").cloned().unwrap_or(Value::Bool(false)), + ); + } + people.push(Value::Object(item)); + } + obj([ + ("runs", Value::Array(people)), + ("families", Value::Array(fams)), + ]) + } +} + +/// Whether an overview counts a result to act on. +pub fn has_act(o: Option<&Value>) -> bool { + o.and_then(|o| get(o, "levels")) + .and_then(|l| get(l, "act")) + .is_some_and(truthy) +} + +/// The folder of the `bin/` tables the summary reads: og-summary's own +/// choice (`OG_BIN_TABLES`, else the checkout the binary sits in), else the +/// `bin/` beside the site's `assets/`. An og built outside a checkout would +/// otherwise read `bin/` under the current folder and fail the repeat +/// expansion headline. +fn bin_tables(default: String, repo_assets: &Path) -> String { + if Path::new(&default).join("hla_alleles.tsv").is_file() { + return default; + } + match repo_assets.parent().map(|r| r.join("bin")) { + Some(b) if b.join("hla_alleles.tsv").is_file() => b.to_string_lossy().into_owned(), + _ => default, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_bin_tables_fall_back_to_the_checkout_of_the_assets() { + let repo = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let got = bin_tables("bin-nowhere".into(), &repo.join("assets")); + assert!( + Path::new(&got) + .join("str_intermediate_ranges.tsv") + .is_file(), + "{got}" + ); + let mine = repo.join("bin").to_string_lossy().into_owned(); + assert_eq!(bin_tables(mine.clone(), Path::new("/nowhere/assets")), mine); + assert_eq!( + bin_tables("bin-nowhere".into(), Path::new("/nowhere/assets")), + "bin-nowhere" + ); + } + + fn ev(name: &str) -> Value { + obj([("event", s(name))]) + } + + #[test] + fn the_newest_run() { + let done = [ev("started"), ev("pgp_run_finished")]; + assert!(run_finished(&done)); + let rerun = [ev("started"), ev("pgp_run_finished"), ev("pgp_step")]; + assert!(!run_finished(&rerun) && !run_finished(&[])); + assert_eq!(current_run(&rerun), &rerun[2..]); + let bare = [ + ev("started"), + ev("completed"), + ev("started"), + ev("process_started"), + ]; + assert_eq!(current_run(&bare), &bare[2..]); + assert_eq!(current_run(&done), &done[..]); + } + + #[test] + fn task_names() { + let t = |n: &str| task_name(n); + assert_eq!( + t("SV:MANTA (s1)"), + (Some("MANTA".into()), Some("s1".into())) + ); + assert_eq!( + t(" A:B:CLINVAR_SCREEN "), + (Some("CLINVAR_SCREEN".into()), None) + ); + assert_eq!(t("ROH(x y)"), (Some("ROH".into()), Some("x y".into()))); + assert_eq!(t("a b"), (None, None)); + assert_eq!(t("A:"), (Some(String::new()), None)); + assert_eq!(t(""), (None, None)); + assert_eq!(t("P (a) (b)"), (Some("P".into()), Some("a) (b".into()))); + } + + #[test] + fn utc_times() { + assert_eq!(utc_seconds("1970-01-01T00:01:00Z"), Some(60.0)); + assert_eq!( + utc_seconds("2026-01-02T10:07:00.123"), + utc_seconds("2026-01-02T10:07:00") + ); + assert_eq!(utc_seconds("2026-01-02 10:07:00"), None); + assert_eq!(utc_seconds(""), None); + } + + #[test] + fn a_note_joined_as_two_sentences() { + let l = |a: &[&str]| a.iter().map(|x| (*x).to_owned()).collect::>(); + assert_eq!(with_note(&l(&["A", "N."]), "N.").unwrap(), "A. N."); + assert_eq!(with_note(&l(&["A.", "N."]), "N.").unwrap(), "A. N."); + assert_eq!(with_note(&l(&["A", "B"]), "N.").unwrap(), "A"); + assert_eq!(with_note(&[], "N."), None); + } +} diff --git a/rust/crates/og-live/src/static_files.rs b/rust/crates/og-live/src/static_files.rs index 71d0806aa..2897856f6 100644 --- a/rust/crates/og-live/src/static_files.rs +++ b/rust/crates/og-live/src/static_files.rs @@ -1,2 +1,305 @@ -//! The live site's static files. Empty until its unit of the 1.0.0 plan -//! lands. +//! The live site's static files: the assets under `assets/live` (the +//! scripts, the style and the plugin scripts) and the page templates there, +//! which `og live serve` fills in and never serves as they are. +//! +//! Also the small text rules the pages need: a file's content type, HTML +//! escaping (Python's `html.escape`), `html.unescape` of the report's +//! links, `os.path.normpath` and a query string's values. + +use pycompat::text::html_escape; +use std::fs; +use std::path::Path; + +/// The page templates of `assets/live`: served filled in, never as an asset. +pub const PAGES: [&str; 5] = [ + "index.html", + "login.html", + "home.html", + "family.html", + "finding.html", +]; + +/// What a browser's page shows while there is no run under the root. +pub const NO_RUN: &str = "
  • No run under this folder yet.
  • "; + +/// The content type of a file by its extension (with the dot, as given: +/// the assets keep its case, a run file is asked with it lowered); `None` +/// for one the site has no type for. +pub fn content_type(ext: &str) -> Option<&'static str> { + Some(match ext { + ".html" => "text/html; charset=utf-8", + ".js" => "text/javascript; charset=utf-8", + ".css" => "text/css; charset=utf-8", + ".json" => "application/json", + ".txt" | ".tsv" | ".md" => "text/plain; charset=utf-8", + ".svg" => "image/svg+xml", + ".png" => "image/png", + ".pdf" => "application/pdf", + _ => return None, + }) +} + +/// The type of an asset or run file: [`content_type`], else bytes. +pub fn type_or_bytes(ext: &str) -> &'static str { + content_type(ext).unwrap_or("application/octet-stream") +} + +/// `os.path.splitext(p)[1]`: the last `.` of the file name and what +/// follows, `""` when the name has none or only starts with dots. +pub fn ext(path: &str) -> &str { + let name = path.rsplit('/').next().unwrap_or(path); + let lead = name.len() - name.trim_start_matches('.').len(); + match name[lead..].rfind('.') { + Some(i) => &name[lead + i..], + None => "", + } +} + +/// An asset's name as the site serves it: a file of `assets/live`, or of +/// its `plugins/` folder, whose name starts with a letter or digit and +/// holds only letters, digits, `.`, `_` and `-`. +pub fn valid_asset(name: &str) -> bool { + let rest = name.strip_prefix("plugins/").unwrap_or(name); + crate::runs::valid_run(rest) +} + +/// The bytes and type of asset `name` under `dir`; `None` for a name that +/// is not an asset's, a page template, or a file that is not inside `dir`. +pub fn asset(dir: &Path, name: &str) -> Option<(&'static str, Vec)> { + if !valid_asset(name) || PAGES.contains(&name) { + return None; + } + let p = dir.join(name); + if !p.is_file() || !crate::family::inside(&p, dir) { + return None; + } + let body = fs::read(&p).ok()?; + Some((type_or_bytes(ext(name)), body)) +} + +/// `html.escape(text)`: `&`, `<`, `>`, `"` and `'`. +pub fn esc(text: &str) -> String { + html_escape(text).into_owned() +} + +/// The script tags of the plugin scripts in `assets/plugins`, by name. +pub fn plugin_tags(assets: &Path) -> String { + let mut js: Vec = fs::read_dir(assets.join("plugins")) + .map(|rd| { + rd.filter_map(|e| e.ok()?.file_name().into_string().ok()) + .filter(|f| valid_asset(f) && f.ends_with(".js")) + .collect() + }) + .unwrap_or_default(); + js.sort(); + js.iter() + .map(|f| format!("", esc(f))) + .collect::>() + .join("\n") +} + +/// The page of run `name`: `family.html` for a family (when the assets have +/// it), else `index.html`, filled in for the live site. +pub fn run_page(assets: &Path, name: &str, family: bool) -> std::io::Result { + let fam = assets.join("family.html"); + let page = if family && fam.is_file() { + fs::read_to_string(fam)? + } else { + fs::read_to_string(assets.join("index.html"))? + }; + Ok(page + .replace("{{assets}}", "/assets") + .replace("{{run}}", &esc(name)) + .replace("{{mode}}", "live") + .replace("{{plugins}}", &plugin_tags(assets))) +} + +/// The home page: `home.html` with the plain list of runs a browser without +/// scripts shows; `runs` is (name, kind) by name. +pub fn home_page(assets: &Path, runs: &[(String, String)]) -> std::io::Result { + let items = runs + .iter() + .map(|(n, kind)| { + format!( + "
  • {0} {1}
  • ", + esc(n), + esc(kind) + ) + }) + .collect::>() + .join("\n"); + let page = fs::read_to_string(assets.join("home.html"))?; + Ok(page + .replace("{{assets}}", "/assets") + .replace("{{data}}", "/runs.json") + .replace("{{runs}}", if items.is_empty() { NO_RUN } else { &items })) +} + +/// The login page with `message` and the address to go back to. +pub fn login_page(assets: &Path, message: &str, next: &str) -> std::io::Result { + let page = fs::read_to_string(assets.join("login.html"))?; + Ok(page + .replace("{{message}}", &esc(message)) + .replace("{{next}}", &esc(next))) +} + +/// A finding's page (`finding.html`) for run `name` and finding `id`. +pub fn finding_page(assets: &Path, name: &str, id: &str) -> std::io::Result { + let page = fs::read_to_string(assets.join("finding.html"))?; + Ok(page + .replace("{{assets}}", "/assets") + .replace("{{run}}", &esc(name)) + .replace("{{finding}}", &esc(id)) + .replace("{{mode}}", "live")) +} + +/// `html.unescape` of the references a report's link can hold: the five +/// named ones HTML escaping writes and the numeric ones. Any other `&...;` +/// stays as it is. +pub fn unescape(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(i) = rest.find('&') { + out.push_str(&rest[..i]); + rest = &rest[i..]; + let end = rest[1..].find(';').map(|j| j + 1); + let decoded = end.and_then(|e| { + let name = &rest[1..e]; + let c = match name { + "amp" => Some('&'), + "lt" => Some('<'), + "gt" => Some('>'), + "quot" => Some('"'), + "apos" => Some('\''), + _ => name.strip_prefix('#').and_then(|n| { + let code = match n.strip_prefix(['x', 'X']) { + Some(h) => u32::from_str_radix(h, 16).ok(), + None => n.parse::().ok(), + }; + code.and_then(char::from_u32) + }), + }; + c.map(|c| (c, e)) + }); + match decoded { + Some((c, e)) => { + out.push(c); + rest = &rest[e + 1..]; + } + None => { + out.push('&'); + rest = &rest[1..]; + } + } + } + out.push_str(rest); + out +} + +/// `posixpath.normpath`: `.` and empty parts dropped, `..` taken against +/// the part before it (kept at the start of a relative path), `.` for +/// nothing left. +pub fn normpath(path: &str) -> String { + if path.is_empty() { + return ".".into(); + } + let lead = if path.starts_with("//") && !path.starts_with("///") { + "//" + } else if path.starts_with('/') { + "/" + } else { + "" + }; + let mut parts: Vec<&str> = Vec::new(); + for p in path.split('/') { + match p { + "" | "." => {} + ".." => { + if !lead.is_empty() && parts.is_empty() { + continue; + } + if parts.last().is_some_and(|l| *l != "..") { + parts.pop(); + } else { + parts.push(".."); + } + } + _ => parts.push(p), + } + } + let joined = format!("{lead}{}", parts.join("/")); + if joined.is_empty() { + ".".into() + } else { + joined + } +} + +/// The first value of `key` in a query string, as `urllib.parse.parse_qs` +/// reads it (`+` is a space, `%XX` a byte, an empty value left out). +pub fn query_value(query: &str, key: &str) -> Option { + query + .split('&') + .filter_map(|pair| { + let (k, v) = pair.split_once('=')?; + let k = crate::viewer::unquote(&k.replace('+', " ")); + let v = crate::viewer::unquote(&v.replace('+', " ")); + (k == key && !v.is_empty()).then_some(v) + }) + .next() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extensions_as_splitext_gives_them() { + assert_eq!(ext("a/b.tsv"), ".tsv"); + assert_eq!(ext("x.tar.gz"), ".gz"); + assert_eq!(ext(".hidden"), ""); + assert_eq!(ext("dir.d/noext"), ""); + assert_eq!(type_or_bytes(".HTML"), "application/octet-stream"); + } + + #[test] + fn asset_names() { + assert!(valid_asset("live.css") && valid_asset("plugins/ops.js")); + assert!(!valid_asset("../x") && !valid_asset("plugins/") && !valid_asset(".x")); + assert!(!valid_asset("a/b.js") && !valid_asset("plugins/plugins/a.js")); + } + + #[test] + fn normpath_as_python() { + for (a, b) in [ + ("a/./b", "a/b"), + ("a/../b", "b"), + ("../a", "../a"), + ("a/../../b", "../b"), + ("/../a", "/a"), + ("", "."), + ("a/..", "."), + ("cpsr/", "cpsr"), + ("//a", "//a"), + ("///a", "/a"), + ] { + assert_eq!(normpath(a), b, "{a}"); + } + } + + #[test] + fn unescape_the_references_of_a_link() { + assert_eq!(unescape("a&b/c/d"), "a&b/c/d"); + assert_eq!(unescape("  & &zz"), "  & &zz"); + } + + #[test] + fn query_values() { + assert_eq!( + query_value("next=%2Fruns%2FX%2F&a=1", "next").as_deref(), + Some("/runs/X/") + ); + assert_eq!(query_value("next=", "next"), None); + assert_eq!(query_value("a=b+c", "a").as_deref(), Some("b c")); + } +} diff --git a/rust/crates/og-live/tests/port_live_server.rs b/rust/crates/og-live/tests/port_live_server.rs new file mode 100644 index 000000000..271105180 --- /dev/null +++ b/rust/crates/og-live/tests/port_live_server.rs @@ -0,0 +1,782 @@ +//! `og live serve` on the synthetic DEMO-001 run (tests/fixtures/live): the +//! pages, `state.json`, `/runs.json`, the event stream, the run files, the +//! policy on every answer, the access log, the plugins' parts of +//! `state.json` and the family pages. These are the checks of the Python +//! server's tests that need no login (`tests/test_owngenome_live.py` +//! keeps the login, the sessions and the ingest listener until they are +//! ported). + +#[path = "port_live_server_util.rs"] +mod util; + +use og_live::server::{self, Server}; +use og_live::state; +use og_runlog::state::{RunFiles, Sources}; +use pycompat::json::Value; +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::net::TcpStream; +use std::path::Path; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; +use util::*; + +fn get_key<'a>(v: &'a Value, k: &str) -> &'a Value { + match v { + Value::Object(o) => o.get(k).unwrap_or(&Value::Null), + _ => &Value::Null, + } +} + +fn s(v: &Value) -> &str { + match v { + Value::Str(x) => x, + _ => "", + } +} + +fn steps_of(v: &Value) -> Vec<(String, String)> { + match get_key(v, "steps") { + Value::Array(a) => a + .iter() + .map(|x| { + ( + s(get_key(x, "step")).to_owned(), + s(get_key(x, "state")).to_owned(), + ) + }) + .collect(), + _ => Vec::new(), + } +} + +const CLINVAR_DONE: &str = r#"{"event": "process_completed", "utcTime": "2026-01-02T10:06:30Z", "trace": {"task_id": 9, "name": "CLINICAL:CLINVAR_SCREEN (DEMO-001)", "process": "CLINICAL:CLINVAR_SCREEN", "tag": "DEMO-001", "status": "COMPLETED", "exit": 0, "start": 1767348360000, "complete": 1767348390000}}"#; +const STEP06_OK: &str = + r#"{"event": "pgp_step", "step": "06", "state": "ok", "utcTime": "2026-01-02T10:06:40Z"}"#; + +/// The genome folder with DEMO-001's messages stored, and the site on it. +fn site(name: &str) -> (Work, Server) { + let w = Work::new(name); + make_root(&w); + let ev = fixture_events(); + store(&w, RUN, &ev.iter().map(String::as_str).collect::>()); + store(&w, RUN, &[CLINVAR_DONE, STEP06_OK]); + let srv = start(config(&w)); + (w, srv) +} + +#[test] +fn state_json_is_the_run_state_with_anchors_headlines_and_plugins() { + let (w, srv) = site("state"); + let p = srv.port(); + let r = get(p, &format!("/runs/{RUN}/state.json")); + assert_eq!(r.status, 200); + assert!(policy_ok(&r), "{:?}", r.headers); + let v = r.json(); + // The same files read by og-runlog directly give the same step states. + let steps = og_runlog::steps::read_steps(&repo().join("assets/steps.tsv")).unwrap(); + let ev = events_path(&w, RUN); + let files = RunFiles::read( + &w.root().join(RUN), + &Sources { + traces: &[], + events: Some(&ev), + stale: false, + }, + ) + .unwrap(); + let want = files + .state(RUN, &steps, state::now(), &jiff::tz::TimeZone::system()) + .to_json(); + assert_eq!(steps_of(&v), steps_of(&want)); + let mine: std::collections::HashMap<_, _> = steps_of(&v).into_iter().collect(); + assert_eq!( + ( + mine["06"].as_str(), + mine["07"].as_str(), + mine["11"].as_str(), + mine["12"].as_str() + ), + ("done", "waiting", "done", "skipped") + ); + let anchors = get_key(&v, "anchors"); + assert_eq!(s(get_key(anchors, "clinvar")), "clinvar"); + assert_eq!(s(get_key(anchors, "carrier")), "carrier-list"); + assert_eq!(s(get_key(anchors, "roh")), "roh"); + assert_eq!( + get_key(&v, "events"), + &Value::Int(fixture_events().len() as i128 + 2) + ); + let Value::Object(plugins) = get_key(&v, "plugins") else { + panic!("no plugins: {v:?}") + }; + let names: Vec<&str> = plugins.keys().map(String::as_str).collect(); + assert_eq!(names, ["charts", "family", "ops", "recheck", "search"]); + assert_eq!(get_key(get_key(&v, "plugins"), "family"), &Value::Null); + assert!(matches!( + get_key(get_key(get_key(&v, "plugins"), "ops"), "run"), + Value::Object(_) + )); + assert!(matches!(get_key(&v, "headlines"), Value::Object(_))); + assert_eq!(s(get_key(get_key(&v, "run"), "state")), "running"); +} + +#[test] +fn a_run_is_finished_while_pgp_run_finished_is_its_newest_message() { + let w = Work::new("finished"); + make_root(&w); + store( + &w, + RUN, + &[ + r#"{"event": "started"}"#, + r#"{"event": "pgp_run_finished", "exit": 0}"#, + ], + ); + let srv = start(config(&w)); + let state_of = || { + s(get_key( + get_key( + &get(srv.port(), &format!("/runs/{RUN}/state.json")).json(), + "run", + ), + "state", + )) + .to_owned() + }; + assert_eq!(state_of(), "finished"); + store( + &w, + RUN, + &[r#"{"event": "pgp_step", "step": "14", "state": "started"}"#], + ); + assert_ne!(state_of(), "finished"); +} + +#[test] +fn a_rerun_is_the_new_run_with_every_message_counted() { + let w = Work::new("rerun"); + let again = "DEMO-RERUN"; + let d = w.root().join(again); + let now = state::now() as i64; + write( + &d.join("logs/run_status.tsv"), + &format!("meta\tstarted_epoch\t{now}\nstep\t11\tplanned\n"), + ); + let roh = r#"{"task_id": 8, "name": "CLINICAL:ROH (DEMO-RERUN)", "process": "CLINICAL:ROH", "tag": "DEMO-RERUN", "status": "COMPLETED", "exit": 0, "submit": 1767348300000, "start": 1767348302000, "complete": 1767348340000}"#; + let done = format!(r#"{{"event": "process_completed", "trace": {roh}}}"#); + let old = [ + r#"{"event": "started"}"#, + done.as_str(), + r#"{"event": "completed"}"#, + r#"{"event": "pgp_run_finished", "exit": 0}"#, + r#"{"event": "started"}"#, + ]; + store(&w, again, &old); + let srv = start(config(&w)); + let v = get(srv.port(), &format!("/runs/{again}/state.json")).json(); + let st: std::collections::HashMap<_, _> = steps_of(&v).into_iter().collect(); + assert_eq!(s(get_key(get_key(&v, "run"), "state")), "running"); + assert_ne!(st["11"], "done"); + assert_eq!(get_key(&v, "events"), &Value::Int(5)); + // The control: the whole file read as one run marks step 11 done. + let all: Vec = old + .iter() + .map(|l| pycompat::json::loads(l).unwrap()) + .collect(); + assert_eq!(state::current_run(&all).len(), 1); + assert_eq!(state::current_run(&all[..4]).len(), 4); +} + +#[test] +fn headlines_read_as_the_report_does_and_say_nothing_is_fine() { + let sm = |sec: &str| { + og_render::model::Summary::parse(&format!( + r#"{{"databases": {{}}, "sections": {{{sec}}}}}"# + )) + .unwrap() + }; + let one = |key: &str, sec: &str| { + let summary = sm(&format!(r#""{key}": {sec}"#)); + let raw = pycompat::json::loads(sec).unwrap(); + state::headline(key, Some(&raw), &summary) + }; + let cp = |plp: &str| { + format!( + r#"{{"state": "ok", "title": "CPSR", "data": {{"plp": {plp}, "call_floor_dp": 10, "low_confidence": 0}}}}"# + ) + }; + let tc: Vec<_> = ["[]", r#"[{"gene": "GENEA"}]"#] + .iter() + .map(|p| one("cpsr", &cp(p)).unwrap()) + .collect(); + let mito = |n: u8| { + format!( + r#"{{"state": "ok", "title": "Mito", "data": {{"annotation": {{"source": "VEP", "clinvar_plp": {n}}}}}}}"# + ) + }; + let tm: Vec<_> = [0, 2] + .iter() + .map(|n| one("mito", &mito(*n)).unwrap()) + .collect(); + let tones: Vec<&str> = tc.iter().chain(&tm).map(|t| t.1).collect(); + assert_eq!(tones, ["info", "attention", "info", "attention"]); + assert!( + tc[1].0.contains("Pathogenic/Likely Pathogenic rows: 1"), + "{}", + tc[1].0 + ); + assert_eq!(tm[1].0, "ClinVar pathogenic or likely pathogenic: 2"); + let cv: Vec<_> = [0, 3] + .iter() + .map(|c| one("clinvar", &format!(r#"{{"state": "ok", "title": "ClinVar screen", "data": {{"count": {c}, "hits": []}}}}"#)).unwrap()) + .collect(); + assert_eq!((cv[0].1, cv[1].1), ("info", "attention")); + assert!( + cv.iter() + .all(|t| t.0.contains(og_render::notes::CLINVAR_FREQ_NOTE)) + ); + assert_eq!(one("roh", r#"{"state": "missing"}"#), None); + let failed = one( + "roh", + r#"{"state": "failed", "title": "ROH", "note": "the step failed"}"#, + ) + .unwrap(); + assert_eq!(failed, ("ROH: the step failed".to_owned(), "attention")); + // A helper that cannot read its data: the title and a pointer to the report. + let odd = one( + "clinvar", + r#"{"state": "ok", "title": "ClinVar screen", "data": {"count": "x"}}"#, + ) + .unwrap(); + assert_eq!( + odd, + ( + "ClinVar screen: finished, see the report".to_owned(), + "info" + ) + ); +} + +#[test] +fn the_run_files_the_report_links_and_nothing_else() { + let (w, srv) = site("files"); + let p = srv.port(); + let mut seen = Vec::new(); + let mut go = |path: &str| { + let r = get(p, path); + seen.push((path.to_owned(), r.clone())); + r + }; + let r = go(&format!("/runs/{RUN}/report.html")); + assert_eq!(r.status, 200); + assert!(r.text().contains("clinvar/hits.tsv")); + assert_eq!( + r.header("content-security-policy"), + Some(og_live::csp::RUN_HTML_CSP) + ); + let r = go(&format!("/runs/{RUN}/clinvar/hits.tsv?x=1")); + assert_eq!( + (r.status, r.header("content-type")), + (200, Some("text/plain; charset=utf-8")) + ); + assert!(r.body.starts_with(b"gene\t")); + for bad in [ + format!("/runs/{RUN}/clinvar/unlinked.tsv"), + format!("/runs/{RUN}/clinvar/escape.tsv"), + format!("/runs/{RUN}/logs/run_status.tsv"), + format!("/runs/{RUN}/..%2F..%2Foutside.txt"), + format!("/runs/{RUN}/cpsr/{RUN}.cpsr.grch38.classification.tsv.gz"), + format!("/runs/{RUN}/twopages/"), + format!("/runs/{RUN}/clinvar/"), + format!("/runs/{RUN}/twopages/a.html"), + "/runs/NO-SUCH-RUN/".to_owned(), + "/runs/../x".to_owned(), + "/nothing/here".to_owned(), + ] { + assert_eq!(go(&bad).status, 404, "{bad}"); + } + let r = go(&format!("/runs/{RUN}/cpsr/")); + let page = format!("/runs/{RUN}/cpsr/{RUN}.cpsr.grch38.html"); + assert_eq!((r.status, r.header("location")), (302, Some(page.as_str()))); + let r = go(&page); + assert_eq!(r.status, 200); + assert!(r.text().contains("CPSR")); + assert_eq!( + r.header("content-security-policy"), + Some(og_live::csp::RUN_HTML_CSP) + ); + let r = go(&format!("/runs/{RUN}")); + assert_eq!( + (r.status, r.header("location")), + (301, Some("/runs/DEMO-001/")) + ); + let r = go("/"); + assert_eq!(r.status, 200); + assert!(r.text().contains("/runs/DEMO-001/")); + let r = go(&format!("/runs/{RUN}/")); + assert_eq!(r.status, 200); + assert!(r.text().contains("data-mode=\"live\"") || r.text().contains("live")); + let r = go("/runs.json"); + let v = r.json(); + let Value::Array(people) = get_key(&v, "runs") else { + panic!("{v:?}") + }; + assert_eq!(s(get_key(&people[0], "name")), RUN); + assert_eq!(s(get_key(&people[0], "state")), "running"); + assert_eq!(get_key(&people[0], "attention"), &Value::Null); + assert_eq!(get_key(&v, "families"), &Value::Array(Vec::new())); + let r = go("/assets/live.css"); + assert_eq!( + (r.status, r.header("content-type")), + (200, Some("text/css; charset=utf-8")) + ); + assert_eq!( + go("/assets/index.html").status, + 404, + "a page template is never an asset" + ); + assert_eq!(go("/assets/..%2Findex.html").status, 404); + let r = go("/login?next=%2Fruns%2F"); + assert_eq!(r.status, 200); + let r = go(&format!("/runs/{RUN}/results.json")); + assert_eq!(r.status, 200); + assert_eq!( + s(get_key(&r.json(), "note")), + og_live::plugins::NO_RESULTS_NOTE + ); + assert_eq!(go(&format!("/runs/{RUN}/finding/clinvar-x-1/")).status, 404); + // Every answer carries the policy, nosniff, no-referrer and no-store. + let bad: Vec<&str> = seen + .iter() + .filter(|(_, r)| { + !policy_ok(r) && r.header("content-security-policy") != Some(og_live::csp::RUN_HTML_CSP) + }) + .map(|(p, _)| p.as_str()) + .collect(); + assert!(bad.is_empty(), "no policy on {bad:?}"); + // The access log: a row per page read, no query string. + let rows = log_rows(&w); + let has = |a: &str, run: &str, sec: &str, st: &str| { + rows.iter() + .any(|r| r.len() == 7 && r[3] == a && r[4] == run && r[5] == sec && r[6] == st) + }; + assert!(has("report", RUN, "-", "200"), "{rows:?}"); + assert!(has("file", RUN, "clinvar/hits.tsv", "200"), "{rows:?}"); + assert!(has("file", RUN, "clinvar/escape.tsv", "404"), "{rows:?}"); + assert!(has("index", "-", "-", "200") && has("runs", "-", "-", "200")); + assert!( + !fs::read_to_string(w.state().join("access.log")) + .unwrap() + .contains('?') + ); +} + +#[test] +fn a_foreign_host_and_a_post_from_another_site_are_refused() { + let (w, srv) = site("host"); + let p = srv.port(); + let r = request(p, "GET", "/", &[("Host", "evil.example.net")]); + assert_eq!(r.status, 403); + assert!(policy_ok(&r)); + assert_eq!( + request(p, "GET", "/", &[("Host", "localhost:8443")]).status, + 200 + ); + assert_eq!(request(p, "GET", "/", &[("Host", "[::1]:1")]).status, 200); + assert_eq!( + request(p, "GET", "/", &[("Host", "127.0.0.1:x")]).status, + 403 + ); + let evil = request( + p, + "POST", + "/logout", + &[ + ("Origin", "https://evil.example.net"), + ("Content-Length", "0"), + ], + ); + assert_eq!(evil.status, 403); + let site = request( + p, + "POST", + "/logout", + &[("Sec-Fetch-Site", "cross-site"), ("Content-Length", "0")], + ); + assert_eq!(site.status, 403); + let own = request( + p, + "POST", + "/logout", + &[("Origin", "null"), ("Content-Length", "0")], + ); + assert_eq!((own.status, own.header("location")), (303, Some("/login"))); + assert!(own.header("set-cookie").unwrap_or("").contains("Max-Age=0")); + let login = request(p, "POST", "/login", &[("Content-Length", "10")]); + assert_eq!((login.status, login.header("location")), (303, Some("/"))); + assert_eq!( + request( + p, + "POST", + &format!("/runs/{RUN}/map.json"), + &[("Content-Length", "0")] + ) + .status, + 404 + ); + assert_eq!(request(p, "PUT", "/", &[]).status, 501); + let head = request(p, "HEAD", "/", &[]); + assert_eq!(head.status, 200); + assert!(head.body.is_empty()); + let rows = log_rows(&w); + assert!(rows.iter().any(|r| r[3] == "host-refused" && r[6] == "403")); + assert!( + rows.iter() + .any(|r| r[3] == "origin-refused" && r[6] == "403") + ); +} + +#[test] +fn the_run_page_gets_one_script_tag_per_plugin_script() { + let w = Work::new("plugin-tags"); + make_root(&w); + let assets = w.dir.join("assets/live"); + fs::create_dir_all(assets.join("plugins")).unwrap(); + for f in ["index.html", "home.html", "login.html", "live.css"] { + fs::copy(repo().join("assets/live").join(f), assets.join(f)).unwrap(); + } + for f in ["a.js", "b.js", "notes.txt", ".hidden.js"] { + write(&assets.join("plugins").join(f), "// plugin\n"); + } + let mut c = config(&w); + c.assets = assets; + let srv = start(c); + let page = get(srv.port(), &format!("/runs/{RUN}/")).text(); + let tags: Vec<&str> = page + .split("<script src=\"/assets/plugins/") + .skip(1) + .filter_map(|x| x.split('"').next()) + .collect(); + assert_eq!(tags, ["a.js", "b.js"]); + assert_eq!(get(srv.port(), "/assets/plugins/a.js").status, 200); + assert_eq!(get(srv.port(), "/assets/plugins/.hidden.js").status, 404); +} + +#[test] +fn search_charts_and_recheck_files_are_served_and_named_in_state_json() { + let (w, srv) = site("run-globs"); + let d = w.root().join(RUN); + write( + &d.join(format!("{RUN}_search_index.json")), + "{\"genes\": {}}\n", + ); + write(&d.join(format!("{RUN}_charts.json")), "{}\n"); + write(&d.join("OTHER_charts.json"), "{}\n"); + write( + &d.join("recheck/state.json"), + r#"{"checked_utc": "2026-10-06T00:00:00Z", "items": []}"#, + ); + let p = srv.port(); + let v = get(p, &format!("/runs/{RUN}/state.json")).json(); + let pl = get_key(&v, "plugins"); + assert_eq!( + s(get_key(get_key(pl, "search"), "file")), + format!("{RUN}_search_index.json") + ); + assert_eq!( + s(get_key(get_key(pl, "charts"), "file")), + format!("{RUN}_charts.json") + ); + assert!(matches!(get_key(get_key(pl, "search"), "mtime"), Value::Int(n) if *n > 0)); + assert_eq!( + s(get_key(get_key(pl, "recheck"), "banner")), + "Database re-check of 2026-10-06T00:00:00Z: nothing newer than this run" + ); + for f in [ + format!("{RUN}_search_index.json"), + format!("{RUN}_charts.json"), + "OTHER_charts.json".into(), + "recheck/state.json".into(), + ] { + let r = get(p, &format!("/runs/{RUN}/{f}")); + assert_eq!( + (r.status, r.header("content-type")), + (200, Some("application/json")), + "{f}" + ); + } + assert_eq!(get(p, &format!("/runs/{RUN}/recheck/log.tsv")).status, 404); +} + +/// An open `/events` stream, read in a thread. +struct Stream { + items: Arc<Mutex<Vec<(usize, Value)>>>, + comments: Arc<AtomicUsize>, + closed: Arc<AtomicBool>, + status: Arc<Mutex<String>>, + sock: TcpStream, +} + +impl Stream { + fn open(port: u16, last: Option<usize>) -> Stream { + let mut sock = TcpStream::connect(("127.0.0.1", port)).unwrap(); + let mut head = format!("GET /runs/{RUN}/events HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\n"); + if let Some(l) = last { + head.push_str(&format!("Last-Event-ID: {l}\r\n")); + } + sock.write_all(format!("{head}\r\n").as_bytes()).unwrap(); + let (items, comments, closed, status) = ( + Arc::new(Mutex::new(Vec::new())), + Arc::new(AtomicUsize::new(0)), + Arc::new(AtomicBool::new(false)), + Arc::new(Mutex::new(String::new())), + ); + let r = sock.try_clone().unwrap(); + let (i2, c2, d2, s2) = ( + items.clone(), + comments.clone(), + closed.clone(), + status.clone(), + ); + thread::spawn(move || { + let mut lines = BufReader::new(r).lines(); + if let Some(Ok(l)) = lines.next() { + *s2.lock().unwrap() = l; + } + for l in lines.by_ref() { + match l { + Ok(l) if l.is_empty() => break, + Ok(_) => {} + Err(_) => break, + } + } + let mut id = 0; + let mut data = None; + for l in lines { + let Ok(l) = l else { break }; + if let Some(x) = l.strip_prefix("id: ") { + id = x.parse().unwrap(); + } else if let Some(x) = l.strip_prefix("data: ") { + data = Some(pycompat::json::loads(x).unwrap()); + } else if l.starts_with(": heartbeat") { + c2.fetch_add(1, Ordering::SeqCst); + } else if l.is_empty() + && let Some(d) = data.take() + { + i2.lock().unwrap().push((id, d)); + } + } + d2.store(true, Ordering::SeqCst); + }); + Stream { + items, + comments, + closed, + status, + sock, + } + } + + fn wait(&self, n: usize, t: Duration) -> bool { + let end = Instant::now() + t; + while Instant::now() < end && self.items.lock().unwrap().len() < n { + thread::sleep(Duration::from_millis(20)); + } + self.items.lock().unwrap().len() >= n + } + + fn items(&self) -> Vec<(usize, Value)> { + self.items.lock().unwrap().clone() + } +} + +impl Drop for Stream { + fn drop(&mut self) { + let _ = self.sock.shutdown(std::net::Shutdown::Both); + } +} + +#[test] +fn the_event_stream_sends_each_stored_message_and_resumes_after_last_event_id() { + let w = Work::new("stream"); + make_root(&w); + let ev = fixture_events(); + store(&w, RUN, &ev.iter().map(String::as_str).collect::<Vec<_>>()); + let mut c = config(&w); + c.heartbeat = Duration::from_millis(300); + let srv = start(c); + let p = srv.port(); + let n0 = ev.len(); + let st = Stream::open(p, None); + assert!(st.wait(n0, Duration::from_secs(5)), "{:?}", st.items()); + assert!(st.status.lock().unwrap().contains(" 200 ")); + let t0 = Instant::now(); + store(&w, RUN, &[CLINVAR_DONE]); + assert!(st.wait(n0 + 1, Duration::from_secs(2)), "not within 2 s"); + let last = st.items().last().unwrap().1.clone(); + assert_eq!( + (s(get_key(&last, "event")), s(get_key(&last, "step"))), + ("process_completed", "06") + ); + assert!(t0.elapsed() < Duration::from_secs(2)); + // What the stream sent is messages() of the stored file. + let steps = og_runlog::steps::read_steps(&repo().join("assets/steps.tsv")).unwrap(); + let stored = og_runlog::files::read_events(Some(&events_path(&w, RUN))).unwrap(); + let want = state::messages(&stored, &steps, 0); + assert_eq!(st.items(), want); + let text = state::dumps(&Value::Array(want.iter().map(|(_, v)| v.clone()).collect())); + for leak in [ + "/data/", + "workDir", + "outdir", + "\"tag\"", + "HP:0001250", + "bcftools", + ] { + assert!(!text.contains(leak), "{leak} in {text}"); + } + thread::sleep(Duration::from_millis(800)); + assert!(st.comments.load(Ordering::SeqCst) >= 1, "no heartbeat"); + assert!( + !st.closed.load(Ordering::SeqCst), + "a quiet stream was closed" + ); + let st2 = Stream::open(p, Some(n0 - 1)); + assert!(st2.wait(2, Duration::from_secs(5))); + let ids: Vec<usize> = st2.items().iter().map(|(i, _)| *i).collect(); + assert_eq!(ids, [n0, n0 + 1]); + // Every slot taken: 503, and the stream is not opened. + srv.live().max_streams.store(2, Ordering::SeqCst); + let r = get(p, &format!("/runs/{RUN}/events")); + assert_eq!( + (r.status, r.text().as_str()), + (503, "too many open event streams\n") + ); + drop(st); + drop(st2); + let end = Instant::now() + Duration::from_secs(5); + while srv.live().streams.load(Ordering::SeqCst) > 0 && Instant::now() < end { + thread::sleep(Duration::from_millis(50)); + } + assert_eq!( + srv.live().streams.load(Ordering::SeqCst), + 0, + "a closed stream kept its slot" + ); + let rows = log_rows(&w); + assert!(rows.iter().any(|r| r[3] == "events" && r[6] == "503")); +} + +#[test] +fn the_command_line_refuses_what_it_cannot_do_yet() { + let w = Work::new("cli"); + make_root(&w); + let root = w.root().to_string_lossy().into_owned(); + let run = |a: &[&str]| { + let args: Vec<String> = a.iter().map(|x| (*x).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = server::cli(&args, &mut out, &mut err); + (code, String::from_utf8_lossy(&err).into_owned()) + }; + let (code, err) = run(&["--root", &root, "--bind", "0.0.0.0", "--port", "0"]); + assert!(code == 2 && err.contains("password"), "{code} {err}"); + let (code, err) = run(&["--root", &root, "--password-file", "/nonexistent"]); + assert!(code == 2 && err.contains("not ported yet"), "{code} {err}"); + let (code, err) = run(&["--port", "0"]); + assert!( + code == 2 && err.contains("--root is required"), + "{code} {err}" + ); + let (code, err) = run(&["--root", "/nonexistent/genome", "--port", "0"]); + assert!(code == 1 && err.contains("not a folder"), "{code} {err}"); + let conf = w.dir.join("live.conf"); + write(&conf, &format!("root = {root}\nmax-streams = 3\n")); + let c = server::configure(&["--config".into(), conf.to_string_lossy().into_owned()]).unwrap(); + assert_eq!((c.root.as_path(), c.max_streams), (Path::new(&root), 3)); + assert_eq!(c.state_dir, w.root().join(".owngenome-live")); +} + +#[test] +fn a_family_folder_is_a_run_with_its_own_page_and_files() { + let w = Work::new("family"); + make_root(&w); + let fam = "DEMO-FAMILY"; + let fd = w.root().join(fam); + fs::create_dir_all(&fd).unwrap(); + for e in fs::read_dir(repo().join("tests/fixtures/live/demo").join(fam)).unwrap() { + let e = e.unwrap(); + fs::copy(e.path(), fd.join(e.file_name())).unwrap(); + } + let srv = start(config(&w)); + let p = srv.port(); + let v = get(p, "/runs.json").json(); + let Value::Array(fams) = get_key(&v, "families") else { + panic!("{v:?}") + }; + assert_eq!(fams.len(), 1); + assert_eq!(s(get_key(&fams[0], "name")), fam); + assert_eq!(s(get_key(&fams[0], "url")), "/runs/DEMO-FAMILY/"); + assert!(matches!(get_key(&fams[0], "members"), Value::Array(m) if !m.is_empty())); + let page = get(p, &format!("/runs/{fam}/")).text(); + let want = fs::read_to_string(repo().join("assets/live/family.html")).unwrap(); + assert!( + page.starts_with(&want[..want.find("{{").unwrap()]), + "not the family page" + ); + let st = get(p, &format!("/runs/{fam}/state.json")).json(); + assert_eq!( + s(get_key(get_key(get_key(&st, "plugins"), "family"), "kind")), + "family" + ); + let r = get(p, &format!("/runs/{fam}/family.json")); + assert_eq!(r.status, 200, "{}", r.text()); + let page = r.json(); + assert!( + matches!(get_key(&page, "layout"), Value::Object(_)), + "{page:?}" + ); + let runs = get_key(&page, "runs"); + assert_eq!( + s(get_key(get_key(runs, RUN), "url")), + "/runs/DEMO-001/", + "{runs:?}" + ); + assert_eq!(get_key(get_key(runs, RUN), "child"), &Value::Bool(false)); + let r = get(p, &format!("/family/{fam}/pedigree.svg")); + assert_eq!( + (r.status, r.header("content-type")), + (200, Some("image/svg+xml")) + ); + assert!(r.text().contains("<svg")); + let r = get(p, &format!("/family/{fam}/relationships.tsv")); + assert_eq!( + (r.status, r.header("content-type")), + (200, Some("text/plain; charset=utf-8")) + ); + for bad in [ + format!("/family/{fam}/de_novo.tsv"), + format!("/family/{fam}/nothing.tsv"), + format!("/family/{RUN}/pedigree.svg"), + format!("/runs/{RUN}/family.json"), + "/family/NO-SUCH/pedigree.svg".into(), + ] { + assert_eq!(get(p, &bad).status, 404, "{bad}"); + } + // A member's page names its family; DEMO-001 has no parents listed. + let st = get(p, &format!("/runs/{RUN}/state.json")).json(); + let m = get_key(get_key(&st, "plugins"), "family"); + assert_eq!(s(get_key(m, "kind")), "member", "{m:?}"); + assert_eq!(get_key(m, "child"), &Value::Bool(false)); + let Value::Array(listed) = get_key(m, "families") else { + panic!("{m:?}") + }; + assert_eq!(s(get_key(&listed[0], "url")), "/runs/DEMO-FAMILY/"); + let rows = log_rows(&w); + assert!( + rows.iter() + .any(|r| r[3] == "family" && r[4] == fam && r[5] == "pedigree.svg" && r[6] == "200") + ); +} diff --git a/rust/crates/og-live/tests/port_live_server_browser.rs b/rust/crates/og-live/tests/port_live_server_browser.rs new file mode 100644 index 000000000..9f76765f9 --- /dev/null +++ b/rust/crates/og-live/tests/port_live_server_browser.rs @@ -0,0 +1,152 @@ +//! The live run page served by `og live serve`, in a headless Chrome +//! (og-browser; skipped without Chrome, except in CI): the step map, a box +//! that repaints after a stored message, a stream refused with 503 and +//! opened again, a box's detail with its tasks and its report anchors, and +//! no policy violation. + +#[path = "port_live_server_util.rs"] +mod util; + +use og_browser::{Browser, Value, chrome_or_skip}; +use std::sync::atomic::Ordering; +use std::thread; +use std::time::Duration; +use util::*; + +fn text(v: Option<Value>) -> String { + match v { + Some(Value::Str(s)) => s, + Some(Value::Int(n)) => n.to_string(), + Some(Value::Bool(b)) => b.to_string(), + _ => String::new(), + } +} + +fn box_state(b: &mut Browser, step: &str) -> String { + text(b.eval(&format!("(function(){{var g=document.querySelector('g.box[data-step=\"{step}\"]');return g?g.dataset.state:'';}})()")).ok()) +} + +#[test] +fn the_run_page_draws_the_steps_and_follows_the_stream() { + let Some(chrome) = chrome_or_skip("load the live run page") else { + return; + }; + let w = Work::new("browser"); + make_root(&w); + let ev = fixture_events(); + store(&w, RUN, &ev.iter().map(String::as_str).collect::<Vec<_>>()); + store( + &w, + RUN, + &[ + r#"{"event": "process_completed", "utcTime": "2026-01-02T10:06:30Z", "trace": {"task_id": 9, "name": "CLINICAL:CLINVAR_SCREEN (DEMO-001)", "process": "CLINICAL:CLINVAR_SCREEN", "tag": "DEMO-001", "status": "COMPLETED", "exit": 0, "start": 1767348360000, "complete": 1767348390000}}"#, + r#"{"event": "pgp_step", "step": "06", "state": "ok", "utcTime": "2026-01-02T10:06:40Z"}"#, + ], + ); + let srv = start(config(&w)); + let base = format!("http://127.0.0.1:{}", srv.port()); + let mut b = Browser::launch(&chrome, &w.dir.join("chrome")).unwrap(); + let wait = |b: &mut Browser, expr: &str, secs: u64| b.wait(expr, Duration::from_secs(secs)); + + b.go(&format!("{base}/runs/{RUN}/")).unwrap(); + let ready = wait( + &mut b, + "document.documentElement.dataset.pgpStatus === 'ready'", + 30, + ); + assert!( + ready.is_some(), + "the run page did not load: {}", + text( + b.eval("location.href + ' ' + document.documentElement.outerHTML.slice(0, 600)") + .ok() + ) + ); + let n = text( + b.eval("document.querySelectorAll('#map g.box').length") + .ok(), + ); + assert!(n.parse::<usize>().is_ok_and(|n| n >= 4), "boxes: {n}"); + assert_eq!(box_state(&mut b, "06"), "done"); + + // A stored message repaints its box. + assert_eq!(box_state(&mut b, "07"), "waiting"); + store( + &w, + RUN, + &[ + r#"{"event": "process_started", "utcTime": "2026-01-02T10:07:00Z", "trace": {"task_id": 20, "name": "PGX:PHARMCAT_PREPROCESS (DEMO-001)", "process": "PGX:PHARMCAT_PREPROCESS", "tag": "DEMO-001", "status": "RUNNING", "start": 1767348420000}}"#, + ], + ); + let after = wait( + &mut b, + "document.querySelector('g.box[data-step=\"07\"]').dataset.state === 'running' && 'running'", + 10, + ); + assert_eq!(text(after), "running", "07 did not repaint"); + + // Every stream slot taken: the stream answers 503, and the page opens it + // again once a slot is free. + let live = srv.live(); + let keep = live.max_streams.swap(0, Ordering::SeqCst); + b.go(&format!("{base}/runs/{RUN}/")).unwrap(); + let refused = wait( + &mut b, + "document.documentElement.dataset.pgpStatus === 'offline' || \ + (document.documentElement.dataset.pgpStreams === '1' && \ + document.documentElement.dataset.pgpStatus === 'ready' && window.PGP.state && 'loaded')", + 10, + ); + thread::sleep(Duration::from_secs(1)); + live.max_streams.store(keep, Ordering::SeqCst); + let reopened = wait( + &mut b, + "Number(document.documentElement.dataset.pgpStreams) >= 2 && 'reopened'", + 15, + ); + let n0 = text(b.eval("window.PGP.state.events").ok()); + thread::sleep(Duration::from_secs(1)); + store( + &w, + RUN, + &[ + r#"{"event": "pgp_step", "step": "07", "state": "ok", "utcTime": "2026-01-02T10:07:30Z"}"#, + ], + ); + let n1 = wait( + &mut b, + &format!("window.PGP.state && window.PGP.state.events > {n0} && window.PGP.state.events"), + 10, + ); + assert!( + refused.is_some() && text(reopened.clone()) == "reopened" && n1.is_some(), + "{refused:?} {reopened:?} {n0} {n1:?}" + ); + + // A box's detail: its tasks, and a link to each section's report anchor. + b.eval("document.querySelector('g.box[data-step=\"11\"]').dispatchEvent(new MouseEvent('click', {bubbles: true})); true").unwrap(); + let detail = text(wait( + &mut b, + "document.getElementById('detail').textContent.indexOf('Step 11') >= 0 && document.getElementById('detail').textContent", + 20, + )); + assert!(detail.contains("ROH"), "{detail}"); + b.eval("document.querySelector('g.box[data-step=\"06\"]').dispatchEvent(new MouseEvent('click', {bubbles: true})); true").unwrap(); + let links = text(wait( + &mut b, + "document.getElementById('detail').textContent.indexOf('Step 06') >= 0 && \ + Array.from(document.querySelectorAll('#detail a')).map(function (a) { return a.getAttribute('href'); }).join(' ')", + 20, + )); + assert!( + links.contains("report.html#clinvar") && links.contains("report.html#carrier-list"), + "{links}" + ); + let blocked = text( + b.eval("document.documentElement.dataset.pgpBlocked || ''") + .ok(), + ); + assert_eq!(blocked, "", "the policy blocked what the page needed"); + let errors = b.page_errors(); + assert!(errors.is_empty(), "page errors: {errors:?}"); +} diff --git a/rust/crates/og-live/tests/port_live_server_map.rs b/rust/crates/og-live/tests/port_live_server_map.rs new file mode 100644 index 000000000..564e9d4b2 --- /dev/null +++ b/rust/crates/og-live/tests/port_live_server_map.rs @@ -0,0 +1,382 @@ +//! The genome map on `og live serve`, with a fake og that records its +//! arguments and stdin and answers as told (the mode file beside it). +//! +//! 1. Without the results the map is never built: 503, og not run (the +//! results hold the veil a child's page keeps closed); +//! 2. with them, `/runs/<run>/map.json` is og's bytes unchanged (equal to a +//! direct og call with the same arguments and stdin), as +//! application/gzip with the site's headers; +//! 3. og gets `map build --root ROOT --run RUN --assets <repo>/assets +//! --cache-dir STATE/map`, and its stdin is `{"results": the body +//! /runs/<run>/results.json sends}`; +//! 4. the gene files are og's bytes of `og map genes --chrom` and +//! `--names`; +//! 5. og exit 3 is 404 with og's reason, 127 or no og 503 with the setup +//! line, any other exit 500 with none of og's bytes; +//! 6. a family's run and an unknown run are 404 and og is not run; +//! 7. one access-log row `map` per request, naming only the run or +//! `genes`; there is no POST route. + +#[path = "port_live_server_util.rs"] +mod util; + +use og_live::plugins::ResultsFns; +use pycompat::json::{Value, loads}; +use std::collections::BTreeSet; +use std::fs; +use std::io::Write; +use std::os::unix::fs::PermissionsExt; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use util::*; + +const FAKE_OG: &str = r#"#!/bin/sh +d=$(dirname "$0") +n=$(ls "$d/calls" | wc -l | tr -d ' ') +mkdir -p "$d/calls/$n" +for a in "$@"; do printf '%s\n' "$a"; done > "$d/calls/$n/args" +cat > "$d/calls/$n/stdin" +mode=$(cat "$d/mode" 2>/dev/null || echo ok) +case "$mode" in +nomap) echo "og map build: no map for this run: the planted reason" >&2; exit 3;; +notbuilt) echo "ERROR: og is not built for this host; run scripts/setup.sh --og" >&2; exit 127;; +crash) printf 'half a document'; printf 'og map build: the planted first line\nand a second line\n' >&2; exit 1;; +esac +cat "$d/payload.bin" +cat "$d/calls/$n/args" "$d/calls/$n/stdin" | cksum +printf '\r\n\000end' +"#; + +const STEP06_OK: &str = + r#"{"event": "pgp_step", "step": "06", "state": "ok", "utcTime": "2026-01-02T10:06:40Z"}"#; + +const FINDING: &str = "clinvar-plantgene1-1"; + +/// The stub results: their answer carries every argument, so a site that +/// built them unlike results.json sends og another body. +fn build(name: &str, summary: &Value, files: Option<&BTreeSet<String>>) -> Result<Value, String> { + let files: Vec<String> = files + .map(|f| f.iter().cloned().collect()) + .unwrap_or_default(); + let text = format!( + r#"{{"schema_version": 1, "sample": {}, "chapters": [], "note": null, + "findings": [{{"id": "{FINDING}", "gene": "PLANTGENE1", "title": "planted finding", "level": "info"}}], + "summary": {}, "files": {}}}"#, + og_live::state::dumps(&Value::Str(name.into())), + og_live::state::dumps(summary), + og_live::state::dumps(&Value::Array(files.into_iter().map(Value::Str).collect())), + ); + loads(&text).map_err(|e| e.to_string()) +} + +fn overview(_: &Value) -> Result<Value, String> { + loads(r#"{"attention": 0, "levels": {}}"#).map_err(|e| e.to_string()) +} + +struct Map { + w: Work, + og: PathBuf, +} + +impl Map { + fn new(name: &str) -> Map { + let w = Work::new(name); + make_root(&w); + let d = w.root().join(RUN); + write( + &d.join(format!("{RUN}_report.html")), + "<a href=\"tables/linked.tsv\">a linked table</a>", + ); + let ev = fixture_events(); + store(&w, RUN, &ev.iter().map(String::as_str).collect::<Vec<_>>()); + store(&w, RUN, &[STEP06_OK]); + write( + &d.join("summary.json"), + r#"{"sections": {}, "planted": true}"#, + ); + let fam = w.root().join("FAM-A"); + fs::create_dir_all(&fam).unwrap(); + fs::copy( + repo().join("tests/fixtures/live/demo/DEMO-FAMILY/members.tsv"), + fam.join("members.tsv"), + ) + .unwrap(); + let bin = w.dir.join("bin"); + fs::create_dir_all(bin.join("calls")).unwrap(); + let og = bin.join("og"); + fs::write(&og, FAKE_OG).unwrap(); + fs::set_permissions(&og, fs::Permissions::from_mode(0o755)).unwrap(); + let mut payload = b"\x1f\x8b".to_vec(); + for _ in 0..1200 { + payload.extend(0u8..=255); + } + fs::write(bin.join("payload.bin"), payload).unwrap(); + Map { w, og } + } + + fn config(&self, results: bool) -> og_live::server::Config { + let mut c = config(&self.w); + c.og = self.og.clone(); + c.results = results.then_some(ResultsFns { build, overview }); + c + } + + fn mode(&self, m: &str) { + fs::write(self.og.with_file_name("mode"), m).unwrap(); + } + + /// The fake og's calls since the last read: (arguments, stdin). + fn calls(&self) -> Vec<(Vec<String>, String)> { + let d = self.og.with_file_name("calls"); + let mut names: Vec<usize> = fs::read_dir(&d) + .unwrap() + .filter_map(|e| e.ok()?.file_name().to_str()?.parse().ok()) + .collect(); + names.sort(); + let out = names + .iter() + .map(|n| { + let c = d.join(n.to_string()); + let args = fs::read_to_string(c.join("args")) + .unwrap() + .lines() + .map(str::to_owned) + .collect(); + (args, fs::read_to_string(c.join("stdin")).unwrap()) + }) + .collect(); + fs::remove_dir_all(&d).unwrap(); + fs::create_dir_all(&d).unwrap(); + out + } + + /// The fake og run directly with `args` and `stdin`: its stdout. + fn direct(&self, args: &[String], stdin: &str) -> Vec<u8> { + let mut p = Command::new(&self.og) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + p.stdin.take().unwrap().write_all(stdin.as_bytes()).unwrap(); + let out = p.wait_with_output().unwrap().stdout; + self.calls(); + out + } +} + +fn sha(b: &[u8]) -> String { + // A content digest without a hashing crate: equal bytes, equal length. + format!( + "{}:{:x}", + b.len(), + b.iter() + .fold(0xcbf29ce484222325u64, |h, x| (h ^ u64::from(*x)) + .wrapping_mul(0x100000001b3)) + ) +} + +#[test] +fn without_the_results_the_map_is_never_built() { + let m = Map::new("map-veil"); + let srv = start(m.config(false)); + let r = get(srv.port(), &format!("/runs/{RUN}/map.json")); + assert_eq!(r.status, 503, "{}", r.text()); + assert_eq!(r.text(), og_live::map::NO_RESULTS); + assert!(m.calls().is_empty(), "og ran without the results' veil"); + // The gene files hold no result: they are og's as ever. + assert_eq!(get(srv.port(), "/map/genes/chr7.json").status, 200); + assert_eq!(m.calls().len(), 1); +} + +#[test] +fn the_map_is_ogs_bytes_with_the_sites_headers() { + let m = Map::new("map"); + let srv = start(m.config(true)); + let p = srv.port(); + let root = m.w.root(); + let state = m.w.state(); + let n_rows = log_rows(&m.w).len(); + + // 2, 3. the person's document + let r = get(p, &format!("/runs/{RUN}/map.json")); + let calls = m.calls(); + let results = get(p, &format!("/runs/{RUN}/results.json")); + assert_eq!(r.status, 200, "{}", r.text()); + assert_eq!(calls.len(), 1); + let direct = m.direct(&calls[0].0, &calls[0].1); + assert!(r.body.len() > 300_000); + assert_eq!( + sha(&r.body), + sha(&direct), + "the map is not og's bytes unchanged" + ); + assert_eq!(r.header("content-type"), Some("application/gzip")); + assert_eq!(r.header("content-security-policy"), Some(og_live::csp::CSP)); + assert!(policy_ok(&r)); + let want: Vec<String> = [ + "map", + "build", + "--root", + &root.to_string_lossy(), + "--run", + RUN, + "--assets", + &repo().join("assets").to_string_lossy(), + "--cache-dir", + &state.join("map").to_string_lossy(), + ] + .iter() + .map(|x| (*x).to_owned()) + .collect(); + assert_eq!(calls[0].0, want); + let body = results.json(); + let mut o = indexmap::IndexMap::new(); + o.insert("results".to_owned(), body.clone()); + assert_eq!( + loads(&calls[0].1).unwrap(), + Value::Object(o), + "og's stdin is not results.json's body" + ); + let text = og_live::state::dumps(&body); + assert!( + text.contains(FINDING) + && text.contains("\"planted\": true") + && text.contains("tables/linked.tsv"), + "{text}" + ); + + // 4. the gene files + for (path, extra) in [ + ("/map/genes/chr7.json", vec!["--chrom", "chr7"]), + ("/map/genes/names.json", vec!["--names"]), + ] { + let r = get(p, path); + let calls = m.calls(); + assert_eq!( + (r.status, r.header("content-type")), + (200, Some("application/gzip")), + "{path}" + ); + assert!(policy_ok(&r)); + let mut want: Vec<String> = [ + "map", + "genes", + "--root", + &root.to_string_lossy(), + "--cache-dir", + &state.join("map").to_string_lossy(), + ] + .iter() + .map(|x| (*x).to_owned()) + .collect(); + want.extend(extra.iter().map(|x| (*x).to_owned())); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].0, want, "{path}"); + assert_eq!(sha(&r.body), sha(&m.direct(&calls[0].0, "")), "{path}"); + } + for bad in [ + "/map/genes/chr123.json", + "/map/genes/x.json", + "/map/genes/chr1.txt", + ] { + assert_eq!(get(p, bad).status, 404, "{bad}"); + } + + // 5. og's exit codes + m.mode("nomap"); + let r = get(p, &format!("/runs/{RUN}/map.json")); + assert_eq!( + (r.status, r.text().as_str()), + ( + 404, + "og map build: no map for this run: the planted reason\n" + ) + ); + m.mode("notbuilt"); + let r = get(p, "/map/genes/chr7.json"); + assert_eq!((r.status, r.text().as_str()), (503, og_live::map::NO_OG)); + fs::rename(&m.og, m.og.with_extension("away")).unwrap(); + let r = get(p, "/map/genes/names.json"); + assert_eq!((r.status, r.text().as_str()), (503, og_live::map::NO_OG)); + fs::rename(m.og.with_extension("away"), &m.og).unwrap(); + m.mode("crash"); + let r = get(p, &format!("/runs/{RUN}/map.json")); + assert_eq!((r.status, r.text().as_str()), (500, og_live::map::FAILED)); + assert!(!r.text().contains("half a document")); + m.mode("ok"); + m.calls(); + + // 6. runs with no person's map + for run in ["FAM-A", "NO-SUCH-RUN"] { + assert_eq!( + get(p, &format!("/runs/{run}/map.json")).status, + 404, + "{run}" + ); + } + assert!( + m.calls().is_empty(), + "og ran for a run with no person's map" + ); + + // 7. the access log, and no POST route + let post = request( + p, + "POST", + &format!("/runs/{RUN}/map.json"), + &[("Content-Length", "0")], + ); + assert_eq!(post.status, 404); + let rows: Vec<Vec<String>> = log_rows(&m.w)[n_rows..] + .iter() + .filter(|r| r[3] == "map") + .cloned() + .collect(); + assert_eq!(rows.len(), 9, "{rows:?}"); + for r in &rows { + let pair = (r[4].as_str(), r[5].as_str()); + assert!( + [ + (RUN, "-"), + ("-", "genes"), + ("FAM-A", "-"), + ("NO-SUCH-RUN", "-") + ] + .contains(&pair), + "{r:?}" + ); + } + let log = fs::read_to_string(state.join("access.log")).unwrap(); + for w in ["PLANTGENE1", FINDING, "chr7", "planted"] { + assert!(!log.contains(w), "{w} in the access log"); + } +} + +#[test] +fn a_finding_page_is_served_only_for_a_finding_of_the_run() { + let m = Map::new("finding"); + let srv = start(m.config(true)); + let p = srv.port(); + let r = get(p, &format!("/runs/{RUN}/finding/{FINDING}/")); + assert_eq!(r.status, 200); + assert!(r.text().contains(FINDING) || r.text().contains(RUN)); + for bad in [ + format!("/runs/{RUN}/finding/clinvar-other-1/"), + format!("/runs/{RUN}/finding/NOT-WELL-FORMED/"), + format!("/runs/FAM-A/finding/{FINDING}/"), + format!("/runs/NO-SUCH/finding/{FINDING}/"), + ] { + assert_eq!(get(p, &bad).status, 404, "{bad}"); + } + let rows = log_rows(&m.w); + assert!( + rows.iter() + .any(|r| r[3] == "finding" && r[4] == RUN && r[6] == "200") + ); + assert!( + !fs::read_to_string(m.w.state().join("access.log")) + .unwrap() + .contains(FINDING) + ); +} diff --git a/rust/crates/og-live/tests/port_live_server_recheck.rs b/rust/crates/og-live/tests/port_live_server_recheck.rs new file mode 100644 index 000000000..b62808ca3 --- /dev/null +++ b/rust/crates/og-live/tests/port_live_server_recheck.rs @@ -0,0 +1,223 @@ +//! The database re-check on the live site, and the ClinVar lock that +//! `og recheck --refresh-clinvar` interlocks with. The check itself (its +//! classes, steps, exit codes and files, `--affected`) is tested by `cargo +//! test -p og-recheck`. +//! +//! 1. `plugins.recheck` of `state.json` gives each step its badges ("newer +//! ClinVar available", "redone <date>") and the banner, `{}` without a +//! check, nothing from a rerun block that is not an object, and nothing +//! from a check older than the run's `run_manifest.tsv`; +//! 2. the ClinVar lock block of `scripts/run-all.sh` and of step 06 waits +//! while a refresh holds the lock and goes on once it ends (needs +//! `flock`; CI fails without it). Its control: the block with its `flock` +//! call taken out must fail the check. + +#[path = "port_live_server_util.rs"] +mod util; + +use og_live::recheck; +use pycompat::json::{Value, loads}; +use std::fs; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; +use util::*; + +fn steps(v: &Value) -> Value { + match v { + Value::Object(o) => o.get("steps").cloned().unwrap_or(Value::Null), + _ => Value::Null, + } +} + +fn banner(v: &Value) -> String { + match v { + Value::Object(o) => match o.get("banner") { + Some(Value::Str(b)) => b.clone(), + _ => String::new(), + }, + _ => String::new(), + } +} + +fn empty() -> Value { + Value::Object(indexmap::IndexMap::new()) +} + +#[test] +fn each_step_gets_its_badges_and_the_banner() { + let w = Work::new("recheck-badges"); + let run = w.dir.join("S1"); + fs::create_dir_all(&run).unwrap(); + assert_eq!(recheck::state(&run), empty()); + write( + &run.join("recheck/state.json"), + r#"{"checked_utc": "2026-10-06T00:00:00Z", "items": [ + {"item": "clinvar_file_date", "label": "ClinVar (step 06's file)", "class": "refresh-available", "steps": ["06", "24"]}, + {"item": "VEP_CACHE_RELEASE", "label": "Ensembl / VEP cache", "class": "no-newer-release", "steps": ["13"]}], + "rerun": {"finished_utc": "2026-10-05T10:00:00Z", "predicted_vs_redone": {"redone": ["06"]}}}"#, + ); + let got = recheck::state(&run); + assert_eq!( + steps(&got), + loads(r#"{"06": ["newer ClinVar available", "redone 2026-10-05"], "24": ["newer ClinVar available"]}"#).unwrap() + ); + assert_eq!( + banner(&got), + "Database re-check of 2026-10-06T00:00:00Z: newer ClinVar available" + ); + let Value::Object(o) = &got else { panic!() }; + assert_eq!(o["file"], Value::Str("recheck/state.json".into())); + assert_eq!(o["checked_utc"], Value::Str("2026-10-06T00:00:00Z".into())); + assert!(matches!(o["mtime"], Value::Int(n) if n > 0)); +} + +#[test] +fn a_rerun_block_that_is_not_an_object_gives_no_badge() { + let w = Work::new("recheck-odd"); + let run = w.dir.join("S1"); + for rr in [ + "[1]", + r#"{"finished_utc": "2026-10-05T10:00:00Z", "predicted_vs_redone": ["06"]}"#, + ] { + write( + &run.join("recheck/state.json"), + &format!(r#"{{"checked_utc": "2026-10-06T00:00:00Z", "items": [], "rerun": {rr}}}"#), + ); + let got = recheck::state(&run); + assert_eq!(steps(&got), empty(), "{rr}"); + assert!(banner(&got).contains("nothing newer"), "{rr}"); + } + // Other classes are named in the banner, with no badge. + write( + &run.join("recheck/state.json"), + r#"{"checked_utc": "2026-10-06", "items": [{"item": "PGS", "class": "error", "steps": ["30"]}, 5]}"#, + ); + let got = recheck::state(&run); + assert_eq!( + (steps(&got), banner(&got)), + ( + empty(), + "Database re-check of 2026-10-06: PGS: error".to_owned() + ) + ); + write(&run.join("recheck/state.json"), "not json"); + assert_eq!(recheck::state(&run), empty()); +} + +#[test] +fn a_check_older_than_the_runs_manifest_is_left_out() { + let w = Work::new("recheck-stale"); + let run = w.dir.join("S1"); + write( + &run.join("recheck/state.json"), + r#"{"checked_utc": "2026-10-06T00:00:00Z", "items": [ + {"item": "clinvar_file_date", "label": "ClinVar", "class": "refresh-available", "steps": ["06"]}]}"#, + ); + write( + &run.join("run_manifest.tsv"), + "run\twritten_utc\t2026-10-07 09:00:00 UTC\r\n", + ); + assert_eq!(recheck::state(&run), empty()); + write( + &run.join("run_manifest.tsv"), + "run\twritten_utc\t2026-10-05 09:00:00 UTC\n", + ); + assert_eq!( + steps(&recheck::state(&run)), + loads(r#"{"06": ["newer ClinVar available"]}"#).unwrap() + ); +} + +/// The ClinVar lock block of a script: from `CVLOCK=` through its closing +/// `fi`. +fn lock_block(path: &Path) -> String { + let text = fs::read_to_string(path).unwrap(); + let re = regex::Regex::new(r"(?ms)^CVLOCK=.*?\n fi\nfi\n").unwrap(); + re.find(&text) + .map(|m| m.as_str().to_owned()) + .unwrap_or_default() +} + +/// `block`, run in bash while another process holds the lock exclusive: it +/// must wait, then go on once the lock is let go. +fn lock_waits(block: &str, name: &str) -> (bool, String) { + let w = Work::new(name); + let g = w.dir.join("genome"); + fs::create_dir_all(g.join("clinvar")).unwrap(); + let lock = g.join("clinvar/.refresh.lock"); + fs::write(&lock, "").unwrap(); + let mut holder = Command::new("flock") + .arg("-x") + .arg(&lock) + .args(["sleep", "3"]) + .spawn() + .unwrap(); + // Until the holder has it: a shared try fails while it is held. + let end = Instant::now() + Duration::from_secs(5); + while Instant::now() < end + && Command::new("flock") + .args(["-s", "-n"]) + .arg(&lock) + .arg("true") + .status() + .is_ok_and(|s| s.success()) + { + thread::sleep(Duration::from_millis(50)); + } + let mut p = Command::new("bash") + .args(["-c", &format!("set -euo pipefail\n{block}echo through\n")]) + .env("G", &g) + .env("GENOME_DIR", &g) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + thread::sleep(Duration::from_millis(1500)); + let waited = p.try_wait().unwrap().is_none(); + let _ = holder.wait(); + let out = p.wait_with_output().unwrap(); + let text = String::from_utf8_lossy(&out.stdout).into_owned(); + let ok = !block.is_empty() + && waited + && out.status.success() + && text.contains("Waiting") + && text.trim_end().ends_with("through"); + (ok, format!("waited {waited}, {:?}, {text}", out.status)) +} + +#[test] +fn the_clinvar_lock_block_waits_for_a_refresh_then_goes_on() { + let have = Command::new("flock") + .arg("-h") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok(); + if !have { + let ci = std::env::var("CI").is_ok_and(|v| v == "true"); + assert!( + !ci, + "[FAIL] flock is not installed: CI must run the ClinVar lock checks" + ); + println!("[SKIP] flock is not installed: the ClinVar lock block is not run here"); + return; + } + let flock_line = regex::Regex::new(r"(?m)^ flock -s -n 7 .*$").unwrap(); + for (i, script) in ["scripts/run-all.sh", "scripts/06-clinvar-screen.sh"] + .iter() + .enumerate() + { + let block = lock_block(&repo().join(script)); + assert!(!block.is_empty(), "no ClinVar lock block in {script}"); + let (ok, detail) = lock_waits(&block, &format!("cvlock-{i}")); + assert!(ok, "{script}: {detail}"); + let planted = flock_line.replace(&block, " :").into_owned(); + assert_ne!(planted, block, "the control's line is not in {script}"); + let (still, _) = lock_waits(&planted, &format!("cvlock-planted-{i}")); + assert!( + !still, + "control: {script}'s block with no flock call passed the check" + ); + } +} diff --git a/rust/crates/og-live/tests/port_live_server_util.rs b/rust/crates/og-live/tests/port_live_server_util.rs new file mode 100644 index 000000000..4fbd8d6bd --- /dev/null +++ b/rust/crates/og-live/tests/port_live_server_util.rs @@ -0,0 +1,256 @@ +//! What the `port_live_server*` tests share: the synthetic DEMO-001 run of +//! `tests/fixtures/live` in a scratch genome folder, `og live serve` on it +//! in this process, and plain HTTP requests to it. Each test file includes +//! it with `#[path]`; on its own it holds no test. +#![allow(dead_code)] + +use og_live::server::{Config, Server}; +use std::fs; +use std::io::{Read, Write}; +use std::net::TcpStream; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +pub const RUN: &str = "DEMO-001"; + +/// The repository's root: four levels above this crate's tests. +pub fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(3) + .expect("rust/crates/og-live sits three levels below the root") + .to_path_buf() +} + +/// A test's own folder, removed when it is dropped (also on a panic). +pub struct Work { + pub dir: PathBuf, +} + +impl Work { + pub fn new(name: &str) -> Work { + let d = Path::new(env!("CARGO_TARGET_TMPDIR")) + .join(format!("og-live-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(&d).unwrap(); + Work { + dir: fs::canonicalize(&d).unwrap(), + } + } + + pub fn root(&self) -> PathBuf { + self.dir.join("genome") + } + + pub fn state(&self) -> PathBuf { + self.dir.join("state") + } +} + +impl Drop for Work { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.dir); + } +} + +fn copy_dir(from: &Path, to: &Path) { + fs::create_dir_all(to).unwrap(); + for e in fs::read_dir(from).unwrap() { + let e = e.unwrap(); + let p = e.path(); + if p.is_dir() { + copy_dir(&p, &to.join(e.file_name())); + } else { + fs::copy(&p, to.join(e.file_name())).unwrap(); + } + } +} + +pub fn write(p: &Path, text: &str) { + fs::create_dir_all(p.parent().unwrap()).unwrap(); + fs::write(p, text).unwrap(); +} + +/// The genome folder of the checks: DEMO-001 with a report that links a +/// table, a symlink out of the root, CPSR's folder and a folder with two +/// pages, and files it does not link. +pub fn make_root(w: &Work) -> PathBuf { + let root = w.root(); + let d = root.join(RUN); + copy_dir(&repo().join("tests/fixtures/live").join(RUN), &d); + write( + &d.join("clinvar/hits.tsv"), + "gene\tvariant\nGENEA\tchr1:100 A>G\n", + ); + write(&d.join("clinvar/unlinked.tsv"), "not linked\n"); + write(&w.dir.join("outside.txt"), "outside the root\n"); + std::os::unix::fs::symlink(w.dir.join("outside.txt"), d.join("clinvar/escape.tsv")).unwrap(); + write( + &d.join(format!("cpsr/{RUN}.cpsr.grch38.html")), + "<!doctype html><title>CPSR\n", + ); + write( + &d.join(format!("cpsr/{RUN}.cpsr.grch38.classification.tsv.gz")), + "not linked\n", + ); + write(&d.join("twopages/a.html"), "\n"); + write(&d.join("twopages/b.html"), "\n"); + write( + &d.join(format!("{RUN}_report.html")), + "\ + hits escape \ + the cpsr folder two pages \ + outside\n", + ); + root +} + +/// The site's config for `w`: the repository's assets and steps.tsv, its +/// own state folder, port 0, no request lines. +pub fn config(w: &Work) -> Config { + let r = repo(); + let mut c = Config::new(w.root()); + c.state_dir = w.state(); + c.access_log = w.state().join("access.log"); + c.port = 0; + c.assets = r.join("assets/live"); + c.repo_assets = r.join("assets"); + c.steps = r.join("assets/steps.tsv"); + c.quiet = true; + c +} + +/// The events file of `run` under the state folder. +pub fn events_path(w: &Work, run: &str) -> PathBuf { + w.state().join("runs").join(run).join("events.jsonl") +} + +/// Appends messages to `run`'s events file, as the ingest listener stores +/// them. +pub fn store(w: &Work, run: &str, lines: &[&str]) { + let p = events_path(w, run); + fs::create_dir_all(p.parent().unwrap()).unwrap(); + let mut f = fs::OpenOptions::new() + .create(true) + .append(true) + .open(p) + .unwrap(); + for l in lines { + f.write_all(l.trim().as_bytes()).unwrap(); + f.write_all(b"\n").unwrap(); + } +} + +/// The fixture's messages (tests/fixtures/live/events.jsonl). +pub fn fixture_events() -> Vec { + fs::read_to_string(repo().join("tests/fixtures/live/events.jsonl")) + .unwrap() + .lines() + .filter(|l| !l.trim().is_empty()) + .map(str::to_owned) + .collect() +} + +pub fn start(c: Config) -> Server { + Server::start(c).unwrap() +} + +/// One answer: status, headers (names lowered) and body. +#[derive(Debug, Clone)] +pub struct Resp { + pub status: u16, + pub headers: Vec<(String, String)>, + pub body: Vec, +} + +impl Resp { + pub fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(k, _)| k == name) + .map(|(_, v)| v.as_str()) + } + + pub fn text(&self) -> String { + String::from_utf8_lossy(&self.body).into_owned() + } + + pub fn json(&self) -> pycompat::json::Value { + pycompat::json::loads(&self.text()) + .unwrap_or_else(|e| panic!("not JSON ({e}): {}", self.text())) + } +} + +/// Sends `head` (with its blank line) and `body` as they are; reads the +/// whole answer. +pub fn raw(port: u16, head: &str, body: &[u8]) -> Resp { + let mut s = TcpStream::connect(("127.0.0.1", port)).unwrap(); + s.set_read_timeout(Some(Duration::from_secs(30))).unwrap(); + s.write_all(head.as_bytes()).unwrap(); + s.write_all(body).unwrap(); + let mut all = Vec::new(); + let _ = s.read_to_end(&mut all); + parse(&all) +} + +pub fn parse(all: &[u8]) -> Resp { + let split = all + .windows(4) + .position(|w| w == b"\r\n\r\n") + .unwrap_or_else(|| panic!("no head in {:?}", String::from_utf8_lossy(all))); + let head = String::from_utf8_lossy(&all[..split]).into_owned(); + let mut lines = head.split("\r\n"); + let status = lines + .next() + .and_then(|l| l.split(' ').nth(1)) + .and_then(|c| c.parse().ok()) + .unwrap_or(0); + let headers = lines + .filter_map(|l| l.split_once(':')) + .map(|(k, v)| (k.trim().to_ascii_lowercase(), v.trim().to_owned())) + .collect(); + Resp { + status, + headers, + body: all[split + 4..].to_vec(), + } +} + +/// A request with `Host: 127.0.0.1:` unless `headers` names one. +pub fn request(port: u16, method: &str, path: &str, headers: &[(&str, &str)]) -> Resp { + let mut head = format!("{method} {path} HTTP/1.1\r\n"); + if !headers.iter().any(|(k, _)| k.eq_ignore_ascii_case("host")) { + head.push_str(&format!("Host: 127.0.0.1:{port}\r\n")); + } + for (k, v) in headers { + head.push_str(&format!("{k}: {v}\r\n")); + } + head.push_str("Connection: close\r\n\r\n"); + raw(port, &head, b"") +} + +pub fn get(port: u16, path: &str) -> Resp { + request(port, "GET", path, &[]) +} + +/// Whether an answer carries the site's policy: nothing from outside, no +/// framing, no sniffing, no cache. +pub fn policy_ok(r: &Resp) -> bool { + let csp = r.header("content-security-policy").unwrap_or(""); + csp.contains("default-src 'none'") + && csp.contains("frame-ancestors 'none'") + && r.header("x-frame-options") == Some("DENY") + && r.header("x-content-type-options") == Some("nosniff") + && r.header("cache-control") == Some("no-store") + && r.header("referrer-policy") == Some("no-referrer") +} + +/// The access log's rows, without the header. +pub fn log_rows(w: &Work) -> Vec> { + fs::read_to_string(w.state().join("access.log")) + .unwrap_or_default() + .lines() + .skip(1) + .map(|l| l.split('\t').map(str::to_owned).collect()) + .collect() +} diff --git a/rust/crates/og-lookup/src/lib.rs b/rust/crates/og-lookup/src/lib.rs index 1fbadd818..7fe5a34b9 100644 --- a/rust/crates/og-lookup/src/lib.rs +++ b/rust/crates/og-lookup/src/lib.rs @@ -14,10 +14,11 @@ //! place is looked for in the annotated VCF in one pass for every such //! rsID of the command line. //! -//! The output is what the Python wrote, byte for byte (`tests/parity`), -//! with one deliberate change: a gene's "bases have a genotype call" count -//! is the union of the called records, so overlapping records (a deletion -//! and the reference block that follows it) no longer count a base twice. +//! Parity tests in `tests/parity` compare the output with the former Python +//! lookup. The Rust lookup counts each base in a gene span once, even when +//! called records overlap. It also reads negative genotype indexes as missing +//! alleles. A record counts toward gene coverage only when at least one index +//! names an allele. mod query; mod report; diff --git a/rust/crates/og-lookup/src/report.rs b/rust/crates/og-lookup/src/report.rs index 196332cc9..42d4722c9 100644 --- a/rust/crates/og-lookup/src/report.rs +++ b/rust/crates/og-lookup/src/report.rs @@ -361,8 +361,7 @@ fn covered(gvcf: &Region, chrom: &str, start: u64, end: u64) -> Result end { continue; } - let (idx, _) = r.gt()?; - if idx.iter().any(Option::is_some) { + if r.has_call()? { let (b, e) = (r.pos.max(start), r.end.min(end)); if b <= e { spans.push((b, e)); diff --git a/rust/crates/og-lookup/src/vcf.rs b/rust/crates/og-lookup/src/vcf.rs index e21b280ad..0fb0d6aaa 100644 --- a/rust/crates/og-lookup/src/vcf.rs +++ b/rust/crates/og-lookup/src/vcf.rs @@ -174,31 +174,32 @@ impl Rec { Ok((idx, g.contains('|'))) } - /// GT as alleles (`None` for a no-call or an index past the alleles), - /// and whether it is phased. + fn allele(&self, index: i128) -> Option<&str> { + let index = usize::try_from(index).ok()?; + if index == 0 { + Some(&self.ref_) + } else { + self.alts.get(index - 1).map(String::as_str) + } + } + + /// GT as alleles (`None` for a no-call or an index outside the allele + /// list), and whether it is phased. pub fn bases(&self) -> Result<(Vec>, bool), Error> { let (idx, phased) = self.gt()?; - let n = 1 + self.alts.len() as i128; - let allele = |i: i128| -> Result, Error> { - // Python's list index: a negative one counts from the end. - let j = if i < 0 { n + i } else { i }; - if i >= n { - Ok(None) - } else if j < 0 { - Err(Error::other("list index out of range")) - } else if j == 0 { - Ok(Some(self.ref_.as_str())) - } else { - Ok(Some(self.alts[(j - 1) as usize].as_str())) - } - }; let bases = idx .into_iter() - .map(|i| i.map_or(Ok(None), allele)) - .collect::>()?; + .map(|i| i.and_then(|i| self.allele(i))) + .collect(); Ok((bases, phased)) } + /// A partial genotype counts as called if any index names an allele. + pub fn has_call(&self) -> Result { + let (idx, _) = self.gt()?; + Ok(idx.into_iter().flatten().any(|i| self.allele(i).is_some())) + } + /// Whether GT calls a variant allele. pub fn has_alt_call(&self) -> Result { let (idx, _) = self.gt()?; @@ -448,16 +449,19 @@ mod tests { } #[test] - fn genotypes_read_as_python_read_them() { + fn genotypes_read_as_vcf_numbers_them() { + // A negative index names no allele (the Python read -1 as the last). let r = rec("chr7\t1\t.\tC\tT,<*>\t.\t.\t.\tGT:DP\t1|-1:9"); let (b, phased) = r.bases().unwrap(); - assert_eq!(b, vec![Some("T"), Some("<*>")]); + assert_eq!(b, vec![Some("T"), None]); assert!(phased); assert!(r.has_alt_call().unwrap()); let r = rec("chr7\t1\t.\tC\tT\t.\t.\t.\tGT\t0/"); assert_eq!(r.gt().unwrap_err().code, 2); let r = rec("chr7\t1\t.\tC\tT\t.\t.\t.\tGT\t0/5"); assert_eq!(r.bases().unwrap().0, vec![Some("C"), None]); + let r = rec("chr7\t1\t.\tC\tT\t.\t.\t.\tGT\t-3/0"); + assert_eq!(r.bases().unwrap().0, vec![None, Some("C")]); } #[test] diff --git a/rust/crates/og-lookup/tests/rv_gt_negative_index.rs b/rust/crates/og-lookup/tests/rv_gt_negative_index.rs new file mode 100644 index 000000000..c114efc62 --- /dev/null +++ b/rust/crates/og-lookup/tests/rv_gt_negative_index.rs @@ -0,0 +1,118 @@ +//! A genotype index below zero names no allele. VCF writes GT as allele +//! numbers from 0 (REF) up, or `.`; a reader that indexed `[REF] + ALT` as a +//! Python list read `-1` as the last ALT, so `GT -1/-1` at REF A, ALT T +//! answered `T/T`. The lookup reads it as a no-call, as it already read an +//! index past the last allele. + +use std::fs; +use std::path::{Path, PathBuf}; + +fn input(name: &str) -> String { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../tests/parity/inputs/lookup") + .join(name) + .to_string_lossy() + .into_owned() +} + +/// Runs `og lookup query` for each of `queries` over `gvcf`: stdout. +fn lookup(gvcf: &str, queries: &[&str]) -> String { + let mut args: Vec = [ + "query", + "--sites", + &input("sites.tsv.gz"), + "--genes", + &input("gencode.v50.basic.genes.gtf"), + "--table", + &input("traits.tsv"), + "--gvcf", + gvcf, + "--", + ] + .map(String::from) + .to_vec(); + args.extend(queries.iter().map(|q| (*q).to_owned())); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let rc = og_lookup::cli(&args, &mut out, &mut err); + let (out, err) = ( + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ); + assert_eq!(rc, 0, "{out}{err}"); + out +} + +#[test] +fn a_negative_genotype_index_is_no_call() { + let dir: PathBuf = Path::new(env!("CARGO_TARGET_TMPDIR")).join("rv_gt_negative_index"); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).unwrap(); + let gvcf = dir.join("g.vcf"); + let body = [ + "chr20\t100\t.\tA\tT\t30\tPASS\t.\tGT:GQ\t-1/-1:30", + "chr20\t200\t.\tA\tT\t30\tPASS\t.\tGT:GQ\t1|-1:30", + "chr20\t300\t.\tA\tT,<*>\t30\tPASS\t.\tGT:GQ\t0/1:30", + "chr20\t400\t.\tA\tT,<*>\t30\tPASS\t.\tGT:GQ\t1/1:30", + ]; + let text = format!( + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n{}\n", + body.join("\n") + ); + fs::write(&gvcf, text).unwrap(); + let gvcf = gvcf.to_string_lossy().into_owned(); + + let genotype = |q: &str| -> String { + let out = lookup(&gvcf, &[q]); + out.lines() + .find(|l| l.trim_start().starts_with("genotype: ")) + .unwrap_or_else(|| panic!("no genotype line for {q}: {out}")) + .trim_start() + .to_owned() + }; + + // -1 is no allele: a no-call, never the ALT T. + assert_eq!( + genotype("chr20:100"), + "genotype: ./. (no call; REF A, ALT T)" + ); + assert_eq!( + genotype("chr20:200"), + "genotype: T|. (half call; REF A, ALT T)" + ); + // The answers of readable genotypes do not change. + assert_eq!( + genotype("chr20:300"), + "genotype: A/T (heterozygous; REF A, ALT T)" + ); + assert_eq!( + genotype("chr20:400"), + "genotype: T/T (homozygous; REF A, ALT T)" + ); + + fs::remove_dir_all(&dir).unwrap(); +} + +#[test] +fn gene_coverage_requires_an_index_that_names_an_allele() { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("rv_gt_coverage_lookup"); + fs::create_dir_all(&dir).unwrap(); + let gvcf = dir.join("g.vcf"); + fs::write( + &gvcf, + "##fileformat=VCFv4.2\n\ + #CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n\ + chr7\t100\t.\tA\tT\t30\tPASS\tEND=109\tGT\t-1/-1\n\ + chr7\t200\t.\tA\tT\t30\tPASS\tEND=219\tGT\t2/2\n\ + chr7\t300\t.\tA\t.\t30\tPASS\tEND=302\tGT\t1/1\n\ + chr7\t400\t.\tA\tT,G\t30\tPASS\tEND=404\tGT\t2|-1\n\ + chr7\t500\t.\tA\t<*>\t30\tPASS\tEND=506\tGT\t0/.\n", + ) + .unwrap(); + let out = lookup(gvcf.to_str().unwrap(), &["GENEA"]); + + assert!( + out.contains("12 of 611 bases have a genotype call"), + "{out}" + ); + fs::remove_dir_all(&dir).unwrap(); +} diff --git a/rust/crates/og-pgx/src/cli.rs b/rust/crates/og-pgx/src/cli.rs index fa82d7a0b..eb729f57e 100644 --- a/rust/crates/og-pgx/src/cli.rs +++ b/rust/crates/og-pgx/src/cli.rs @@ -361,7 +361,7 @@ fn contraindication_section(a: &CpicArgs, table_path: &str) -> Vec { .medications .as_deref() .and_then(|m| read_medications(m).ok()) - .map(|m| m.into_iter().map(|m| m.name).collect()) + .map(|m| m.into_iter().flat_map(|m| m.names).collect()) .unwrap_or_default(); let found = contraindication_lines(&hits, &table, &med_names); if found.is_empty() { diff --git a/rust/crates/og-pgx/src/sections.rs b/rust/crates/og-pgx/src/sections.rs index 8bd5cd0b3..fb1af5113 100644 --- a/rust/crates/og-pgx/src/sections.rs +++ b/rust/crates/og-pgx/src/sections.rs @@ -444,8 +444,9 @@ pub struct Phenoconversion { pub lines: Vec, } -/// Phenoconversion by an opt-in medication list. A line naming no drug of -/// the inhibitor table is listed as not checked. The genotype phenotype is +/// Phenoconversion by an opt-in medication list. Every drug of a line +/// counts, so a combination product converts by each of its drugs; a line +/// none of whose drugs is in the inhibitor table is listed as not checked. The genotype phenotype is /// never changed: the converted one sits beside it. With several inhibitors /// of one gene the strongest counts (CPIC gives no rule for combining them). /// @@ -465,11 +466,18 @@ pub fn phenoconversion( let mut unknown: Vec<&str> = Vec::new(); let mut names: Vec<&str> = Vec::new(); for m in meds { - if !inhibitors.keys().any(|(d, _)| *d == m.name) && !unknown.contains(&m.text.as_str()) { + // A line is unchecked only when none of its drugs is in the table. + let known = m + .names + .iter() + .any(|n| inhibitors.keys().any(|(d, _)| d == n)); + if !known && !unknown.contains(&m.text.as_str()) { unknown.push(&m.text); } - if !names.contains(&m.name.as_str()) { - names.push(&m.name); + for n in &m.names { + if !names.contains(&n.as_str()) { + names.push(n); + } } } for gene in PHENOCONVERSION_GENES { @@ -576,9 +584,14 @@ pub fn phenoconversion( format!("CPIC gives no rule that converts the {gene} phenotype") }; let inhibiting = hits.iter().map(|h| h.0).collect::>().join(", "); + let verb = if hits.len() > 1 { + "inhibit" + } else { + "inhibits" + }; column.insert( gene, - format!("not computed: {inhibiting} inhibits {gene}; {why}"), + format!("not computed: {inhibiting} {verb} {gene}; {why}"), ); let mut first = why.chars(); let why_cap: String = first diff --git a/rust/crates/og-pgx/src/tables.rs b/rust/crates/og-pgx/src/tables.rs index ef8455d7e..8fe0c5e0c 100644 --- a/rust/crates/og-pgx/src/tables.rs +++ b/rust/crates/og-pgx/src/tables.rs @@ -93,9 +93,14 @@ pub fn read_pypgx(path: &str) -> Result, PyErr> { Ok(out) } -/// A diplotype pypgx actually called. +/// A diplotype pypgx actually called. `Indeterminate` is no call, in any +/// case and in any form: pypgx writes it for a gene it could not call, +/// step 32 writes it for CYP2D6 when the depth check fails, and a +/// diplotype with an Indeterminate allele (`*1/Indeterminate`) names no +/// diplotype either. pub fn pypgx_called(diplotype: &str) -> bool { - !matches!(diplotype, "" | "FAILED" | "N/A" | "Not called") + let d = strip(diplotype, None); + !matches!(d, "" | "FAILED" | "N/A" | "Not called") && !lower(d).contains("indeterminate") } // --- step 36 ------------------------------------------------------------- @@ -198,20 +203,56 @@ pub fn read_inhibitors(path: &str) -> Result { Ok(out) } -/// One line of a medication list: as written, and the drug it names. +/// One line of a medication list: as written, and every drug name in it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Medication { pub text: String, - pub name: String, + /// The words of the line that can name a drug, lower case, in order, + /// each once. Never empty. + pub names: Vec, } -const DRUG_SEPARATORS: &str = " \t,;/+"; +/// Words of a medication line that name no drug: units, dose forms and how +/// often it is taken. +const DOSE_WORDS: &[&str] = &[ + "mg", "mcg", "µg", "ug", "g", "kg", "ml", "l", "iu", "unit", "units", "%", "tablet", "tablets", + "tab", "tabs", "capsule", "capsules", "cap", "caps", "er", "xr", "sr", "xl", "daily", "once", + "twice", "a", "day", "per", "weekly", "morning", "evening", "night", +]; + +/// A word that is a dose, a unit or a frequency, not a drug: it starts with +/// a digit or a point (`20`, `400mg`, `.5`), or it is one of the dose words. +fn dose_word(w: &str) -> bool { + w.starts_with(|c: char| c.is_ascii_digit() || c == '.') || DOSE_WORDS.contains(&w) +} + +/// The parts of a hyphenated word that can name a drug, in order: each part +/// that starts with a letter, has three or more characters and is no dose +/// word. `olanzapine-fluoxetine` gives olanzapine and fluoxetine; the `co` +/// of `co-trimoxazole`, the `l` of `l-dopa` and the `5` of `5-htp` are +/// never drugs, so they give nothing. A word without a hyphen has no parts. +fn hyphen_parts(w: &str) -> impl Iterator { + w.split('-').filter(move |p| { + *p != w && p.chars().count() >= 3 && p.starts_with(char::is_alphabetic) && !dose_word(p) + }) +} -/// The lines of a medication list, one drug per line; `#` starts a comment -/// and blank lines are skipped. The name is the first word, lower case, so -/// a dose after it (`20 mg`) does not hide the drug. A slash or a plus also -/// ends a word, so `lopinavir/ritonavir` names lopinavir: the first drug of -/// a combination is the one checked. +/// The lines of a medication list, one product per line; `#` starts a +/// comment and blank lines are skipped. Every word of a line can name a +/// drug, so each drug of a combination product is checked: whitespace, +/// commas, semicolons, `/`, `+` and brackets split the line, punctuation at +/// a word's ends is dropped (`paroxetine.` is paroxetine), and the word +/// `and` is dropped (`olanzapine/fluoxetine`, `naltrexone/bupropion 8/90 mg` +/// and `lopinavir and ritonavir` each name two drugs; `Contrave +/// (naltrexone/bupropion)` names three words). Doses, units and frequencies +/// (`20`, `mg`, `daily`) are dropped too. A hyphenated word names itself +/// first, so a hyphenated name of a table still matches whole, then its +/// parts (`hyphen_parts`), so `olanzapine-fluoxetine` is checked for +/// fluoxetine. A part is only a name: it hits only when it is itself a name +/// of a table, and no table name is a part of another one (a test checks +/// the shipped tables). A line of nothing but those keeps +/// its first word, so it is still listed as not checked; a line of only +/// separators and punctuation is skipped. pub fn read_medications(path: &str) -> Result, PyErr> { let text = read_text(path)?; let mut out = Vec::new(); @@ -220,15 +261,32 @@ pub fn read_medications(path: &str) -> Result, PyErr> { if t.is_empty() { continue; } - let first = strip(t, Some(DRUG_SEPARATORS)) - .split(|c: char| pycompat::text::is_space(c) || ",;/+".contains(c)) - .next() - .unwrap_or(""); - let name = norm_drug(first); - if !name.is_empty() { + let words: Vec = t + .split(|c: char| pycompat::text::is_space(c) || ",;/+()[]".contains(c)) + .map(|w| w.trim_matches(|c: char| !c.is_alphanumeric())) + .filter(|w| !w.is_empty()) + .map(lower) + .collect(); + let mut names: Vec = Vec::new(); + for w in &words { + if w == "and" || dose_word(w) { + continue; + } + for n in std::iter::once(w.as_str()).chain(hyphen_parts(w)) { + if !names.iter().any(|m| m == n) { + names.push(n.to_owned()); + } + } + } + if names.is_empty() + && let Some(first) = words.first() + { + names.push(first.clone()); + } + if !names.is_empty() { out.push(Medication { text: t.to_owned(), - name, + names, }); } } @@ -388,23 +446,117 @@ mod tests { } #[test] - fn medication_lines_name_their_first_drug() { + fn medication_lines_name_every_drug() { let p = tmp( "meds.txt", - "Paroxetine 20 mg daily\n, paroxetine\n;\n# a comment\n\nlopinavir/ritonavir 400/100 mg\r\nValproate # evening\n", + "Paroxetine 20 mg daily\n, paroxetine\n;\n# a comment\n\nlopinavir/ritonavir 400/100 mg\r\nValproate # evening\nnaltrexone/bupropion 8/90 mg\nOlanzapine + fluoxetine; olanzapine\nlopinavir and ritonavir\n20 mg\nContrave (naltrexone/bupropion) 8/90 mg\nEpilim (sodium valproate) 500 mg\nDepakote(divalproex sodium) [delayed-release]\nparoxetine.\nolanzapine-fluoxetine 6/25 mg\nCo-trimoxazole 960 mg\nL-dopa 100 mg\n5-HTP 100 mg\n( )\n", ); - let got: Vec<(String, String)> = read_medications(&p) + let got: Vec<(String, Vec)> = read_medications(&p) .unwrap() .into_iter() - .map(|m| (m.text, m.name)) + .map(|m| (m.text, m.names)) .collect(); - let want = [ - ("Paroxetine 20 mg daily", "paroxetine"), - (", paroxetine", "paroxetine"), - ("lopinavir/ritonavir 400/100 mg", "lopinavir"), - ("Valproate", "valproate"), + let want: [(&str, &[&str]); 16] = [ + ("Paroxetine 20 mg daily", &["paroxetine"]), + (", paroxetine", &["paroxetine"]), + ( + "lopinavir/ritonavir 400/100 mg", + &["lopinavir", "ritonavir"], + ), + ("Valproate", &["valproate"]), + ("naltrexone/bupropion 8/90 mg", &["naltrexone", "bupropion"]), + ( + "Olanzapine + fluoxetine; olanzapine", + &["olanzapine", "fluoxetine"], + ), + ("lopinavir and ritonavir", &["lopinavir", "ritonavir"]), + ("20 mg", &["20"]), + // Brackets split a line, and punctuation at a word's ends is + // dropped, so a brand name with its generic in brackets names + // both, and a line of only brackets names nothing. + ( + "Contrave (naltrexone/bupropion) 8/90 mg", + &["contrave", "naltrexone", "bupropion"], + ), + ( + "Epilim (sodium valproate) 500 mg", + &["epilim", "sodium", "valproate"], + ), + ( + "Depakote(divalproex sodium) [delayed-release]", + &[ + "depakote", + "divalproex", + "sodium", + "delayed-release", + "delayed", + "release", + ], + ), + ("paroxetine.", &["paroxetine"]), + // A hyphenated word names itself, then each part that can name + // a drug: never a prefix, a lone letter or a number. + ( + "olanzapine-fluoxetine 6/25 mg", + &["olanzapine-fluoxetine", "olanzapine", "fluoxetine"], + ), + ("Co-trimoxazole 960 mg", &["co-trimoxazole", "trimoxazole"]), + ("L-dopa 100 mg", &["l-dopa", "dopa"]), + ("5-HTP 100 mg", &["5-htp"]), ]; - assert_eq!(got, want.map(|(a, b)| (a.to_owned(), b.to_owned()))); + let want: Vec<(String, Vec)> = want + .iter() + .map(|(t, n)| ((*t).to_owned(), n.iter().map(|s| (*s).to_owned()).collect())) + .collect(); + assert_eq!(got, want); + } + + /// The names of `names` that are a hyphen part of another of them. + fn part_clashes(names: &[String]) -> Vec { + names + .iter() + .flat_map(|n| hyphen_parts(n).filter(|p| names.iter().any(|m| m == p))) + .map(str::to_owned) + .collect() + } + + #[test] + fn no_table_name_is_a_hyphen_part_of_another() { + // A hyphen part of a medication word is checked even when the whole + // word is a table name; that adds no hit only while no name of the + // tables is a part of another one. + let root = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../assets/"); + let inh = read_inhibitors(&format!("{root}pgx_inhibitors.tsv")).unwrap(); + let contra = read_contraindications(&format!("{root}pgx_contraindications.tsv")).unwrap(); + let mut names: Vec = inh.keys().map(|(d, _)| d.clone()).collect(); + for r in &contra { + names.extend(r.names.split(',').map(norm_drug).filter(|n| !n.is_empty())); + } + assert!(names.len() > 40, "{names:?}"); + assert_eq!(part_clashes(&names), Vec::::new()); + // Control: a planted clash is found. + let planted = ["co-trimoxazole".to_owned(), "trimoxazole".to_owned()]; + assert_eq!(part_clashes(&planted), ["trimoxazole"]); + } + + #[test] + fn indeterminate_is_no_pypgx_call() { + for d in ["*1/*4", " *1/*1 ", "Reference/rs9923231"] { + assert!(pypgx_called(d), "{d}"); + } + for d in [ + "", + "FAILED", + "N/A", + "Not called", + "Indeterminate", + "indeterminate", + " INDETERMINATE ", + "*1/Indeterminate", + "Indeterminate/Indeterminate", + ] { + assert!(!pypgx_called(d), "{d}"); + } } #[test] diff --git a/rust/crates/og-pgx/tests/i411_pypgx_nocall.rs b/rust/crates/og-pgx/tests/i411_pypgx_nocall.rs new file mode 100644 index 000000000..d7e82ea4c --- /dev/null +++ b/rust/crates/og-pgx/tests/i411_pypgx_nocall.rs @@ -0,0 +1,393 @@ +//! `og pgx cpic-report` with pypgx's `Indeterminate` and with combination +//! products on the medication list (issue #411, pgx-06, pgx-30, pgx-16): +//! +//! - pypgx writes `Indeterminate` for a gene it could not call, and step 32 +//! writes it for CYP2D6 when the depth check fails. In any case, and as +//! one allele (`*1/Indeterminate`), it is no call: a gene PharmCAT could +//! not call gets no comparison row and no warning, and a gene PharmCAT +//! did call is `PharmCAT only`, never a mismatch; +//! - every drug of a combination product is checked: `olanzapine/fluoxetine` +//! converts CYP2D6 by fluoxetine, `naltrexone/bupropion 8/90 mg` by +//! bupropion and is not listed as unchecked, and the second drug of a +//! combination is marked as on the medication list in the drug +//! contraindications. +//! +//! Each case has its control: a real pypgx call still warns, and a list +//! without the inhibitor converts nothing. + +use std::fs; +use std::path::PathBuf; + +const ASSETS: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../../assets/"); + +struct Scratch(PathBuf); + +impl Scratch { + fn new(name: &str) -> Scratch { + let d = + std::env::temp_dir().join(format!("og-pgx-i411-nocall-{name}-{}", std::process::id())); + let _ = fs::remove_dir_all(&d); + fs::create_dir_all(&d).unwrap(); + Scratch(d) + } + + fn put(&self, name: &str, body: &str) -> String { + let p = self.0.join(name); + fs::write(&p, body).unwrap(); + p.to_string_lossy().into_owned() + } + + fn read(&self, name: &str) -> String { + fs::read_to_string(self.0.join(name)).unwrap_or_default() + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +/// One gene of a PharmCAT 3.x report with a single diplotype. +fn gene(name: &str, a1: &str, a2: &str, phenotype: &str, score: Option<&str>) -> String { + let score = score.map_or_else(String::new, |s| format!(r#", "activityScore": "{s}""#)); + format!( + r#""{name}": {{"sourceDiplotypes": [{{"allele1": {{"name": "{a1}"}}, "allele2": {{"name": "{a2}"}}, "label": "{a1}/{a2}", "phenotypes": ["{phenotype}"]{score}}}], "relatedDrugs": [{{"name": "codeine"}}]}}"# + ) +} + +fn report(genes: &[String]) -> String { + format!( + r#"{{"pharmcatVersion": "3.4.0", "genes": {{{}}}}}"#, + genes.join(", ") + ) +} + +/// `og pgx cpic-report` on `report` with the extra options: (exit code, +/// recommendations text, phenotype rows, comparison rows). +fn run( + s: &Scratch, + report: &str, + extra: &[&str], +) -> (u8, String, Vec>, Vec>) { + let rep = s.put("report.json", report); + let out = s.0.join("out").to_string_lossy().into_owned(); + let mut args: Vec = [ + "cpic-report", + "--sample", + "S", + "--report", + &rep, + "--outdir", + &out, + "--date", + "2026-01-01", + ] + .map(str::to_owned) + .to_vec(); + args.extend(extra.iter().map(|a| (*a).to_owned())); + let rc = og_pgx::main(&args); + let rows = |text: String| -> Vec> { + text.lines() + .skip(1) + .map(|l| l.split('\t').map(str::to_owned).collect()) + .collect() + }; + ( + rc, + s.read("out/S_cpic_recommendations.txt"), + rows(s.read("out/S_phenotypes.tsv")), + rows(s.read("comparison.tsv")), + ) +} + +fn row<'a>(rows: &'a [Vec], gene: &str) -> Option<&'a Vec> { + rows.iter().find(|r| r.first().is_some_and(|g| g == gene)) +} + +/// CYP2D6 not called by PharmCAT, CYP2C9 called; pypgx's CYP2D6 and CYP2C9 +/// diplotypes as given. +fn pypgx_run(name: &str, cyp2d6: &str, cyp2c9: &str) -> (String, Vec>) { + let s = Scratch::new(name); + let rep = report(&[ + gene("CYP2D6", "Unknown", "Unknown", "No Result", None), + gene("CYP2C9", "*1", "*1", "Normal Metabolizer", None), + ]); + let pg = s.put( + "pypgx.tsv", + &format!( + "Gene\tDiplotype\tPhenotype\tCNV_call\tSource\nCYP2D6\t{cyp2d6}\tPoor Metabolizer\t.\tbam\nCYP2C9\t{cyp2c9}\tIndeterminate\t.\tvcf\n" + ), + ); + let comp = s.0.join("comparison.tsv").to_string_lossy().into_owned(); + let (rc, rec, _, comp) = run(&s, &rep, &["--pypgx", &pg, "--comparison", &comp]); + assert_eq!(rc, 0, "{rec}"); + (rec, comp) +} + +#[test] +fn indeterminate_pypgx_row_is_no_call() { + for indeterminate in [ + "Indeterminate", + "indeterminate", + "*1/Indeterminate", + "Indeterminate/Indeterminate", + ] { + let (rec, comp) = pypgx_run("indet", indeterminate, indeterminate); + assert!( + row(&comp, "CYP2D6").is_none(), + "{indeterminate}: no caller called CYP2D6, so no comparison row: {comp:?}" + ); + let c9 = row(&comp, "CYP2C9").expect("PharmCAT called CYP2C9"); + assert_eq!( + (c9[3].as_str(), c9[4].as_str()), + ("PharmCAT only", "PharmCAT only"), + "{indeterminate}: never a mismatch" + ); + assert!( + !rec.contains("pypgx (step 32) called") && !rec.contains("Drugs affected by CYP2D6"), + "{indeterminate}: no warning reads it as a pypgx call: {rec}" + ); + assert!( + rec.contains("No gene that PharmCAT could not call has a pypgx call."), + "{indeterminate}: {rec}" + ); + } +} + +#[test] +fn a_real_pypgx_call_still_warns() { + let (rec, comp) = pypgx_run("control", "*4/*4", "*1/*2"); + assert!( + rec.contains( + "WARNING: PharmCAT has no result for CYP2D6, but pypgx (step 32) called *4/*4 (Poor Metabolizer)." + ), + "{rec}" + ); + assert_eq!( + row(&comp, "CYP2D6").map(|r| r[3].as_str()), + Some("pypgx only") + ); + assert_eq!(row(&comp, "CYP2C9").map(|r| r[3].as_str()), Some("No")); +} + +/// The phenoconversion column of CYP2D6 and the recommendations text for a +/// CYP2D6 call and a medication list. +fn convert(name: &str, cyp2d6: &[&str; 4], meds: &str) -> (String, String) { + let s = Scratch::new(name); + let rep = report(&[gene( + "CYP2D6", + cyp2d6[0], + cyp2d6[1], + cyp2d6[2], + Some(cyp2d6[3]), + )]); + let list = s.put("meds.txt", meds); + let inh = format!("{ASSETS}pgx_inhibitors.tsv"); + let (rc, rec, rows, _) = run(&s, &rep, &["--medications", &list, "--inhibitors", &inh]); + assert_eq!(rc, 0, "{rec}"); + let col = row(&rows, "CYP2D6") + .and_then(|r| r.get(4).cloned()) + .unwrap_or_default(); + (col, rec) +} + +const POOR: [&str; 4] = ["*4", "*4", "Poor Metabolizer", "0.0"]; +const NORMAL: [&str; 4] = ["*1", "*1", "Normal Metabolizer", "2.0"]; + +#[test] +fn olanzapine_fluoxetine_is_checked_for_fluoxetine() { + let (col, rec) = convert("olafluo", &POOR, "olanzapine/fluoxetine 6/25 mg\n"); + assert_eq!( + col, "Poor Metabolizer (phenoconverted by fluoxetine)", + "{rec}" + ); + assert!(!rec.contains("Not in the inhibitor table"), "{rec}"); + // Control: the first drug alone inhibits nothing in the table. + let (col, rec) = convert("ola", &POOR, "olanzapine 5 mg\n"); + assert_eq!(col, "-", "{rec}"); + assert!( + rec.contains("Not in the inhibitor table: olanzapine 5 mg."), + "{rec}" + ); +} + +#[test] +fn naltrexone_bupropion_converts_by_bupropion() { + let (col, rec) = convert("nalbup", &NORMAL, "naltrexone/bupropion 8/90 mg\n"); + assert_eq!( + col, "Poor Metabolizer (phenoconverted by bupropion)", + "{rec}" + ); + assert!( + rec.contains( + "WARNING: CYP2D6 genotype phenotype Normal Metabolizer; phenoconverted phenotype Poor Metabolizer (phenoconverted by bupropion), activity score adjusted to 0." + ), + "{rec}" + ); + assert!(!rec.contains("Not in the inhibitor table"), "{rec}"); + // `and`, `+`, `;` and brackets split a line too, and a trailing stop + // is not part of the drug's name. + for line in [ + "naltrexone and bupropion", + "naltrexone + bupropion", + "naltrexone; bupropion", + "Contrave (naltrexone/bupropion) 8/90 mg", + "naltrexone, bupropion.", + ] { + let (col, rec) = convert("nalbup2", &NORMAL, &format!("{line}\n")); + assert_eq!( + col, "Poor Metabolizer (phenoconverted by bupropion)", + "{line}: {rec}" + ); + } +} + +/// The contraindication section for one pathogenic POLG hit and a list. +fn contra(name: &str, meds: &str) -> String { + let s = Scratch::new(name); + let rep = report(&[gene( + "CYP2D6", + "*1", + "*1", + "Normal Metabolizer", + Some("2.0"), + )]); + let list = s.put("meds.txt", meds); + let hits = s.put( + "hits.tsv", + "chrom\tpos\tref\talt\tgenotype\tclinvar_id\tgeneinfo\tclnsig\tclnrevstat\nchr15\t89300000\tC\tT\t0/1\t1\tPOLG:1\tLikely_pathogenic\tx\n", + ); + let (inh, table) = ( + format!("{ASSETS}pgx_inhibitors.tsv"), + format!("{ASSETS}pgx_contraindications.tsv"), + ); + let (rc, rec, _, _) = run( + &s, + &rep, + &[ + "--medications", + &list, + "--inhibitors", + &inh, + "--contraindications", + &table, + "--clinvar-hits", + &hits, + ], + ); + assert_eq!(rc, 0, "{rec}"); + rec +} + +#[test] +fn a_combinations_second_drug_is_on_the_list() { + let want = " WARNING: valproic acid -- POLG chr15:89300000 C>T, genotype 0/1, ClinVar Likely_pathogenic; ON THE MEDICATION LIST."; + let rec = contra("contra", "lamotrigine/valproate 100/500 mg\n"); + assert!(rec.lines().any(|l| l == want), "{rec}"); + // A brand name with its generic in brackets. + for line in [ + "Epilim (sodium valproate) 500 mg", + "Depakote (divalproex sodium)", + ] { + let rec = contra("contra-brand", &format!("{line}\n")); + assert!(rec.lines().any(|l| l == want), "{line}: {rec}"); + } + // Control: without valproate the same hit warns, not as on the list. + let rec = contra("contra-control", "lamotrigine 100 mg\n"); + assert!( + rec.contains("WARNING: valproic acid -- POLG") && !rec.contains("ON THE MEDICATION LIST"), + "{rec}" + ); +} + +// --- a combination written with a hyphen ------------------------------------ + +/// `convert` with an inhibitor table of the given CYP2D6 drugs, each a +/// strong inhibitor; the rows are invented for the test. +fn convert_with(name: &str, drugs: &[&str], meds: &str) -> (String, String) { + let s = Scratch::new(name); + let rep = report(&[gene( + "CYP2D6", + "*1", + "*1", + "Normal Metabolizer", + Some("2.0"), + )]); + let list = s.put("meds.txt", meds); + let mut table = + "Drug\tGene\tStrength\tActivity_score_factor\tClass_source\tRule_source\n".to_owned(); + for d in drugs { + table.push_str(&format!("{d}\tCYP2D6\tstrong\t0\tx\tx\n")); + } + let inh = s.put("inhibitors.tsv", &table); + let (rc, rec, rows, _) = run(&s, &rep, &["--medications", &list, "--inhibitors", &inh]); + assert_eq!(rc, 0, "{rec}"); + let col = row(&rows, "CYP2D6") + .and_then(|r| r.get(4).cloned()) + .unwrap_or_default(); + (col, rec) +} + +#[test] +fn a_hyphenated_combination_is_checked_for_each_drug() { + let (col, rec) = convert("olafluo-hyphen", &POOR, "olanzapine-fluoxetine 6/25 mg\n"); + assert_eq!( + col, "Poor Metabolizer (phenoconverted by fluoxetine)", + "{rec}" + ); + assert!(!rec.contains("Not in the inhibitor table"), "{rec}"); + let (col, rec) = convert("nalbup-hyphen", &NORMAL, "Naltrexone-Bupropion 8/90 mg\n"); + assert_eq!( + col, "Poor Metabolizer (phenoconverted by bupropion)", + "{rec}" + ); + assert!(!rec.contains("Not in the inhibitor table"), "{rec}"); + // The second drug of a hyphenated combination is on the list for the + // drug contraindications too. + let want = " WARNING: valproic acid -- POLG chr15:89300000 C>T, genotype 0/1, ClinVar Likely_pathogenic; ON THE MEDICATION LIST."; + let rec = contra("contra-hyphen", "lamotrigine-valproate 100/500 mg\n"); + assert!(rec.lines().any(|l| l == want), "{rec}"); +} + +#[test] +fn a_hyphenated_drug_name_of_the_table_still_matches_whole() { + let (col, rec) = convert_with("whole", &["co-trimoxazole"], "Co-trimoxazole 960 mg\n"); + assert_eq!( + col, "Poor Metabolizer (phenoconverted by co-trimoxazole)", + "{rec}" + ); + // A control, green before and after hyphen parts were added: the + // whole name matches, and its parts add nothing. + assert!( + rec.contains( + "WARNING: CYP2D6 genotype phenotype Normal Metabolizer; phenoconverted phenotype Poor Metabolizer (phenoconverted by co-trimoxazole), activity score adjusted to 0." + ) && !rec.contains("Not in the inhibitor table"), + "{rec}" + ); +} + +#[test] +fn a_hyphen_part_gives_no_false_hit() { + // With the shipped table none of these names an inhibitor, whole or by + // its parts, so each line stays unchecked. + for line in [ + "co-trimoxazole 960 mg", + "L-dopa 100 mg", + "L 5 mg", + "5-HTP 100 mg", + ] { + let (col, rec) = convert("parts", &NORMAL, &format!("{line}\n")); + assert_eq!(col, "-", "{line}: {rec}"); + assert!( + rec.contains(&format!("Not in the inhibitor table: {line}.")), + "{line}: {rec}" + ); + } + // A prefix, a lone letter and a number are never drugs: a table that + // lists them as names still gives no hit for these lines. + for line in ["co-trimoxazole 960 mg", "L-dopa 100 mg", "5-HTP 100 mg"] { + let (col, rec) = convert_with("short-parts", &["co", "l", "5"], &format!("{line}\n")); + assert_eq!(col, "-", "{line}: {rec}"); + } +} diff --git a/rust/crates/og-pgx/tests/port_tests_pgx_cpic.rs b/rust/crates/og-pgx/tests/port_tests_pgx_cpic.rs index 08684f735..67aa9ff15 100644 --- a/rust/crates/og-pgx/tests/port_tests_pgx_cpic.rs +++ b/rust/crates/og-pgx/tests/port_tests_pgx_cpic.rs @@ -2015,9 +2015,10 @@ fn cyp2c9_and_cyp3a5_inhibitors_give_a_note() { &meds("lopinavir/ritonavir 400/100 mg\n"), ); assert!( - cell(&r.rows, "CYP3A5", 4).starts_with("not computed: lopinavir inhibits CYP3A5") + cell(&r.rows, "CYP3A5", 4) + .starts_with("not computed: lopinavir, ritonavir inhibit CYP3A5;") && !section(&r.rec).contains("Not in the inhibitor table"), - "a boosted combination written 'lopinavir/ritonavir' is matched as lopinavir" + "a boosted combination written 'lopinavir/ritonavir' is matched as both drugs" ); let r = run_report(&work, "PHENO2C9NONE", &rep, &meds("paroxetine\n")); assert!( diff --git a/rust/crates/og-render/src/chapters.rs b/rust/crates/og-render/src/chapters.rs index 7ea79341b..042e07641 100644 --- a/rust/crates/og-render/src/chapters.rs +++ b/rust/crates/og-render/src/chapters.rs @@ -152,9 +152,73 @@ pub type TextFn = fn(&Summary) -> Result, Malformed>; /// A section's HTML part: its cards, none when the section is left out. pub type HtmlFn = fn(&Summary) -> Result; -/// The writers of section `sid`, or `None` for a section not ported yet, -/// which both reports leave out. One arm per section, a blank line apart, so -/// the units that port the others each change their own line. +/// Where a report finds each section's writers: [`writer`] for the reports +/// a run writes, another table in a test that plants a fault. +pub type WriterFn = fn(&str) -> Option<(TextFn, HtmlFn)>; + +/// A part of a report that could not be written: the section's anchor (or +/// `charts`, `terms`), and why. The rest of the report is still written, +/// and the part says why in place of its result. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Failed { + pub part: String, + pub why: String, +} + +/// Runs one part of a report: its error, or a panic inside it, comes back +/// as the error that part shows, so one bad part never stops the report. +/// While it runs, the hook of [`abort_outside_guarded`] lets a panic on +/// this thread unwind to here instead of ending the process. +pub fn guarded(f: impl FnOnce() -> Result) -> Result { + GUARDED.with(|g| g.set(g.get() + 1)); + let _depth = Depth; + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) { + Ok(r) => r, + Err(p) => { + let msg = p + .downcast_ref::<&str>() + .map(|m| (*m).to_owned()) + .or_else(|| p.downcast_ref::().cloned()) + .unwrap_or_else(|| "no message".to_owned()); + Err(Malformed(format!("internal error: {msg}"))) + } + } +} + +thread_local! { + /// How many [`guarded`] calls are running on this thread. + static GUARDED: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Leaves one [`guarded`] call, also when its closure panics. +struct Depth; + +impl Drop for Depth { + fn drop(&mut self) { + GUARDED.with(|g| g.set(g.get().saturating_sub(1))); + } +} + +/// Makes a panic end the process at once, as `panic = "abort"` does, +/// unless it happens inside [`guarded`] on the same thread. og is built +/// with `panic = "unwind"` only so that [`guarded`] can catch a panic in +/// one part of a report; everywhere else an unwinding panic could end a +/// worker thread quietly (a reader whose channel closes reads as the end +/// of its input) and og would write a cut file and exit 0. og's main calls +/// this once, first. The panic message is printed as always. +pub fn abort_outside_guarded() { + let default = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + default(info); + if GUARDED.try_with(std::cell::Cell::get).unwrap_or(0) == 0 { + std::process::abort(); + } + })); +} + +/// The writers of section `sid`, or `None` for an anchor no section has. +/// One arm per section, a blank line apart, so the units that change one +/// section each change their own line. pub fn writer(sid: &str) -> Option<(TextFn, HtmlFn)> { match sid { "qc" => Some((sections::qc::text, sections::qc::html)), @@ -173,7 +237,7 @@ pub fn writer(sid: &str) -> Option<(TextFn, HtmlFn)> { "exomiser" => Some((sections::exomiser::text, sections::exomiser::html)), - "structural-variants" => None, + "structural-variants" => Some((sections::structural::text, sections::structural::html)), "mei" => Some((sections::mei::text, sections::mei::html)), @@ -187,7 +251,7 @@ pub fn writer(sid: &str) -> Option<(TextFn, HtmlFn)> { "chip" => Some((sections::chip::text, sections::chip::html)), - "mosaic" => None, + "mosaic" => Some((sections::sex_loss::text, sections::sex_loss::html)), "mosaic-mca" => Some((sections::mocha::text, sections::mocha::html)), @@ -232,7 +296,7 @@ pub fn writer(sid: &str) -> Option<(TextFn, HtmlFn)> { "viral" => Some((sections::viral::text, sections::viral::html)), - "consumer-export" => None, + "consumer-export" => Some((sections::consumer::text, sections::consumer::html)), _ => None, } diff --git a/rust/crates/og-render/src/charts/mod.rs b/rust/crates/og-render/src/charts/mod.rs index 1bf958ea1..c612e5251 100644 --- a/rust/crates/og-render/src/charts/mod.rs +++ b/rust/crates/og-render/src/charts/mod.rs @@ -314,11 +314,16 @@ pub fn figure(spec: &Spec) -> Option { /// `_charts.json`: the schema version, the sample, when the summary /// was made and the charts. pub fn json(s: &Summary) -> Result { + json_of(s, &build(s)?) +} + +/// The same file around charts already built. +pub fn json_of(s: &Summary, specs: &[Spec]) -> Result { let mut m = IndexMap::new(); m.insert("schema_version".into(), Value::Int(SCHEMA_VERSION)); m.insert("sample".into(), s.need("sample")?.clone()); m.insert("generated_utc".into(), s.need("generated_utc")?.clone()); - let charts = build(s)?.iter().map(Spec::to_value).collect(); + let charts = specs.iter().map(Spec::to_value).collect(); m.insert("charts".into(), Value::Array(charts)); Ok(Value::Object(m)) } diff --git a/rust/crates/og-render/src/html.rs b/rust/crates/og-render/src/html.rs index efccdf40c..d964b219c 100644 --- a/rust/crates/og-render/src/html.rs +++ b/rust/crates/og-render/src/html.rs @@ -6,14 +6,20 @@ //! newest file date and "New since" under the first card title, each card's //! caveats under its title, the cards, then links to the step's own files. -use crate::chapters::{CHAPTERS, FrameState, LIMITS, Sec, step_of, writer}; +use pycompat::json::Value; + +use crate::chapters::{ + CHAPTERS, Failed, FrameState, LIMITS, Sec, TERMS, WriterFn, guarded, section_of, step_of, + writer, +}; +use crate::charts; use crate::css::CSS; use crate::escape::e; use crate::model::{ Card, Cell, Colour, HtmlPart, Inline, Layout, Line, Malformed, Section, State, Summary, Table, py, }; -use crate::notes::{VIEWER_NOTE, stale_note}; +use crate::notes::{NOTES, VIEWER_NOTE, stale_note}; use crate::sections::run; /// The Content-Security-Policy: nothing is fetched, images only inline, @@ -322,8 +328,195 @@ fn chapter_open(body: &mut Vec, cid: &str, title: &str, question: &str) body.push("
    ".to_owned()); } -/// The HTML report of one summary. +/// The caveats a note line is made of: one, or several joined by spaces +/// (step 11's F_ROH and band notes); empty when the line holds anything +/// else. +fn notes_in(line: &str) -> Vec<&'static str> { + let Some(mut rest) = line + .strip_prefix("

    ") + .and_then(|r| r.strip_suffix("

    ")) + else { + return Vec::new(); + }; + let mut found = Vec::new(); + while !rest.is_empty() { + let next = NOTES.iter().map(|(_, n)| *n).find_map(|n| { + let en = e(n); + if rest == en { + Some((n, "")) + } else { + rest.strip_prefix(&*en) + .and_then(|r| r.strip_prefix(' ')) + .map(|r| (n, r)) + } + }); + let Some((n, r)) = next else { + return Vec::new(); + }; + found.push(n); + rest = r; + } + found +} + +/// The sections whose card prints its caveat as its last paragraph, before +/// the caveat moves up under the title: the chart goes where that paragraph +/// was, at the end of the card (render_report.py's ancestry card ends with +/// `

    {ANCESTRY_FINE_NOTE}

    `). +const FIGURE_LAST: &[&str] = &["ancestry-fine"]; + +/// Puts a chart's figure line into its section's lines, under its caveat: +/// right after the line whose sentence the caveat is part of (the PRS +/// caption), at the end of the first card for the sections of +/// [`FIGURE_LAST`], else under the first card's title, after that card's +/// step line and caveats. +fn place_figure(lines: &mut Vec, sid: &str, caveat: &str, figure: String) { + if FIGURE_LAST.contains(&sid) { + let h2 = lines.iter().position(|x| x.trim().starts_with("

    ")); + let end = h2.and_then(|h| { + lines[h..] + .iter() + .position(|x| x == "

    ") + .map(|i| h + i) + }); + if let Some(i) = end { + lines.insert(i, figure); + return; + } + } + let cav = e(caveat); + let inside = lines.iter().position(|x| { + let t = x.trim(); + t.contains(&*cav) && !t.starts_with("

    ") && notes_in(t).is_empty() + }); + let at = match inside { + Some(i) => i + 1, + None => { + let Some(h2) = lines.iter().position(|x| x.trim().starts_with("

    ")) else { + return; + }; + let mut i = h2 + 1; + if lines + .get(i) + .is_some_and(|x| x.starts_with("

    ")) + { + i += 1; + } + while lines + .get(i) + .is_some_and(|x| x.starts_with("

    ")) + { + i += 1; + } + i + } + }; + lines.insert(at, figure); +} + +/// The closing appendix: one line per gene symbol or rsID of the search +/// index, with a link to each section that names it when the report shows +/// that section. No script: the browser's find searches it; the live site's +/// search box filters the same index. +fn terms_appendix(index: &Value, shown: &[&str]) -> Result, Malformed> { + let mut body = vec![ + "

    ".to_owned(), + "

    Every gene symbol and rsID a section of this report names, with the sections \ + that name it and how many times. A gene listed here is a gene the results mention, not a finding: open \ + the section to see what it says. The genes of the clinical filter (step 23) and variant prioritization \ + (step 31) come from their tables, which the report counts but does not list.

    " + .to_owned(), + ]; + let path = "index"; + let terms = crate::model::get(index, "terms", path)?.filter(|v| crate::model::truthy(v)); + let terms = match terms { + None => None, + Some(Value::Object(m)) => Some(m), + Some(_) => return Err(Malformed("index.terms is not an object".to_owned())), + }; + match terms { + None => { + body.push("

    No section of this report names a gene or an rsID.

    ".to_owned()) + } + Some(terms) => { + body.push("
      ".to_owned()); + for (term, items) in terms { + let Value::Array(items) = items else { + return Err(Malformed(format!("index.terms.{term} is not a list"))); + }; + let mut links = Vec::new(); + for it in items { + let ipath = format!("index.terms.{term}[]"); + let need = |k: &str| crate::model::need(it, k, &ipath); + let mut label = e(&py(need("title")?)?).into_owned(); + if crate::model::is_str(Some(need("state")?), "stale") { + label.push_str(" (stale)"); + } + let anchor = crate::model::get(it, "anchor", &ipath)? + .filter(|a| crate::model::truthy(a)) + .map(py) + .transpose()?; + let mut link = match anchor { + Some(a) if shown.contains(&a.as_str()) => { + format!("{label}", e(&a)) + } + _ => label, + }; + let count = need("count")?; + if crate::model::num(Some(count)).is_some_and(|n| n > 1.0) { + link.push_str(&format!(" ({})", py(count)?)); + } + links.push(link); + } + body.push(format!( + "
    • {}: {}
    • ", + e(term), + links.join(", ") + )); + } + body.push("
    ".to_owned()); + } + } + body.push("
    ".to_owned()); + Ok(body) +} + +/// The search index of a summary read alone: the step 23 and 31 tables, +/// which need the sample folder, are left out. +pub fn summary_index(s: &Summary) -> Result { + crate::index::build(s.root(), None, &[], &mut std::io::sink()).map_err(Malformed) +} + +/// The HTML report of one summary, with its closing appendix from the +/// summary's own search index. pub fn report(s: &Summary) -> Result { + let idx = guarded(|| summary_index(s)).ok(); + report_using(s, idx.as_ref(), writer).map(|(page, _)| page) +} + +/// The HTML report of one summary with the section writers of `writers`, +/// the closing appendix listing `index` (none without one), and the parts +/// that could not be written. A section whose data cannot be read, +/// or whose writer stops on an internal error, shows as failed with why; +/// charts or an appendix that cannot be made are left out; the rest of the +/// report is still written. +pub fn report_using( + s: &Summary, + index: Option<&Value>, + writers: WriterFn, +) -> Result<(String, Vec), Malformed> { + let mut failed = Vec::new(); + let mut fail = |part: &str, why: &Malformed| { + failed.push(Failed { + part: part.to_owned(), + why: why.to_string(), + }); + }; + let specs = guarded(|| charts::build(s)).unwrap_or_else(|why| { + fail("charts", &why); + Vec::new() + }); + let mut shown_ids: Vec<&str> = Vec::new(); let ns = s.new_since(); let mut toc = vec![ "

    " + ) + && !tx.contains("percentile 94.4") + && !ht.contains(""), + "4c: declared female: both reports print not applicable, no percentile:\n{tx}" + ); + } else { + assert!( + tx.contains("percentile 94.4 (EUR)") + && ht.contains("") + && tx.contains( + "No sex was declared, so these sex-specific scores are shown for any sex: PGS000662 (male)." + ) + && ht.contains("PGS000662 (male)."), + "4c: no sex: both reports keep the percentile and name the score in a note:\n{tx}" + ); + } + } + // Red control: the score list without its applies_to column. + let two: String = std::fs::read_to_string(&assets) + .unwrap() + .lines() + .map(|l| { + if l.starts_with('#') { + format!("{l}\n") + } else { + format!("{}\n", l.split('\t').take(2).collect::>().join("\t")) + } + }) + .collect(); + put(&wp("two_columns.tsv"), &two); + let (_, ctl, _) = sex_rows(&wp("two_columns.tsv"), "female", "ctl", "", ""); + assert_eq!( + [g(&ctl, "Percentile"), g(&ctl, "Sex_Check")], + ["94.4", ""], + "4c: control: without applies_to the female sample would get the percentile" + ); + + // 4d. the Ambiguous column + formatted( + &wp("amb_off"), + "PGS000003", + "PGS000003", + &[ + "1\t2\tA\tG\t1", + "1\t3\tA\tT\t1", + "1\t4\tC\tG\t1", + "1\t5\tT\tC\t1", + ], + 0, + "left out", + "", + "2\tdropped", + "", + ); + put( + &wp("amb_res/sample/match/sample_summary.csv"), + "dataset,accession,ambiguous,is_multiallelic,match_flipped,duplicate_best_match,duplicate_ID,match_IDs,match_status,count,score_pass,match_rate\n\ + sample,PGS000003,false,false,false,false,false,NA,matched,2,true,0.5\n\ + sample,PGS000003,true,false,false,false,false,NA,excluded,2,true,0.5\n", + ); + put_gz( + &wp("amb_res/sample/score/aggregated_scores.txt.gz"), + "sampleset\tFID\tIID\tPGS\tSUM\tDENOM\nsample\tHG\tHG\tPGS000003\t2\t2\n", + ); + let (ar, ao, at_) = (wp("amb_res"), wp("amb_off"), wp("amb.tsv")); + let code = prs_table(&[ + "--sample", + "S", + "--results", + &ar, + "--sampleset", + "sample", + "--scores", + &ao, + "--input-kind", + "gvcf", + "--out", + &at_, + ]); + assert_eq!( + (code, g(&first_row(&at_), "Ambiguous")), + (0, "2 dropped".to_owned()), + "4d: the Ambiguous column" + ); + + // 4e. a chrX row and the panel's intersection: pgsc_calc's match log + let mut rx = ROWS1.to_vec(); + rx.push("X\t51000000\tA\tG\t1"); + formatted( + &wp("pgsx_panel"), + "PGS000001", + "Coronary artery disease", + &rx, + 1, + "scored", + "", + "0\tdropped", + "", + ); + pgsc_results(&wp("xpanel"), true, "PGS000001"); + let log = + "row_nr,accession,chr_name,chr_position,effect_allele,other_allele,match_status,dataset\n"; + let chrx_of = || { + let (res, sc, o) = (wp("xpanel"), wp("pgsx_panel"), wp("xp.tsv")); + let code = prs_table(&[ + "--sample", + "S", + "--results", + &res, + "--sampleset", + "sample", + "--scores", + &sc, + "--input-kind", + "gvcf", + "--panel", + "pgsc_1000G_v1", + "--out", + &o, + ]); + assert_eq!(code, 0, "og prs table on the chrX panel"); + g(&first_row(&o), "ChrX") + }; + put_gz( + &wp("xpanel/sample/match/sample_log.csv.gz"), + &format!( + "{log}0,PGS000001,1,1100,A,G,matched,sample\n1,PGS000001,X,51000000,A,G,excluded,sample\n" + ), + ); + assert_eq!( + chrx_of(), + "1 left out (not matched in the panel run)", + "4e: the chrX row excluded" + ); + put_gz( + &wp("xpanel/sample/match/sample_log.csv.gz"), + &format!( + "{log}0,PGS000001,1,1100,A,G,matched,sample\n1,PGS000001,X,51000000,A,G,matched,sample\n" + ), + ); + assert_eq!(chrx_of(), "1 scored", "4e: the chrX row matched"); + put( + &wp("pgsx_panel/chrx_rows.tsv"), + "pgs_id\tchrx_rows\tchrx\nPGS000001\t3\tscored\n", + ); + assert_eq!( + chrx_of(), + "1 scored, 2 left out (not matched in the panel run)", + "4e: 1 of 3 chrX rows matched" + ); +} + +#[test] +fn the_prs_card_and_the_fine_ancestry_section_in_both_reports() { + let w = Scratch::new("prs-reports"); + let cols = PRS_COLUMNS.join("\t"); + let d = w.p("sample_raw/S"); + put( + &format!("{d}/prs/S_prs_summary.tsv"), + &format!( + "{cols}\nCoronary artery disease\tPGS000001\t62\t2\t3\t66.7\tNA\tNA\t8 left out\tgvcf\n\ + Trait two\tPGS000002\t1\t5\t5\t100.0\tNA\tNA\t3 scored\tgvcf\n" + ), + ); + let (summ, txt, html) = render(&d); + assert!( + txt.contains("Raw score only") && html.contains("Raw score only"), + "no panel: the raw-score line" + ); + assert!( + txt.contains("Rows on chrX were left out, because the sample's sex was not given: PGS000001 (8).") + && html.contains("Rows on chrX were left out, because the sample's sex was not given: PGS000001 (8).") + && !txt.contains("PGS000002 (3)"), + "chrX left out: both reports name the score and its row count, not the scored one:\n{txt}" + ); + assert!( + html.contains(""), + "no panel: the HTML row says raw score only" + ); + let na = not_assessed(&summ); + assert!( + na.iter().any(|x| x == PRS_NOT_ADJUSTED) && !na.iter().any(|x| x == PRS_WITHHELD), + "no panel: not-assessed names the missing percentiles: {na:?}" + ); + let caveat = PRS_ANCESTRY_NOTE; + assert!( + txt.contains(caveat) && html.contains(caveat), + "no panel: the ancestry-transfer caveat in both reports" + ); + + // A panel run whose input lists variants only: the percentiles withheld. + let d = w.p("sample_withheld/S"); + let withheld = "percentile withheld: the input lists variants only"; + put( + &format!("{d}/prs/S_prs_summary.tsv"), + &format!( + "{}\tPercentile_Note\tInput\nCoronary artery disease\tPGS000001\t62\t2\t3\t66.7\tNA\tEUR\t0\t0\t\t\t{withheld}\tvcf\n", + PRS_COLUMNS[..12].join("\t") + ), + ); + let (summ, txt, html) = render(&d); + assert!( + txt.contains("Percentile withheld: the input lists variants only") + && html.contains("Percentile withheld: the input lists variants only"), + "withheld: both reports say why:\n{txt}" + ); + let both = format!("{txt}{html}"); + assert!( + !both.contains("no ancestry reference panel was installed") + && !both.contains("--ancestry-panel adds it"), + "withheld: no report says no panel was installed, nor asks for one:\n{txt}" + ); + let na = not_assessed(&summ); + assert!( + na.iter().any(|x| x == PRS_WITHHELD) && !na.iter().any(|x| x == PRS_NOT_ADJUSTED), + "withheld: not-assessed gives the true reason: {na:?}" + ); + + let d = w.p("sample_adj/S"); + put( + &format!("{d}/prs/S_prs_summary.tsv"), + &format!( + "{cols}\nCoronary artery disease\tPGS000001\t1.25\t2\t3\t66.7\t94.4\tEUR\t0\tgvcf\n" + ), + ); + put( + &format!("{d}/ancestry/S_ancestry.tsv"), + "key\tvalue\nsample\tS\nreference_panel\tpgsc_1000G_v1\npopulation\tEUR\n", + ); + let (summ, txt, html) = render(&d); + assert!( + txt.contains("percentile 94.4 (EUR)"), + "panel: the percentile with its group in the text:\n{txt}" + ); + assert!( + html.contains(""), + "panel: the percentile with its group in the HTML" + ); + assert!( + txt.contains("among the EUR samples of the pgsc_1000G_v1 reference panel") + && !txt.contains("Raw score only") + && !html.contains("Raw score only") + && !txt.contains("Rows on chrX"), + "panel: the note names the group and the panel, no raw-score line, no chrX line:\n{txt}" + ); + assert!( + !not_assessed(&summ).iter().any(|x| x == PRS_NOT_ADJUSTED), + "panel: no not-assessed line" + ); + assert!( + txt.contains(caveat) && html.contains(caveat), + "panel: the ancestry-transfer caveat in both reports" + ); + + // 5b. the fine-grained ancestry section (step 26b), beside the PRS card + put( + &format!("{d}/ancestry_fine/S_ancestry_fine.tsv"), + "level\tcode\tgroup\tfraction\tse\tlow\thigh\n\ + continent\tWEA\tWest Eurasia (Europe, Middle East, North Africa)\t0.8200\t0.0300\t0.7600\t0.8800\n\ + continent\tCSA\tCentral and South Asia\t0.1500\t0.0300\t0.0900\t0.2100\n\ + continent\tAFR\tAfrica\t0.0300\t0.0100\t0.0100\t0.0500\n\ + region\tLEV\tLevant\t0.4600\t0.1000\t0.2600\t0.6600\n\ + region\tSAR\tSardinia\t0.0050\t0.0200\t0.0000\t0.0600\n\ + region\tOCE\tOceania\t0.0010\t0.0005\t0.0000\t0.0020\n\ + painting\tWEA\tWest Eurasia (Europe, Middle East, North Africa)\t0.6400\t.\t.\t.\n\ + painting\tAFR\tAfrica\t0.3600\t.\t.\t.\n", + ); + let mut png = b"\x89PNG\r\n\x1a\n".to_vec(); + png.extend([b'0'; 64]); + std::fs::write(format!("{d}/ancestry_fine/S_painting.png"), png).unwrap(); + let (_, txt, html) = render(&d); + assert!( + txt.contains("percentile 94.4 (EUR)") && html.contains(""), + "26b: both reports still render with the PRS card beside this one" + ); + assert!( + txt.contains("## Fine-Grained Ancestry (ADMIXTURE, RFMix)") + && txt + .contains("West Eurasia (Europe, Middle East, North Africa) 82% (76% to 88%)"), + "26b: the text section has the group, its fraction and its bootstrap range:\n{txt}" + ); + let fine = txt + .split("## Fine-Grained") + .nth(1) + .unwrap_or("") + .split("##") + .next() + .unwrap(); + assert!( + txt.contains("Sardinia") && !fine.contains("Oceania"), + "26b: Sardinia shown, Oceania not:\n{fine}" + ); + assert!( + html.contains("

    Fine-Grained Ancestry

    ") + && html.contains("West Eurasia (Europe, Middle East, North Africa) 64%") + && html.contains("src=\"data:image/png;base64,"), + "26b: the painted shares and the picture are in the HTML card" + ); + assert!( + txt.contains("not a nationality") && html.contains("not a nationality"), + "26b: both reports say a group name is not a nationality" + ); +} + +const VCF_HEAD: &str = + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n"; + +#[test] +fn the_clinvar_card_never_shows_green_and_carries_the_cdna_check() { + let w = Scratch::new("clinvar"); + let d = w.p("clinvar_hom/S"); + put( + &format!("{d}/clinvar/S_clinvar_hits.vcf"), + &format!( + "{VCF_HEAD}chr1\t20\t2\tT\tG\t50\tPASS\tGENEINFO=GENEB:22;CLNSIG=Pathogenic;\ + CLNREVSTAT=reviewed_by_expert_panel\tGT\t1/1\n" + ), + ); + let (_, txt, html) = render(&d); + let badge_line = html + .lines() + .find(|l| l.contains("ClinVar matches")) + .unwrap_or(""); + assert!( + !badge_line.contains("badge-green") && badge_line.contains("badge-yellow\">1<"), + "a homozygous 3-star P/LP hit gets no green badge: {badge_line}" + ); + let freq = "a heterozygous hit in a dominant gene as well as a homozygous one: a hit above 1% in gnomAD"; + assert!( + txt.contains(freq) + && html.contains(freq) + && !format!("{txt}{html}").contains("Look up a homozygous P/LP hit"), + "both reports ask for every hit's population frequency:\n{txt}" + ); + assert!( + txt.contains("frequency: gnomAD AF unknown") && html.contains("
    "), + "without step 13 the frequency is unknown:\n{txt}" + ); + put(&format!("{d}/clinvar/S_clinvar_hits.vcf"), VCF_HEAD); + let (_, _, html0) = render(&d); + assert!( + html0.contains( + "ClinVar matches0<" + ), + "no hit gets a gray badge, not a green all-clear" + ); + + // 6b. step 06's cDNA read check beside each hit + let d = w.p("clinvar_cdna/S"); + put( + &format!("{d}/clinvar/S_clinvar_hits.vcf"), + &format!( + "{VCF_HEAD}chr1\t20\t2\tT\tG\t50\tPASS\tGENEINFO=GENEB:22;CLNSIG=Pathogenic;CLNREVSTAT=reviewed_by_expert_panel\tGT\t0/1\n\ + chr2\t30\t3\tC\tA\t50\tPASS\tGENEINFO=GENEC:33;CLNSIG=Pathogenic;CLNREVSTAT=reviewed_by_expert_panel\tGT\t0/1\n" + ), + ); + let (summ, _, html) = render(&d); + assert!( + !html.contains("cDNA read check") + && at(&summ, &["sections", "clinvar", "data", "cdna_flagged"]).is_none(), + "no table, no column (a Nextflow folder)" + ); + let flag = "possible cDNA/transgene contamination: check the reads"; + put( + &format!("{d}/clinvar/S_clinvar_cdna_check.tsv"), + &format!( + "chrom\tpos\tref\talt\talt_reads\talt_clipped\tref_reads\tref_clipped\tcdna_check\tcdna_note\n\ + chr1\t20\tT\tG\t9\t8\t11\t0\tflagged\t{flag}\nchr2\t30\tC\tA\t12\t0\t10\t0\tclear\t\n" + ), + ); + let (summ, txt, html) = render(&d); + let shown = format!("{flag} (8 of 9 alt reads and 0 of 11 ref reads clipped at an exon edge)"); + assert!( + txt.contains("chr1:20 T>G") + && txt.contains(&format!("reads: {shown}")) + && txt.contains("reads: clear"), + "the text report prints the flag under the flagged hit:\n{txt}" + ); + assert!( + html.contains("") + && html.contains(&format!("")) + && html.contains(""), + "the HTML hits table has the column, the flag and clear" + ); + assert!( + html.contains("Flagged by the cDNA read check1<"), + "the card counts one flagged hit" + ); + let p = w.p("summary.json"); + put(&p, &json::dumps(&summ, &Dump::default()).unwrap()); + assert_eq!( + rc(&[ + "dev", + "schema", + &repo("tests/schema/summary.schema.json"), + &p + ]), + 0, + "the summary validates against the schema" + ); +} + +#[test] +fn the_telomere_mosaic_paralog_and_export_cards() { + let w = Scratch::new("cards"); + // 7. telomere card + let d = w.p("telomere/S"); + put( + &format!("{d}/telomere/S/S_summary.tsv"), + "PID\tsample\tintratelomeric_reads\ttotal_reads_with_tel_gc\ttel_content\nS\ttumor\t61873\t185204533\t334.08\n", + ); + let (_, txt, html) = render(&d); + assert!( + txt.contains("## Telomere content (relative, TelomereHunter)") + && txt.contains( + "Telomere content: 334.08 intratelomeric reads per million reads with 48-52% GC" + ), + "telomere: the text heading and the value with its unit:\n{txt}" + ); + assert!( + html.contains("

    Telomere content (relative)

    ") + && html.contains("intratelomeric reads per million reads with 48-52% GC") + && html.contains("not a telomere length"), + "telomere: the HTML card title and the unit" + ); + let mut said = Vec::new(); + for dir in [ + repo("bin"), + repo("rust/crates/og-render/src"), + repo("rust/crates/og-render/src/sections"), + ] { + for f in std::fs::read_dir(&dir).unwrap() { + let p = f.unwrap().path(); + if p.is_file() + && std::fs::read_to_string(&p) + .unwrap_or_default() + .contains("Telomere Length") + { + said.push(p); + } + } + } + assert!( + said.is_empty(), + "no report source says 'Telomere Length': {said:?}" + ); + + // 8. mosaic loss of chrY or chrX + let d = w.p("mosaic/S"); + let mosaic_table = |rows: &[(&str, &str)]| { + let body: String = rows + .iter() + .map(|(k, v)| format!("mosaic_{k}\t{v}\n")) + .collect(); + put( + &format!("{d}/qc/S_sample_qc.tsv"), + &format!("key\tvalue\nsample\tS\ninferred_sex\tmale\n{body}"), + ); + let (_, t, h) = render(&d); + (t, h) + }; + let male = [ + ("sex", "male"), + ("chrom", "chrY"), + ("depth_ratio", "0.351"), + ("depth_ratio_expected", "0.5"), + ("loss_depth", "0.298"), + ("loss_depth_low", "0.293"), + ("loss_depth_high", "0.303"), + ("floor", "0.100"), + ("call", "detected"), + ("reason", "the low end of the 95% range is above 10%"), + ]; + let with = |over: &[(&'static str, &'static str)]| -> Vec<(&'static str, &'static str)> { + let mut v: Vec<(&str, &str)> = male.to_vec(); + for (k, x) in over { + match v.iter_mut().find(|y| y.0 == *k) { + Some(y) => y.1 = x, + None => v.push((k, x)), + } + } + v + }; + let (txt, html) = mosaic_table(&male); + let line = "From depth: estimated share of the cells that lost the Y 29.8% (95% range 29.3% to 30.3%); \ + chrY to autosome depth ratio 0.351"; + let card_m = card(&html, "Mosaic Loss of Chromosome Y or X"); + assert!( + txt.contains("## Mosaic loss of chromosome Y or X") + && txt.contains("chrY: detected (the low end") + && txt.contains(line), + "mosaic: the text report heading, call and estimate:\n{txt}" + ); + assert!( + card_m.contains("badge-yellow\">detected<") && card_m.contains(line), + "mosaic: the HTML card:\n{card_m}" + ); + for t in [txt.as_str(), card_m] { + assert!( + t.contains("population-level association, not a diagnosis") + && t.contains("Below about 10% of the cells"), + "mosaic: population-level, not a diagnosis, and the 10% floor" + ); + } + assert!( + !txt.contains("allele fractions at"), + "mosaic: a man has no allele-fraction line" + ); + let (txt, html) = mosaic_table(&with(&[ + ("sex", "female"), + ("chrom", "chrX"), + ("depth_ratio", "0.849"), + ("depth_ratio_expected", "1.0"), + ("x_het_sites", "600"), + ("loss_af", "0.316"), + ("loss_af_low", "0.288"), + ("loss_af_high", "0.340"), + ])); + assert!( + txt.contains("of the cells that lost one X") + && txt.contains("From chrX allele fractions at 600 heterozygous sites: 31.6% (95% range 28.8% to 34.0%)") + && html.contains("31.6%"), + "mosaic: a woman shows one X and the allele-fraction line:\n{txt}" + ); + let (txt, html) = mosaic_table(&with(&[ + ("depth_ratio", ""), + ("loss_depth", "0.190"), + ("loss_depth_low", "-0.002"), + ("loss_depth_high", "0.382"), + ("call", "not_detected"), + ])); + let card_m = card(&html, "Mosaic Loss of Chromosome Y or X"); + assert!( + txt.contains("chrY: not detected above 10%") + && !txt.contains("below 10%") + && txt.contains("depth ratio ?,") + && card_m.contains("badge-gray\">not detected above 10%<"), + "mosaic: not detected says 'above 10%', a missing depth ratio prints '?':\n{txt}" + ); + put( + &format!("{d}/qc/S_sample_qc.tsv"), + "key\tvalue\nsample\tS\ninferred_sex\tmale\n", + ); + let (_, txt, html) = render(&d); + assert!( + card(&html, "Mosaic Loss of Chromosome Y or X").contains("badge-gray\">Not run<") + && !txt.contains("## Mosaic loss"), + "mosaic: no estimate, the card says Not run and the text has no section" + ); + + // 9. paralog card (step 35b) + assert!( + !html.contains("Paralog Genes (Paraphase)"), + "paralogs: no card when step 35b did not run" + ); + let d = w.p("paralogs/S"); + let f = format!("{d}/paralogs/S_paraphase_copy_number.tsv"); + put( + &f, + "region\tgene\tcopy_number\tstatus\tdetail\ttotal_cn\tregion_depth\tgenome_depth\npms2\tPMS2\t2\tcalled\t.\t4\t60\t30\n", + ); + std::fs::File::options() + .write(true) + .open(&f) + .unwrap() + .set_modified(UNIX_EPOCH + Duration::from_secs(1)) + .unwrap(); + put( + &format!("{d}/logs/run_status.tsv"), + "meta\tstarted_epoch\t1000\nstep\t35b\tfailed\n", + ); + let (summ, _, html) = render(&d); + let card_p = card(&html, "Paralog Genes (Paraphase)"); + assert!( + at(&summ, &["sections", "paralogs", "state"]) == Some(&s("stale")) + && card_p.contains("badge-yellow\">Stale<") + && card_p.contains("
    "), + "paralogs: a result older than a run where step 35b failed is stale, with its PMS2 row:\n{card_p}" + ); + + // 12. raw-data export card + let d = w.p("consumer/S"); + std::fs::create_dir_all(&d).unwrap(); + let (_, txt, html) = render(&d); + assert!( + !txt.contains("Raw-Data Export") && !html.contains("Raw-Data Export"), + "raw-data export: no card and no text section without the step's table" + ); + put( + &format!("{d}/consumer_export/S_consumer_export.tsv"), + "key\tvalue\nsites\t200\ncalled\t150\nno_call\t50\nfrom_reference_blocks\t120\nfile_23andme\tS_23andme_v5.zip\n\ + file_ancestrydna\tS_ancestrydna_layout.zip\nno_call: no gVCF record\t30\nno_call: filtered call\t20\n", + ); + let (_, txt, html) = render(&d); + let want = [ + "Sites with a genotype: 150 of 200 (75.0%); '--' at 50", + "Homozygous reference from the gVCF's reference blocks: 120", + "consumer_export/S_23andme_v5.zip and consumer_export/S_ancestrydna_layout.zip", + "'--' because: no gVCF record 30; filtered call 20", + "Uploading one to a service shares your genotypes", + ]; + assert!( + txt.contains("## Raw-Data Export (23andMe v5 layout)") + && want.iter().all(|x| txt.contains(x)), + "raw-data export: the text section has every line:\n{txt}" + ); + assert!( + html.contains("

    Raw-Data Export (23andMe v5 layout)

    ") + && want.iter().all(|x| html.contains(&*e(x))), + "raw-data export: the HTML card has every line" + ); +} + +#[test] +fn the_blood_group_and_couple_cards() { + let w = Scratch::new("blood-couple"); + // 10. blood groups + let d = w.p("blood/S"); + put( + &format!("{d}/bloodtype/S_bloodtype.tsv"), + "system\tphenotype\tgenotype\tconfidence\tnote\nABO\tO\tO.01/O.01\thigh\t\n\ + RhD\tD+\tRHD/RHD (RHD copies by read depth 1.78)\thigh\t\nDuffy\tFy(a+b-)\tFY*A/FY*B GATA\thigh\tc.-67T>C\n\ + HPA-1\tHPA-1(a+b+)\t1a/1b\thigh\t\n", + ); + let (_, txt, html) = render(&d); + assert!( + txt.contains("## Blood Groups (predicted from the genome)") + && [ + "ABO O ", + "RhD D+ ", + "Duffy Fy(a+b-) ", + "HPA-1 HPA-1(a+b+) " + ] + .iter() + .all(|x| txt.contains(x)), + "blood groups: the text section has every system:\n{txt}" + ); + assert!( + html.contains("

    Blood Groups (predicted)

    ") + && html.contains("
    ") + && html.contains("") + && html.contains(">D+"), + "blood groups: the HTML card has ABO, RhD and the table" + ); + assert_eq!( + ( + txt.matches("not typed from blood").count(), + html.matches("not typed from blood").count() + ), + (1, 1), + "blood groups: both reports say it is predicted, not typed from blood" + ); + let empty = w.p("noblood/S"); + std::fs::create_dir_all(&empty).unwrap(); + assert!( + !render(&empty).2.contains("Blood Groups"), + "blood groups: no card without step 41" + ); + let words = |t: &str| { + let l = t.to_lowercase(); + ["transfus", "donat", "donor"].iter().any(|x| l.contains(x)) + }; + let src = repo("rust/crates/og-bloodtype/src"); + let mut names: Vec = std::fs::read_dir(&src) + .unwrap() + .map(|f| f.unwrap().path()) + .collect(); + names.sort(); + let code: String = names + .iter() + .filter(|p| p.extension().is_some_and(|x| x == "rs")) + .map(|p| std::fs::read_to_string(p).unwrap()) + .collect(); + let said: Vec<&str> = [ + ("text report", &txt), + ("HTML report", &html), + ("og bloodtype", &code), + ] + .iter() + .filter(|(_, t)| words(t)) + .map(|(n, _)| *n) + .collect(); + assert!( + said.is_empty(), + "blood groups: nothing about transfusion or donation: {said:?}" + ); + assert!( + words("for a Transfusion"), + "control: the wording check finds the word" + ); + + // 13. couple card (step 39) + let genes = std::fs::read_to_string(repo("assets/carrier_genes.tsv")).unwrap(); + let g: Vec<&str> = genes + .lines() + .find(|x| x.starts_with("NAGA\t")) + .unwrap() + .split('\t') + .collect(); + let pos = (g[2].parse::().unwrap() + g[3].parse::().unwrap()) / 2; + let cv = |p: i64| { + format!( + "{VCF_HEAD}chr22\t{p}\t1\tA\tG\t.\tPASS\tGENEINFO=NAGA:4668;CLNSIG=Pathogenic;\ + CLNREVSTAT=reviewed_by_expert_panel\tGT\t0/1\n" + ) + }; + let c = w.p("couple"); + put( + &format!("{c}/ped.tsv"), + "sample\tsex\tfather\tmother\tpartner\nP1\tfemale\t.\t.\tP2\nP2\tmale\t.\t.\tP1\n", + ); + put(&format!("{c}/P1.vcf"), &cv(pos)); + put(&format!("{c}/P2.vcf"), &cv(pos + 5)); + let out = og(&[ + "couple", + "screen", + "--pedigree", + &format!("{c}/ped.tsv"), + "--genes", + &repo("assets/carrier_genes.tsv"), + "--clinvar", + &format!("P1={c}/P1.vcf"), + "--clinvar", + &format!("P2={c}/P2.vcf"), + "--out-dir", + &format!("{c}/out"), + ]); + assert!( + out.status.success(), + "og couple screen: {}", + String::from_utf8_lossy(&out.stderr) + ); + let d = format!("{c}/P2"); + let table = format!("{d}/couple/P2_couple_carrier.tsv"); + put( + &table, + &std::fs::read_to_string(format!("{c}/out/P2_couple_carrier.tsv")).unwrap(), + ); + let (txt, html) = reports(&summary_of("P2", &d)); + let card_c = card(&html, "Couple Carrier Screen"); + let want = [ + "Partner: P1".to_owned(), + "Shared: NAGA (Schindler disease): 1 in 4 per pregnancy".to_owned(), + format!("This sample: chr22:{} A>G het", pos + 5), + format!("P1: chr22:{pos} A>G het, Pathogenic, 3 stars"), + "genetic counsellor".to_owned(), + ]; + assert!( + want.iter().all(|x| card_c.contains(x.as_str())), + "couple: the card:\n{card_c}" + ); + assert!( + want.iter().all(|x| txt.contains(&x.replace(">", ">"))) + && txt.contains("## Couple Carrier Screen (step 39)"), + "couple: the text report says the same:\n{txt}" + ); + std::fs::remove_file(&table).unwrap(); + let (txt, html) = reports(&summary_of("P2", &d)); + assert!( + !html.contains("Couple Carrier Screen") && !txt.contains("Couple Carrier Screen"), + "couple: without the table there is no card and no text section" + ); +} + +/// The card titles of DEMO-001's HTML report: the chapters move cards, they +/// never add, drop or rename one. +const GOLDEN_TITLES: [&str; 47] = [ + "Quality Control", + "Mosaic Loss of Chromosome Y or X", + "Variant Calling", + "ClinVar Screening", + "Carrier List (this sample)", + "Couple Carrier Screen", + "Pharmacogenomics", + "pypgx Pharmacogenomics", + "PGx Copy Number", + "CPIC Drug Recommendations", + "Genes With a Non-Normal or Unresolved Phenotype (CPIC)", + "CYP2D6 Across Callers", + "HLA Typing", + "Structural Variants", + "Structural Variants, Class 4 and 5 (AnnotSV)", + "Cancer Predisposition", + "Repeat Expansions", + "Runs of Homozygosity", + "Mitochondrial Haplogroup", + "Paralog Genes (Paraphase)", + "SMN1/SMN2 Copy Number (Parascopy)", + "Y-Chromosome Haplogroup", + "Y Placement on the Current Public Tree", + "Y-STR Markers (HipSTR)", + "Raw-Data Export (23andMe v5 layout)", + "Blood Groups (predicted)", + "Archaic Ancestry (Neanderthal, Denisovan)", + "Lp(a): LPA KIV-2 Repeat Count", + "MUC1 VNTR Frameshifts (VNtyper)", + "Locityper Second Opinion at Hard Loci", + "Viral DNA in the Unmapped Reads", + "Clonal Haematopoiesis (CHIP)", + "Mosaic Chromosomal Alterations (MoChA)", + "Mobile Element Insertions", + "Telomere content (relative)", + "Mitochondrial Analysis", + "Clinical Variant Filter", + "Variant Prioritization (slivar)", + "Symptom-Driven Ranking (Exomiser)", + "Polygenic Risk Scores", + "Fine-Grained Ancestry", + "Traits", + "Ancient Populations", + "Genes This Genome Could Not Read Reliably", + "ClinVar Hits (best-reviewed first, 3 of 3)", + "Steps Not Run", + "Not Assessed by This Pipeline", +]; + +/// Every card title of an HTML page, sorted: each `

    `, whatever its +/// attributes, except the chapter and contents headings. +fn titles(html: &str) -> Vec { + let mut t: Vec = html + .split("')) + .filter(|(attrs, _)| !attrs.contains("chapter-title") && !attrs.contains("toc-title")) + .filter_map(|(_, x)| x.split_once("

    ").map(|y| y.0)) + .filter(|x| !x.contains('<')) + .map(str::to_owned) + .collect(); + t.sort(); + t +} + +fn golden() -> Vec { + let mut g: Vec = GOLDEN_TITLES.iter().map(|x| (*x).to_owned()).collect(); + g.sort(); + g +} + +/// Every http(s) src or href that is not the loopback viewer and not inside +/// a link marked as opening a website. +fn external_problems(html: &str) -> Vec { + const OPEN: &str = "
    ") { + Some(end) + if tail[..end].contains(" (opens a website") => + { + rest = &tail[end + "".len()..]; + } + _ => { + kept.push_str(OPEN); + rest = &tail[OPEN.len()..]; + } + } + } + kept.push_str(rest); + // ASCII only, so a byte offset in `low` is the same offset in `kept`. + let low = kept.to_ascii_lowercase(); + let mut found = Vec::new(); + for attr in ["src", "href"] { + let mut from = 0; + while let Some(i) = low[from..].find(attr) { + let mut k = from + i + attr.len(); + from = k; + let b = low.as_bytes(); + while k < b.len() && b[k].is_ascii_whitespace() { + k += 1; + } + if b.get(k) != Some(&b'=') { + continue; + } + k += 1; + while k < b.len() && b[k].is_ascii_whitespace() { + k += 1; + } + if matches!(b.get(k), Some(b'"' | b'\'')) { + k += 1; + } + if low[k..].starts_with("http:") || low[k..].starts_with("https:") { + let end = kept[k..] + .find(['"', '\'', ' ', '>']) + .map_or(kept.len(), |x| k + x); + let url = &kept[k..end]; + if !url.starts_with(og_render::html::VIEWER_URL) { + found.push(url.to_owned()); + } + } + } + } + found +} + +fn frame_of<'a>(html: &'a str, sid: &str) -> &'a str { + html.split_once(&format!("id=\"{sid}\">")) + .map_or("", |x| x.1.split("").next().unwrap()) +} + +/// The text report's chapter titles, in order, each with the text under it. +fn text_chapters(txt: &str) -> Vec<(String, String)> { + let mut out: Vec<(String, String)> = Vec::new(); + for l in txt.split_inclusive('\n') { + let head = l + .strip_prefix("# ") + .and_then(|r| { + r.split_once(". ") + .filter(|(n, _)| !n.is_empty() && n.bytes().all(|b| b.is_ascii_digit())) + }) + .and_then(|(_, t)| t.split_once(": ").map(|x| x.0.to_owned())); + match head { + Some(t) => out.push((t, String::new())), + None => { + if let Some(last) = out.last_mut() { + last.1.push_str(l); + } + } + } + } + out +} + +fn under<'a>(txt: &'a str, title: &str) -> &'a str { + txt.find(title).map_or("", |i| { + let end = (i + 120).min(txt.len()); + let end = (end..=txt.len()) + .find(|&k| txt.is_char_boundary(k)) + .unwrap(); + &txt[i..end] + }) +} + +#[test] +fn the_chapters_on_the_demo_sample() { + let w = Scratch::new("chapters"); + let g = w.p("demo"); + std::fs::create_dir_all(&g).unwrap(); + let out = og(&[ + "dev", + "demo-sample", + "--genome-dir", + &g, + "--records", + "20000", + "--repo", + REPO, + ]); + assert!( + out.status.success(), + "og dev demo-sample: {}", + String::from_utf8_lossy(&out.stderr) + ); + let sd = format!("{g}/DEMO-001"); + let summ = summary_of("DEMO-001", &sd); + let (txt, html) = reports(&summ); + + // The haplogroup links: FamilyTreeDNA and YFull say the address carries + // the haplogroup, the docs link does not. + for (kind, hg) in [("y", "R-M269*(xR-L23)"), ("mt", "H1a+16189")] { + let hd = w.p(&format!("hg-{kind}/S")); + let title = if kind == "y" { + put( + &format!("{hd}/y_haplogroup/S_y_haplogroup.txt"), + &format!("Sample_name\tHg\tValid_markers\tQC-score\nS\t{hg}\t9\t1\n"), + ); + "Y-Chromosome Haplogroup" + } else { + put( + &format!("{hd}/mito/S_haplogroup.txt"), + &format!("SampleID\tHaplogroup\nS\t{hg}\n"), + ); + "Mitochondrial Haplogroup" + }; + let (_, _, h) = render(&hd); + let c = card(&h, title); + let marks: Vec<&str> = c + .split("(") + .skip(1) + .map(|x| x.split(")").next().unwrap()) + .collect(); + let mut want = + vec![og_render::sections::ancestry::OFFSITE_HAPLOGROUP; marks.len().saturating_sub(1)]; + want.push(og_render::html::OFFSITE); + assert!( + marks.len() >= 3 && marks == want, + "{hg}: the offsite notes: {marks:?}" + ); + } + + assert_eq!( + external_problems(&html), + Vec::::new(), + "no off-machine address but the marked links" + ); + for u in [ + "https://discover.familytreedna.com/", + "https://www.yfull.com/", + "https://github.com/", + ] { + assert!( + html.contains(&format!("x", + "", + "y", + ] { + assert!( + !external_problems(&format!("{html}{bad}")).is_empty(), + "red control: {bad} is caught" + ); + } + + let sids: Vec<&str> = og_render::chapters::CHAPTERS + .iter() + .flat_map(|c| c.sections.iter().map(|x| x.id)) + .collect(); + let cv = frame_of(&html, "clinvar"); + assert!( + cv.contains("

    ClinVar Screening

    ") + && cv.contains("

    ClinVar Hits") + && !cv.contains("badge-green"), + "the ClinVar section has its two cards and no green badge" + ); + assert!( + frame_of(&html, "qc").contains("badge-green"), + "red control: the detector sees green in the QC block" + ); + let mut nohit = summ.clone(); + set( + &mut nohit, + &["sections", "clinvar", "data", "count"], + Value::Int(0), + ); + set( + &mut nohit, + &["sections", "clinvar", "data", "hits"], + j("[]"), + ); + let nohit_html = reports(&nohit).1; + let nv = frame_of(&nohit_html, "clinvar"); + assert!( + nv.contains("

    ClinVar Screening

    ") + && nv.contains("0") + && !nv.contains("badge-green"), + "no ClinVar hit: the screen card with a gray 0, still no green:\n{nv}" + ); + assert!( + frame_of(&html, "cpic").contains("CPIC Drug Recommendations"), + "the CPIC table sits in the CPIC section" + ); + + let heads = [ + ("QUALITY", "## Quality control"), + ("CLINICAL", "## ClinVar Pathogenic Screen"), + ("CARRIER", "## Carrier List (this sample)"), + ("MEDICINES", "## Pharmacogenomics (PharmCAT)"), + ("ANCESTRY", "## Mitochondrial Haplogroup"), + ("TRAITS AND SCORES", "## Polygenic Risk Scores"), + ( + "OTHER AND EXPORTS", + "## Raw-Data Export (23andMe v5 layout)", + ), + ]; + let chunks = text_chapters(&txt); + for (c, h) in heads { + let body = chunks + .iter() + .find(|x| x.0 == c) + .map_or("", |x| x.1.as_str()); + assert!(body.contains(h), "text chapter {c} holds {h}"); + } + + assert_eq!( + titles(&html), + golden(), + "the card titles are the golden list" + ); + assert_ne!( + titles(&html.replace("

    Traits

    ", "")), + golden(), + "red control: a dropped card is caught" + ); + assert_ne!( + titles(&html.replace( + "

    Traits

    ", + "

    Traits

    Extra

    " + )), + golden(), + "red control: an extra card with attributes on its heading is caught" + ); + + // "New since" and the database re-check + let badge = "New since 2026-01-01T00:00:00Z"; + let mut since = summ.clone(); + set( + &mut since, + &["new_since"], + j(r#"{"previous_utc": "2026-01-01T00:00:00Z", "sections": ["clinvar"]}"#), + ); + set( + &mut since, + &["recheck"], + j( + r#"{"checked_utc": "2026-02-02T00:00:00Z", "affected_steps": ["06", "24"], "rerun": null, + "items": [{"item": "clinvar_file_date", "label": "ClinVar", "class": "refresh-available"}, + {"item": "VEP_CACHE_RELEASE", "label": "VEP", "class": "no-newer-release"}]}"#, + ), + ); + let (t2, h2) = reports(&since); + let badge_problems = |page: &str, want: &[&str]| -> Vec { + let got: Vec<&str> = sids + .iter() + .copied() + .filter(|sid| frame_of(page, sid).contains(badge)) + .collect(); + let n = page.matches(badge).count(); + if got == want && n == want.len() { + Vec::new() + } else { + vec![format!("the badge in {got:?}, {n} on the page")] + } + }; + assert_ne!( + badge_problems(&h2.replace(badge, ""), &["clinvar"]), + Vec::::new(), + "red control: a missing badge is caught" + ); + assert_eq!( + badge_problems(&h2, &["clinvar"]), + Vec::::new(), + "the badge only where the result changed" + ); + assert!( + !html.contains("New since") && !txt.contains("New since"), + "no badge without a previous run" + ); + let mut every = since.clone(); + let keys: Vec = og_render::chapters::CHAPTERS + .iter() + .flat_map(|c| c.sections.iter().flat_map(|x| x.keys.iter().map(|k| s(k)))) + .collect(); + set(&mut every, &["new_since", "sections"], Value::Array(keys)); + assert!( + !badge_problems(&reports(&every).1, &["clinvar"]).is_empty(), + "red control: a badge on every section" + ); + let text_with = |sections: &[&str], missing: Option<&str>| -> (String, String) { + let mut x = since.clone(); + if let Some(m) = missing { + set(&mut x, &["sections", m, "state"], s("missing")); + } + set( + &mut x, + &["new_since", "sections"], + Value::Array(sections.iter().map(|k| s(k)).collect()), + ); + let (t, h) = reports(&x); + (h, t) + }; + let cyp = |h: &str, t: &str| { + let _ = h; + under(t, "## CYP2D6 across callers").contains(&format!("[{badge}]")) + }; + let (h, t) = text_with(&["pypgx"], None); + assert!( + cyp(&h, &t), + "the text CYP2D6 block carries the badge when pypgx changed:\n{}", + under(&t, "## CYP2D6") + ); + let (h, t) = text_with(&["clinvar"], None); + assert!( + !cyp(&h, &t), + "red control: the CYP2D6 check fails when only ClinVar changed" + ); + let (h, t) = text_with(&["roh"], Some("roh")); + assert!( + frame_of(&h, "roh").contains(badge) + && frame_of(&h, "roh").contains("Not run") + && !t.contains("## Runs of Homozygosity") + && !t.contains(badge), + "a section gone missing keeps its badge on its HTML card, the text leaves the section out" + ); + assert_eq!( + titles(&h2), + golden(), + "new since: the card titles stay the golden list" + ); + + let hist = format!("{sd}/history/2026-01-01T000000Z"); + std::fs::create_dir_all(&hist).unwrap(); + let mut old = summ.clone(); + set(&mut old, &["generated_utc"], s("2026-01-01T00:00:00Z")); + let has_hits = matches!(at(&summ, &["sections", "clinvar", "data", "hits"]), Some(Value::Array(a)) if !a.is_empty()); + if let Some(Value::Object(sec)) = match &mut old { + Value::Object(m) => m.get_mut("sections"), + _ => None, + } && let Some(Value::Object(cvs)) = sec.get_mut("clinvar") + && let Some(Value::Object(data)) = cvs.get_mut("data") + && let Some(Value::Array(hits)) = data.get_mut("hits") + && let Some(Value::Object(h0)) = hits.first_mut() + { + h0.insert("significance".to_owned(), s("Uncertain significance")); + } + put( + &format!("{hist}/summary.json"), + &json::dumps(&old, &Dump::default()).unwrap(), + ); + if std::path::Path::new(&format!("{sd}/run_manifest.tsv")).is_file() { + std::fs::copy( + format!("{sd}/run_manifest.tsv"), + format!("{hist}/run_manifest.tsv"), + ) + .unwrap(); + } + let again = summary_of("DEMO-001", &sd); + std::fs::remove_dir_all(format!("{sd}/history")).unwrap(); + assert!( + has_hits + && at(&summ, &["new_since"]).is_none() + && at(&again, &["new_since"]) + == Some(&j( + r#"{"previous_utc": "2026-01-01T00:00:00Z", "sections": ["clinvar"]}"# + )), + "new since: og summary collect compares with the newest history/ folder: {:?}", + at(&again, &["new_since"]) + ); + assert!( + t2.matches(&format!("[{badge}]")).count() == 1 + && t2.contains(&format!("## ClinVar Pathogenic Screen\n---\n [{badge}]")), + "new since: the text marks the ClinVar section once" + ); + + let shown: Vec<&str> = NOTES + .iter() + .map(|x| x.1) + .filter(|n| html.contains(&*e(n))) + .collect(); + // Every caveat shown is its own caveat paragraph, at most once a card + // (one note may sit on several cards: notes.rs lists where). + let loose_of = |page: &str| -> Vec { + shown + .iter() + .copied() + .filter(|n| { + let wrapped = format!("

    {}

    ", e(n)); + *n != PRS_ANCESTRY_NOTE + && (page.matches(&*e(n)).count() != page.matches(&wrapped).count() + || page + .split(" 1)) + }) + .map(|n| n[..40].to_owned()) + .collect() + }; + let loose = loose_of(&html); + assert!( + shown.len() > 20 && loose.is_empty(), + "every caveat shown is under a card title, once a card: {loose:?} {}", + shown.len() + ); + let eh = format!("

    {}", e(EH_NOTE)); + assert!(html.contains(&eh), "the ExpansionHunter caveat is shown"); + let moved = html.replace(&eh, &format!("

    {}", e(EH_NOTE))); + assert_eq!( + loose_of(&moved), + vec![EH_NOTE[..40].to_owned()], + "red control: a caveat left in a body is caught" + ); + let twice = html.replacen(&eh, &format!("{eh}

    \n{eh}"), 1); + assert_eq!( + loose_of(&twice), + vec![EH_NOTE[..40].to_owned()], + "red control: a caveat shown twice is caught" + ); + let mo = html + .split_once("

    Mosaic Loss of Chromosome Y or X

    ") + .map_or("", |x| x.1) + .trim_start(); + let after_meta = mo + .strip_prefix("

    Step 33 ") + .and_then(|r| r.split_once("

    ")) + .filter(|(m, _)| !m.contains('\n')) + .map(|x| x.1.trim_start()); + assert!( + after_meta + .is_some_and(|r| r.starts_with(&format!("

    {}", e(MOSAIC_NOTE)))), + "the caveat comes right after the title and the step line:\n{}", + &mo[..mo.len().min(400)] + ); +} diff --git a/scripts/06-clinvar-screen.sh b/scripts/06-clinvar-screen.sh index df812e75a..c0d68b3f9 100755 --- a/scripts/06-clinvar-screen.sh +++ b/scripts/06-clinvar-screen.sh @@ -67,7 +67,7 @@ done echo "=== ClinVar Pathogenic Screen: ${SAMPLE} ===" -# bin/owngenome_recheck.py --refresh-clinvar holds this lock while it replaces the ClinVar files every sample +# bin/og recheck --refresh-clinvar holds this lock while it replaces the ClinVar files every sample # shares; the screen holds it shared on fd 7 until it ends, so it waits for a refresh going on and a refresh # refuses while it reads. Without flock (macOS: brew install flock) nothing waits here. CVLOCK="${GENOME_DIR}/clinvar/.refresh.lock" diff --git a/scripts/24-html-report.sh b/scripts/24-html-report.sh index 1d7f0c3f9..510e9a7ac 100755 --- a/scripts/24-html-report.sh +++ b/scripts/24-html-report.sh @@ -2,14 +2,16 @@ # 24-html-report.sh — Generate a self-contained HTML report of all pipeline results # Usage: ./scripts/24-html-report.sh # -# Reads every step's output once into ${SAMPLE}/summary.json (og summary -# collect, run by bin/render_report.py) and renders the HTML report from it. -# generate-report.sh renders the text report from the same summary when -# nothing has changed since, so the two reports show the same values. A result from an -# earlier run whose step was skipped or failed in the latest run-all.sh run is -# marked stale with its date. No internet connection is needed to view it. -# ${SAMPLE}_search_index.json lists every gene and rsID the results mention -# (bin/results_index.py), for the report's closing list and the live search. +# og report render reads every step's output once into ${SAMPLE}/summary.json +# (og summary collect) and renders the HTML report from it. generate-report.sh +# renders the text report from the same summary when nothing has changed +# since, so the two reports show the same values. A result from an earlier +# run whose step was skipped or failed in the latest run-all.sh run is marked +# stale with its date. A section og cannot write says so in its place, and +# the rest of the report is still written. No internet connection is needed +# to view it. ${SAMPLE}_search_index.json lists every gene and rsID the +# results mention (og report index), for the report's closing list and the +# live search. set -euo pipefail SAMPLE=${1:?Usage: $0 } @@ -41,8 +43,9 @@ fi echo "Reading pipeline outputs and writing the report..." rm -f "$OUTPUT" "${SAMPLE_DIR}/${SAMPLE}_search_index.json" "${SAMPLE_DIR}/${SAMPLE}_charts.json" # assets/: the ACMG SF rules (acmg_sf.tsv) and the carrier list (carrier_genes.tsv). +# og is the static binary, so the python image runs it. run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" -v "${PGP_ROOT}/assets:/pgp-assets:ro" "${PYTHON_IMAGE}" \ - python3 /pgp-bin/render_report.py \ + /pgp-bin/og report render \ --assets /pgp-assets \ --sample "$SAMPLE" \ --sample-dir "/genome/${SAMPLE}" \ diff --git a/scripts/ci/e2e-case-paths.tsv b/scripts/ci/e2e-case-paths.tsv index 1241a5e03..1f9306253 100644 --- a/scripts/ci/e2e-case-paths.tsv +++ b/scripts/ci/e2e-case-paths.tsv @@ -128,7 +128,7 @@ nf-wiring-2-sv-str-tools bin/og modules/local/cnvpytor/* modules/local/duphold/* output-contract-1-hg002 assets/* bin/* modules/* scripts/* paralog-catalog-1-gba assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og modules/local/parascopy/* modules/local/vcf_precheck/* scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs paralog-checks-and-doc-fixes-1-hba assets/par_grch38.bed assets/paralog_loci.tsv bin/cn_loci.tsv bin/collect_summary.py bin/og modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/paraphase/* modules/local/parascopy/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/35-paralogs.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs -paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs +paralog-rows-1 assets/paralog_loci.tsv assets/parascopy_models_v1.7.txt bin/cn_loci.tsv bin/og scripts/35-paralogs.sh scripts/ci/fake-docker/lib.sh scripts/setup.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs paralogs-paraphase-1 assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/stub/sample.vcf.gz bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/align_minimap2/* modules/local/deepvariant/* modules/local/fastp/* modules/local/html_report/* modules/local/paraphase/* modules/local/vcf_precheck/* scripts/01b-fastp-qc.sh scripts/24-html-report.sh scripts/35b-paraphase.sh scripts/ci/check-images.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh rust/crates/og-dev/src/fixture.rs paralogs-report-section-1-nf-report-tables assets/acmg_sf.tsv assets/callability_genes.tsv assets/carrier_genes.tsv assets/construct_variants.tsv assets/deep_intronic_sites.tsv assets/mito_confirmed.tsv assets/par_grch38.bed assets/pgx_genes.tsv assets/stub/sample.bam assets/stub/sample.bam.bai assets/stub/sample.vcf.gz bin/cn_loci.tsv bin/collect_summary.py bin/og bin/render_report.py bin/results_index.py modules/local/clinvar_screen/* modules/local/export_consumer/* modules/local/html_report/* modules/local/pypgx/* modules/local/slivar/* modules/local/vcf_precheck/* scripts/06-clinvar-screen.sh scripts/24-html-report.sh scripts/31-slivar.sh scripts/32-pypgx.sh scripts/36-pgx-consensus.sh scripts/40-export-consumer.sh scripts/ci/fake-docker/lib.sh scripts/validate-setup.sh tests/fixtures/consumer_export/sites.tsv.gz pgx-1-depth-check bin/og scripts/21-cyrius.sh scripts/32-pypgx.sh scripts/ci/alt-depth-ab.sh scripts/ci/fake-docker/lib.sh diff --git a/scripts/ci/e2e-case-times.tsv b/scripts/ci/e2e-case-times.tsv index ab5d048e2..c08ec51d8 100644 --- a/scripts/ci/e2e-case-times.tsv +++ b/scripts/ci/e2e-case-times.tsv @@ -17,6 +17,9 @@ # missing. A run of the E2E workflow by hand with trace: true measures the # times again and prints a new file in its summary; its traced needs are an # upper bound to check by hand (docs/testing.md). +# prs-3-panel-measure takes 0 s except on a run dispatched on a branch other +# than main, where it measures the 1000 Genomes panel for about 2900 s; there +# .github/workflows/e2e.yml gives it that time through E2E_SECONDS. 01-fixture base 17 - 10-validate-setup base 7 - 20-align-02 base 116 - diff --git a/scripts/ci/e2e-shards.sh b/scripts/ci/e2e-shards.sh index 99741c04e..7abd0d3e5 100755 --- a/scripts/ci/e2e-shards.sh +++ b/scripts/ci/e2e-shards.sh @@ -7,6 +7,7 @@ # scripts/ci/e2e-shards.sh cover FILE # scripts/ci/e2e-shards.sh merge EXPECTED RESULTS... # scripts/ci/e2e-shards.sh times TRACE... +# scripts/ci/e2e-shards.sh seconds # scripts/ci/e2e-shards.sh check # # base prints the base cases, one per line, in name order. The job @@ -36,6 +37,8 @@ # scripts/ci/e2e-run.sh with E2E_TRACE=true). Keeps each case's # stage and needs, adds the traced needs. Exits 1 when the trace # saw no case read another case's files, which means it saw nothing. +# seconds prints "CASESECONDS" for every case, the time plan and +# count weigh it by: its row, or E2E_SECONDS where that names it. # check exits 1 unless the table is well formed and every case has a # row, naming each case with none (tests/test_e2e_shards.sh runs it # on the real table). The split itself still runs a case with no @@ -46,6 +49,11 @@ # E2E_SELECT a file of shard case names (scripts/ci/e2e-scope.sh # select): plan and count only those. Every case one of # them needs must be listed too. +# E2E_SECONDS "CASE=SECONDS ..." times that replace the table's for +# this run, for a case whose cost depends on the run: +# prs-3-panel-measure measures only on a dispatched run on +# a branch, which is where e2e.yml sets it. Every job of a +# run must see the same value, or their splits differ. # E2E_SHARD_FLOOR the least seconds of cases worth a shard job of its # own (default 900: a shard job spends 18 to 20 minutes # besides its cases, run 41847167444) @@ -57,6 +65,7 @@ TIMES=${E2E_TIMES:-${REPO}/scripts/ci/e2e-case-times.tsv} DEFAULT_SECONDS=300 SELECT=${E2E_SELECT:-} SHARD_FLOOR=${E2E_SHARD_FLOOR:-900} +OVERRIDE=${E2E_SECONDS:-} export LC_ALL=C die() { echo "ERROR: $*" >&2; exit 1; } @@ -71,8 +80,36 @@ cases() { done | sort } -# rows: the table without comments and blank lines. -rows() { grep -vE '^[[:space:]]*(#|$)' "$TIMES" || true; } +# table_rows: the table without comments and blank lines. +table_rows() { grep -vE '^[[:space:]]*(#|$)' "$TIMES" || true; } + +# check_override: a name in E2E_SECONDS that has no row, or a time that is +# not a whole number, stops the script. Runs in the main shell, before any +# command, since rows runs in subshells where a stop would go unseen. +check_override() { + local bad + [ -n "$OVERRIDE" ] || return 0 + bad=$(table_rows | awk -F'\t' -v o="$OVERRIDE" ' + { have[$1] = 1 } + END { + n = split(o, kv, " ") + for (i = 1; i <= n; i++) { + if (kv[i] !~ /^[^=]+=[0-9]+$/) { print "E2E_SECONDS: " kv[i] " is not CASE=SECONDS"; continue } + split(kv[i], p, "=") + if (!(p[1] in have)) print "E2E_SECONDS names " p[1] ", which has no row in the table" + } + }') + [ -z "$bad" ] || die "$bad" +} + +# rows: table_rows with the seconds of E2E_SECONDS. +rows() { + [ -n "$OVERRIDE" ] || { table_rows; return; } + table_rows | awk -F'\t' -v OFS='\t' -v o="$OVERRIDE" ' + BEGIN { n = split(o, kv, " "); for (i = 1; i <= n; i++) { split(kv[i], p, "="); s[p[1]] = p[2] } } + ($1 in s) { $3 = s[$1] } + { print }' +} # check_table: a malformed, duplicate or stale row, or a need that cannot be # met, stops the split instead of silently moving a case. @@ -283,7 +320,14 @@ times() { } printf "%s\t%s\t%d\t%s\n", c, st, s, (out == "" ? "-" : out) } - }' <(cases) <(rows) "$@" + }' <(cases) <(table_rows) "$@" +} + +# seconds: the weight of every case, as groups() reads it. +seconds() { + check_table + awk -F'\t' -v def="$DEFAULT_SECONDS" 'FILENAME == ARGV[1] { s[$1] = $3; next } + { printf "%s\t%d\n", $1, ($1 in s) ? s[$1] : def }' <(rows) <(cases) } # check: the table checks of every command, plus a row for every case. @@ -299,6 +343,7 @@ check() { cmd=${1:-} [ "$#" -gt 0 ] && shift +[ -r "$TIMES" ] && check_override case "$cmd" in base) base ;; plan) plan "${1:?Usage: $0 plan N}" ;; @@ -306,6 +351,7 @@ case "$cmd" in cover) cover "${1:?Usage: $0 cover FILE}" ;; merge) [ "$#" -ge 1 ] || die "Usage: $0 merge EXPECTED RESULTS..."; merge "$@" ;; times) times "$@" ;; + seconds) seconds ;; check) check ;; - *) sed -n '2,51p' "$0" >&2; exit 2 ;; + *) sed -n '2,59p' "$0" >&2; exit 2 ;; esac diff --git a/scripts/generate-report.sh b/scripts/generate-report.sh index 646615f5d..6c2b688fc 100755 --- a/scripts/generate-report.sh +++ b/scripts/generate-report.sh @@ -2,7 +2,7 @@ # generate-report.sh — Create a plain-text summary report of all pipeline results # Usage: ./scripts/generate-report.sh # -# Renders the text report (bin/render_report.py) from ${SAMPLE}/summary.json, +# Renders the text report (og report render) from ${SAMPLE}/summary.json, # the summary og summary collect reads from every step's output. When step 24 # wrote that summary and nothing it reads has changed since (in a run-all.sh # run, step 24 ran just before), the text report is rendered from it as it is: @@ -81,8 +81,9 @@ else # assets/: the ACMG SF rules (acmg_sf.tsv) and the carrier list (carrier_genes.tsv). from=(--assets /pgp-assets --json "/genome/${SAMPLE}/summary.json") fi +# og is the static binary, so the python image runs it. run_in -v "${PGP_ROOT}/bin:/pgp-bin:ro" -v "${PGP_ROOT}/assets:/pgp-assets:ro" "${PYTHON_IMAGE}" \ - python3 /pgp-bin/render_report.py \ + /pgp-bin/og report render \ "${from[@]}" \ --sample "$SAMPLE" \ --sample-dir "/genome/${SAMPLE}" \ diff --git a/scripts/run-all.sh b/scripts/run-all.sh index dfa11e549..6c73b137c 100755 --- a/scripts/run-all.sh +++ b/scripts/run-all.sh @@ -44,8 +44,9 @@ # The other members are never run here: run run-all.sh for each, and the last one runs the family steps. # A step without its data or BAM is skipped. A report-only tool whose task fails is skipped, and its step is marked # failed (logs/run_status.tsv). Results are published by hard link when work/ and GENOME_DIR share a filesystem, -# you passed no -w, -c or --publish_dir_mode, and you run as root or Linux's fs.protected_hardlinks is 0 (then a user -# may link the tasks' root-owned files). +# you passed no -w, -c or --publish_dir_mode, and you run as root or Linux's fs.protected_hardlinks is 0. Every +# published file is yours (tasks run as you; Nextflow gives the CPSR and CNVPYTOR work files back), but this +# condition is kept as it was. # logs/run_status.tsv keeps this command with its switches: bin/og recheck --rerun replays it when a newer # ClinVar or catalog changes a result (docs/recheck.md). set -euo pipefail @@ -384,9 +385,11 @@ if [ -n "$EX" ]; then [ "$XC" = "$NC" ] && [ "$XM" = "$M" ] || echo "[Resources] the pool follows a task cap you set above the engine's ${NC} CPUs and ${M} GB, or one not in whole CPUs or GB: a task can then be killed inside the engine (exit 137)" else rm -f "$EXC"; fi # Publish by hard link when it is sure to work: the BAM and every other output are then stored once, not twice. -# Nextflow does not fall back to a copy when a link fails; the run stops. The tasks run as root (the docker profile) -# and their files can stay root's, and Linux lets another user link a file it does not own only when -# fs.protected_hardlinks is 0. So: root or protected_hardlinks 0, and a probe link from work/ to GENOME_DIR that works. +# Nextflow does not fall back to a copy when a link fails; the run stops. The docker profile runs tasks as you, and +# Nextflow's docker.fixOwnership gives the work files of CPSR and CNVPYTOR (which run as root) back to you, so every +# published file is yours. Linux lets another user link a file it does not own only when fs.protected_hardlinks is 0; +# the condition from when tasks ran as root is kept as it was: root or protected_hardlinks 0, and a probe link from +# work/ to GENOME_DIR that works. # The probe only sees the default work/: with -w, -work-dir, NXF_WORK, or a -c/-config that may set workDir, keep the copy. if [[ " $* " != *" --publish_dir_mode"* && " $* " != *" -w "* && " $* " != *" -work-dir"* && " $* " != *" -c "* \ && " $* " != *" -config "* ]] && [ -z "${NXF_WORK:-}" ] \ diff --git a/tests/browser/README.md b/tests/browser/README.md index 6d23bddd6..141b1a4a1 100644 --- a/tests/browser/README.md +++ b/tests/browser/README.md @@ -22,7 +22,10 @@ Without a Chrome a check prints `[SKIP] no Chrome or Chromium to ...` and passes | `tests/live_replay.rs` | the live page in replay mode on `tests/fixtures/live/frames.json`: it opens at the last frame, Play replays from the first, the map's roles and labels, a headline opens its step with Enter, the policy blocked nothing, a running replay opens at its first frame. Red control: without `frames.json` the page gets a 404 for it and the first check fails. The policy it is served with must equal the live site's `CSP` | | `tests/live_assets.rs` | `assets/live`, plugins included, has no outside URL, inline script, `on*` attribute or `eval`, with two planted mistakes the scan must catch | | `tests/rv_live_reads_sketch_label.rs` | the reads offers of the live demo, built by `tests/demo/build_live_demo.py` with the og of the test run: a finding's reads drawing says it is an illustration before the picture, and no reads offer in the demo (the finding page, a card's links, a chart mark's panel, a search hit with a place, the map's card) links to the viewer page; each says "Reads: not in the demo". Outside the demo the finding page keeps its link. After a search for CYP2D6 and the card's reads button, the top of the reads panel is in view, on a wide screen and on a phone. Red control: with a map.js that does not scroll to the panel, the panel stays out of view | +| `tests/rv_live_load_errors.rs` | a live page that cannot load its data says so, on a small demo site served by a stub that can fail one path, with the live site's policy: a person page whose `results.json` answers 500, a home page whose `runs.json` fails and a family page whose `family.json` answers `{}` each say what could not be loaded and offer Try again, which draws the page once the stub recovers; a closed page whose state cannot be read says "Results not loaded" and never "child", while DEMO-003, a real child, still reads "Results about a child"; while `frames.json` is held, the wire, run health and re-check panels say they wait for the run's data, and a state with no re-check then reads "No re-check yet". Each case was red on main before the fix | +| `tests/rv_viewer_shell.rs` | the genome viewer's page (`assets/viewer`) on a synthetic sample, served by a stub that answers the paths and the policy of `og view serve`: the page is dark at 1440x900 and 390x844, the igv.js search box and track label are readable, the variant track fits the live map's short frame, on a phone the page is no wider than the screen, it has one h1 naming the sample and one main landmark, and a 500 from `config.json` shows an alert with Try again, which starts the viewer once the server answers. `the_policy_is_og_view_serves` keeps the stub's policy equal to `og view serve`'s | | `rust/crates/og-live/tests/port_viewer.rs` (`cargo test -p og-live`) | the local genome viewer, served by `og view serve` itself on a synthetic sample: igv.js starts on the config (status ready), the policy blocks nothing, and igv.js reads the reference, the BAM, its index and the VCF. Red control: with an empty igv.min.js the page reports an error | +| `rust/crates/og-live/tests/port_live_server_browser.rs` (`cargo test -p og-live`) | the live run page, served by `og live serve` on the synthetic DEMO-001 run: the step map, a box that repaints after a stored message, a stream refused with 503 and opened again, a box's detail with its tasks and its report anchors, and no policy violation or page error | The other browser checks still run in the Python tests that start the live site's Python server (`tests/test_owngenome_live.py`, `test_live_demo.py`, `test_live_viewer.py`, `test_owngenome_live_family.py`, `test_report_charts.py`). Each moves here when its server is ported to Rust. diff --git a/tests/demo/make_demo_sample.py b/tests/demo/make_demo_sample.py index b12892ddd..8ff2de870 100755 --- a/tests/demo/make_demo_sample.py +++ b/tests/demo/make_demo_sample.py @@ -1092,7 +1092,7 @@ def bloodtype(d): "rs17822931": 0, "rs10427255": 1, "rs1815739": 1, "rs12913832": 1, "rs1800407": 0, "rs16891982": 2, "rs1426654": 0, "rs12203592": 0, "rs1805007": 0, "rs1805008": 0, "rs1805009": 0, "rs3827760": 0, "rs1801133": 1, "rs601338": 1, "rs2282679": 1, "rs887829": 1, "rs1801272": 0, "rs429358": 0, "rs7412": 0, - "rs10455872": 0, "rs3798220": 0, "rs78655421": 0, + "rs10455872": 0, "rs3798220": 0, "rs78655421": 0, "rs869051967": 0, } diff --git a/tests/e2e/nextflow-hardening-2-nf.sh b/tests/e2e/nextflow-hardening-2-nf.sh index daff2f705..8c8d839cf 100755 --- a/tests/e2e/nextflow-hardening-2-nf.sh +++ b/tests/e2e/nextflow-hardening-2-nf.sh @@ -193,7 +193,7 @@ check "the CYP2D6 card reads step 36's table (result: ${CONS_RESULT:-none})" tes check "the Structural Variants card counts the consensus SVs" \ has 'Consensus SVs \(2\+ callers\)(<[^>]*>)?[1-9][0-9]*<' "$(cat "${R}/${SAMPLE}_report.html" 2>/dev/null)" check "the Steps Not Run list names none of the SV callers" \ - lacks '

  • Structural variants \((Manta|Delly)\)
  • |
  • SV consensus merge
  • ' "$(cat "${R}/${SAMPLE}_report.html" 2>/dev/null)" + lacks '
  • Structural variants \((Manta|Delly)\)( \([^)]*\))?
  • |
  • SV consensus merge( \([^)]*\))?
  • ' "$(cat "${R}/${SAMPLE}_report.html" 2>/dev/null)" # --- The bash steps 22 and 37 on the same calls and BAM, for parity-diff -------------------- # A GENOME_DIR of hard links: the reference, Yleaf's tables, step 04's Manta diff --git a/tests/e2e/paralog-rows-1.sh b/tests/e2e/paralog-rows-1.sh index b5a98d2e8..1dcf956a1 100755 --- a/tests/e2e/paralog-rows-1.sh +++ b/tests/e2e/paralog-rows-1.sh @@ -130,7 +130,7 @@ echo "paralog-rows: step 35 on 37 rows in ${ELAPSED} s, ${called} called, ${nass # The reports read every table: both, on this sample REP="${CASE_TMP}/report_all" mkdir -p "$REP" -python3 "${REPO}/bin/render_report.py" --assets "${REPO}/assets" --sample "$P" --sample-dir "${GENOME_DIR}/${P}" \ +"${REPO}/bin/og" report render --assets "${REPO}/assets" --sample "$P" --sample-dir "${GENOME_DIR}/${P}" \ --json "${REP}/summary.json" -o "${REP}/report.html" > "${REP}/render.log" 2>&1 loci_in_summary() { jq '.sections.parascopy.data.loci // [] | length' "${REP}/summary.json" 2>/dev/null || echo 0; } check_eq "summary.json has the 36 loci other than SMN1" "$(loci_in_summary)" 36 @@ -153,7 +153,7 @@ check "no SMN table without the SMN1 row" test ! -e "${GENOME_DIR}/${Q}/paralogs check "the HBA table is written" test -s "${GENOME_DIR}/${Q}/paralogs/${Q}_hba_copy_number.tsv" render_q() { rm -rf "$1"; mkdir -p "$1" - python3 "${REPO}/bin/render_report.py" --assets "${REPO}/assets" --sample "$Q" --sample-dir "${GENOME_DIR}/${Q}" \ + "${REPO}/bin/og" report render --assets "${REPO}/assets" --sample "$Q" --sample-dir "${GENOME_DIR}/${Q}" \ --json "$1/summary.json" -o "$1/report.html" > "$1/render.log" 2>&1 } render_q "${CASE_TMP}/report_nosmn" diff --git a/tests/e2e/prs-1-agreement.sh b/tests/e2e/prs-1-agreement.sh index 1cd31e37a..da0df96ba 100644 --- a/tests/e2e/prs-1-agreement.sh +++ b/tests/e2e/prs-1-agreement.sh @@ -20,12 +20,12 @@ # match log, and its sum less the autosomal sum above equals plink2's # --score of those rows on the same genotypes with the sex set. # Writes nothing other cases read; removes what it made. When case prs-3 -# measures (a run dispatched on a branch other than main) it also starts that -# case's download of the 1000 Genomes panel in the background, so the 7.4 GB -# arrive while cases prs-1 and prs-2 run. +# measures (a run dispatched on a branch other than main, not traced) it +# also starts that case's download of the 1000 Genomes panel in the +# background, so the 7.4 GB arrive while cases prs-1 and prs-2 run. . "$(dirname "$0")/lib.sh" -if [ "${PGSC_MEASURE:-}" = 1 ] || { [ "${GITHUB_EVENT_NAME:-}" = workflow_dispatch ] && [ "${GITHUB_REF:-}" != refs/heads/main ]; }; then +if [ "${PGSC_MEASURE:-}" = 1 ] || { [ "${GITHUB_EVENT_NAME:-}" = workflow_dispatch ] && [ "${GITHUB_REF:-}" != refs/heads/main ] && [ "${E2E_TRACE:-false}" != true ]; }; then rm -f "${E2E_WORK}/panel-download.status" # Its own session, so the case's timeout does not stop it; it ends on its own. setsid nohup timeout 2700 bash "$(dirname "$0")/prs-3-panel-measure.sh" --download \ diff --git a/tests/e2e/prs-3-panel-measure.sh b/tests/e2e/prs-3-panel-measure.sh index 8e6e76a1b..484dfa150 100644 --- a/tests/e2e/prs-3-panel-measure.sh +++ b/tests/e2e/prs-3-panel-measure.sh @@ -2,10 +2,10 @@ # What the 1000 Genomes ancestry panel costs on a GitHub-hosted runner, for # docs/25-prs.md: the download, the disk at its fullest and the memory at its # highest while pgsc_calc projects a genome-wide target onto the panel and -# scores it. Runs on an E2E run dispatched on a branch other than main, or -# with PGSC_MEASURE=1: it adds about 40 minutes, more than the monthly, pull -# request and main dispatched runs have left in their 150 (a dispatched run -# that carried it hit that limit twice). +# scores it. Runs on an E2E run dispatched on a branch other than main and not +# traced, or with PGSC_MEASURE=1: it adds 35 to 80 minutes to its shard, which +# the monthly, pull request and main runs do not pay. A traced run skips it, so +# the time table keeps its row at 0 s and the one traced leg stays in its limit. # # --download (started in the background by case prs-1, so the download # runs while other cases do) fetches the panel with 8 byte @@ -25,9 +25,13 @@ # used until then. # Case prs-2 projects HG002 onto the small synthetic panel on every run. # Removes the panel and everything else it made. +# Its own limit, above the runner's 3600 s: the wait for the download (up to +# 30 minutes, when EBI is slow), the site list, the target and pgsc_calc's 35 +# minutes add up to about 80. +# case-timeout: 5400 . "$(dirname "$0")/lib.sh" -measure_on() { [ "${PGSC_MEASURE:-}" = 1 ] || { [ "${GITHUB_EVENT_NAME:-}" = workflow_dispatch ] && [ "${GITHUB_REF:-}" != refs/heads/main ]; }; } +measure_on() { [ "${PGSC_MEASURE:-}" = 1 ] || { [ "${GITHUB_EVENT_NAME:-}" = workflow_dispatch ] && [ "${GITHUB_REF:-}" != refs/heads/main ] && [ "${E2E_TRACE:-false}" != true ]; }; } RES=https://ftp.ebi.ac.uk/pub/databases/spot/pgs/resources DIR="${GENOME_DIR}/reference/pgsc_calc" PANEL="${DIR}/${PGSC_PANEL}.tar.zst" @@ -65,7 +69,7 @@ if [ "${1:-}" = --download ]; then fi if ! measure_on; then - NO_CHECKS_REASON="the 1000 Genomes panel measurement runs on an E2E run dispatched on a branch other than main, or with PGSC_MEASURE=1" + NO_CHECKS_REASON="the 1000 Genomes panel measurement runs on an E2E run dispatched on a branch other than main and not traced, or with PGSC_MEASURE=1" finish fi command -v nextflow >/dev/null || { fail "nextflow is not on PATH"; finish; } diff --git a/tests/e2e/prs-runopts-and-trace-1.sh b/tests/e2e/prs-runopts-and-trace-1.sh index 4113fe055..06408719a 100755 --- a/tests/e2e/prs-runopts-and-trace-1.sh +++ b/tests/e2e/prs-runopts-and-trace-1.sh @@ -5,9 +5,13 @@ # docker.runOptions adds --cpu-shares 300 and a label and whose # process.containerOptions adds another label. The images.config PRS # writes for pgsc_calc must carry both after -u and --network none (and -# the sex mount), without the outer --user root:root, and Docker's create +# the sex mount), without a word of the outer --user root:root or +# -u $(id -u):$(id -g), and Docker's create # events must show pgsc_calc's own containers (its pgscatalog images) # with both labels. +# - Nextflow, the same row stubbed under the docker profile alone: every +# task's docker run line, expanded by bash as its .command.run does, +# runs the container as the caller, as the bash steps' run_in does. # - Nextflow and step 25: pgsc_calc's trace, report and timeline are # published as prs/pgsc_calc/pipeline_info/execution_{trace.txt, # report.html,timeline.html}, and nothing else is there. The trace has @@ -74,11 +78,12 @@ SHEET="${CASE_TMP}/samplesheet.csv" rm -rf "$NF_OUT" "$WORK" printf 'sample,vcf,vcf_index,gvcf,gvcf_index,sex\n%s,%s,%s,%s,%s,male\n' "$P" \ "${D}/vcf/${P}.vcf.gz" "${D}/vcf/${P}.vcf.gz.tbi" "${D}/vcf/${P}.g.vcf.gz" "${D}/vcf/${P}.g.vcf.gz.tbi" > "$SHEET" -# The outer flags: a low-priority share and a label each way. The docker -# profile's --user root:root and --network none are repeated, as a user's -# config would, and must not reach pgsc_calc. +# The outer flags: a low-priority share and a label each way. A user's +# --user root:root, the common -u $(id -u):$(id -g) (the last -u, so the +# outer tasks run as the caller) and --network none, as a config could +# carry, must not reach pgsc_calc, not one word of them. cat > "${CASE_TMP}/outer.config" <<'NFCONF' -docker.runOptions = '--user root:root --cpu-shares 300 --label pgp.runopts=e2e' +docker.runOptions = '--user root:root -u $(id -u):$(id -g) --cpu-shares 300 --label pgp.runopts=e2e' process.containerOptions = '--network none --label pgp.containeropts=e2e' NFCONF @@ -126,6 +131,8 @@ sed "s/^docker\.runOptions = .*/docker.runOptions = '-u 1001:118 --network none check "control: the line before this package (no outer flags) is refused" refused runopts_ok "${CASE_TMP}/main.config" sed "s/'\$/ --user root:root'/" "${CONFIG:-/dev/null}" > "${CASE_TMP}/root.config" check "control: a line that keeps the outer --user root:root is refused" refused runopts_ok "${CASE_TMP}/root.config" +sed "s/ --cpu-shares 300/ -g) --cpu-shares 300/" "${CONFIG:-/dev/null}" > "${CASE_TMP}/split.config" +check "control: a line with a stray word of the outer -u \$(id -u):\$(id -g) is refused" refused runopts_ok "${CASE_TMP}/split.config" # Docker's view: pgsc_calc's containers got both labels. echo "containers created: $(wc -l < "$EVENTS" | tr -d ' '); events errors: $(head -c 300 "${CASE_TMP}/events.err")" @@ -141,6 +148,65 @@ awk -F'\t' -v OFS='\t' '{$2 = ""; $3 = ""; print}' "$EVENTS" > "${CASE_TMP}/even check "control: containers without the labels are counted as unlabelled" \ test "$(labelled "${CASE_TMP}/events_unlabelled.tsv")" -eq 0 +# --- Nextflow, the docker profile alone: every task runs as the caller ---------------- +# The same row, stubbed, with no outer.config: the docker run line of each +# task's .command.run, expanded by bash as the task does, gives the container +# the caller's ids. Before this check's package the profile ran every task as +# --user root:root. +STUB_WORK="${E2E_WORK}/nf-work-prs-stub" +rm -rf "$STUB_WORK" +echo "+ nextflow run main.nf -profile docker -stub --tools prs (the same row, no outer config)" +STUB_RC=0 +( + cd "$CASE_TMP" && nextflow run "${REPO}/main.nf" -profile docker -stub -ansi-log false \ + -work-dir "$STUB_WORK" \ + --input "$SHEET" \ + --reference "${G}/reference/GRCh38_no_alt_analysis_set.fasta" \ + --sex_check warn \ + --tools prs \ + --pgs_scoring "$PGS" \ + --outdir "${CASE_TMP}/stub-out" \ + --max_cpus 4 --max_memory 14.GB +) > "${CASE_TMP}/stub.log" 2>&1 || STUB_RC=$? +tail -n 5 "${CASE_TMP}/stub.log" +check_eq "the stub run under the docker profile exits 0" "$STUB_RC" 0 +# task_user RUN: the user the docker run line of RUN (a .command.run) gives +# its container, the last -u or --user as in docker run, after bash expands +# the line's options as the task does; empty without a user. +task_user() { + local line opts a user='' + local -a w=() + line=$(grep -m1 'docker run' "$1") || return 0 + # shellcheck disable=SC2016 # the line's own text, not an expansion + opts=${line#*'-w "$NXF_TASK_WORKDIR" '} + opts=${opts%% --name *} + eval "w=($opts)" + set -- ${w[@]+"${w[@]}"} + while [ $# -gt 0 ]; do + a=$1; shift + case "$a" in + -u|--user) user=${1-}; [ $# -eq 0 ] || shift ;; + -u?*) user=${a#-u} ;; + --user=*) user=${a#--user=} ;; + esac + done + printf '%s\n' "$user" +} +CALLER="$(id -u):$(id -g)" +TASKS=0 MINE=0 ONE='' +while IFS= read -r run; do + grep -q 'docker run' "$run" || continue + TASKS=$((TASKS + 1)); ONE=${ONE:-$run} + u=$(task_user "$run") + if [ "$u" = "$CALLER" ]; then MINE=$((MINE + 1)); else echo " not the caller: ${u:-no user} in ${run#"$STUB_WORK"/}"; fi +done < <(find "$STUB_WORK" -name .command.run 2>/dev/null) +echo "stub tasks in a container: ${TASKS}, run as the caller (${CALLER}): ${MINE}; first line: $(grep -m1 'docker run' "${ONE:-/dev/null}" 2>/dev/null)" +check_ge "stub tasks that ran in a container" "$TASKS" 2 +check_eq "stub tasks whose container runs as the caller (${CALLER})" "$MINE" "$TASKS" +sed 's/ --name / --user 0:0 --name /' "${ONE:-/dev/null}" > "${CASE_TMP}/root.command.run" 2>/dev/null +check "control: a docker run line that ends on --user 0:0 is read as root" test "$(task_user "${CASE_TMP}/root.command.run")" = 0:0 +rm -rf "$STUB_WORK" "${CASE_TMP}/stub-out" + check_info "nextflow" "${NF_OUT}/${P}/prs/pgsc_calc/pipeline_info" check_eq "PRS scores in the summary (one per id of assets/pgs_scores.tsv)" "$(awk 'NR > 1' "${NF_OUT}/${P}/prs/${P}_prs_summary.tsv" 2>/dev/null | wc -l | tr -d ' ')" \ "$(awk -F'\t' '$1 ~ /^PGS[0-9]+$/' "${REPO}/assets/pgs_scores.tsv" | wc -l | tr -d ' ')" diff --git a/tests/e2e/qc-2-nextflow-sample-qc.sh b/tests/e2e/qc-2-nextflow-sample-qc.sh index 11b9742fe..5b164d725 100755 --- a/tests/e2e/qc-2-nextflow-sample-qc.sh +++ b/tests/e2e/qc-2-nextflow-sample-qc.sh @@ -145,7 +145,7 @@ card() { }' 2>/dev/null < "$1" || echo absent } check_eq "pypgx ran: its section is filled" "$(section "$JSON" pypgx)" ok -check "the Steps Not Run list does not name pypgx" lacks '
  • pypgx
  • ' "$(cat "$HTML" 2>/dev/null)" +check "the Steps Not Run list does not name pypgx" lacks '
  • pypgx( \([^)]*\))?
  • ' "$(cat "$HTML" 2>/dev/null)" # The indexcov sex check: HTML_REPORT gets INDEXCOV's folder from UPSTREAM. # The report must show the sex INDEXCOV's own table holds, not "not available". diff --git a/tests/fake-docker/delta-report-and-manifest-history.sh b/tests/fake-docker/delta-report-and-manifest-history.sh index fbe8c6bb7..6a90f2f38 100755 --- a/tests/fake-docker/delta-report-and-manifest-history.sh +++ b/tests/fake-docker/delta-report-and-manifest-history.sh @@ -9,9 +9,9 @@ # 4. a newer history folder without a summary.json (a copy that stopped # half way) is passed over: the delta is against the newest folder that # holds one, and the report does not fail. -# The report renderer runs with the host's python3 (bin/render_report.py, -# until og report render replaces it) and og run delta with the host's -# bin/og, on the host side of the call's mounts, so both are the real code. +# The report renderer (og report render) and og run delta run with the +# host's bin/og, on the host side of the call's mounts, so both are the real +# code. # The Guard fake-docker job builds og into bin/.og. The case reads the JSON # with jq. # shellcheck source=../../scripts/ci/fake-docker/lib.sh @@ -35,7 +35,7 @@ cat > "${CASE_WORK}/tools-hook" <<'HOOK' set -euo pipefail . "${CASE_WORK:?}/host-path.sh" case "${*:2}" in - "python3 /pgp-bin/"* | "/pgp-bin/og "*) + "/pgp-bin/og "*) # The real tool, on the host paths of every /... argument. shift mapped=() diff --git a/tests/fake-docker/reports-stale-step.sh b/tests/fake-docker/reports-stale-step.sh index c13dce8d3..71360792d 100755 --- a/tests/fake-docker/reports-stale-step.sh +++ b/tests/fake-docker/reports-stale-step.sh @@ -4,8 +4,9 @@ # old slivar result stale, with its date, instead of showing it as current. # # The tools are faked as in base-run-all-default.sh, except the report -# renderer: the hook runs bin/render_report.py with the host's python3 on the -# host side of the call's mounts, so the reports are the real ones. +# renderer: the hook runs og report render with the host's bin/og (the Guard +# fake-docker job builds it into bin/.og) on the host side of the call's +# mounts, so the reports are the real ones. # Also checks the run manifest and the step status file run-all.sh writes. # shellcheck source=../../scripts/ci/fake-docker/lib.sh . "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" @@ -48,7 +49,7 @@ put() { printf '%b' "$2" > "$h" } case "$args" in - "python3 /pgp-bin/render_report.py"*) + "/pgp-bin/og report render"*) # The real renderer, on the host paths of every /... argument. shift mapped=() diff --git a/tests/fake-docker/run-all-ownhealth-stale.sh b/tests/fake-docker/run-all-ownhealth-stale.sh index 6ac222d7e..84ff12f54 100644 --- a/tests/fake-docker/run-all-ownhealth-stale.sh +++ b/tests/fake-docker/run-all-ownhealth-stale.sh @@ -23,7 +23,7 @@ cat > "${CASE_WORK}/tools-hook" <<'HOOK' #!/usr/bin/env bash case "${*:2}" in "cat /usr/local/share/ExpansionHunter/variant_catalog/grch38/variant_catalog.json") echo '[{"LocusId": "HTT"}]' ;; - *render_report.py*) + "/pgp-bin/og report render"*) [ ! -e "${GENOME_DIR}/sample1/facts/sample1_genome_facts.json" ] || touch "${CASE_WORK}/facts-seen-at-24" [ ! -e "${CASE_WORK}/fail-24" ] || { touch "${CASE_WORK}/render-failed"; exit 1; } ;; esac diff --git a/tests/fake-docker/run-all-partial-failure.sh b/tests/fake-docker/run-all-partial-failure.sh index 00a53e4dc..f4750b497 100644 --- a/tests/fake-docker/run-all-partial-failure.sh +++ b/tests/fake-docker/run-all-partial-failure.sh @@ -22,7 +22,8 @@ # modules/local, which is how a row finds its step. # A fake nextflow stands in; a wrapper in front of it writes the two files the # completion handler would. The reports are the real ones: the hook runs -# bin/render_report.py with the host's python3. +# og report render with the host's bin/og (the Guard fake-docker job builds +# it into bin/.og). # shellcheck source=../../scripts/ci/fake-docker/lib.sh . "${REPO_ROOT:?}/scripts/ci/fake-docker/lib.sh" @@ -55,7 +56,7 @@ cat > "${CASE_WORK}/render-hook" <<'HOOK' set -euo pipefail . "${CASE_WORK:?}/host-path.sh" case "${*:2}" in - "python3 /pgp-bin/render_report.py"*) + "/pgp-bin/og report render"*) shift mapped=() for a in "$@"; do diff --git a/tests/fixtures/live/demo/README.md b/tests/fixtures/live/demo/README.md index 6c4519168..8184785f0 100644 --- a/tests/fixtures/live/demo/README.md +++ b/tests/fixtures/live/demo/README.md @@ -3,9 +3,9 @@ Invented inputs of the live-site demo (`tests/demo/build_live_demo.py`), added to the invented sample DEMO-001 that `og dev demo-sample` writes. No real sample, person or run. -- `DEMO-001/recheck/state.json`: a database re-check as `bin/owngenome_recheck.py` writes it: a newer +- `DEMO-001/recheck/state.json`: a database re-check as `bin/og recheck` writes it: a newer ClinVar release and an Ensembl cache with nothing newer. The build sets `checked_utc` to one hour - after the replayed run ends and each item's `steps` with `owngenome_recheck.affected_steps`, so the + after the replayed run ends and each item's `steps` with `og recheck --affected`, so the badges name the steps the real check would name. - `DEMO-FAMILY/members.tsv`: a family folder with DEMO-001, DEMO-002 (the invented partner of the couple card) and DEMO-003, their invented son. The build writes DEMO-002 and DEMO-003 with diff --git a/tests/fixtures/recheck/README.md b/tests/fixtures/recheck/README.md index 8fb967623..ce157af15 100644 --- a/tests/fixtures/recheck/README.md +++ b/tests/fixtures/recheck/README.md @@ -1,6 +1,6 @@ -# Fixtures of bin/owngenome_recheck.py +# Fixtures of og recheck -Invented data, read by `tests/test_owngenome_recheck.py` and `tests/fake-docker/run-all-recheck.sh`. No real sample. +Invented data, read by `tests/fake-docker/run-all-recheck.sh`. No real sample. - `run_manifest.tsv`: the rows `bin/write_manifest.sh` writes that the check reads, for a made-up run. - `offline.json`: upstream answers in the `--offline` format, with a newer ClinVar, VEP 117 and a newer PGS file. diff --git a/tests/parity/README.md b/tests/parity/README.md index dc5817068..60e49dea2 100644 --- a/tests/parity/README.md +++ b/tests/parity/README.md @@ -49,4 +49,4 @@ duplicate records, TAS2R38 phase blocks, phase-site VCFs and catalog refusals. The oracle adapter maps the three Rust verbs to the old flags. The Python report keeps only the metadata and table reader listed in `remnants.tsv`. -The `report-render` cases render the summaries that `gen/report-corpus.sh` writes: each summary-collect folder collected whole by `og summary collect`, and 28 changes of one summary made with `jq`. `og report render` writes only the frame and the sections ported so far, so the generator's `oracle.sh` keeps those parts of `bin/render_report.py`'s reports (awk, by each section's title in the text report and its anchor in the HTML report) and drops the rest; what is kept compares byte for byte. Six summaries whose PRS scores the Python's HTML report cannot render run as text-only cases. +The `report-render` cases render the summaries that `gen/report-corpus.sh` writes: each summary-collect folder collected whole by `og summary collect`, and changes of a few of those summaries made with `jq`. Both reports are compared whole, byte for byte, against `bin/render_report.py`'s: every section, the charts and the closing gene list. The generator's `oracle.sh` reads a null PRS count as og's "not available", and its copy of `results_index.py` leaves out the `acmg_sf` section as og's index does. Six summaries whose PRS scores the Python's HTML report cannot render run as text-only cases. One case, `m-optin-viral-na`, is compared with its committed goldens instead: og's viral line differs from the Python's on purpose there. The Python still runs, so a golden it matches fails as stale. diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index a728067fc..fdc0742b8 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -686,6 +686,7 @@ sv-postfilter checks gen:sv-oracle.sh,sv-postfilter/c45.tsv,sv-postfilter/genes. sv-postfilter bare gen:sv-oracle.sh,sv-postfilter/c45.tsv --class45 c45.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE sv-postfilter consensus-gone gen:sv-oracle.sh,sv-postfilter/c45.tsv --class45 c45.tsv --consensus none.vcf --out out.tsv out.tsv bytes PYTHON_IMAGE sv-postfilter consensus-empty gen:sv-oracle.sh,sv-postfilter/c45.tsv,sv-postfilter/consensus_empty.vcf.gz --class45 c45.tsv --consensus consensus_empty.vcf.gz --out out.tsv out.tsv bytes PYTHON_IMAGE +sv-postfilter consensus-ids gen:sv-oracle.sh,sv-postfilter/c45.tsv,sv-postfilter/consensus_ids.vcf --class45 c45.tsv --consensus consensus_ids.vcf --out out.tsv out.tsv bytes PYTHON_IMAGE sv-postfilter empty gen:sv-oracle.sh,sv-postfilter/empty.tsv,sv-postfilter/genes.bed --class45 empty.tsv --genes genes.bed --out out.tsv out.tsv bytes PYTHON_IMAGE sv-postfilter header-only gen:sv-oracle.sh,sv-postfilter/header_only.tsv --class45 header_only.tsv --rules rules.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE sv-postfilter rules gen:sv-oracle.sh,sv-postfilter/rows7.tsv,sv-postfilter/genes7.bed --class45 rows7.tsv --genes genes7.bed --rules rules.tsv --out out.tsv out.tsv bytes PYTHON_IMAGE @@ -1325,6 +1326,17 @@ report-render a-prs-sex gen:report-corpus.sh --summary a-prs-sex.json -o a-prs-s report-render a-y-calls gen:report-corpus.sh --summary a-y-calls.json -o a-y-calls.txt -o a-y-calls.html a-y-calls.txt,a-y-calls.html bytes PYTHON_IMAGE report-render a-since gen:report-corpus.sh --summary a-since.json -o a-since.txt -o a-since.html a-since.txt,a-since.html bytes PYTHON_IMAGE report-render a-numbers gen:report-corpus.sh --summary a-numbers.json -o a-numbers.txt a-numbers.txt bytes PYTHON_IMAGE +# report-render, the structural-variant, mosaic sex-loss and raw-data export sections: the s-* summaries of gen/report-corpus.sh. +report-render s-sexloss-detected gen:report-corpus.sh --summary s-sexloss-detected.json -o s-sexloss-detected.txt -o s-sexloss-detected.html s-sexloss-detected.txt,s-sexloss-detected.html bytes PYTHON_IMAGE +report-render s-sexloss-chrx gen:report-corpus.sh --summary s-sexloss-chrx.json -o s-sexloss-chrx.txt -o s-sexloss-chrx.html s-sexloss-chrx.txt,s-sexloss-chrx.html bytes PYTHON_IMAGE +report-render s-sexloss-odd gen:report-corpus.sh --summary s-sexloss-odd.json -o s-sexloss-odd.txt -o s-sexloss-odd.html s-sexloss-odd.txt,s-sexloss-odd.html bytes PYTHON_IMAGE +report-render s-sexloss-not-estimated gen:report-corpus.sh --summary s-sexloss-not-estimated.json -o s-sexloss-not-estimated.txt -o s-sexloss-not-estimated.html s-sexloss-not-estimated.txt,s-sexloss-not-estimated.html bytes PYTHON_IMAGE +report-render s-sexloss-gain-stale gen:report-corpus.sh --summary s-sexloss-gain-stale.json -o s-sexloss-gain-stale.txt -o s-sexloss-gain-stale.html s-sexloss-gain-stale.txt,s-sexloss-gain-stale.html bytes PYTHON_IMAGE +report-render s-sexloss-failed gen:report-corpus.sh --summary s-sexloss-failed.json -o s-sexloss-failed.txt -o s-sexloss-failed.html s-sexloss-failed.txt,s-sexloss-failed.html bytes PYTHON_IMAGE +report-render s-consumer-reasons gen:report-corpus.sh --summary s-consumer-reasons.json -o s-consumer-reasons.txt -o s-consumer-reasons.html s-consumer-reasons.txt,s-consumer-reasons.html bytes PYTHON_IMAGE +report-render s-consumer-stale gen:report-corpus.sh --summary s-consumer-stale.json -o s-consumer-stale.txt -o s-consumer-stale.html s-consumer-stale.txt,s-consumer-stale.html bytes PYTHON_IMAGE +report-render s-sv-stale gen:report-corpus.sh --summary s-sv-stale.json -o s-sv-stale.txt -o s-sv-stale.html s-sv-stale.txt,s-sv-stale.html bytes PYTHON_IMAGE +report-render s-sv-mixed gen:report-corpus.sh --summary s-sv-mixed.json -o s-sv-mixed.txt -o s-sv-mixed.html s-sv-mixed.txt,s-sv-mixed.html bytes PYTHON_IMAGE # report-render, the traits, copy-number and repeat sections: every branch of each in each state (m-genome-*). report-render m-genome-ok gen:report-corpus.sh --summary m-genome-ok.json -o m-genome-ok.txt -o m-genome-ok.html m-genome-ok.txt,m-genome-ok.html bytes PYTHON_IMAGE report-render m-genome-stale gen:report-corpus.sh --summary m-genome-stale.json -o m-genome-stale.txt -o m-genome-stale.html m-genome-stale.txt,m-genome-stale.html bytes PYTHON_IMAGE @@ -1481,6 +1493,16 @@ report-charts a-prs-sex gen:report-corpus.sh --summary a-prs-sex.json --charts-j report-charts a-y-calls gen:report-corpus.sh --summary a-y-calls.json --charts-json a-y-calls_charts.json a-y-calls_charts.json json PYTHON_IMAGE report-charts a-since gen:report-corpus.sh --summary a-since.json --charts-json a-since_charts.json a-since_charts.json json PYTHON_IMAGE report-charts a-numbers gen:report-corpus.sh --summary a-numbers.json --charts-json a-numbers_charts.json a-numbers_charts.json json PYTHON_IMAGE +report-charts s-sexloss-detected gen:report-corpus.sh --summary s-sexloss-detected.json --charts-json s-sexloss-detected_charts.json s-sexloss-detected_charts.json json PYTHON_IMAGE +report-charts s-sexloss-chrx gen:report-corpus.sh --summary s-sexloss-chrx.json --charts-json s-sexloss-chrx_charts.json s-sexloss-chrx_charts.json json PYTHON_IMAGE +report-charts s-sexloss-odd gen:report-corpus.sh --summary s-sexloss-odd.json --charts-json s-sexloss-odd_charts.json s-sexloss-odd_charts.json json PYTHON_IMAGE +report-charts s-sexloss-not-estimated gen:report-corpus.sh --summary s-sexloss-not-estimated.json --charts-json s-sexloss-not-estimated_charts.json s-sexloss-not-estimated_charts.json json PYTHON_IMAGE +report-charts s-sexloss-gain-stale gen:report-corpus.sh --summary s-sexloss-gain-stale.json --charts-json s-sexloss-gain-stale_charts.json s-sexloss-gain-stale_charts.json json PYTHON_IMAGE +report-charts s-sexloss-failed gen:report-corpus.sh --summary s-sexloss-failed.json --charts-json s-sexloss-failed_charts.json s-sexloss-failed_charts.json json PYTHON_IMAGE +report-charts s-consumer-reasons gen:report-corpus.sh --summary s-consumer-reasons.json --charts-json s-consumer-reasons_charts.json s-consumer-reasons_charts.json json PYTHON_IMAGE +report-charts s-consumer-stale gen:report-corpus.sh --summary s-consumer-stale.json --charts-json s-consumer-stale_charts.json s-consumer-stale_charts.json json PYTHON_IMAGE +report-charts s-sv-stale gen:report-corpus.sh --summary s-sv-stale.json --charts-json s-sv-stale_charts.json s-sv-stale_charts.json json PYTHON_IMAGE +report-charts s-sv-mixed gen:report-corpus.sh --summary s-sv-mixed.json --charts-json s-sv-mixed_charts.json s-sv-mixed_charts.json json PYTHON_IMAGE report-charts m-genome-ok gen:report-corpus.sh --summary m-genome-ok.json --charts-json m-genome-ok_charts.json m-genome-ok_charts.json json PYTHON_IMAGE report-charts m-genome-stale gen:report-corpus.sh --summary m-genome-stale.json --charts-json m-genome-stale_charts.json m-genome-stale_charts.json json PYTHON_IMAGE report-charts m-genome-failed gen:report-corpus.sh --summary m-genome-failed.json --charts-json m-genome-failed_charts.json m-genome-failed_charts.json json PYTHON_IMAGE diff --git a/tests/parity/gen/report-corpus.sh b/tests/parity/gen/report-corpus.sh index cc4519b69..29ca21a4e 100644 --- a/tests/parity/gen/report-corpus.sh +++ b/tests/parity/gen/report-corpus.sh @@ -17,12 +17,8 @@ # py/ holds the modules render_report.py imports, as they were at the # oracle's commit (tests/parity/oracle.ref, report-render). # -# oracle.sh renders with render_report.py, then keeps of its two reports -# the parts og report render writes so far: the frame and the sections of -# PORTED (each section's block in the text report, its frame and its line -# in the contents in the HTML report). A chapter left with no section goes, -# and so does the closing gene and rsID appendix, which og report index will -# write. Everything kept is compared byte for byte. +# oracle.sh renders with render_report.py; both reports are compared byte +# for byte, whole. set -euo pipefail export LC_ALL=C TZ=UTC here=$(cd "$(dirname "$0")" && pwd) @@ -37,6 +33,13 @@ mkdir -p py for m in collect_summary results_index genome_lookup report_charts live_results; do git -C "$root" show "${ORACLE_SHA}:bin/${m}.py" > "py/${m}.py" done +# og's index leaves out the acmg_sf section, whose card the clinical section +# already holds (rust/crates/og-render/src/index.rs), so a gene is not listed +# twice in the closing appendix; the oracle's index does the same. +sed 's/^ if sec.get("state") not in ("ok", "stale"):$/ if key == "acmg_sf" or sec.get("state") not in ("ok", "stale"):/' \ + py/results_index.py > py/results_index.py.new +mv py/results_index.py.new py/results_index.py +grep -q 'key == "acmg_sf"' py/results_index.py || { echo "report-corpus.sh: the acmg_sf skip did not apply" >&2; exit 1; } # --- one summary per summary-collect case --------------------------------------------- # Each generator writes its folders under folders//, with the @@ -521,13 +524,41 @@ amut numbers '.sections.y_str.data.markers[0].count = 13 | .sections.y_str.data. | .sections.archaic.data.neanderthal_bp = 51234567.0 | .sections.archaic.data.denisovan_bp = true | .sections.prs.data.scores[0].condition = 7 | .sections.prs.data.scores[1].score = 0.25 | .sections.ancestry_fine.data.continent[1].fraction = 1' +# The structural-variant, mosaic sex-loss and raw-data export sections: +# each call of step 33's sex-loss check, chrX with an allele fraction, the +# no-call reasons with tied counts, and the SV sections stale. +smut() { jq -c "$2" "$abase" > "s-$1.json"; } +sl='.sections.mosaic_sex_loss = {"title": "Mosaic loss of chromosome Y or X", "step": "33", "state": "ok", + "source": "qc/S_sample_qc.tsv", "file_date": "2021-03-04 05:06 UTC", "note": null, "data": {' +smut sexloss-detected "$sl"'"chrom": "chrY", "call": "detected", "reason": "depth below the floor", + "depth_ratio": 0.7123, "loss_depth": 0.2877, "loss_depth_low": 0.25, "loss_depth_high": 0.33, "loss_af": null}}' +smut sexloss-chrx "$sl"'"chrom": "chrX", "call": "detected", "reason": "allele fraction & depth", + "depth_ratio": null, "loss_depth": null, "loss_af": 0.125, "loss_af_low": 0.1, "loss_af_high": null, "x_het_sites": 40}}' +smut sexloss-odd "$sl"'"chrom": "", "call": "maybe ", "reason": null, "loss_depth": 0.05}}' +smut sexloss-not-estimated "$sl"'"chrom": "chrY", "call": "not_estimated", "reason": "too few reads on chrY"}}' +smut sexloss-gain-stale "$sl"'"chrom": "chrY", "call": "gain", "reason": "depth above 1"}} | .sections.mosaic_sex_loss.state = "stale" + | .sections.mosaic_sex_loss.note = "from an older run "' +smut sexloss-failed "$sl"'}} | .sections.mosaic_sex_loss.state = "failed" | .sections.mosaic_sex_loss.note = "step 33 failed"' +smut consumer-reasons '.sections.consumer_export.data.no_call_reasons = {"zeta": 5, "alpha": 5, "low ": 9, "one": 1} + | .sections.consumer_export.data.from_reference_blocks = 1234 | .sections.consumer_export.data.file_ancestrydna = "S_ancestry.txt"' +smut consumer-stale '.sections.consumer_export.state = "stale" | .sections.consumer_export.note = "older export" + | .sections.consumer_export.data.no_call_reasons = {} | .sections.consumer_export.data.sites = 0' +# shellcheck disable=SC2016 # jq's $k, not the shell's +smut sv-stale 'reduce ("manta", "delly", "sv_consensus", "annotsv") as $k (.; .sections[$k].state = "stale" + | .sections[$k].note = "why: &'"'"'\"")' +smut sv-mixed '.sections.manta.state = "failed" | .sections.manta.note = "manta failed" | .sections.delly.state = "missing" + | .sections.annotsv.state = "unreadable" | .sections.annotsv.note = "bad tsv"' rm -f "$abase" # --- the oracle ----------------------------------------------------------------------- cat > oracle.sh << 'ORACLE' #!/bin/sh # The oracle of og report render (tests/parity/oracle.ref): render_report.py -# with the modules of py/, then the parts of its reports og writes so far. +# with the modules of py/. Both reports are compared whole: every section, +# the charts in the HTML report and the closing gene and rsID appendix. +# A PRS row whose counts the summary has as null reads "not available", as +# og writes it; render_report.py writes "None" there (and stops its HTML +# report on that row), so the text report's "None" is read as og's words. # sh ./oracle.sh PROG --summary F -o R.txt -o R.html set -e prog=$1 @@ -537,127 +568,16 @@ cp py/*.py "$d"/ cp "$prog" "$d/render_report.py" PYTHONPATH=$d python3 "$d/render_report.py" "$@" rm -rf "$d" -# The sections og report render writes; the others are left out. -PORTED=" qc variants not-run not-assessed " -PORTED="$PORTED pharmcat cpic cyp2d6 hla mito " -# port-report-optin -PORTED="${PORTED}exomiser mei vntyper chip mosaic-mca locityper roh telomere viral " -PORTED="${PORTED}callability clinvar clinical-filter cpsr slivar carrier-list couple " -PORTED="${PORTED}ancestry-fine mt-haplogroup y-chromosome archaic ancient prs " -PORTED="${PORTED}expansions lpa paralogs traits-panel bloodtype " while [ $# -gt 0 ]; do case "$1" in -o) - out=$2 - shift - case "$out" in - *.html) awk -v keep="$PORTED" -f html.awk "$out" > "$out.kept" ;; - *.txt) awk -v keep="$PORTED" -f text.awk "$out" > "$out.kept" ;; + case "$2" in + *.txt) + sed 's| None (None/None variants matched) | not available (not available/not available variants matched) |' "$2" > "$2.kept" + mv "$2.kept" "$2" ;; esac - mv "$out.kept" "$out" ;; + shift ;; esac shift done ORACLE - -# html.awk: drops each section not in keep (its line in the contents and its -# frame) and every chart (og report charts will write them), then each -# chapter with no section left (its contents entry and its body), and the -# gene and rsID appendix. -cat > html.awk << 'AWK' -function wanted(line, id) { - if (!match(line, /id="[^"]*"/) && !match(line, /href="#[^"]*"/)) return 1 - id = substr(line, RSTART, RLENGTH) - sub(/^(id="|href="#)/, "", id); sub(/"$/, "", id) - return index(keep, " " id " ") > 0 -} -{ lines[++n] = $0 } -END { - # Pass 1: drop the sections not kept. - m = 0 - for (i = 1; i <= n; i++) { - l = lines[i] - # The charts are not ported yet (port-report-charts): drop their figures. - if (l ~ /^
    /) continue - if (l ~ /^
  • ") i++ - continue - } - out[++m] = l - } - # Pass 2: drop the chapters with no section left, and the appendix. - for (i = 1; i <= m; i++) { - l = out[i] - if (l ~ /^
  • /) continue - if (l ~ /^
  • " && out[i + 2] == " ") { i += 3; continue } - if (l ~ /^
    ") { - while (out[i] != "
    ") i++ - continue - } - } - print l - } -} -AWK - -# text.awk: drops each section block not in keep, then each chapter left -# with no block. A block starts at its "## " title; the titles of the kept -# sections are listed here, as render_report.py writes them. A PRS row -# whose counts the summary has as null reads "not available", as og writes -# it; render_report.py writes "None" there (and stops its HTML report on -# that row). -cat > text.awk << 'AWK' -BEGIN { - rule = sprintf("%80s", ""); gsub(/ /, "#", rule) - title["## Quality control"] = "qc"; title["## Variant Calling (DeepVariant)"] = "variants" - title["## Pharmacogenomics (PharmCAT)"] = "pharmcat"; title["## CPIC Drug-Gene Recommendations"] = "cpic" - title["## pypgx Pharmacogenomics"] = "cyp2d6"; title["## PGx copy number"] = "cyp2d6" - title["## CYP2D6 across callers"] = "cyp2d6"; title["## HLA Typing (T1K)"] = "hla" - title["## Mitochondrial Variants (Mutect2)"] = "mito" - # port-report-optin - title["## Symptom-Driven Ranking (Exomiser, step 48)"] = "exomiser"; title["## Mobile Element Insertions (step 50)"] = "mei" - title["## MUC1 VNTR Frameshifts (ADTKD-MUC1, VNtyper)"] = "vntyper"; title["## Clonal Haematopoiesis (CHIP, step 49)"] = "chip" - title["## Mosaic Chromosomal Alterations (MoChA, step 51)"] = "mosaic-mca"; title["## Runs of Homozygosity"] = "roh" - title["## Locityper Second Opinion at Hard Loci (step 53)"] = "locityper" - title["## Telomere content (relative, TelomereHunter)"] = "telomere"; title["## Viral DNA in the Unmapped Reads (step 47)"] = "viral" - title["## Genes This Genome Could Not Read Reliably"] = "callability" - title["## ClinVar Pathogenic Screen"] = "clinvar"; title["## Clinical Variant Filter"] = "clinical-filter" - title["## Cancer Predisposition Screening (CPSR)"] = "cpsr"; title["## Variant Prioritization (slivar)"] = "slivar" - title["## Carrier List (this sample)"] = "carrier-list"; title["## Couple Carrier Screen (step 39)"] = "couple" - title["## Fine-Grained Ancestry (ADMIXTURE, RFMix)"] = "ancestry-fine"; title["## Mitochondrial Haplogroup"] = "mt-haplogroup" - title["## Y-Chromosome Haplogroup (Yleaf)"] = "y-chromosome"; title["## Y Placement on the Current Public Tree"] = "y-chromosome" - title["## Y-STR Markers (HipSTR)"] = "y-chromosome"; title["## Archaic Ancestry (IBDmix, step 43)"] = "archaic" - title["## Ancient Populations (AADR PCA, step 44)"] = "ancient"; title["## Polygenic Risk Scores"] = "prs" - title["## Repeat Expansions (ExpansionHunter)"] = "expansions" - title["## Lp(a): LPA KIV-2 Repeat Count and Risk Variants"] = "lpa" - title["## Paralog Genes (Paraphase, PacBio HiFi)"] = "paralogs" - title["## SMN1/SMN2 Copy Number (Parascopy, short reads)"] = "paralogs" - title["## Paralog Copy Number"] = "paralogs" - title["## Traits Panel (step 42)"] = "traits-panel" - title["## Blood Groups (predicted from the genome)"] = "bloodtype" -} -{ sub(/ None \(None\/None variants matched\) /, " not available (not available/not available variants matched) "); lines[++n] = $0 } -END { - # The end part (steps not run onward) and the head (before the first - # banner) stay; chapters are banner, title, banner, blank, then blocks. - for (i = 1; i <= n; i++) if (lines[i] == "## Steps Not Run") { stop = i; break } - for (i = 1; i < stop; i++) { - if (lines[i] == rule && lines[i + 1] ~ /^# [0-9]+\. /) { - head = lines[i] "\n" lines[i + 1] "\n" lines[i + 2] "\n" lines[i + 3] - i += 3; body = ""; on = 0 - while (i + 1 < stop && !(lines[i + 1] == rule && lines[i + 2] ~ /^# [0-9]+\. /)) { - i++ - if (lines[i] ~ /^## /) on = (lines[i] in title) && index(keep, " " title[lines[i]] " ") > 0 - if (on) body = body lines[i] "\n" - } - if (body != "") printf "%s\n%s", head, body - continue - } - print lines[i] - } - for (i = stop; i <= n; i++) print lines[i] -} -AWK diff --git a/tests/parity/gen/summary-collect-c.sh b/tests/parity/gen/summary-collect-c.sh index 51af0da38..741dbc593 100644 --- a/tests/parity/gen/summary-collect-c.sh +++ b/tests/parity/gen/summary-collect-c.sh @@ -100,7 +100,7 @@ put $f/prs/S_prs_summary.tsv "Condition\tPGS_ID\tScore_SUM\tVariants_Matched\tVa CAD\tPGS000018\t0.5\t100\t120\t0\t12 dropped\tall\tok\t75\tEUR\tEUR;;AFR\tHR 1.5 (EUR)\t T2D\tPGS000014\t0.1\t10\t20\t0\t0\tall\tok\tNA\tNA\t\t\tpercentile withheld: the input lists variants only Short\tPGS1\n" -put $f/ancestry/S_ancestry.tsv "key\tvalue\npopulation\tEUR\nreference_panel\tHGDP+1kGP\n\tskipped\npopulation_low_confidence\n" +put $f/ancestry/S_ancestry.tsv "key\tvalue\npopulation\tEUR\nreference_panel\tHGDP+1kGP\n\tskipped\npopulation_low_confidence\nprobability_AFR\t0.4\nprobability_EUR\t0.6\n" put $f/ancestry_fine/S_ancestry_fine.tsv "level\tcode\tgroup\tfraction\tse\tlow\thigh continent\tWEA\tWest Eurasia\t0.95\t0.01\t0.93\t0.97 continent\tAFR\tAfrica\t0.05\t0.01\tnan\tx diff --git a/tests/parity/golden/consumer/export-edge-female/out/S_consumer_export.tsv b/tests/parity/golden/consumer/export-edge-female/out/S_consumer_export.tsv new file mode 100644 index 000000000..d151b3968 --- /dev/null +++ b/tests/parity/golden/consumer/export-edge-female/out/S_consumer_export.tsv @@ -0,0 +1,21 @@ +key value +sites 16 +called 5 +no_call 11 +from_reference_blocks 3 +from_variant_records 3 +reference_checked 4 +reference_mismatch 1 +template_sites 99 +table_dropped 0 +sex female +file_23andme S_23andme_v5.zip +file_ancestrydna S_ancestrydna_layout.zip +no_call: chrY site of a female sample 1 +no_call: inside a deletion or a longer allele 1 +no_call: no gVCF record 3 +no_call: no genotype 1 +no_call: no-call in the gVCF 1 +no_call: reference base differs from the site table 1 +no_call: symbolic allele 1 +no_call: unreadable genotype 2 diff --git a/tests/parity/golden/consumer/export-edge-female/stdout b/tests/parity/golden/consumer/export-edge-female/stdout new file mode 100644 index 000000000..42faa7bd0 --- /dev/null +++ b/tests/parity/golden/consumer/export-edge-female/stdout @@ -0,0 +1,11 @@ +Sites: 16 (of 99 on the chip list; 0 dropped when the table was built) +Called: 5 (31.2%), 3 of them from reference blocks; no-call (--): 11 + chrY site of a female sample: 1 + inside a deletion or a longer allele: 1 + no gVCF record: 3 + no genotype: 1 + no-call in the gVCF: 1 + reference base differs from the site table: 1 + symbolic allele: 1 + unreadable genotype: 2 +Reference bases checked at 4 variant records, 1 differ from the table diff --git a/tests/parity/golden/consumer/export-edge-male/out/S_consumer_export.tsv b/tests/parity/golden/consumer/export-edge-male/out/S_consumer_export.tsv new file mode 100644 index 000000000..8a8e855b5 --- /dev/null +++ b/tests/parity/golden/consumer/export-edge-male/out/S_consumer_export.tsv @@ -0,0 +1,21 @@ +key value +sites 16 +called 5 +no_call 11 +from_reference_blocks 3 +from_variant_records 3 +reference_checked 4 +reference_mismatch 1 +template_sites 99 +table_dropped 0 +sex male +file_23andme S_23andme_v5.zip +file_ancestrydna S_ancestrydna_layout.zip +no_call: inside a deletion or a longer allele 1 +no_call: no gVCF record 3 +no_call: no genotype 1 +no_call: no-call in the gVCF 1 +no_call: reference base differs from the site table 1 +no_call: symbolic allele 1 +no_call: two alleles on a haploid site (chrM heteroplasmy, or male chrX/chrY) 1 +no_call: unreadable genotype 2 diff --git a/tests/parity/golden/consumer/export-edge-male/stdout b/tests/parity/golden/consumer/export-edge-male/stdout new file mode 100644 index 000000000..49c4865d5 --- /dev/null +++ b/tests/parity/golden/consumer/export-edge-male/stdout @@ -0,0 +1,11 @@ +Sites: 16 (of 99 on the chip list; 0 dropped when the table was built) +Called: 5 (31.2%), 3 of them from reference blocks; no-call (--): 11 + inside a deletion or a longer allele: 1 + no gVCF record: 3 + no genotype: 1 + no-call in the gVCF: 1 + reference base differs from the site table: 1 + symbolic allele: 1 + two alleles on a haploid site (chrM heteroplasmy, or male chrX/chrY): 1 + unreadable genotype: 2 +Reference bases checked at 4 variant records, 1 differ from the table diff --git a/tests/parity/golden/pgx-parse/pheno-boosted/out/PHENO_BOOSTED_cpic_recommendations.txt b/tests/parity/golden/pgx-parse/pheno-boosted/out/PHENO_BOOSTED_cpic_recommendations.txt new file mode 100644 index 000000000..3a4a5b002 --- /dev/null +++ b/tests/parity/golden/pgx-parse/pheno-boosted/out/PHENO_BOOSTED_cpic_recommendations.txt @@ -0,0 +1,46 @@ +Pharmacogenomic Drug Recommendations +===================================== +Sample: PHENO_BOOSTED +Date: 2026-11-20 +Source: PharmCAT 3.4.0 report (pheno-boosted.json) + +Gene Results: +------------------------------------------------------------------------ +Gene Diplotype Phenotype +CYP2D6 *1/*2 Normal Metabolizer +CYP2C19 *1/*1 Normal Metabolizer +CYP2C9 *1/*1 Normal Metabolizer +CYP3A5 *3/*3 Poor Metabolizer + +Medications and Phenoconversion (CYP2D6, CYP2C19, CYP2C9, CYP3A4, CYP3A5): +------------------------------------------------------------------------ + + Medication list: pheno-boosted.txt (1 line). + CYP3A4 -- on lopinavir (strong CYP3A4 inhibitor), ritonavir (strong CYP3A4 inhibitor): not converted, the report has no result for the gene, so there is no genotype phenotype to convert. + WARNING: CYP3A5 genotype phenotype Poor Metabolizer; on lopinavir (strong CYP3A5 inhibitor), ritonavir (strong CYP3A5 inhibitor). CPIC gives no rule that converts the CYP3A5 phenotype, so no converted phenotype is computed. + Drugs affected by CYP3A5: tacrolimus. + The genotype phenotype in the table above is unchanged. A phenoconverted phenotype + holds while the inhibitor is taken. This is an aid for your clinician, not advice. + +Affected Medications: +------------------------------------------------------------------------ + + CYP3A5 -- Poor Metabolizer: + Diplotype: *3/*3 + Drugs (pipeline fallback table; the report names none): tacrolimus + Action: Consult CPIC guidelines at https://cpicpgx.org/guidelines/ + +Uncallable Genes: +------------------------------------------------------------------------ + + None -- all genes were successfully called. + +NOTE: Uncallable genes may lack coverage, have complex structural +variants, or require data not present in your VCF. Their absence +from the recommendations section does NOT mean normal function. + +------------------------------------------------------------------------ +DISCLAIMER: These recommendations are based on CPIC clinical +guidelines. Always consult your healthcare provider before +making any medication changes. This is NOT medical advice. +------------------------------------------------------------------------ diff --git a/tests/parity/golden/pgx-parse/pheno-boosted/out/PHENO_BOOSTED_phenotypes.tsv b/tests/parity/golden/pgx-parse/pheno-boosted/out/PHENO_BOOSTED_phenotypes.tsv new file mode 100644 index 000000000..e5dfce12a --- /dev/null +++ b/tests/parity/golden/pgx-parse/pheno-boosted/out/PHENO_BOOSTED_phenotypes.tsv @@ -0,0 +1,5 @@ +Gene Diplotype Phenotype Status Phenoconverted_phenotype +CYP2D6 *1/*2 Normal Metabolizer normal - +CYP2C19 *1/*1 Normal Metabolizer normal - +CYP2C9 *1/*1 Normal Metabolizer normal - +CYP3A5 *3/*3 Poor Metabolizer non-normal not computed: lopinavir, ritonavir inhibit CYP3A5; CPIC gives no rule that converts the CYP3A5 phenotype diff --git a/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.html b/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.html index 13c4013f4..8edc364e6 100644 --- a/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.html +++ b/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.html @@ -119,8 +119,10 @@

    Contents

  • Clinical filter and secondary findings
  • Cancer predisposition
  • Variant prioritization
  • +
  • Structural variants
  • Repeat expansions
  • Mitochondrial variants
  • +
  • Mosaic loss of chromosome Y or X
  • Carrier @@ -159,6 +161,7 @@

    Contents

  • Not assessed
  • +
  • Genes and rsIDs in these results
  • @@ -234,6 +237,18 @@

    Variant Prioritization (slivar)

    StatusNot run
    +
    +
    +

    Structural Variants

    +

    Steps 04, 19, 18, 22, 05 · Not run · link to this section

    +

    Short reads find large deletions and duplications better than insertions and inversions, and the callers disagree on many events. The totals count every call, most of them common in the population; only the AnnotSV class 4 and 5 rows are graded for disease.

    +
    Manta SVs (total / PASS)N/A / N/A
    +
    Delly SVs (total / PASS)N/A / N/A
    +
    CNVpytor CNVsN/A
    +
    CNVpytor significant (e-val < 0.01)N/A
    +
    Consensus SVs (2+ callers)N/A
    +
    +

    Repeat Expansions

    @@ -248,6 +263,16 @@

    Mitochondrial Analysis

    StatusNot run
    +
    +
    +

    Mosaic Loss of Chromosome Y or X

    +

    Step 33 · Complete · file date 2021-03-04 05:06 UTC · link to this section

    +

    Loss of chrY in a share of the blood cells of men, and of one chrX in women, becomes more common with age. Population studies link loss of Y with smoking and with higher rates of heart disease and some cancers; loss of X is less studied. That is a population-level association, not a diagnosis for one person. Below about 10% of the cells nothing can be told at 30x (docs/33-sample-qc.md).

    +
    chrY or chrXnot detected above 10%
    +
    Why
    +

    From depth: estimated share of the cells that lost one X 2.0% (95% range -1.0% to 5.0%); None to autosome depth ratio 0.980, 1.0 expected

    +
    +
    @@ -426,6 +451,18 @@

    Not Assessed by This Pipeline

    +
    +

    Genes and rsIDs in these results

    +

    Where in this report does a gene or an rsID come up?

    +
    +
    +
    +

    Every gene symbol and rsID a section of this report names, with the sections that name it and how many times. A gene listed here is a gene the results mention, not a finding: open the section to see what it says. The genes of the clinical filter (step 23) and variant prioritization (step 31) come from their tables, which the report counts but does not list.

    +

    No section of this report names a gene or an rsID.

    +
    +
    +
    +
    Disclaimer: This report is for educational and research purposes only. It is not a clinical diagnosis. Always discuss genomic findings with a qualified healthcare diff --git a/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.txt b/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.txt index 4d011bee3..56de04fa1 100644 --- a/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.txt +++ b/tests/parity/golden/report-render/m-optin-viral-na/m-optin-viral-na.txt @@ -26,6 +26,16 @@ SNPs: 11111 Indels: 6667 Counts of the calls DeepVariant made against the reference. A whole genome usually holds 4 to 5 million, most of them common. PASS calls passed the caller's filter; a count says nothing about health on its own. +################################################################################ +# 2. CLINICAL: Is there anything to take to a doctor? +################################################################################ + +## Mosaic loss of chromosome Y or X +--- + chrY or chrX: not detected above 10% (None) + From depth: estimated share of the cells that lost one X 2.0% (95% range -1.0% to 5.0%); None to autosome depth ratio 0.980, 1.0 expected + Loss of chrY in a share of the blood cells of men, and of one chrX in women, becomes more common with age. Population studies link loss of Y with smoking and with higher rates of heart disease and some cancers; loss of X is less studied. That is a population-level association, not a diagnosis for one person. Below about 10% of the cells nothing can be told at 30x (docs/33-sample-qc.md). + ################################################################################ # 3. CARRIER: What could I pass on to a child? ################################################################################ diff --git a/tests/parity/golden/summary-collect/all-history/all-history.json b/tests/parity/golden/summary-collect/all-history/all-history.json index 6df9f687b..d74efe9f7 100644 --- a/tests/parity/golden/summary-collect/all-history/all-history.json +++ b/tests/parity/golden/summary-collect/all-history/all-history.json @@ -13723,7 +13723,9 @@ "ancestry": { "population": "EUR", "panel": "HGDP+1kGP", - "low_confidence": null + "low_confidence": "True", + "mixed": "True", + "second_population": "AFR" } }, "partial": "partial: covers 1 of 22 autosomes" @@ -13896,11 +13898,8 @@ "GENE2" ], "caller": "manta", - "support": 2, - "support_callers": [ - "manta", - "delly" - ], + "support": null, + "support_callers": [], "exon_edge": "yes", "exon_edge_detail": "exon 2", "population_af": 0.001, @@ -13921,13 +13920,8 @@ "GENE3" ], "caller": "delly", - "support": 3, - "support_callers": [ - "manta", - "delly", - "tiddit", - "caller 4" - ], + "support": null, + "support_callers": [], "exon_edge": "no", "exon_edge_detail": "", "population_af": null, @@ -13948,10 +13942,8 @@ "G9" ], "caller": "tiddit", - "support": 1, - "support_callers": [ - "tiddit" - ], + "support": null, + "support_callers": [], "exon_edge": "no", "exon_edge_detail": "", "population_af": 0.2, @@ -13972,10 +13964,8 @@ "acmg_class": "3", "genes": [], "caller": "manta", - "support": 1, - "support_callers": [ - "manta" - ], + "support": null, + "support_callers": [], "exon_edge": "not checked", "exon_edge_detail": "", "population_af": null, @@ -13988,7 +13978,7 @@ ], "consensus": "sv_merged/S_sv_consensus.vcf.gz", "consensus_note": null, - "single_caller": 1, + "single_caller": 0, "exon_edge": 1, "exon_edge_checked": true, "common": 1 diff --git a/tests/parity/golden/summary-collect/all/all.json b/tests/parity/golden/summary-collect/all/all.json index 467431b39..32f877405 100644 --- a/tests/parity/golden/summary-collect/all/all.json +++ b/tests/parity/golden/summary-collect/all/all.json @@ -13728,7 +13728,9 @@ "ancestry": { "population": "EUR", "panel": "HGDP+1kGP", - "low_confidence": null + "low_confidence": "True", + "mixed": "True", + "second_population": "AFR" } }, "partial": "partial: covers 1 of 22 autosomes" @@ -13901,11 +13903,8 @@ "GENE2" ], "caller": "manta", - "support": 2, - "support_callers": [ - "manta", - "delly" - ], + "support": null, + "support_callers": [], "exon_edge": "yes", "exon_edge_detail": "exon 2", "population_af": 0.001, @@ -13926,13 +13925,8 @@ "GENE3" ], "caller": "delly", - "support": 3, - "support_callers": [ - "manta", - "delly", - "tiddit", - "caller 4" - ], + "support": null, + "support_callers": [], "exon_edge": "no", "exon_edge_detail": "", "population_af": null, @@ -13953,10 +13947,8 @@ "G9" ], "caller": "tiddit", - "support": 1, - "support_callers": [ - "tiddit" - ], + "support": null, + "support_callers": [], "exon_edge": "no", "exon_edge_detail": "", "population_af": 0.2, @@ -13977,10 +13969,8 @@ "acmg_class": "3", "genes": [], "caller": "manta", - "support": 1, - "support_callers": [ - "manta" - ], + "support": null, + "support_callers": [], "exon_edge": "not checked", "exon_edge_detail": "", "population_af": null, @@ -13993,7 +13983,7 @@ ], "consensus": "sv_merged/S_sv_consensus.vcf.gz", "consensus_note": null, - "single_caller": 1, + "single_caller": 0, "exon_edge": 1, "exon_edge_checked": true, "common": 1 diff --git a/tests/parity/golden/summary-collect/c-float--inf/float--inf.json b/tests/parity/golden/summary-collect/c-float--inf/float--inf.json index ebeaa0777..e82f79333 100644 --- a/tests/parity/golden/summary-collect/c-float--inf/float--inf.json +++ b/tests/parity/golden/summary-collect/c-float--inf/float--inf.json @@ -519,7 +519,9 @@ "ancestry": { "population": "EUR", "panel": "HGDP+1kGP", - "low_confidence": null + "low_confidence": "True", + "mixed": "True", + "second_population": "AFR" } } }, @@ -690,11 +692,8 @@ "GENE2" ], "caller": "manta", - "support": 2, - "support_callers": [ - "manta", - "delly" - ], + "support": null, + "support_callers": [], "exon_edge": "yes", "exon_edge_detail": "exon 2", "population_af": 0.001, @@ -715,13 +714,8 @@ "GENE3" ], "caller": "delly", - "support": 3, - "support_callers": [ - "manta", - "delly", - "tiddit", - "caller 4" - ], + "support": null, + "support_callers": [], "exon_edge": "no", "exon_edge_detail": "", "population_af": null, @@ -742,10 +736,8 @@ "G9" ], "caller": "tiddit", - "support": 1, - "support_callers": [ - "tiddit" - ], + "support": null, + "support_callers": [], "exon_edge": "no", "exon_edge_detail": "", "population_af": 0.2, @@ -766,10 +758,8 @@ "acmg_class": "3", "genes": [], "caller": "manta", - "support": 1, - "support_callers": [ - "manta" - ], + "support": null, + "support_callers": [], "exon_edge": "not checked", "exon_edge_detail": "", "population_af": null, @@ -782,7 +772,7 @@ ], "consensus": "sv_merged/S_sv_consensus.vcf.gz", "consensus_note": null, - "single_caller": 1, + "single_caller": 0, "exon_edge": 1, "exon_edge_checked": true, "common": 1 diff --git a/tests/parity/golden/summary-collect/c-float-inf/float-inf.json b/tests/parity/golden/summary-collect/c-float-inf/float-inf.json index fefaebc93..2a6f6524a 100644 --- a/tests/parity/golden/summary-collect/c-float-inf/float-inf.json +++ b/tests/parity/golden/summary-collect/c-float-inf/float-inf.json @@ -519,7 +519,9 @@ "ancestry": { "population": "EUR", "panel": "HGDP+1kGP", - "low_confidence": null + "low_confidence": "True", + "mixed": "True", + "second_population": "AFR" } } }, @@ -690,11 +692,8 @@ "GENE2" ], "caller": "manta", - "support": 2, - "support_callers": [ - "manta", - "delly" - ], + "support": null, + "support_callers": [], "exon_edge": "yes", "exon_edge_detail": "exon 2", "population_af": 0.001, @@ -715,13 +714,8 @@ "GENE3" ], "caller": "delly", - "support": 3, - "support_callers": [ - "manta", - "delly", - "tiddit", - "caller 4" - ], + "support": null, + "support_callers": [], "exon_edge": "no", "exon_edge_detail": "", "population_af": null, @@ -742,10 +736,8 @@ "G9" ], "caller": "tiddit", - "support": 1, - "support_callers": [ - "tiddit" - ], + "support": null, + "support_callers": [], "exon_edge": "no", "exon_edge_detail": "", "population_af": 0.2, @@ -766,10 +758,8 @@ "acmg_class": "3", "genes": [], "caller": "manta", - "support": 1, - "support_callers": [ - "manta" - ], + "support": null, + "support_callers": [], "exon_edge": "not checked", "exon_edge_detail": "", "population_af": null, @@ -782,7 +772,7 @@ ], "consensus": "sv_merged/S_sv_consensus.vcf.gz", "consensus_note": null, - "single_caller": 1, + "single_caller": 0, "exon_edge": 1, "exon_edge_checked": true, "common": 1 diff --git a/tests/parity/golden/summary-collect/c-float-nan/float-nan.json b/tests/parity/golden/summary-collect/c-float-nan/float-nan.json index 5580a7bd1..463d3f193 100644 --- a/tests/parity/golden/summary-collect/c-float-nan/float-nan.json +++ b/tests/parity/golden/summary-collect/c-float-nan/float-nan.json @@ -519,7 +519,9 @@ "ancestry": { "population": "EUR", "panel": "HGDP+1kGP", - "low_confidence": null + "low_confidence": "True", + "mixed": "True", + "second_population": "AFR" } } }, @@ -690,11 +692,8 @@ "GENE2" ], "caller": "manta", - "support": 2, - "support_callers": [ - "manta", - "delly" - ], + "support": null, + "support_callers": [], "exon_edge": "yes", "exon_edge_detail": "exon 2", "population_af": 0.001, @@ -715,13 +714,8 @@ "GENE3" ], "caller": "delly", - "support": 3, - "support_callers": [ - "manta", - "delly", - "tiddit", - "caller 4" - ], + "support": null, + "support_callers": [], "exon_edge": "no", "exon_edge_detail": "", "population_af": null, @@ -742,10 +736,8 @@ "G9" ], "caller": "tiddit", - "support": 1, - "support_callers": [ - "tiddit" - ], + "support": null, + "support_callers": [], "exon_edge": "no", "exon_edge_detail": "", "population_af": 0.2, @@ -766,10 +758,8 @@ "acmg_class": "3", "genes": [], "caller": "manta", - "support": 1, - "support_callers": [ - "manta" - ], + "support": null, + "support_callers": [], "exon_edge": "not checked", "exon_edge_detail": "", "population_af": null, @@ -782,7 +772,7 @@ ], "consensus": "sv_merged/S_sv_consensus.vcf.gz", "consensus_note": null, - "single_caller": 1, + "single_caller": 0, "exon_edge": 1, "exon_edge_checked": true, "common": 1 diff --git a/tests/parity/golden/summary-collect/c-full/full.json b/tests/parity/golden/summary-collect/c-full/full.json index 6021171e3..9b03fa5c0 100644 --- a/tests/parity/golden/summary-collect/c-full/full.json +++ b/tests/parity/golden/summary-collect/c-full/full.json @@ -622,7 +622,9 @@ "ancestry": { "population": "EUR", "panel": "HGDP+1kGP", - "low_confidence": null + "low_confidence": "True", + "mixed": "True", + "second_population": "AFR" } } }, @@ -793,11 +795,8 @@ "GENE2" ], "caller": "manta", - "support": 2, - "support_callers": [ - "manta", - "delly" - ], + "support": null, + "support_callers": [], "exon_edge": "yes", "exon_edge_detail": "exon 2", "population_af": 0.001, @@ -818,13 +817,8 @@ "GENE3" ], "caller": "delly", - "support": 3, - "support_callers": [ - "manta", - "delly", - "tiddit", - "caller 4" - ], + "support": null, + "support_callers": [], "exon_edge": "no", "exon_edge_detail": "", "population_af": null, @@ -845,10 +839,8 @@ "G9" ], "caller": "tiddit", - "support": 1, - "support_callers": [ - "tiddit" - ], + "support": null, + "support_callers": [], "exon_edge": "no", "exon_edge_detail": "", "population_af": 0.2, @@ -869,10 +861,8 @@ "acmg_class": "3", "genes": [], "caller": "manta", - "support": 1, - "support_callers": [ - "manta" - ], + "support": null, + "support_callers": [], "exon_edge": "not checked", "exon_edge_detail": "", "population_af": null, @@ -885,7 +875,7 @@ ], "consensus": "sv_merged/S_sv_consensus.vcf.gz", "consensus_note": null, - "single_caller": 1, + "single_caller": 0, "exon_edge": 1, "exon_edge_checked": true, "common": 1 diff --git a/tests/parity/golden/summary-collect/cb-full/cb-full.json b/tests/parity/golden/summary-collect/cb-full/cb-full.json index 86fc5522d..9715c2989 100644 --- a/tests/parity/golden/summary-collect/cb-full/cb-full.json +++ b/tests/parity/golden/summary-collect/cb-full/cb-full.json @@ -921,7 +921,7 @@ "variant": "DEL chr2:47386000-47388000", "acmg_class": "4", "caller": "Delly", - "support": 1, + "support": null, "via": "EPCAM exons 8 and 9, the 3' end (ENST00000263735)", "rule": "All P and LP", "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", @@ -932,7 +932,7 @@ "variant": "DUP chr17:43000000-43100000", "acmg_class": "4", "caller": "CNVpytor", - "support": 1, + "support": null, "via": null, "rule": "All P and LP", "rule_note": null, diff --git a/tests/parity/golden/summary-collect/cb-quoted-dosage/cb-quoted-dosage.json b/tests/parity/golden/summary-collect/cb-quoted-dosage/cb-quoted-dosage.json index 86fc5522d..9715c2989 100644 --- a/tests/parity/golden/summary-collect/cb-quoted-dosage/cb-quoted-dosage.json +++ b/tests/parity/golden/summary-collect/cb-quoted-dosage/cb-quoted-dosage.json @@ -921,7 +921,7 @@ "variant": "DEL chr2:47386000-47388000", "acmg_class": "4", "caller": "Delly", - "support": 1, + "support": null, "via": "EPCAM exons 8 and 9, the 3' end (ENST00000263735)", "rule": "All P and LP", "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", @@ -932,7 +932,7 @@ "variant": "DUP chr17:43000000-43100000", "acmg_class": "4", "caller": "CNVpytor", - "support": 1, + "support": null, "via": null, "rule": "All P and LP", "rule_note": null, diff --git a/tests/parity/golden/summary-collect/cb-quoted-interval/cb-quoted-interval.json b/tests/parity/golden/summary-collect/cb-quoted-interval/cb-quoted-interval.json index 86fc5522d..9715c2989 100644 --- a/tests/parity/golden/summary-collect/cb-quoted-interval/cb-quoted-interval.json +++ b/tests/parity/golden/summary-collect/cb-quoted-interval/cb-quoted-interval.json @@ -921,7 +921,7 @@ "variant": "DEL chr2:47386000-47388000", "acmg_class": "4", "caller": "Delly", - "support": 1, + "support": null, "via": "EPCAM exons 8 and 9, the 3' end (ENST00000263735)", "rule": "All P and LP", "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", @@ -932,7 +932,7 @@ "variant": "DUP chr17:43000000-43100000", "acmg_class": "4", "caller": "CNVpytor", - "support": 1, + "support": null, "via": null, "rule": "All P and LP", "rule_note": null, diff --git a/tests/parity/golden/summary-collect/cb-quoted-smn/cb-quoted-smn.json b/tests/parity/golden/summary-collect/cb-quoted-smn/cb-quoted-smn.json index 86fc5522d..9715c2989 100644 --- a/tests/parity/golden/summary-collect/cb-quoted-smn/cb-quoted-smn.json +++ b/tests/parity/golden/summary-collect/cb-quoted-smn/cb-quoted-smn.json @@ -921,7 +921,7 @@ "variant": "DEL chr2:47386000-47388000", "acmg_class": "4", "caller": "Delly", - "support": 1, + "support": null, "via": "EPCAM exons 8 and 9, the 3' end (ENST00000263735)", "rule": "All P and LP", "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", @@ -932,7 +932,7 @@ "variant": "DUP chr17:43000000-43100000", "acmg_class": "4", "caller": "CNVpytor", - "support": 1, + "support": null, "via": null, "rule": "All P and LP", "rule_note": null, diff --git a/tests/parity/golden/summary-collect/cb-unicode-contig/cb-unicode-contig.json b/tests/parity/golden/summary-collect/cb-unicode-contig/cb-unicode-contig.json index 86fc5522d..9715c2989 100644 --- a/tests/parity/golden/summary-collect/cb-unicode-contig/cb-unicode-contig.json +++ b/tests/parity/golden/summary-collect/cb-unicode-contig/cb-unicode-contig.json @@ -921,7 +921,7 @@ "variant": "DEL chr2:47386000-47388000", "acmg_class": "4", "caller": "Delly", - "support": 1, + "support": null, "via": "EPCAM exons 8 and 9, the 3' end (ENST00000263735)", "rule": "All P and LP", "rule_note": "A deletion of the 3' end of EPCAM silences MSH2 and counts as an MSH2 finding.", @@ -932,7 +932,7 @@ "variant": "DUP chr17:43000000-43100000", "acmg_class": "4", "caller": "CNVpytor", - "support": 1, + "support": null, "via": null, "rule": "All P and LP", "rule_note": null, diff --git a/tests/parity/golden/sv-add-callers/rdcn-female/stdout b/tests/parity/golden/sv-add-callers/rdcn-female/stdout new file mode 100644 index 000000000..7f2a96897 --- /dev/null +++ b/tests/parity/golden/sv-add-callers/rdcn-female/stdout @@ -0,0 +1,21 @@ +##fileformat=VCFv4.1 +##contig= +##contig= +##INFO= +##INFO= +##FORMAT= +##svcallers_added="Callers beside manta: delly: 5 PASS DEL/DUP calls, 5 added, and 4 PASS calls longer than 1000 bp set aside because read depth (RDCN) does not show the change" +##INFO= +##INFO= +##INFO= +##INFO= +##INFO= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT s1 +chr1 10000000 DEL_rd1 N 60 PASS END=60000000;SVTYPE=DEL;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:1 +chr1 120000000 DUP_rd3 N 60 PASS END=120500000;SVTYPE=DUP;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:3 +chr1 150000000 DEL_short N 60 PASS END=150000500;SVTYPE=DEL;SVCALLER=delly GT:RDCN 0/1:2 +chr20 1000000 MantaDEL:1 N 500 PASS END=1100000;SVTYPE=DEL;SVLEN=-100000;SVCALLER=manta GT 0/1 +chr20 3000000 MantaINV:2 N 500 PASS END=4000000;SVTYPE=INV;SVLEN=1000000;INV3;SVCALLER=manta GT 0/1 +chrX 1000000 DEL_par1 N 60 PASS END=1500000;SVTYPE=DEL;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:1 +chrX 50000000 DEL_x1 N 60 PASS END=51000000;SVTYPE=DEL;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:1 diff --git a/tests/parity/golden/sv-add-callers/rdcn-male/stdout b/tests/parity/golden/sv-add-callers/rdcn-male/stdout new file mode 100644 index 000000000..99a96c560 --- /dev/null +++ b/tests/parity/golden/sv-add-callers/rdcn-male/stdout @@ -0,0 +1,21 @@ +##fileformat=VCFv4.1 +##contig= +##contig= +##INFO= +##INFO= +##FORMAT= +##svcallers_added="Callers beside manta: delly: 5 PASS DEL/DUP calls, 5 added, and 4 PASS calls longer than 1000 bp set aside because read depth (RDCN) does not show the change" +##INFO= +##INFO= +##INFO= +##INFO= +##INFO= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT s1 +chr1 10000000 DEL_rd1 N 60 PASS END=60000000;SVTYPE=DEL;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:1 +chr1 120000000 DUP_rd3 N 60 PASS END=120500000;SVTYPE=DUP;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:3 +chr1 150000000 DEL_short N 60 PASS END=150000500;SVTYPE=DEL;SVCALLER=delly GT:RDCN 0/1:2 +chr20 1000000 MantaDEL:1 N 500 PASS END=1100000;SVTYPE=DEL;SVLEN=-100000;SVCALLER=manta GT 0/1 +chr20 3000000 MantaINV:2 N 500 PASS END=4000000;SVTYPE=INV;SVLEN=1000000;INV3;SVCALLER=manta GT 0/1 +chrX 1000000 DEL_par1 N 60 PASS END=1500000;SVTYPE=DEL;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:1 +chrX 60000000 DUP_x2 N 60 PASS END=60500000;SVTYPE=DUP;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:2 diff --git a/tests/parity/golden/sv-add-callers/rdcn-no-sex/stdout b/tests/parity/golden/sv-add-callers/rdcn-no-sex/stdout new file mode 100644 index 000000000..8775c4975 --- /dev/null +++ b/tests/parity/golden/sv-add-callers/rdcn-no-sex/stdout @@ -0,0 +1,22 @@ +##fileformat=VCFv4.1 +##contig= +##contig= +##INFO= +##INFO= +##FORMAT= +##svcallers_added="Callers beside manta: delly: 6 PASS DEL/DUP calls, 6 added, and 3 PASS calls longer than 1000 bp set aside because read depth (RDCN) does not show the change" +##INFO= +##INFO= +##INFO= +##INFO= +##INFO= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT s1 +chr1 10000000 DEL_rd1 N 60 PASS END=60000000;SVTYPE=DEL;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:1 +chr1 120000000 DUP_rd3 N 60 PASS END=120500000;SVTYPE=DUP;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:3 +chr1 150000000 DEL_short N 60 PASS END=150000500;SVTYPE=DEL;SVCALLER=delly GT:RDCN 0/1:2 +chr20 1000000 MantaDEL:1 N 500 PASS END=1100000;SVTYPE=DEL;SVLEN=-100000;SVCALLER=manta GT 0/1 +chr20 3000000 MantaINV:2 N 500 PASS END=4000000;SVTYPE=INV;SVLEN=1000000;INV3;SVCALLER=manta GT 0/1 +chrX 1000000 DEL_par1 N 60 PASS END=1500000;SVTYPE=DEL;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:1 +chrX 50000000 DEL_x1 N 60 PASS END=51000000;SVTYPE=DEL;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:1 +chrX 60000000 DUP_x2 N 60 PASS END=60500000;SVTYPE=DUP;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:2 diff --git a/tests/parity/golden/sv-add-callers/traps/stdout b/tests/parity/golden/sv-add-callers/traps/stdout new file mode 100644 index 000000000..60fcc59ab --- /dev/null +++ b/tests/parity/golden/sv-add-callers/traps/stdout @@ -0,0 +1,26 @@ +##fileformat=VCFv4.2 +##contig= +##contig= +##contig= +##svcallers_added="Callers beside x: delly: 5 PASS DEL/DUP calls, 5 added, and 2 PASS calls longer than 1000 bp set aside because read depth (RDCN) does not show the change; cnvpytor: 2 DEL/DUP calls with e-val < 0.01 and q0 < 0.5, 1 added" +##INFO= +##INFO= +##INFO= +##INFO= +##INFO= +##INFO= +##INFO= +##FORMAT= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT A B +chr1 200 M_c N A 1 PASS SVTYPE=INS;END=200;SVCALLER=x GT:GQ 0/1:3 0/0:1 +chr1 300 M_a N 1 PASS SVCALLER=x GT 0/1 0/0 +chr1 10000 D_flagend N 60 PASS SVTYPE=DEL;SVCALLER=delly GT 0/1 ./. +chr1 20000 D_imp N 60 PASS END=20500;SVTYPE=DEL;IMPRECISE;SVCALLER=delly GT:RDCN 0/1:. ./. +chr1 100000 M_long N 1 PASS SVTYPE=DEL;END=110000;SVCALLER=x GT 0/1 0/0 +chr1 100000 D_inside N 60 PASS END=104000;SVTYPE=DEL;SVCALLER=delly GT:RDCN 0/1:1 ./. +chr1 110000 C_ok N . PASS END=115000;SVTYPE=DUP;SVCALLER=cnvpytor GT ./. ./. +chr2 300 M_b N 1 . SVCALLER=x GT 1/1 0/0 +chr9 500 M_un N 1 PASS END=900;SVTYPE=DEL;SVCALLER=x GT 0/1 0/0 +chr2 50000 D_dup N 60 PASS END=60000;SVTYPE=DUP:TANDEM;SVCALLER=delly GT:RDCN 1/1:3 ./. +chr3 70000 D_sites N 60 PASS END=70500;SVTYPE=DEL;SVCALLER=delly GT ./. ./. diff --git a/tests/parity/golden/sv-postfilter/bare/out.tsv b/tests/parity/golden/sv-postfilter/bare/out.tsv new file mode 100644 index 000000000..69ae98f98 --- /dev/null +++ b/tests/parity/golden/sv-postfilter/bare/out.tsv @@ -0,0 +1,7 @@ +AnnotSV_ID ID SV_chrom SV_start SV_end SV_length SV_type Gene_name ACMG_class AnnotSV_ranking_score Caller Exon_edge Exon_edge_detail Population_AF Population_source Common AnnotSV_class Depth_support Rule Rule_detail +20_1000200_1009800_DEL_1 SV1 20 1000200 1009800 -9600 DEL GENE1 4 0.95 manta not checked 0.05 gnomAD-SV_v4.1_DEL_fixture yes 4 not checked +20_10100999_10102999_DEL_1 SV2 20 10100999 10102999 -2000 DEL GENE2 5 0.95 manta not checked no 5 not checked +20_20000000_20450000_DEL_1 SV3 20 20000000 20450000 -450000 DEL GENE3 4 0.95 manta not checked no 4 not checked +20_30000000_30500000_DUP_1 SV4 20 30000000 30500000 500000 DUP GENE4 4 0.95 delly . no 4 unknown +20_1000190_1009800_DEL_1 SV5 20 1000190 1009800 -9610 DEL GENE5 4 0.95 cnvpytor not checked no 4 yes +20_3000100_3005000_DEL_1 SV6 20 3000100 3005000 -4900 DEL GENE6 4 0.95 manta not checked no 4 not checked diff --git a/tests/parity/golden/sv-postfilter/bare/stderr b/tests/parity/golden/sv-postfilter/bare/stderr new file mode 100644 index 000000000..fdc4a5947 --- /dev/null +++ b/tests/parity/golden/sv-postfilter/bare/stderr @@ -0,0 +1 @@ +Class 4-5 SVs checked: 6; deletions on exon edges: not checked; common (benign-set AF >= 1%): 1; moved to class 3: 0 (no rule table); depth support no: 0, not checked: 4, unknown: 1 diff --git a/tests/parity/golden/sv-postfilter/checks/out.tsv b/tests/parity/golden/sv-postfilter/checks/out.tsv new file mode 100644 index 000000000..b61ea34ef --- /dev/null +++ b/tests/parity/golden/sv-postfilter/checks/out.tsv @@ -0,0 +1,7 @@ +AnnotSV_ID ID SV_chrom SV_start SV_end SV_length SV_type Gene_name ACMG_class AnnotSV_ranking_score Caller Exon_edge Exon_edge_detail Population_AF Population_source Common AnnotSV_class Depth_support Rule Rule_detail Support Support_callers +20_1000200_1009800_DEL_1 SV1 20 1000200 1009800 -9600 DEL GENE1 4 0.95 manta yes NM_900001 (FIXG1): exon 1 and exon 3 edges 0.05 gnomAD-SV_v4.1_DEL_fixture yes 4 not checked +20_10100999_10102999_DEL_1 SV2 20 10100999 10102999 -2000 DEL GENE2 5 0.95 manta no no 5 not checked +20_20000000_20450000_DEL_1 SV3 20 20000000 20450000 -450000 DEL GENE3 4 0.95 manta no no 4 not checked +20_30000000_30500000_DUP_1 SV4 20 30000000 30500000 500000 DUP GENE4 4 0.95 delly . no 4 unknown +20_1000190_1009800_DEL_1 SV5 20 1000190 1009800 -9610 DEL GENE5 4 0.95 cnvpytor no no 4 yes +20_3000100_3005000_DEL_1 SV6 20 3000100 3005000 -4900 DEL GENE6 4 0.95 manta yes NM_900005 (MING1): exon 1 and exon 2 edges no 4 not checked diff --git a/tests/parity/golden/sv-postfilter/checks/stderr b/tests/parity/golden/sv-postfilter/checks/stderr new file mode 100644 index 000000000..ec9d46257 --- /dev/null +++ b/tests/parity/golden/sv-postfilter/checks/stderr @@ -0,0 +1 @@ +Class 4-5 SVs checked: 6; deletions on exon edges: 2; common (benign-set AF >= 1%): 1; moved to class 3: 0 (no rule table); depth support no: 0, not checked: 4, unknown: 1 diff --git a/tests/parity/golden/sv-postfilter/consensus-empty/out.tsv b/tests/parity/golden/sv-postfilter/consensus-empty/out.tsv new file mode 100644 index 000000000..aee04b905 --- /dev/null +++ b/tests/parity/golden/sv-postfilter/consensus-empty/out.tsv @@ -0,0 +1,7 @@ +AnnotSV_ID ID SV_chrom SV_start SV_end SV_length SV_type Gene_name ACMG_class AnnotSV_ranking_score Caller Exon_edge Exon_edge_detail Population_AF Population_source Common AnnotSV_class Depth_support Rule Rule_detail Support Support_callers +20_1000200_1009800_DEL_1 SV1 20 1000200 1009800 -9600 DEL GENE1 4 0.95 manta not checked 0.05 gnomAD-SV_v4.1_DEL_fixture yes 4 not checked 1 manta +20_10100999_10102999_DEL_1 SV2 20 10100999 10102999 -2000 DEL GENE2 5 0.95 manta not checked no 5 not checked 1 manta +20_20000000_20450000_DEL_1 SV3 20 20000000 20450000 -450000 DEL GENE3 4 0.95 manta not checked no 4 not checked 1 manta +20_30000000_30500000_DUP_1 SV4 20 30000000 30500000 500000 DUP GENE4 4 0.95 delly . no 4 unknown 1 delly +20_1000190_1009800_DEL_1 SV5 20 1000190 1009800 -9610 DEL GENE5 4 0.95 cnvpytor not checked no 4 yes +20_3000100_3005000_DEL_1 SV6 20 3000100 3005000 -4900 DEL GENE6 4 0.95 manta not checked no 4 not checked 1 manta diff --git a/tests/parity/golden/sv-postfilter/consensus-empty/stderr b/tests/parity/golden/sv-postfilter/consensus-empty/stderr new file mode 100644 index 000000000..fdc4a5947 --- /dev/null +++ b/tests/parity/golden/sv-postfilter/consensus-empty/stderr @@ -0,0 +1 @@ +Class 4-5 SVs checked: 6; deletions on exon edges: not checked; common (benign-set AF >= 1%): 1; moved to class 3: 0 (no rule table); depth support no: 0, not checked: 4, unknown: 1 diff --git a/tests/parity/golden/sv-postfilter/consensus-gone/out.tsv b/tests/parity/golden/sv-postfilter/consensus-gone/out.tsv new file mode 100644 index 000000000..c6368b44a --- /dev/null +++ b/tests/parity/golden/sv-postfilter/consensus-gone/out.tsv @@ -0,0 +1,7 @@ +AnnotSV_ID ID SV_chrom SV_start SV_end SV_length SV_type Gene_name ACMG_class AnnotSV_ranking_score Caller Exon_edge Exon_edge_detail Population_AF Population_source Common AnnotSV_class Depth_support Rule Rule_detail Support Support_callers +20_1000200_1009800_DEL_1 SV1 20 1000200 1009800 -9600 DEL GENE1 4 0.95 manta not checked 0.05 gnomAD-SV_v4.1_DEL_fixture yes 4 not checked +20_10100999_10102999_DEL_1 SV2 20 10100999 10102999 -2000 DEL GENE2 5 0.95 manta not checked no 5 not checked +20_20000000_20450000_DEL_1 SV3 20 20000000 20450000 -450000 DEL GENE3 4 0.95 manta not checked no 4 not checked +20_30000000_30500000_DUP_1 SV4 20 30000000 30500000 500000 DUP GENE4 4 0.95 delly . no 4 unknown +20_1000190_1009800_DEL_1 SV5 20 1000190 1009800 -9610 DEL GENE5 4 0.95 cnvpytor not checked no 4 yes +20_3000100_3005000_DEL_1 SV6 20 3000100 3005000 -4900 DEL GENE6 4 0.95 manta not checked no 4 not checked diff --git a/tests/parity/golden/sv-postfilter/consensus-gone/stderr b/tests/parity/golden/sv-postfilter/consensus-gone/stderr new file mode 100644 index 000000000..fdc4a5947 --- /dev/null +++ b/tests/parity/golden/sv-postfilter/consensus-gone/stderr @@ -0,0 +1 @@ +Class 4-5 SVs checked: 6; deletions on exon edges: not checked; common (benign-set AF >= 1%): 1; moved to class 3: 0 (no rule table); depth support no: 0, not checked: 4, unknown: 1 diff --git a/tests/parity/golden/sv-postfilter/consensus-ids/out.tsv b/tests/parity/golden/sv-postfilter/consensus-ids/out.tsv new file mode 100644 index 000000000..3b18ba247 --- /dev/null +++ b/tests/parity/golden/sv-postfilter/consensus-ids/out.tsv @@ -0,0 +1,7 @@ +AnnotSV_ID ID SV_chrom SV_start SV_end SV_length SV_type Gene_name ACMG_class AnnotSV_ranking_score Caller Exon_edge Exon_edge_detail Population_AF Population_source Common AnnotSV_class Depth_support Rule Rule_detail Support Support_callers +20_1000200_1009800_DEL_1 SV1 20 1000200 1009800 -9600 DEL GENE1 4 0.95 manta not checked 0.05 gnomAD-SV_v4.1_DEL_fixture yes 4 not checked 1 manta +20_10100999_10102999_DEL_1 SV2 20 10100999 10102999 -2000 DEL GENE2 5 0.95 manta not checked no 5 not checked 2 manta,delly +20_20000000_20450000_DEL_1 SV3 20 20000000 20450000 -450000 DEL GENE3 4 0.95 manta not checked no 4 not checked 1 manta +20_30000000_30500000_DUP_1 SV4 20 30000000 30500000 500000 DUP GENE4 4 0.95 delly . no 4 unknown 2 manta,delly +20_1000190_1009800_DEL_1 SV5 20 1000190 1009800 -9610 DEL GENE5 4 0.95 cnvpytor not checked no 4 yes +20_3000100_3005000_DEL_1 SV6 20 3000100 3005000 -4900 DEL GENE6 4 0.95 manta not checked no 4 not checked 1 manta diff --git a/tests/parity/golden/sv-postfilter/consensus-ids/stderr b/tests/parity/golden/sv-postfilter/consensus-ids/stderr new file mode 100644 index 000000000..fdc4a5947 --- /dev/null +++ b/tests/parity/golden/sv-postfilter/consensus-ids/stderr @@ -0,0 +1 @@ +Class 4-5 SVs checked: 6; deletions on exon edges: not checked; common (benign-set AF >= 1%): 1; moved to class 3: 0 (no rule table); depth support no: 0, not checked: 4, unknown: 1 diff --git a/tests/parity/golden/sv-postfilter/no-rules/out.tsv b/tests/parity/golden/sv-postfilter/no-rules/out.tsv new file mode 100644 index 000000000..49bc7759f --- /dev/null +++ b/tests/parity/golden/sv-postfilter/no-rules/out.tsv @@ -0,0 +1,19 @@ +AnnotSV_ID ID SV_chrom SV_start SV_end SV_length SV_type Gene_name ACMG_class AnnotSV_ranking_score Caller Exon_edge Exon_edge_detail Population_AF Population_source Common AnnotSV_class Depth_support Rule Rule_detail FORMAT S1 +4_87614600_87615500_DEL_1 TR_IN 4 87614600 87615500 -900 DEL DSPP 4 0.99 manta no no 4 yes GT:DHFFC 0/1:0.45 +4_87614600_87616700_DEL_1 TR_OUT 4 87614600 87616700 -2100 DEL DSPP 4 0.99 manta no no 4 yes GT:DHFFC 0/1:0.45 +14_106000000_106100000_DEL_1 IGH_IN 14 106000000 106100000 -100000 DEL IGHV3-23 4 0.99 manta no no 4 yes GT:DHFFC 0/1:0.5 +14_105400000_105700000_DEL_1 IGH_EDGE 14 105400000 105700000 -300000 DEL TMEM121 4 0.99 manta no no 4 yes GT:DHFFC 0/1:0.5 +14_105550000_105650000_DEL_1 IGH_MOST 14 105550000 105650000 -100000 DEL IGHA2 4 0.99 manta no no 4 yes GT:DHFFC 0/1:0.5 +6_31000000_31100000_DEL_1 MHC_IN 6 31000000 31100000 -100000 DEL HLA-C 4 0.99 manta no no 4 yes GT:DHFFC 0/1:0.5 +10_46634800_46635600_DEL_1 ALIAS 10 46634800 46635600 -800 DEL RHEB 5 0.99 manta no no 5 yes GT:DHFFC 0/1:0.3 +7_151470000_151480000_DEL_1 REAL 7 151470000 151480000 -10000 DEL RHEB 5 0.99 manta no no 5 yes GT:DHFFC 0/1:0.3 +20_40000000_40300000_DEL_1 D_LOW 20 40000000 40300000 -300000 DEL G7 3 0.99 manta no no 4 no no_depth_support read depth does not show the change (duphold DHFFC 0.95) GT:DHFFC 0/1:0.95 +20_40000000_40300000_DEL_1 D_OK 20 40000000 40300000 -300000 DEL G8 4 0.99 manta no no 4 yes GT:DHFFC 0/1:0.5 +20_41000000_41300000_DUP_1 DUP_LOW 20 41000000 41300000 300000 DUP G9 3 0.99 manta . no 4 no no_depth_support read depth does not show the change (duphold DHBFC 1.1) GT:DHBFC 0/1:1.1 +20_42000000_42300000_DEL_1 DELLY2 20 42000000 42300000 -300000 DEL G10 3 0.99 delly no no 4 no no_depth_support read depth does not show the change (Delly RDCN 2) GT:RDCN 0/1:2 +20_43000000_43300000_DEL_1 DELLY1 20 43000000 43300000 -300000 DEL G11 4 0.99 delly no no 4 yes GT:RDCN 0/1:1 +X_50000000_50300000_DUP_1 XDUP X 50000000 50300000 300000 DUP G12 4 0.99 delly . no 4 yes GT:RDCN 0/1:2 +20_44000000_44300000_DEL_1 NODEPTH 20 44000000 44300000 -300000 DEL G13 4 0.99 manta no no 4 not checked GT 0/1 +20_45000000_45300000_DEL_1 CNV 20 45000000 45300000 -300000 DEL G14 4 0.99 cnvpytor no no 4 yes GT 0/1 +20_46000000_46300000_DEL_1 DELLY_GT 20 46000000 46300000 -300000 DEL G15 4 0.99 delly no no 4 unknown GT 0/1 +20_47000000_47000800_DEL_1 DELLY_SHORT 20 47000000 47000800 -800 DEL G16 4 0.99 delly no no 4 not checked GT 0/1 diff --git a/tests/parity/golden/sv-postfilter/no-rules/stderr b/tests/parity/golden/sv-postfilter/no-rules/stderr new file mode 100644 index 000000000..ce8ef2d16 --- /dev/null +++ b/tests/parity/golden/sv-postfilter/no-rules/stderr @@ -0,0 +1 @@ +Class 4-5 SVs checked: 18; deletions on exon edges: 0; common (benign-set AF >= 1%): 0; moved to class 3: 3 (no rule table); depth support no: 3, not checked: 2, unknown: 1 diff --git a/tests/parity/golden/sv-postfilter/rules/out.tsv b/tests/parity/golden/sv-postfilter/rules/out.tsv new file mode 100644 index 000000000..8fb9a26f7 --- /dev/null +++ b/tests/parity/golden/sv-postfilter/rules/out.tsv @@ -0,0 +1,19 @@ +AnnotSV_ID ID SV_chrom SV_start SV_end SV_length SV_type Gene_name ACMG_class AnnotSV_ranking_score Caller Exon_edge Exon_edge_detail Population_AF Population_source Common AnnotSV_class Depth_support Rule Rule_detail FORMAT S1 +4_87614600_87615500_DEL_1 TR_IN 4 87614600 87615500 -900 DEL DSPP 3 0.99 manta no no 4 yes tandem_repeat inside the coding tandem repeat DSPP:18bp motif at 4:87614315-87616446 (TRExplorer): a change in the repeat's length, which SV callers do not size; read it with a repeat genotyper GT:DHFFC 0/1:0.45 +4_87614600_87616700_DEL_1 TR_OUT 4 87614600 87616700 -2100 DEL DSPP 4 0.99 manta no no 4 yes GT:DHFFC 0/1:0.45 +14_106000000_106100000_DEL_1 IGH_IN 14 106000000 106100000 -100000 DEL IGHV3-23 3 0.99 manta no no 4 yes ig_tr_locus 100% inside the IGH locus (14:105583731-106879812), where blood cells cut DNA out when they rearrange their receptor genes (V(D)J recombination) GT:DHFFC 0/1:0.5 +14_105400000_105700000_DEL_1 IGH_EDGE 14 105400000 105700000 -300000 DEL TMEM121 4 0.99 manta no no 4 yes GT:DHFFC 0/1:0.5 +14_105550000_105650000_DEL_1 IGH_MOST 14 105550000 105650000 -100000 DEL IGHA2 3 0.99 manta no no 4 yes ig_tr_locus 66% inside the IGH locus (14:105583731-106879812), where blood cells cut DNA out when they rearrange their receptor genes (V(D)J recombination) GT:DHFFC 0/1:0.5 +6_31000000_31100000_DEL_1 MHC_IN 6 31000000 31100000 -100000 DEL HLA-C 3 0.99 manta no no 4 yes mhc 100% inside the MHC (6:28510120-33480577), whose haplotypes differ widely between people; gnomAD-SV marks such SVs low confidence (IGH_MHC_OVERLAP) GT:DHFFC 0/1:0.5 +10_46634800_46635600_DEL_1 ALIAS 10 46634800 46635600 -800 DEL RHEBP1 3 0.99 manta no no 5 yes gene_alias RHEB here is the pseudogene RHEBP1 (HGNC:10010), which carries RHEB as an alias; the gene RHEB (HGNC:10011) lies elsewhere GT:DHFFC 0/1:0.3 +7_151470000_151480000_DEL_1 REAL 7 151470000 151480000 -10000 DEL RHEB 5 0.99 manta no no 5 yes GT:DHFFC 0/1:0.3 +20_40000000_40300000_DEL_1 D_LOW 20 40000000 40300000 -300000 DEL G7 3 0.99 manta no no 4 no no_depth_support read depth does not show the change (duphold DHFFC 0.95) GT:DHFFC 0/1:0.95 +20_40000000_40300000_DEL_1 D_OK 20 40000000 40300000 -300000 DEL G8 4 0.99 manta no no 4 yes GT:DHFFC 0/1:0.5 +20_41000000_41300000_DUP_1 DUP_LOW 20 41000000 41300000 300000 DUP G9 3 0.99 manta . no 4 no no_depth_support read depth does not show the change (duphold DHBFC 1.1) GT:DHBFC 0/1:1.1 +20_42000000_42300000_DEL_1 DELLY2 20 42000000 42300000 -300000 DEL G10 3 0.99 delly no no 4 no no_depth_support read depth does not show the change (Delly RDCN 2) GT:RDCN 0/1:2 +20_43000000_43300000_DEL_1 DELLY1 20 43000000 43300000 -300000 DEL G11 4 0.99 delly no no 4 yes GT:RDCN 0/1:1 +X_50000000_50300000_DUP_1 XDUP X 50000000 50300000 300000 DUP G12 4 0.99 delly . no 4 yes GT:RDCN 0/1:2 +20_44000000_44300000_DEL_1 NODEPTH 20 44000000 44300000 -300000 DEL G13 4 0.99 manta no no 4 not checked GT 0/1 +20_45000000_45300000_DEL_1 CNV 20 45000000 45300000 -300000 DEL G14 4 0.99 cnvpytor no no 4 yes GT 0/1 +20_46000000_46300000_DEL_1 DELLY_GT 20 46000000 46300000 -300000 DEL G15 4 0.99 delly no no 4 unknown GT 0/1 +20_47000000_47000800_DEL_1 DELLY_SHORT 20 47000000 47000800 -800 DEL G16 4 0.99 delly no no 4 not checked GT 0/1 diff --git a/tests/parity/golden/sv-postfilter/rules/stderr b/tests/parity/golden/sv-postfilter/rules/stderr new file mode 100644 index 000000000..c9724f472 --- /dev/null +++ b/tests/parity/golden/sv-postfilter/rules/stderr @@ -0,0 +1 @@ +Class 4-5 SVs checked: 18; deletions on exon edges: 0; common (benign-set AF >= 1%): 0; moved to class 3: 8; depth support no: 3, not checked: 2, unknown: 1 diff --git a/tests/parity/golden/sv-postfilter/traps/out.tsv b/tests/parity/golden/sv-postfilter/traps/out.tsv new file mode 100644 index 000000000..3d5a530f9 --- /dev/null +++ b/tests/parity/golden/sv-postfilter/traps/out.tsv @@ -0,0 +1,9 @@ +AnnotSV_ID ID SV_chrom SV_start SV_end SV_length SV_type Gene_name ACMG_class AnnotSV_ranking_score Caller Exon_edge Exon_edge_detail Population_AF Population_source Common AnnotSV_class Depth_support Rule Rule_detail Support Support_callers FORMAT S1 +20_1000200_1009800_DEL_1 "Q""UOTE" 20 1000200 1009800 -9600 DEL "G""1;G2" 4 0.99 manta yes NM_900001 (FIXG1): exon 1 and exon 3 edges no 4 yes GT:DHFFC:DHFFC 0/1:0.9:0.5 +20_40000000_40300000_DEL_1 D075 20 40000000 40300000 -300000 DEL G9 3 0.99 manta no no 4 no no_depth_support read depth does not show the change (duphold DHFFC 0.75) GT:DHFFC 0/1:0.75 +chr20_1000200_1009800_del_1 CHR chr20 1000200 1009800 9600 del FIXG1 3 0.99 manta yes NM_900001 (FIXG1): exon 1 and exon 3 edges no 4 no no_depth_support read depth does not show the change (duphold DHFFC inf) GT:DHFFC 0/1:inf +20_5000_5000_INS_1 INS 20 5000 5000 0 INS G3 5 0.99 manta . no 5 . GT 0/1 +6_31000000_31100000_INV_1 INV 6 31000000 31100000 100000 INV HLA-C 3 0.99 delly . no 5 . mhc 100% inside the MHC (6:28510120-33480577), whose haplotypes differ widely between people; gnomAD-SV marks such SVs low confidence (IGH_MHC_OVERLAP) GT 0/1 +20_5000_9000_DUP_1 NAN 20 5000 9000 4000 DUP G4 nan 0.99 manta . no nan no no_depth_support read depth does not show the change (duphold DHBFC 1) GT:DHBFC 0/1:1.0 +X_2781000_2900000_DEL_1 XDEL X 2781000 2900000 -119000 DEL G5 4 0.99 delly no no 4 yes GT:RDCN 0/1:1 +20_1000_3000_DUP_1 EXTRA 20 1000 3000 2000 DUP G6 4 0.99 manta . no 4 not checked GT 0/1 diff --git a/tests/parity/golden/traits/shared-vocabulary/rules-out.tsv b/tests/parity/golden/traits/shared-vocabulary/rules-out.tsv index c65e252d3..2e7934105 100644 --- a/tests/parity/golden/traits/shared-vocabulary/rules-out.tsv +++ b/tests/parity/golden/traits/shared-vocabulary/rules-out.tsv @@ -1,10 +1,10 @@ rule gene trait rsids chrom pos genotypes gq phase result status reading ancestry note pmids -APOE APOE APOE alleles (e2, e3, e4) rs429358,rs7412 chr19 44908684,44908822 T/G,C/T 40,40 . . other_allele Not read: needs a genotype at both rs429358 and rs7412. yes Odds ratios from a meta-analysis of people of European descent (Farrer 1997); the e4 effect was weaker in African American and Hispanic people and stronger in Japanese people. Step 25's Alzheimer's score (PGS000334) includes the same two sites; docs/42-traits.md says why step 06 does not list APOE. 8346443,9343467 +APOE APOE APOE alleles (e2, e3, e4, and the rare e1) rs429358,rs7412 chr19 44908684,44908822 T/G,C/T 40,40 . . other_allele Not read: needs a genotype at both rs429358 and rs7412. yes Odds ratios of Alzheimer's disease against e3/e3 from a meta-analysis of clinic and autopsy studies of people of European descent (Farrer 1997), adjusted for age: odds ratios of those case-control studies, not a lifetime risk. The e4 effect was seen at every age from 40 to 90 but was smaller after 70, and the risk of a genotype differed between men and women. The e4 effect was weaker in African American and Hispanic people and stronger in Japanese people. The two sites also make the rare e1, whose pairs are not read. Step 25's Alzheimer's score (PGS000334) includes the same two sites; docs/42-traits.md says why step 06 does not list APOE. 8346443,9343467 TAS2R38 TAS2R38 Bitter taste haplotype (TAS2R38, PTC and PROP) rs713598,rs1726866,rs10246939 chr7 141973545,141972905,141972804 .,.,. .,.,. . . not_covered Not read: needs a genotype at every one of rs713598, rs1726866 and rs10246939. no The three TAS2R38 sites of the traits table read together: PAV (Pro49, Ala262, Val296) is the taster form and AVI (Ala49, Val262, Ile296) the non-taster form. Kim 2003 found five haplotypes worldwide; this rule reads the two common ones, and a copy with another (AAV, AAI, PVI or a rarer one) is not read. TAS2R38 lies on the minus strand, so the alleles here are the complement of the codon bases. 12595690,15723792 APOL1 APOL1 APOL1 kidney risk variants (G1 read, G2 not read) rs73885319,rs60910145 chr22 36265860,36265988 .,. .,. . . not_covered Not read: needs a genotype at both rs73885319 and rs60910145. yes G1 is two missense changes that sit on one copy, S342G (rs73885319) and I384M (rs60910145), in perfect linkage disequilibrium in Genovese 2010, so they are read together without phase. G2 (rs71785313) is a 6-base deletion of N388 and Y389; this rule table reads single-base sites only, so this rule does not read G2 and a G2 copy reads as G0 here; the HTML report reads G2 from the sample's VCF beside this result (og summary collect). G1 and G2 never share a copy. In African Americans, two risk copies (G1 or G2) went with focal segmental glomerulosclerosis (odds ratio 10.5) and hypertension-attributed end-stage kidney disease (odds ratio 7.3), and one copy did not raise the risk (a recessive pattern). Both variants are common on African chromosomes and were absent from the European chromosomes studied. 20647424 SERPINA1 SERPINA1 Alpha-1 antitrypsin Pi type (M, S, Z) rs17580,rs28929474 chr14 94380925,94378610 .,. .,. . . not_covered Not read: needs a genotype at both rs17580 and rs28929474. yes The two common deficiency alleles of SERPINA1: S (Glu264Val, rs17580) and Z (Glu342Lys, rs28929474). SERPINA1 lies on the minus strand, so the alleles here are the complement of the codon bases. M stands for a copy with neither S nor Z; other deficiency and null alleles are not read. The readings follow the ATS/ERS 2003 standards; a serum alpha-1 antitrypsin level confirms a deficiency. Z is most common in people of northern European descent. 14522813 -HFE HFE HFE haemochromatosis genotype (C282Y, H63D) rs1800562,rs1799945 chr6 26092913,26090951 .,. .,. . . not_covered Not read: needs a genotype at both rs1800562 and rs1799945. yes The two HFE missense changes of Feder 1996: C282Y (rs1800562), found homozygous in 83% of 178 patients with hereditary haemochromatosis, and H63D (rs1799945). The two sites are about 2 kb apart, too far for most read pairs to phase: heterozygous at both reads as C282Y/H63D, one change on each copy, and the reading says it assumes that. Many people with a risk genotype never build up iron; ferritin and transferrin saturation tell. The secondary-findings list reports C282Y homozygotes too (assets/acmg_sf.tsv); this line is the reading of every genotype. C282Y is most common in people of northern European descent. 8696333 -GC GC Vitamin D-binding protein type (Gc1F, Gc1S, Gc2) rs7041,rs4588 chr4 71752617,71752606 .,. .,. . . not_covered Not read: needs a genotype at both rs7041 and rs4588. yes rs7041 (Asp432Glu) and rs4588 (Thr436Lys) together give the three common vitamin D-binding protein types: Gc1F (Asp, Thr), Gc1S (Glu, Thr) and Gc2 (Asp, Lys). GC lies on the minus strand, so the alleles here are the complement of the codon bases. In Powe 2013 the two sites explained 79.4% of the variation in the protein level and 9.9% of that of total 25-hydroxyvitamin D. Gc1F was the most common type in black participants and Gc1S in white ones. The monoclonal assay that study used was questioned in later letters, so the protein levels are those of that assay. A protein type, not a vitamin D deficiency. 24256378 +HFE HFE HFE haemochromatosis genotype (C282Y, H63D) rs1800562,rs1799945 chr6 26092913,26090951 .,. .,. . . not_covered Not read: needs a genotype at both rs1800562 and rs1799945. yes The two HFE missense changes of Feder 1996: C282Y (rs1800562), found homozygous in 83% of 178 patients with hereditary haemochromatosis, and H63D (rs1799945). The two sites are about 2 kb apart, too far for most read pairs to phase: heterozygous at both reads as C282Y/H63D, one change on each copy, and the reading says it assumes that. Many people with a risk genotype never build up iron; ferritin and transferrin saturation tell. How many become ill comes from later cohorts: in a Melbourne cohort of people of northern European descent aged 40 to 69, followed for 12 years, iron-overload disease was documented in 28% of male and 1% of female C282Y homozygotes (Allen 2008) and in 1 of 82 male and none of 95 female C282Y/H63D compound heterozygotes (Gurrin 2009); a US screening of 41,038 clinic attendees put frank haemochromatosis at under 1% of homozygotes (Beutler 2002). The secondary-findings list reports C282Y homozygotes too (assets/acmg_sf.tsv); this line is the reading of every genotype. C282Y is most common in people of northern European descent. 8696333,18199861,19554541,11812557 +GC GC Vitamin D-binding protein type (Gc1F, Gc1S, Gc2) rs7041,rs4588 chr4 71752617,71752606 .,. .,. . . not_covered Not read: needs a genotype at both rs7041 and rs4588. yes rs7041 (Asp432Glu) and rs4588 (Thr436Lys) together give the three common vitamin D-binding protein types: Gc1F (Asp, Thr), Gc1S (Glu, Thr) and Gc2 (Asp, Lys). GC lies on the minus strand, so the alleles here are the complement of the codon bases. In Powe 2013 the two sites explained 9.9% of the variation in total 25-hydroxyvitamin D, and Gc1F was the most common type in black participants and Gc1S in white ones. That study also reported protein levels that differed widely by type, but its monoclonal assay reads the types differently: with mass spectrometry or a polyclonal assay, genotype explained at most 9% of the variation in the protein level, against 85% with the monoclonal assay (Denburg 2016). So the readings name the type and give no protein level. A protein type, not a vitamin D deficiency. 24256378,27250744 CAD9p21 CDKN2B-AS1 Chromosome 9p21 heart disease risk haplotype rs10757278,rs1333049 chr9 22124478,22125504 .,. .,. . . not_covered Not read: needs a genotype at both rs10757278 and rs1333049. yes Two tags of the 9p21 risk haplotype near CDKN2A and CDKN2B: rs10757278 (G, Helgadottir 2007, myocardial infarction) and rs1333049 (C, the strongest coronary artery disease signal of the WTCCC 2007 scan). McPherson 2007 found the same interval, with no link to cholesterol, blood pressure or diabetes. The two are read together as tags of one haplotype; heterozygous at both reads as one risk copy, and the reading says it assumes both risk alleles sit on that copy. A copy with only one of the two risk alleles is not read. The studies were of people of European descent. 17478679,17554300,17478681 MAPT MAPT 17q21.31 inversion lineage (H1, H2) rs1052553,rs8070723 chr17 45996523,46003698 .,. .,. . . not_covered Not read: needs a genotype at both rs1052553 and rs8070723. yes The 17q21.31 region around MAPT comes in two lineages, H1 and H2, that differ by a 900-kb inversion and do not recombine (Stefansson 2005); H2 was found in about 20% of European chromosomes, rarely in Africans and almost never in East Asians. Two H1/H2 tags are read together: rs1052553 in MAPT (A is H1, Wider 2010) and rs8070723 (Hoglinger 2011's proxy for H1/H2, its minor allele G in H2). A copy with H1 at one tag and H2 at the other is not read. GRCh38 carries H1. 15654335,19912324,21685912 SHARE SHARE Shared position test rs999999991,rs999999992 chr19 44908684,44908822 T/G,C/T 40,40 not_used carrier called One alternate copy. Phase unknown: this reading assumes rs999999991 G and rs999999992 T sit on one copy (the copies h0 and h1); no phasing ran (no BAM, or phasing failed), so that is not checked. no Synthetic alternate allele vocabulary. 1 diff --git a/tests/parity/inputs/sv-postfilter/consensus_ids.vcf b/tests/parity/inputs/sv-postfilter/consensus_ids.vcf new file mode 100644 index 000000000..4a2eccb98 --- /dev/null +++ b/tests/parity/inputs/sv-postfilter/consensus_ids.vcf @@ -0,0 +1,10 @@ +##fileformat=VCFv4.1 +##source=SURVIVOR +##FORMAT= +##FORMAT= +##FORMAT= +#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT manta delly +chr20 3000000 OTHER N . PASS SUPP=2;SUPP_VEC=11;SVLEN=-5000;SVTYPE=DEL;END=3005000 GT:TY:ID 0/1:DEL:OTHER 0/1:DEL:DELLY6 +chr20 10101000 SV2 N . PASS SUPP=2;SUPP_VEC=11;SVLEN=-2000;SVTYPE=DEL;SVMETHOD=SURVIVOR1.0.7;CHR2=chr20;END=10103000;STRANDS=+- GT:TY:ID 0/1:DEL:SV2 0/1:DEL:DELLY2 +chr20 20000010 MantaDUP:2 N . PASS SUPP=2;SUPP_VEC=11;SVLEN=450000;SVTYPE=DUP;END=20450010 GT:TY:ID 0/1:DUP:MantaDUP_2 0/1:DUP:DELLY3 +chr20 30000000 SV4 N . PASS SUPP=2;SUPP_VEC=11;SVLEN=500000;SVTYPE=DUP;END=30500000 GT:TY:ID 0/1:DUP:MantaDUP_9 0/1:DUP:SV4 diff --git a/tests/parity/remnants.tsv b/tests/parity/remnants.tsv index 3d6d3a00a..1397862b5 100644 --- a/tests/parity/remnants.tsv +++ b/tests/parity/remnants.tsv @@ -4,4 +4,4 @@ # the importers are exactly these (docs/rust.md). # filenamesimporterslane bin/genome_lookup.py GENE_KEYS,RSID_RE,norm_chrom,parse_query,names_of,walk_mentions,flat bin/results_index.py R5c -bin/collect_summary.py HERE,ASSETS_DIR,SECTIONS,og,collect,STARS,COMMON_AF,RUN_ALL_STEPS,open_text,info_map,read_tsv,hla_quality,HLA_MAX_CANDIDATES,EXPLORE_DOCS,csq_rank bin/owngenome_live.py,bin/render_report.py,bin/results_index.py,tests/demo/make_demo_sample.py,tests/test_collect_summary.py,tests/test_live_journal.py,tests/test_live_results.py,tests/test_render_report.py,tests/test_run_state.py,tests/test_viewer.py R6b +bin/collect_summary.py HERE,ASSETS_DIR,SECTIONS,og,collect,STARS,COMMON_AF,RUN_ALL_STEPS,open_text,info_map,read_tsv,hla_quality,HLA_MAX_CANDIDATES,EXPLORE_DOCS,csq_rank bin/owngenome_live.py,bin/render_report.py,bin/results_index.py,tests/demo/make_demo_sample.py,tests/test_collect_summary.py,tests/test_live_journal.py,tests/test_live_results.py,tests/test_run_state.py,tests/test_viewer.py R6b diff --git a/tests/schema/summary.schema.json b/tests/schema/summary.schema.json index db8383557..f06c11c0a 100644 --- a/tests/schema/summary.schema.json +++ b/tests/schema/summary.schema.json @@ -1881,6 +1881,12 @@ "type": "string" } }, + "selected_no_result": { + "type": "array", + "items": { + "type": "string" + } + }, "not_assessed": { "type": "array", "items": { diff --git a/tests/schema/validate.py b/tests/schema/validate.py index d807fe17c..7a37db985 100644 --- a/tests/schema/validate.py +++ b/tests/schema/validate.py @@ -1,10 +1,9 @@ -"""validate.py: an in-process JSON Schema check for three Python test files. +"""validate.py: an in-process JSON Schema check for one Python test file. The check CI and the e2e cases run is `og dev schema SCHEMA DOC` (rust/crates/og-dev/src/schema.rs). This module stays only because -tests/test_collect_summary.py, tests/test_hla_report.py and -tests/test_render_report.py call validate(schema, doc) in-process, in the -unit-tests job, which has no og. It goes when those tests move to Rust. +tests/test_collect_summary.py calls validate(schema, doc) in-process, from +Python. It goes when that test moves to Rust. It follows og dev schema's rules: the keywords tests/schema/summary.schema.json uses (type, enum, required, properties, additionalProperties as a schema, true diff --git a/tests/test_annotsv_postfilter.sh b/tests/test_annotsv_postfilter.sh index 861c61639..26e4595f5 100755 --- a/tests/test_annotsv_postfilter.sh +++ b/tests/test_annotsv_postfilter.sh @@ -3,7 +3,7 @@ # The class 4 and 5 checks with the real tools: step 05 (AnnotSV, then # og sv postfilter in ANNOTSV_IMAGE) and step 22 (SURVIVOR_IMAGE, # BCFTOOLS_IMAGE) on the made-up data of tests/fixtures/annotsv/class45.sh, -# then both reports from bin/render_report.py. +# then both reports from og report render. # # Checks: # 1. step 05 writes _sv_class4_5_checked.tsv from what AnnotSV @@ -89,7 +89,7 @@ expect "the consensus: DEL2 only, two callers" \ # --- 3. the reports ----------------------------------------------------------------------- report() { - python3 "${REPO}/bin/render_report.py" --sample s1 --sample-dir "$S" --json "${WORK}/summary.json" \ + "${REPO}/bin/og" report render --sample s1 --sample-dir "$S" --json "${WORK}/summary.json" \ -o "${WORK}/report.txt" -o "${WORK}/report.html" > /dev/null } report diff --git a/tests/test_collect_summary.py b/tests/test_collect_summary.py index c13869ea3..3a07a20b0 100644 --- a/tests/test_collect_summary.py +++ b/tests/test_collect_summary.py @@ -1094,26 +1094,9 @@ def annotsv_section(work, schema): av = sec["data"] check("annotsv: the section is read", sec["state"] == "ok" and av["count"] == 3, sec) by = {x["sv"]: x for x in av.get("rows", [])} - check("annotsv: the two-caller deletion has support 2, manta and delly", - (by["20:10100999-10102999"]["support"], by["20:10100999-10102999"]["support_callers"]) == (2, ["manta", "delly"]), - by.get("20:10100999-10102999")) - check("annotsv: the deletion only Manta made has support 1", by["20:20000000-20450000"]["support"] == 1, - by.get("20:20000000-20450000")) check("annotsv: the intron-exact deletion is on exon edges, with its AF and common flag", (by["20:1000200-1009800"]["exon_edge"], by["20:1000200-1009800"]["population_af"], by["20:1000200-1009800"]["common"]) == ("yes", 0.05, True), by.get("20:1000200-1009800")) - check("annotsv: the counts", (av["single_caller"], av["exon_edge"], av["common"]) == (2, 1, 1), av) - lines = ["Class 4-5 SVs: 3 (single caller: 2; deletions on exon edges: 1; common: 1)", - "DEL 20:10100999-10102999 class 5 G1, G2, G3 and 2 more support: 2 callers (manta, delly)", - "DEL 20:20000000-20450000 class 4 G9 support: single caller (manta)", - "on exon edges: NM_900001 (FIXG1): exon 1 and exon 3 edges", - "benign-set AF 0.05 (gnomAD-SV_v4.1_DEL_fixture), common"] - for x in lines: - check(f"text report: {x[:60]}", x in txt, txt) - for x in ("Structural Variants, Class 4 and 5 (AnnotSV)", "2 callers (manta, delly)", "single caller (manta)", - "on exon edges: NM_900001 (FIXG1): exon 1 and exon 3 edges", - "Class 4-5 SVs: 3 (single caller: 2; deletions on exon edges: 1; common: 1)"): - check(f"HTML report: {x[:60]}", x in html_, x) check("annotsv: the summary validates", not validate.validate(schema, json.loads(json.dumps(summ))), validate.validate(schema, json.loads(json.dumps(summ)))) @@ -1137,13 +1120,6 @@ def annotsv_section(work, schema): check("annotsv rules: SV3 is listed apart, AnnotSV class 4, now 3, with the rule and its detail", [(x["sv"], x["annotsv_class"], x["acmg_class"], x["rule"]) for x in moved] == [("20:20000000-20450000", "4", "3", "ig_tr_locus")] and "IGFIX" in moved[0]["rule_detail"], moved) - check("annotsv rules: the counts leave the moved call out (single caller 1, not 2)", - (av["single_caller"], av["exon_edge"], av["common"]) == (1, 1, 1), av) - check("annotsv rules: the text report counts 2 and lists SV3 apart with the reason", - "Class 4-5 SVs: 2 (single caller: 1; deletions on exon edges: 1; common: 1)" in txt - and "Moved to class 3 by step 05's rules, not counted above: 1" in txt - and "DEL 20:20000000-20450000 G9 AnnotSV class 4 -> 3 (ig_tr_locus: " in txt - and "DEL 20:20000000-20450000 class" not in txt, txt) check("annotsv rules: the HTML card has the moved table with the rule and the reason", "

    Moved to class 3 by step 05's rules

    " in html_ and "
    " in html_ diff --git a/tests/test_install_docs_order.sh b/tests/test_install_docs_order.sh new file mode 100644 index 000000000..da79c3b05 --- /dev/null +++ b/tests/test_install_docs_order.sh @@ -0,0 +1,287 @@ +#!/usr/bin/env bash +# A new user must see what to install before the first command to copy, and +# one ordered install path for their platform. +# +# README.md: +# - a "Before you begin" list (Docker, bash, Java, Nextflow, RAM) comes +# before the first fenced code block; +# - its RAM line says a 16 GB machine needs swap for step 25 (25-prs #memory); +# - the first code block clones the repository and checks out the newest +# release tag, so setup downloads og instead of building it. The tag is +# picked with exactly git tag --list 'v[0-9]*' --sort=-v:refname, so a +# fixture-* or e2e-data-* tag never wins. +# docs/getting-started.md: +# - the RAM row of the hardware table names swap for step 25 (25-prs.md#memory); +# - "### Get the code" clones and checks out the release tag the same way; +# - four sections, "### Linux", "### Intel Mac", "### Apple Silicon Mac" and +# "### Windows with WSL2", each installing Java and the pinned Nextflow, +# with Docker named; each section's last code block is the verification +# block (docker info, bash --version, java -version, nextflow -version) +# and a "Next" line follows it; +# - inside each section the steps run in an order that works on a new +# machine: the Java install line comes before the first NXF_VER= line +# (the Nextflow installer needs Java); in the two Mac sections the +# Homebrew install line comes before the first brew command, and on +# Apple Silicon so does the brew shellenv eval (/opt/homebrew is not on +# PATH before it); on Linux and WSL2 the apt-get install line, which +# installs curl, comes before the first curl command; +# - every NXF_VER= in the page is NEXTFLOW_VERSION from versions.env +# (a value such as latest fails too). +# Planted copies with one defect each must fail (a check that cannot fail is +# not a check). +# +# Run: bash tests/test_install_docs_order.sh +set -euo pipefail + +REPO=$(cd "$(dirname "$0")/.." && pwd) +README="${REPO}/README.md" +GUIDE="${REPO}/docs/getting-started.md" +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +FAILS=0 +fail() { echo "FAIL: $*"; FAILS=$((FAILS + 1)); } +pass() { echo "ok: $*"; } + +NXF_PIN=$(sed -n 's/^NEXTFLOW_VERSION="\([0-9.]*\)".*/\1/p' "${REPO}/versions.env") +[ -n "$NXF_PIN" ] || { echo "FAIL: no NEXTFLOW_VERSION in versions.env"; exit 1; } + +SECTIONS=("Linux" "Intel Mac" "Apple Silicon Mac" "Windows with WSL2") +VERIFY_CMDS=("docker info" "bash --version" "java -version" "nextflow -version") +# The newest release tag: only v tags, so fixture-* and e2e-data-* lose. +TAG_PICK="git tag --list 'v[0-9]*' --sort=-v:refname | head -n 1" +# Lines inside a section, as extended regexes. +JAVA_RE='^[[:space:]]*(sudo apt-get install .*openjdk-17|brew install .*temurin@17)' +NXF_RE='NXF_VER=' +BREW_RE='^[[:space:]]*brew ' +SHELLENV_RE='^[[:space:]]*eval "\$\(/opt/homebrew/bin/brew shellenv\)"' +APT_RE='^[[:space:]]*sudo apt-get install ' +CURL_RE='^[[:space:]]*curl ' + +# section FILE TITLE: the lines under "### TITLE" up to the next heading of +# level 1 to 3 outside a code block. +section() { + awk -v h="### $2" ' + /^[ \t]*```/ { fence = !fence } + !fence && $0 == h { on = 1; next } + on && !fence && /^#{1,3} / { exit } + on { print } + ' "$1" +} + +# last_block: the contents of the last fenced code block on stdin. +last_block() { + awk ' + /^[ \t]*```/ { if (fence) { fence = 0 } else { fence = 1; n++; delete b; k = 0 }; next } + fence { b[++k] = $0 } + END { for (i = 1; i <= k; i++) print b[i] } + ' +} + +# first_block: the contents of the first fenced code block on stdin. +first_block() { + awk '/^[ \t]*```/ { if (fence) exit; fence = 1; next } fence { print }' +} + +# first_line REGEX: the number of the first line on stdin matching the +# extended REGEX, or nothing. +first_line() { + grep -n -E -- "$1" | head -n 1 | cut -d: -f1 || true +} + +# check_checkout WHERE BLOCK: BLOCK clones the repository and checks out the +# newest release tag with the release filter. Prints each problem. +check_checkout() { + local where=$1 block=$2 rc=0 + if ! grep -q 'git clone https://github.com/GeiserX/OwnGenome' <<< "$block"; then + echo "${where} does not clone the repository"; rc=1 + fi + # shellcheck disable=SC2016 # the literal command the docs show + if ! grep -q 'git checkout "$(git tag' <<< "$block"; then + echo "${where} does not check out the newest release tag"; rc=1 + elif ! grep -qF "git checkout \"\$(${TAG_PICK})\"" <<< "$block"; then + echo "${where} picks the tag without the release filter (want: ${TAG_PICK})"; rc=1 + fi + return "$rc" +} + +# check_readme FILE: prints each problem; returns 1 when there is one. +check_readme() { + local f=$1 rc=0 first_fence begin before block word + first_fence=$(grep -n '^```' "$f" | head -n 1 | cut -d: -f1) + begin=$(grep -n -i 'before you begin' "$f" | head -n 1 | cut -d: -f1) + if [ -z "$first_fence" ]; then + echo "no fenced code block"; return 1 + fi + if [ -z "$begin" ] || [ "$begin" -ge "$first_fence" ]; then + echo "no \"Before you begin\" list before the first code block (line ${first_fence})" + rc=1 + else + before=$(sed -n "${begin},${first_fence}p" "$f") + for word in Docker bash Java Nextflow RAM; do + grep -q "$word" <<< "$before" || { echo "\"Before you begin\" does not name ${word}"; rc=1; } + done + if ! grep 'RAM' <<< "$before" | grep -q 'swap'; then + echo "the RAM line does not say a 16 GB machine needs swap"; rc=1 + fi + if ! grep 'RAM' <<< "$before" | grep -q '25-prs/#memory'; then + echo "the RAM line does not link step 25's memory section"; rc=1 + fi + fi + block=$(awk -v s="$first_fence" 'NR > s && /^[ \t]*```/ { exit } NR > s { print }' "$f") + check_checkout "the first code block" "$block" || rc=1 + return "$rc" +} + +# check_guide FILE: prints each problem; returns 1 when there is one. +check_guide() { + local f=$1 rc=0 t body block cmd brew_line hb_line env_line java_line nxf_line apt_line curl_line v + if ! grep -E '^\| \*\*RAM\*\*' "$f" | grep -q 'swap'; then + echo "the RAM row does not say a 16 GB machine needs swap"; rc=1 + fi + if ! grep -E '^\| \*\*RAM\*\*' "$f" | grep -q '25-prs.md#memory'; then + echo "the RAM row does not link 25-prs.md#memory"; rc=1 + fi + body=$(section "$f" "Get the code") + if [ -z "$body" ]; then + echo "no \"### Get the code\" section"; rc=1 + else + check_checkout "Get the code: its first code block" "$(first_block <<< "$body")" || rc=1 + fi + for t in "${SECTIONS[@]}"; do + body=$(section "$f" "$t") + if [ -z "$body" ]; then + echo "no \"### ${t}\" section"; rc=1; continue + fi + grep -q 'Docker' <<< "$body" || { echo "${t}: does not install Docker"; rc=1; } + grep -qE 'openjdk-17|temurin@17' <<< "$body" || { echo "${t}: does not install Java"; rc=1; } + grep -q "NXF_VER=" <<< "$body" || { echo "${t}: does not install the pinned Nextflow"; rc=1; } + block=$(last_block <<< "$body") + for cmd in "${VERIFY_CMDS[@]}"; do + grep -qF "$cmd" <<< "$block" || { echo "${t}: the last code block does not run '${cmd}'"; rc=1; } + done + if ! awk '/^[ \t]*```/ { fence = !fence; seen = 1; after = 0; next } !fence && seen && /Next/ { after = 1 } END { exit !after }' <<< "$body"; then + echo "${t}: no \"Next\" line after the verification block"; rc=1 + fi + # Line numbers below count from the top of the section. + java_line=$(first_line "$JAVA_RE" <<< "$body") + nxf_line=$(first_line "$NXF_RE" <<< "$body") + if [ -z "$java_line" ]; then + echo "${t}: no apt-get openjdk-17 or brew temurin@17 install line"; rc=1 + elif [ -n "$nxf_line" ] && [ "$nxf_line" -lt "$java_line" ]; then + echo "${t}: installs Nextflow (line ${nxf_line} of the section) before Java (line ${java_line})"; rc=1 + fi + case "$t" in + *Mac) + brew_line=$(first_line "$BREW_RE" <<< "$body") + hb_line=$(first_line 'raw\.githubusercontent\.com/Homebrew/install' <<< "$body") + if [ -z "$hb_line" ]; then + echo "${t}: no Homebrew install line"; rc=1 + elif [ -n "$brew_line" ] && [ "$brew_line" -lt "$hb_line" ]; then + echo "${t}: runs brew (line ${brew_line} of the section) before installing Homebrew (line ${hb_line})"; rc=1 + fi + ;; + Linux|*WSL2) + apt_line=$(first_line "$APT_RE" <<< "$body") + curl_line=$(first_line "$CURL_RE" <<< "$body") + if [ -z "$apt_line" ]; then + echo "${t}: no apt-get install line"; rc=1 + elif [ -n "$curl_line" ] && [ "$curl_line" -lt "$apt_line" ]; then + echo "${t}: runs curl (line ${curl_line} of the section) before apt-get installs it (line ${apt_line})"; rc=1 + fi + ;; + esac + if [ "$t" = "Apple Silicon Mac" ]; then + env_line=$(first_line "$SHELLENV_RE" <<< "$body") + brew_line=$(first_line "$BREW_RE" <<< "$body") + if [ -z "$env_line" ]; then + echo "${t}: never puts /opt/homebrew on the PATH (no eval of brew shellenv)"; rc=1 + elif [ -n "$brew_line" ] && [ "$brew_line" -lt "$env_line" ]; then + echo "${t}: runs brew (line ${brew_line} of the section) before brew shellenv puts it on the PATH (line ${env_line})"; rc=1 + fi + fi + done + while read -r v; do + [ "$v" = "$NXF_PIN" ] || { echo "NXF_VER=${v} is not NEXTFLOW_VERSION ${NXF_PIN}"; rc=1; } + done < <(grep -oE 'NXF_VER=[^[:space:]]*' "$f" | cut -d= -f2) + return "$rc" +} + +# expect_fail NAME CHECK FILE WANT: a planted copy must fail CHECK, and the +# problems it prints must include WANT (so it fails for the planted defect). +expect_fail() { + local name=$1 check=$2 file=$3 want=$4 out + if out=$("$check" "$file"); then + fail "control: ${name} passed" + elif ! grep -qF -- "$want" <<< "$out"; then + fail "control: ${name} failed, but not with \"${want}\": ${out}" + else + pass "control: ${name} fails: $(grep -F -- "$want" <<< "$out" | head -n 1)" + fi +} + +# move_after FILE TITLE LINE_RE ANCHOR_RE: FILE with, inside "### TITLE", the +# lines matching LINE_RE taken out and put back after the first later line +# matching ANCHOR_RE (a planted step-order defect). +move_after() { + awk -v h="### $2" -v pat="$3" -v anc="$4" ' + $0 == h { on = 1; print; next } + /^[ \t]*```/ { fence = !fence } + on && !fence && /^#{1,3} / { on = 0 } + on && !done && $0 ~ pat { held = held $0 "\n"; next } + on && !done && held != "" && $0 ~ anc { print; printf "%s", held; done = 1; next } + { print } + ' "$1" +} + +# The real files. +if out=$(check_readme "$README"); then + pass "README.md lists the requirements before the first command and checks out a release" +else + fail "README.md:"$'\n'"${out}" +fi +if out=$(check_guide "$GUIDE"); then + pass "docs/getting-started.md has the four ordered install sections" +else + fail "docs/getting-started.md:"$'\n'"${out}" +fi + +# Controls on README.md. +# shellcheck disable=SC2016 # backticks of a Markdown fence, not a command +{ printf '```bash\necho first\n```\n\n'; cat "$README"; } > "${WORK}/readme-early-block.md" +expect_fail "a code block above \"Before you begin\"" check_readme "${WORK}/readme-early-block.md" 'before the first code block' +# shellcheck disable=SC2016 # sed matches the literal $( of the README line +sed '/git checkout "\$(git tag/d' "$README" > "${WORK}/readme-main.md" +expect_fail "a clone that stays on main" check_readme "${WORK}/readme-main.md" 'newest release tag' +sed 's/--list .v\[0-9\]\*. //' "$README" > "${WORK}/readme-nofilter.md" +expect_fail "a release checkout without the v[0-9]* filter" check_readme "${WORK}/readme-nofilter.md" 'the first code block picks the tag without the release filter' +sed '/RAM/s/swap/memory/g' "$README" > "${WORK}/readme-noswap.md" +expect_fail "a RAM line with no swap" check_readme "${WORK}/readme-noswap.md" 'needs swap' + +# Controls on docs/getting-started.md. +sed '/^| \*\*RAM\*\*/s/swap/memory/g' "$GUIDE" > "${WORK}/guide-noswap.md" +expect_fail "a RAM row with no swap" check_guide "${WORK}/guide-noswap.md" 'RAM row does not say' +sed 's/^### Intel Mac$/### Intel Macs/' "$GUIDE" > "${WORK}/guide-nosection.md" +expect_fail "a missing Intel Mac section" check_guide "${WORK}/guide-nosection.md" 'no "### Intel Mac" section' +awk '$0 == "### Apple Silicon Mac" { on = 1 } on && /^nextflow -version$/ { on = 0; next } { print }' "$GUIDE" > "${WORK}/guide-noverify.md" +expect_fail "a verification block without nextflow -version" check_guide "${WORK}/guide-noverify.md" "Apple Silicon Mac: the last code block does not run 'nextflow -version'" +awk '$0 == "### Intel Mac" { on = 1 } on && /^```bash$/ && !done { print; print "brew install wget"; done = 1; next } { print }' "$GUIDE" > "${WORK}/guide-brew-first.md" +expect_fail "a Mac section that runs brew before installing Homebrew" check_guide "${WORK}/guide-brew-first.md" 'Intel Mac: runs brew' +sed 's/--list .v\[0-9\]\*. //' "$GUIDE" > "${WORK}/guide-nofilter.md" +expect_fail "a Get the code checkout without the v[0-9]* filter" check_guide "${WORK}/guide-nofilter.md" 'Get the code: its first code block picks the tag without the release filter' +move_after "$GUIDE" "Intel Mac" 'temurin@17' 'NXF_VER=' > "${WORK}/guide-java-late.md" +expect_fail "a section that installs Nextflow before Java" check_guide "${WORK}/guide-java-late.md" 'Intel Mac: installs Nextflow' +move_after "$GUIDE" "Apple Silicon Mac" 'brew shellenv' 'brew install --cask' > "${WORK}/guide-shellenv-late.md" +expect_fail "an Apple Silicon section that runs brew before brew shellenv" check_guide "${WORK}/guide-shellenv-late.md" 'before brew shellenv puts it on the PATH' +move_after "$GUIDE" "Linux" 'apt-get install' 'get.docker.com' > "${WORK}/guide-curl-first.md" +expect_fail "a Linux section that runs curl before apt-get installs it" check_guide "${WORK}/guide-curl-first.md" 'Linux: runs curl' +awk -v pin="NXF_VER=${NXF_PIN}" '!done && index($0, pin) { sub(pin, "NXF_VER=1.0.0"); done = 1 } { print }' "$GUIDE" > "${WORK}/guide-oldnxf.md" +expect_fail "an NXF_VER that is not the pin" check_guide "${WORK}/guide-oldnxf.md" 'NXF_VER=1.0.0 is not' +awk -v pin="NXF_VER=${NXF_PIN}" '!done && index($0, pin) { sub(pin, "NXF_VER=latest"); done = 1 } { print }' "$GUIDE" > "${WORK}/guide-latestnxf.md" +expect_fail "an unpinned NXF_VER=latest" check_guide "${WORK}/guide-latestnxf.md" 'NXF_VER=latest is not' + +if [ "$FAILS" -gt 0 ]; then + echo "${FAILS} check(s) failed" + exit 1 +fi +echo "All install-docs checks passed." diff --git a/tests/test_owngenome_live.py b/tests/test_owngenome_live.py index 8fcb85e2b..bef434a5f 100644 --- a/tests/test_owngenome_live.py +++ b/tests/test_owngenome_live.py @@ -14,29 +14,25 @@ a negative Content-Length 413, a body that is not JSON 400, a run named '../x' 400; metadata.parameters (an HPO list, a VCF path) and a task's script are not stored; - 5. the event stream sends a posted message within 2 s, Last-Event-ID - resumes after the id given, what it sends equals messages(), and an open - stream ends at logout; - 6. state.json's step states are run_state's for the same files; a run is - finished only while pgp_run_finished is its newest message; ClinVar's - headline is never 'ok' and carries the report's caveat; - 7. every browser response carries the Content-Security-Policy (the scan of - assets/live/ for outside URLs, inline scripts, on* attributes and eval - is cargo test -p og-browser, tests/live_assets.rs); - 8. a plugin (tests/fixtures/live/plugins) adds a flag, a route, a run file + 5. an open event stream ends at logout; + 6. a plugin (tests/fixtures/live/plugins) adds a flag, a route, a run file and a state hook, all behind the session; the page gets one script tag per file in assets/live/plugins; - 9. run files: a file the report links to is served, an unlinked one and a - symlink out of the root are 404; - 10. in a headless Chrome (skipped without Chrome, except in CI): the login - form, the step map, a box that repaints after a posted message, and no - policy violation (the replay mode is cargo test -p og-browser, - tests/live_replay.rs); - 11. with --tls-cert, a client that connects and sends nothing does not hold + 7. in a headless Chrome (skipped without Chrome, except in CI): a page + without a session goes to the login form, which leads back to the run + page; + 8. with --tls-cert, a client that connects and sends nothing does not hold up another client's HTTPS request. -Checks 2, 4, 5, 6, 8 and 11 are also run against a planted mistake and must fail +Checks 2, 4, 6 and 8 are also run against a planted mistake and must fail there: a check that cannot fail is no check. +The rest of the site is checked against og live serve by cargo test -p og-live +(tests/port_live_server*.rs): the event stream's messages and Last-Event-ID, +state.json and the headlines, the policy on every answer, the run files, the +map, the re-check plugin and the step map in Chrome. The scan of assets/live/ +is cargo test -p og-browser (tests/live_assets.rs), the replay mode +tests/live_replay.rs. + Run: python3 tests/test_owngenome_live.py """ import contextlib @@ -65,7 +61,6 @@ RUN = "DEMO-001" PASSWORD = "synthetic test password 7" FAILS = [] -RESPONSES = [] # (path, headers) of every browser response def check(desc, ok, detail=""): @@ -79,7 +74,7 @@ def control(desc, still_passes): check(f"control: {desc}", not still_passes, "the check passed on the planted mistake") -def request(port, method, path, body=None, headers=None, record=True): +def request(port, method, path, body=None, headers=None): c = http.client.HTTPConnection("127.0.0.1", port, timeout=10) h = {"Host": f"127.0.0.1:{port}"} h.update(headers or {}) @@ -88,8 +83,6 @@ def request(port, method, path, body=None, headers=None, record=True): data = r.read() hdr = {k.lower(): v for k, v in r.getheaders()} c.close() - if record: - RESPONSES.append((path, hdr)) return r.status, hdr, data @@ -152,7 +145,7 @@ def login(port, password=PASSWORD): def post(port, run, ev): body = ev if isinstance(ev, (bytes, str)) else json.dumps(ev) - return request(port, "POST", f"/ingest/{run}", body, {"Content-Type": "application/json"}, record=False) + return request(port, "POST", f"/ingest/{run}", body, {"Content-Type": "application/json"}) class Stream: @@ -382,7 +375,7 @@ def close(self): self.p.kill() -def browser_checks(port, work, live): +def browser_checks(port, work): chrome = find_chrome() if not chrome: ci = os.environ.get("CI") == "true" @@ -402,53 +395,6 @@ def browser_checks(port, work, live): ok = b.wait(f"location.pathname === '/runs/{RUN}/' && document.documentElement.dataset.pgpStatus === 'ready'") check("browser: the login form leads back to the run page, which loads", ok, b.eval("location.href + ' ' + document.documentElement.outerHTML.slice(0, 600)")) - n = b.eval("document.querySelectorAll('#map g.box').length") - check("browser: the step map draws one box per step shown", isinstance(n, int) and n >= 4, n) - check("browser: a finished step is painted done", - b.eval("document.querySelector('g.box[data-step=\"06\"]').dataset.state") == "done", - b.eval("document.querySelector('g.box[data-step=\"06\"]') && document.querySelector('g.box[data-step=\"06\"]').dataset.state")) - before = b.eval("document.querySelector('g.box[data-step=\"07\"]').dataset.state") - post(INGEST_PORT, RUN, {"event": "process_started", "utcTime": "2026-01-02T10:07:00Z", - "trace": {"task_id": 20, "name": f"PGX:PHARMCAT_PREPROCESS ({RUN})", - "process": "PGX:PHARMCAT_PREPROCESS", "tag": RUN, "status": "RUNNING", - "start": 1767348420000}}) - after = b.wait("document.querySelector('g.box[data-step=\"07\"]').dataset.state === 'running' && 'running'", 10) - check("browser: a box repaints after a posted message (07 waiting -> running)", - before == "waiting" and after == "running", (before, after)) - - class Refuse: # every stream slot taken: the stream answers 503 - def acquire(self, blocking=True): - return False - - def release(self): - pass - real_streams, live.streams = live.streams, Refuse() - try: - b.go(f"{base}/runs/{RUN}/") - refused = b.wait("document.documentElement.dataset.pgpStatus === 'offline' || " - "(document.documentElement.dataset.pgpStreams === '1' && " - "document.documentElement.dataset.pgpStatus === 'ready' && window.PGP.state && 'loaded')", 10) - time.sleep(1) - finally: - live.streams = real_streams - reopened = b.wait("Number(document.documentElement.dataset.pgpStreams) >= 2 && 'reopened'", 15) - n0 = b.eval("window.PGP.state.events") - time.sleep(1) - post(INGEST_PORT, RUN, {"event": "pgp_step", "step": "06", "state": "ok", "utcTime": "2026-01-02T10:07:30Z"}) - n1 = b.wait(f"window.PGP.state && window.PGP.state.events > {n0} && window.PGP.state.events", 10) - check("browser: a stream refused with 503 is opened again, and its messages update the page", - refused and reopened == "reopened" and n1, (refused, reopened, n0, n1)) - b.eval("document.querySelector('g.box[data-step=\"11\"]').dispatchEvent(new MouseEvent('click', {bubbles: true})); true") - detail = b.wait("document.getElementById('detail').textContent.indexOf('Step 11') >= 0 && " - "document.getElementById('detail').textContent") - check("browser: a click on a box opens its detail with its tasks", detail and "ROH" in detail, detail) - b.eval("document.querySelector('g.box[data-step=\"06\"]').dispatchEvent(new MouseEvent('click', {bubbles: true})); true") - links = b.wait("document.getElementById('detail').textContent.indexOf('Step 06') >= 0 && " - "Array.from(document.querySelectorAll('#detail a')).map(function (a) { return a.getAttribute('href'); }).join(' ')") - check("browser: the detail links each section to its report anchor", - links and "report.html#clinvar" in links and "report.html#carrier-list" in links, links) - blocked = b.eval("document.documentElement.dataset.pgpBlocked || ''") - check("browser: the policy blocked nothing on the live page", not blocked, blocked) finally: b.close() @@ -642,7 +588,7 @@ def dropped(p, handler, t): code2, _, _ = post(INGEST_PORT, "-x", {"event": "started"}) check("a run named '../x' or '-x' is 400", code == 400 and code2 == 400, (code, code2)) code, _, _ = request(INGEST_PORT, "POST", f"/ingest/{RUN}", '{"event":"started"}', - {"Origin": "https://evil.example.net"}, record=False) + {"Origin": "https://evil.example.net"}) check("a browser's post (an Origin header) is 403", code == 403, code) stored = open(live.store.path(RUN)).read() @@ -653,126 +599,25 @@ def stored_leaks(text): control("the raw message stored", not stored_leaks(open(os.path.join(FIX, "events.jsonl")).read())) # 5. the event stream - steps = run_state.read_steps() n0 = len(live.store.events(RUN)) - st = Stream(port, cookie) - got_all = st.wait(n0, 5) - t0 = time.time() - post(INGEST_PORT, RUN, {"event": "process_completed", "utcTime": "2026-01-02T10:06:30Z", - "trace": {"task_id": 9, "name": f"CLINICAL:CLINVAR_SCREEN ({RUN})", - "process": "CLINICAL:CLINVAR_SCREEN", "tag": RUN, "status": "COMPLETED", - "exit": 0, "start": 1767348360000, "complete": 1767348390000}}) - live_ok = st.wait(n0 + 1, 2) - check("the stream sends a posted process_completed within 2 s", - got_all and live_ok and st.items[-1][1].get("event") == "process_completed" and st.items[-1][1].get("step") == "06", - (got_all, live_ok, time.time() - t0, st.items[-1:])) - want = owngenome_live.messages(live.store.events(RUN), steps) - check("what the stream sent equals messages()", st.items == want, (st.items[:3], want[:3])) - st.close() - st2 = Stream(port, cookie, last=n0 - 1) - st2.wait(2, 5) - check("Last-Event-ID resumes after the id given", [i for i, _ in st2.items] == [n0, n0 + 1], - [i for i, _ in st2.items]) - st2.close() _, _, gone = login(port) st3 = Stream(port, gone) - st3.wait(n0 + 1, 5) + st3.wait(n0, 5) k = len(st3.items) live.end_session(__import__("hashlib").sha256(gone.encode()).hexdigest()) post(INGEST_PORT, RUN, {"event": "pgp_step", "step": "06", "state": "ok", "utcTime": "2026-01-02T10:06:40Z"}) end = time.time() + 3 while time.time() < end and not st3.closed: time.sleep(0.02) - check("an open stream ends when its session does, and sends nothing after", st3.closed and len(st3.items) == k, - (st3.closed, len(st3.items), k)) + check("an open stream ends when its session does, and sends nothing after", + k == n0 and st3.closed and len(st3.items) == k, (st3.closed, len(st3.items), k, n0)) st3.close() - check("no message carries a path or a tag", - not any(k in json.dumps(want) for k in ("/data/", "workDir", "outdir", '"tag"')), json.dumps(want)[:300]) - # 6. state.json + # 6. plugins _, _, body = request(port, "GET", f"/runs/{RUN}/state.json", headers=auth) s = json.loads(body) - ref = run_state.collect(RUN, os.path.join(root, RUN), events_path=live.store.path(RUN)) - check("state.json's step states are run_state's", - [(x["step"], x["state"]) for x in s["steps"]] == [(x["step"], x["state"]) for x in ref["steps"]], - [(x["step"], x["state"]) for x in s["steps"] if x["state"] != "not planned"]) - mine = {x["step"]: x["state"] for x in s["steps"]} - check("the fixture's steps: 06 done, 07 waiting, 11 done, 12 skipped", - (mine["06"], mine["07"], mine["11"], mine["12"]) == ("done", "waiting", "done", "skipped"), mine) - check("state.json names each section's report anchor (ClinVar: clinvar, the carrier list: carrier-list)", - s.get("anchors", {}).get("clinvar") == "clinvar" and s["anchors"].get("carrier") == "carrier-list" - and s["anchors"].get("roh") == "roh", s.get("anchors")) - done = [{"event": "started"}, {"event": "pgp_run_finished", "exit": 0}] - rerun = done + [{"event": "pgp_step", "step": "14", "state": "started"}] - check("a run is finished while pgp_run_finished is its newest message, not once a later run posts", - owngenome_live.run_finished(done) and not owngenome_live.run_finished(rerun) and not owngenome_live.run_finished([])) - control("any pgp_run_finished in the file counts", not any(e.get("event") == "pgp_run_finished" for e in rerun)) - # A second run of a sample: events.jsonl still holds the first run, which finished with step 11 done. - again = "DEMO-RERUN" - ad = os.path.join(work, "rerun", again) - os.makedirs(os.path.join(ad, "logs")) - with open(os.path.join(ad, "logs", "run_status.tsv"), "w") as f: - f.write(f"meta\tstarted_epoch\t{int(time.time())}\nstep\t11\tplanned\n") - roh = {"task_id": 8, "name": f"CLINICAL:ROH ({again})", "process": "CLINICAL:ROH", "tag": again, - "status": "COMPLETED", "exit": 0, "submit": 1767348300000, "start": 1767348302000, - "complete": 1767348340000} - old = [{"event": "started"}, {"event": "process_completed", "trace": roh}, {"event": "completed"}, - {"event": "pgp_run_finished", "exit": 0}, {"event": "started"}] - live.store.runs[again] = list(old) - rs = live.state({"name": again, "dir": ad}) - r11 = next(x["state"] for x in rs["steps"] if x["step"] == "11") - check("a rerun's state.json is the new run's: running, step 11 not done, every message still counted", - rs["run"]["state"] == "running" and r11 != "done" and rs["events"] == len(old), (rs["run"], r11)) - evf = os.path.join(work, "rerun", "events.jsonl") - with open(evf, "w") as f: - f.writelines(json.dumps(e) + "\n" for e in old) - whole = run_state.collect(again, ad, events_path=evf) - control("state built from the whole events file", - whole["run"]["state"] == "running" - and next(x["state"] for x in whole["steps"] if x["step"] == "11") != "done") - check("a bare nextflow rerun (no pgp_run_finished) starts at its own 'started'", - owngenome_live.current_run([{"event": "started"}, {"event": "completed"}, {"event": "started"}, - {"event": "process_started"}]) == [{"event": "started"}, {"event": "process_started"}]) - check("a finished run keeps its own messages while pgp_run_finished is the newest", - owngenome_live.current_run(old[:4]) == old[:4]) - cp = {"plp": [{"gene": "GENEA"}], "call_floor_dp": 10, "low_confidence": 0} - tc = [owngenome_live.headline("cpsr", {"state": "ok", "title": "CPSR", "data": dict(cp, plp=p)}, {"databases": {}}) - for p in ([], cp["plp"])] - tm = [owngenome_live.headline("mito", {"state": "ok", "title": "Mito", - "data": {"annotation": {"source": "VEP", "clinvar_plp": n}}}) for n in (0, 2)] - check("CPSR and mito headlines: attention with P/LP rows, info with none, the report's own lines", - [t[1] for t in tc + tm] == ["info", "attention", "info", "attention"] - and "Pathogenic/Likely Pathogenic rows: 1" in tc[1][0] - and tm[1][0] == "ClinVar pathogenic or likely pathogenic: 2", tc + tm) - table = owngenome_live.HEADLINES - owngenome_live.HEADLINES = {k: v for k, v in table.items() if k not in ("cpsr", "mito")} - try: - planted = [owngenome_live.headline(k, {"state": "ok", "title": k, "data": data}, {"databases": {}})[1] - for k, data in (("cpsr", cp), ("mito", {"annotation": {"source": "VEP", "clinvar_plp": 2}}))] - finally: - owngenome_live.HEADLINES = table - control("the headline table without cpsr and mito", planted == ["attention", "attention"]) check("a state hook's answer is under plugins", s["plugins"].get("fixture", {}).get("steps") == len(s["steps"]), s.get("plugins")) - import render_report - tones = [owngenome_live.headline("clinvar", {"state": "ok", "title": "ClinVar screen", "data": {"count": c, "hits": []}}) - for c in (0, 3)] - check("the ClinVar headline is never 'ok' and carries the report's caveat word for word", - all(t[1] in ("info", "attention") for t in tones) and tones[1][1] == "attention" - and all(render_report.CLINVAR_FREQ_NOTE in t[0] for t in tones), tones) - check("a section with no file has no headline", owngenome_live.headline("roh", {"state": "missing"}) is None) - - # 7. the policy - code, _, _ = request(port, "GET", "/login") - code, _, _ = request(port, "GET", "/assets/live.css") - code, _, _ = request(port, "GET", "/nothing/here", headers=auth) - bad = [p for p, hdr in RESPONSES if "default-src 'none'" not in hdr.get("content-security-policy", "") - or "frame-ancestors 'none'" not in hdr.get("content-security-policy", "") - or hdr.get("x-frame-options") != "DENY" - or hdr.get("x-content-type-options") != "nosniff" or hdr.get("cache-control") != "no-store"] - check(f"every browser response ({len(RESPONSES)}) carries the policy, nosniff and no-store", not bad, bad) - - # 8. plugins code, _, body = request(port, "GET", f"/fixture/{RUN}", headers=auth) code2, _, _ = request(port, "GET", f"/fixture/{RUN}") check("a plugin route answers with its own flag behind the session, 401 without", @@ -799,43 +644,10 @@ def stored_leaks(text): check("a plugin script is served", code == 200, code) live.cfg.assets = ASSETS - # 9. run files - code, hd, body = request(port, "GET", f"/runs/{RUN}/report.html", headers=auth) - check("report.html is the run's report, sandboxed", code == 200 and b"clinvar/hits.tsv" in body - and "sandbox" in hd.get("content-security-policy", ""), (code, hd)) - code, _, body = request(port, "GET", f"/runs/{RUN}/clinvar/hits.tsv", headers=auth) - check("a file the report links to is served", code == 200 and body.startswith(b"gene\t"), code) - c1, _, _ = request(port, "GET", f"/runs/{RUN}/clinvar/unlinked.tsv", headers=auth) - c2, _, _ = request(port, "GET", f"/runs/{RUN}/clinvar/escape.tsv", headers=auth) - c3, _, _ = request(port, "GET", f"/runs/{RUN}/logs/run_status.tsv", headers=auth) - c4, _, _ = request(port, "GET", f"/runs/{RUN}/..%2F..%2Foutside.txt", headers=auth) - check("an unlinked file, a symlink out of the root and a path out of the run are 404", - (c1, c2, c3, c4) == (404, 404, 404, 404), (c1, c2, c3, c4)) - code, hd, _ = request(port, "GET", f"/runs/{RUN}/cpsr/", headers=auth) - page = f"/runs/{RUN}/cpsr/{RUN}.cpsr.grch38.html" - check("the report's cpsr/ folder link opens the CPSR HTML report in it", - code == 302 and hd.get("location") == page, (code, hd.get("location"))) - code, hd, body = request(port, "GET", page, headers=auth) - check("the CPSR HTML report in the linked folder is served sandboxed", - code == 200 and b"CPSR" in body - and hd.get("content-security-policy") == owngenome_live.RUN_HTML_CSP, (code, hd)) - c1, _, _ = request(port, "GET", f"/runs/{RUN}/cpsr/{RUN}.cpsr.grch38.classification.tsv.gz", headers=auth) - c2, _, _ = request(port, "GET", f"/runs/{RUN}/twopages/", headers=auth) - c3, _, _ = request(port, "GET", f"/runs/{RUN}/clinvar/", headers=auth) - c4, _, _ = request(port, "GET", f"/runs/{RUN}/twopages/a.html", headers=auth) - check("a linked folder serves only its one HTML; a folder with two HTML files (itself or either file) " - "and an unlinked folder are 404", (c1, c2, c3, c4) == (404, 404, 404, 404), (c1, c2, c3, c4)) - code, _, body = request(port, "GET", "/", headers=auth) - check("the index lists the run", code == 200 and f'/runs/{RUN}/'.encode() in body, code) - log = open(os.path.join(state_dir, "access.log")).read() - check("the access log has a row per page read, with no query string", - "\treport\tDEMO-001\t" in log and "\tfile\tDEMO-001\tclinvar/hits.tsv\t200" in log and "?" not in log, - log[-600:]) - - # 10. the browser - browser_checks(port, work, live) + # 7. the browser + browser_checks(port, work) - # 11. TLS + # 8. TLS tls_checks(root, work, pwfile) finally: stop(*servers) diff --git a/tests/test_owngenome_live_map.py b/tests/test_owngenome_live_map.py deleted file mode 100755 index baf6fb353..000000000 --- a/tests/test_owngenome_live_map.py +++ /dev/null @@ -1,315 +0,0 @@ -#!/usr/bin/env python3 -"""bin/owngenome_live_map.py in a running bin/owngenome_live.py, with a fake og beside it. - -The plugin runs bin/og from its own folder, so the test copies it (and the results plugin) into a -scratch bin/ with a fake og that records its arguments and stdin and answers as told. - -Checks: - 1. without a session every route is 401 and og is not run; - 2. GET /runs/<run>/map.json answers og's bytes unchanged (sha256 equal to a direct og call with - the same arguments and stdin), as application/gzip with the site's headers (CSP, nosniff, - no-referrer, no-store, DENY); - 3. og gets `map build --root ROOT --run RUN --assets <repo>/assets --cache-dir STATE/map`, and its - stdin is {"results": the body GET /runs/<run>/results.json sends}; live_results.build is a stub - whose answer carries its arguments, so a plugin that calls it unlike the results plugin differs; - 4. /map/genes/<chrom>.json and /map/genes/names.json answer og's bytes of `og map genes`; - 5. og exit 3 is 404 with og's reason; exit 127 or no og is 503 with the setup line, and og's - first stderr line on the server's stderr; any other exit is 500 with og's first stderr line - on the server's stderr and none of og's bytes sent; - 6. a family's run or an unknown run is 404 and og is not run; - 7. each request writes one access-log row 'map' whose run and section are only the run or '-' - and 'genes': no finding id, no gene and no chromosome; the server has no POST route for it. -Each check also runs against planted mistakes (mutants of the plugin) and must fail there: a check -that cannot fail is no check. - -Run: python3 tests/test_owngenome_live_map.py -""" -import contextlib -import hashlib -import http.client -import io -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile -import threading -import urllib.parse - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -BIN = os.path.join(REPO, "bin") -sys.path.insert(0, BIN) -import live_results # noqa: E402 -import owngenome_live # noqa: E402 - -FIX = os.path.join(REPO, "tests", "fixtures", "live") -RUN = "DEMO-001" -PASSWORD = "synthetic test password 7" -# A results body with a finding, so a plugin that wrote a finding or a gene to the log is caught. -RESULTS = {"schema_version": 1, "sample": RUN, "chapters": [], "note": None, - "findings": [{"id": "clinvar-PLANTGENE1-1", "gene": "PLANTGENE1", "title": "planted finding"}]} -FAKE_OG = r'''#!PYTHON -import hashlib, json, os, sys -args = sys.argv[1:] -data = sys.stdin.buffer.read() -with open(os.environ["FAKE_OG_LOG"], "a") as f: - f.write(json.dumps({"args": args, "stdin": data.decode()}) + "\n") -mode = os.environ.get("FAKE_OG_MODE", "ok") -if mode == "nomap": - sys.stderr.write("og map build: no map for this run: the planted reason\n") - sys.exit(3) -if mode == "notbuilt": - sys.stderr.write("ERROR: og is not built for x86_64; run scripts/setup.sh --og\n") - sys.exit(127) -if mode == "crash": - sys.stdout.buffer.write(b"half a document") - sys.stderr.write("og map build: the planted first line\nand a second line\n") - sys.exit(1) -# Every byte value, and enough of them to fill a pipe, from the arguments and stdin. -seed = hashlib.sha256(json.dumps(args).encode() + data).digest() -sys.stdout.buffer.write(b"\x1f\x8b" + bytes(range(256)) * 1200 + seed + b"\r\n\x00end") -''' -FAILS = [] - - -def check(desc, ok, detail="", quiet=False): - """Records one check; QUIET runs (the mutants) print nothing.""" - if not quiet: - print(("[PASS] " if ok else "[FAIL] ") + desc + ("" if ok else f" -- {str(detail)[:600]}")) - if not ok: - FAILS.append(desc) - - -def request(port, path, headers=None, method="GET"): - """(status, lower-cased headers, body) of one request to the browser port.""" - c = http.client.HTTPConnection("127.0.0.1", port, timeout=30) - h = {"Host": f"127.0.0.1:{port}"} - h.update(headers or {}) - c.request(method, path, headers=h) - r = c.getresponse() - data = r.read() - hdr = {k.lower(): v for k, v in r.getheaders()} - c.close() - return r.status, hdr, data - - -def login(port): - """The Cookie header of a fresh session.""" - c = http.client.HTTPConnection("127.0.0.1", port, timeout=30) - c.request("POST", "/login", urllib.parse.urlencode({"password": PASSWORD, "next": "/"}), - {"Host": f"127.0.0.1:{port}", "Content-Type": "application/x-www-form-urlencoded"}) - r = c.getresponse() - r.read() - m = re.match(r"owngenome_live=([^;]+)", r.getheader("Set-Cookie") or "") - c.close() - return {"Cookie": f"owngenome_live={m.group(1)}"} if m else {} - - -def og_calls(work): - """The fake og's calls since the last read (arguments and stdin), then forgets them.""" - p = os.path.join(work, "og.log") - calls = [json.loads(x) for x in open(p)] if os.path.exists(p) else [] - if os.path.exists(p): - os.remove(p) - return calls - - -def scenario(plugin_src, quiet=False): - """Runs every check against a server with PLUGIN_SRC as its map plugin; returns the failures.""" - start = len(FAILS) - work = os.path.realpath(tempfile.mkdtemp(prefix="pgp-live-map-")) - srv, err = None, io.StringIO() - keep_build, keep_env = live_results.build, dict(os.environ) - try: - root = os.path.join(work, "genome") - shutil.copytree(os.path.join(FIX, RUN), os.path.join(root, RUN)) - with open(os.path.join(root, RUN, f"{RUN}_report.html"), "w") as f: # gives the run files to build() - f.write('<a href="tables/linked.tsv">a linked table</a>') - os.makedirs(os.path.join(root, "FAM-A")) # a family's run, from the run_kind below - state = os.path.join(work, "state") - plugins = os.path.join(work, "bin") - os.makedirs(plugins) - with open(os.path.join(plugins, "owngenome_live_map.py"), "w") as f: - f.write(plugin_src) - shutil.copy(os.path.join(BIN, "owngenome_live_results.py"), plugins) - og = os.path.join(plugins, "og") - with open(og, "w") as f: - f.write(FAKE_OG.replace("PYTHON", sys.executable)) - os.chmod(og, 0o755) - pw = os.path.join(work, "password") - with open(pw, "w") as f: - f.write(owngenome_live.hash_password(PASSWORD, iterations=1000) + "\n") - os.environ.update(FAKE_OG_LOG=os.path.join(work, "og.log"), FAKE_OG_MODE="ok") - # The stub's answer depends on every argument, so a plugin that calls build() unlike the - # results plugin sends og a different body. - live_results.build = lambda name, summary, files=None: json.loads(json.dumps( - dict(RESULTS, sample=name, summary=summary, files=sorted(files or [])))) - - app = owngenome_live.App() - app.run_kind(lambda a: [{"name": "FAM-A", "dir": os.path.join(root, "FAM-A"), "kind": "family"}]) - with contextlib.redirect_stderr(err): - live, web, ingest = owngenome_live.start( - ["--root", root, "--state-dir", state, "--port", "0", "--ingest-port", "0", "--password-file", pw, - "--plugin-dir", plugins, "--quiet"], app) - srv = (live, web, ingest) - live.summary = lambda name, d, st: {"planted": True} - for s in (web, ingest): - threading.Thread(target=s.serve_forever, daemon=True).start() - port = web.server_address[1] - routes = [f"/runs/{RUN}/map.json", "/map/genes/chr7.json", "/map/genes/names.json"] - log_path = os.path.join(state, "access.log") - - def log_rows(): - """The access log's rows, without the header.""" - return [r.split("\t") for r in open(log_path).read().splitlines()[1:]] - - # 1. the login - codes = [request(port, r)[0] for r in routes] - check("1. without a session every map route is 401 and og is not run", - codes == [401] * 3 and not og_calls(work), codes, quiet) - auth = login(port) - n_rows = len(log_rows()) - - # 2, 3. the person's document - code, hd, body = request(port, routes[0], auth) - calls = og_calls(work) - _, _, results = request(port, f"/runs/{RUN}/results.json", auth) - direct = subprocess.run([og] + calls[0]["args"], input=calls[0]["stdin"].encode(), capture_output=True, - check=False) \ - if calls else None - og_calls(work) - check("2. map.json is og's bytes unchanged (sha256 equal to a direct og call)", - code == 200 and direct is not None and len(body) > 300000 - and hashlib.sha256(body).hexdigest() == hashlib.sha256(direct.stdout).hexdigest(), - (code, len(body), direct and len(direct.stdout)), quiet) - want = {"content-type": "application/gzip", "x-content-type-options": "nosniff", - "referrer-policy": "no-referrer", "cache-control": "no-store", "x-frame-options": "DENY", - "content-security-policy": owngenome_live.CSP} - check("2. map.json has the site's headers", all(hd.get(k) == v for k, v in want.items()), hd, quiet) - check("3. og runs `map build` with the root, the run, the repo's assets and STATE/map", - len(calls) == 1 and calls[0]["args"] == ["map", "build", "--root", os.path.realpath(root), "--run", RUN, - "--assets", os.path.join(work, "assets"), - "--cache-dir", os.path.join(state, "map")], - calls, quiet) - check("3. og's stdin is {\"results\": the body of results.json}", - len(calls) == 1 and json.loads(calls[0]["stdin"]) == {"results": json.loads(results)} - and json.loads(results)["findings"] == RESULTS["findings"] and json.loads(results)["files"], - (calls, results[:200]), quiet) - - # 4. the gene files - ok = True - for path, want_args in ((routes[1], ["--chrom", "chr7"]), (routes[2], ["--names"])): - code, hd, body = request(port, path, auth) - calls = og_calls(work) - direct = subprocess.run([og] + calls[0]["args"], input=b"", capture_output=True, check=False) \ - if calls else None - og_calls(work) - ok = ok and code == 200 and hd.get("content-type") == "application/gzip" \ - and hd.get("cache-control") == "no-store" and direct is not None and body == direct.stdout \ - and calls[0]["args"] == ["map", "genes", "--root", os.path.realpath(root), - "--cache-dir", os.path.join(state, "map")] + want_args - check("4. the gene files are og's bytes of `og map genes --chrom` and `--names`", ok, (code, calls), quiet) - - # 5. og's exit codes - os.environ["FAKE_OG_MODE"] = "nomap" - code, _, body = request(port, routes[0], auth) - check("5. og exit 3 is 404 with og's reason", code == 404 and b"the planted reason" in body, (code, body), quiet) - os.environ["FAKE_OG_MODE"] = "notbuilt" - with contextlib.redirect_stderr(err): - code, _, body = request(port, routes[1], auth) - os.rename(og, og + ".away") - code2, _, body2 = request(port, routes[2], auth) - os.rename(og + ".away", og) - setup = b"The map needs og on this host: run scripts/setup.sh --og" - check("5. og exit 127, or no og at all, is 503 with the setup line", - code == 503 and setup in body and code2 == 503 and setup in body2, (code, body, code2, body2), quiet) - check("5. og exit 127 puts og's first stderr line on the server's stderr", - "og is not built for x86_64" in err.getvalue(), err.getvalue()[-300:], quiet) - os.environ["FAKE_OG_MODE"] = "crash" - with contextlib.redirect_stderr(err): - code, _, body = request(port, routes[0], auth) - check("5. any other og exit is 500, logs og's first stderr line, and sends none of og's bytes", - code == 500 and b"half a document" not in body and "the planted first line" in err.getvalue() - and "and a second line" not in err.getvalue(), (code, body, err.getvalue()[-300:]), quiet) - os.environ["FAKE_OG_MODE"] = "ok" - og_calls(work) - - # 6. runs with no person's map - codes = [request(port, f"/runs/{r}/map.json", auth)[0] for r in ("FAM-A", "NO-SUCH-RUN")] - check("6. a family's run and an unknown run are 404, and og is not run", - codes == [404, 404] and not og_calls(work), codes, quiet) - - # 7. the access log - code, _, _ = request(port, routes[0], auth, method="POST") - rows = log_rows()[n_rows:] - rows = [r for r in rows if r[3] not in ("results",)] - maps = [r for r in rows if r[3] == "map"] - text = open(log_path).read() - check("7. one 'map' row per request, naming only the run or 'genes'", - len(maps) == 9 and all((r[4], r[5]) in ((RUN, "-"), ("-", "genes"), ("FAM-A", "-"), - ("NO-SUCH-RUN", "-")) for r in maps), - (len(maps), maps), quiet) - check("7. the access log names no finding, no gene and no chromosome", - not any(w in text for w in ("PLANTGENE1", "clinvar-PLANTGENE1-1", "chr7", "planted")), text, quiet) - check("7. there is no POST route to the map", code == 404, code, quiet) - finally: - live_results.build = keep_build - os.environ.clear() - os.environ.update(keep_env) - if srv: - live, web, ingest = srv - live.stopping = True - for s in (web, ingest): - s.shutdown() - s.server_close() - shutil.rmtree(work, ignore_errors=True) - return len(FAILS) - start - - -# Planted mistakes: each must make a check fail. -MUTANTS = [ - ("a route without the login", 'app.route("GET", r"/runs/(%s)/map\\.json" % NAME, build)', - 'app.route("GET", r"/runs/(%s)/map\\.json" % NAME, build, auth=False)'), - ("the log row names a finding", 'stdin, run["name"], "-")', - 'stdin, run["name"], body["findings"][0]["id"])'), - ("the log row names the chromosome", 'b"", "-",\n "genes")', - 'b"", "-",\n m.group(1))'), - ("og's bytes changed", "return req.send_body(200, GZIP, p.stdout)", - "return req.send_body(200, GZIP, p.stdout[:-1])"), - ("exit 3 read as a failure", "if code == 3:", "if code == 33:"), - ("og missing read as a failure", "if code == 127:", "if code == 172:"), - ("og's stdout sent on a failure", 'return req.send_body(500, TEXT, "The map could not be built: the server\'s log ' - 'says why.\\n")\n\n\ndef register', "return req.send_body(500, TEXT, p.stdout)\n\n\ndef register"), - ("stdin without the results", 'json.dumps({"results": body})', 'json.dumps({"results": {}})'), - ("og's 127 reason dropped", 'if err: # e.g.', 'if False: # e.g.'), - ("build() given other files", 'live.run_files(run) if run["dir"] else set())', 'set())'), - ("build() given another name", 'return live_results.build(name, summary,', - 'return live_results.build(name + "x", summary,'), - ("a family's run built", 'if not run or run.get("kind") == "family":', "if not run:"), - ("the assets of the wrong folder", 'ASSETS = os.path.join(os.path.dirname(HERE), "assets")', - 'ASSETS = os.path.join(HERE, "assets")'), -] - - -def main(): - """Runs the checks on the plugin, then on each planted mistake.""" - with open(os.path.join(BIN, "owngenome_live_map.py")) as f: - src = f.read() - scenario(src) - real = len(FAILS) - for name, old, new in MUTANTS: - if old not in src: - check(f"mutant '{name}': its line is in owngenome_live_map.py", False, old) - continue - keep = len(FAILS) - n = scenario(src.replace(old, new, 1), quiet=True) - del FAILS[keep:] - check(f"mutant '{name}' fails a check ({n} failed)", n > 0) - print(f"\n{'FAILED: ' + str(len(FAILS)) + ' check(s)' if FAILS else 'All checks passed.'}") - return 1 if FAILS or real else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_owngenome_recheck.py b/tests/test_owngenome_recheck.py deleted file mode 100644 index d68fe4a75..000000000 --- a/tests/test_owngenome_recheck.py +++ /dev/null @@ -1,326 +0,0 @@ -#!/usr/bin/env python3 -"""og recheck from the outside, the live-site plugin that shows its result, and the ClinVar lock -block of run-all.sh and step 06 that `og recheck --refresh-clinvar` interlocks with. No network: -the upstream answers come from tests/fixtures/recheck/offline.json. - -The check itself (classes, steps, failed lookups, the locks, the rerun comparison, --launch-dir, -digests, unknown and skipped pins, steps 50 to 53) is tested by `cargo test -p og-recheck` -(rust/crates/og-recheck/src/tests.rs). Here: - 1. og recheck on the fixture sample: exit 0, ClinVar refresh-available with the steps 06 and - downstream, state.json replaced whole and log.tsv grown by one row per item on a second check; - 2. an offline answer that failed makes og recheck exit 1, and a sample with no manifest exit 2; - 3. --rerun refuses (exit 3) while run_status.tsv has no finished_utc; - 4. --affected prints the steps of a key, and bin/owngenome_recheck.py, kept for crontabs, runs - og recheck with the same arguments and says it is deprecated; - 5. the live-site plugin (bin/owngenome_live_recheck.py) lets the browser fetch - recheck/state.json, and its state hook gives each step its badges ("newer ClinVar - available", "redone <date>") and {} when there is no check; - 6. the ClinVar lock block of run-all.sh and of step 06 waits while a refresh holds the lock and - goes on once it ends (needs flock; it fails in CI without it). -Each check is also run against a planted mistake and must fail there. - -Run: python3 tests/test_owngenome_recheck.py (OG names the og binary, as tests/og_binary.py reads it) -""" -import fcntl -import gzip -import importlib.util -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "tests")) -from og_binary import og_binary # noqa: E402 - -FIX = os.path.join(REPO, "tests", "fixtures", "recheck") -SHIM = os.path.join(REPO, "bin", "owngenome_recheck.py") -FAILS = 0 -CLINVAR_STEPS = ["06", "13", "20", "23", "24", "27", "30", "31", "38", "39"] - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:800]}") - if not ok: - FAILS += 1 - - -def genome(tmp, finished=True): - g = os.path.join(tmp, "genome") - s = os.path.join(g, "S1") - os.makedirs(os.path.join(s, "logs"), exist_ok=True) - shutil.copy(os.path.join(FIX, "run_manifest.tsv"), os.path.join(s, "run_manifest.tsv")) - os.makedirs(os.path.join(g, "clinvar"), exist_ok=True) - with gzip.open(os.path.join(g, "clinvar", "clinvar_pathogenic_chr.vcf.gz"), "wt") as f: - f.write("##fileformat=VCFv4.1\n##fileDate=20260901\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n") - with open(os.path.join(s, "logs", "run_status.tsv"), "w") as f: - f.write("meta\tstarted_epoch\t1000\nmeta\tstarted_utc\t2026-09-02 10:00:00 UTC\n" - "meta\tcommand\techo replayed\nmeta\tswitches\tGENOME_DIR TOOLS\n") - if finished: - f.write("meta\tfinished_utc\t2026-09-02 12:00:00 UTC\n") - return g - - -def og(cmd, *args): - """cmd (og's path, or [interpreter, script]) with 'recheck' and args: (rc, stdout, stderr).""" - argv = list(cmd) + (["recheck"] if len(cmd) == 1 else []) + list(args) - p = subprocess.run(argv, capture_output=True, text=True, check=False, timeout=60) - return p.returncode, p.stdout, p.stderr - - -def cli_checks(cmd): - """Checks 1-3 on og recheck as cmd runs it: {name: (ok, detail)}.""" - out = {} - tmp = tempfile.mkdtemp(prefix="recheck.") - try: - g = genome(tmp) - off = os.path.join(FIX, "offline.json") - base = ["--sample", "S1", "--genome-dir", g, "--offline", off] - rc, o, e = og(cmd, *base) - d = os.path.join(g, "S1", "recheck") - st = json.load(open(os.path.join(d, "state.json"))) if os.path.isfile(os.path.join(d, "state.json")) else {} - items = {i["item"]: i for i in st.get("items", [])} - cv = items.get("clinvar_file_date", {}) - out["1. og recheck: ClinVar refresh-available, its steps 06 and downstream, exit 0"] = ( - rc == 0 and cv.get("class") == "refresh-available" and cv.get("steps") == CLINVAR_STEPS - and st.get("affected_steps") == CLINVAR_STEPS, (rc, e, cv, st.get("affected_steps"))) - log1 = open(os.path.join(d, "log.tsv")).read() if os.path.isfile(os.path.join(d, "log.tsv")) else "" - og(cmd, *base) - log2 = open(os.path.join(d, "log.tsv")).read() if os.path.isfile(os.path.join(d, "log.tsv")) else "" - left = sorted(os.listdir(d)) if os.path.isdir(d) else [] - out["1. state.json is replaced whole and log.tsv grows one row per item"] = ( - log2.startswith(log1) and len(log2.splitlines()) - len(log1.splitlines()) == len(items) > 0 - and left == ["log.tsv", "state.json"], (left, len(log1.splitlines()), len(log2.splitlines()))) - bad = os.path.join(tmp, "a500.json") - data = json.load(open(off)) - data["PHARMCAT_IMAGE"] = {"error": "HTTP 500"} - json.dump(data, open(bad, "w")) - rc5, _, _ = og(cmd, "--sample", "S1", "--genome-dir", g, "--offline", bad) - rc_none, _, e_none = og(cmd, "--sample", "S9", "--genome-dir", g, "--offline", off) - out["2. a failed lookup exits 1, a sample with no manifest exits 2"] = ( - rc5 == 1 and rc_none == 2 and "run_manifest.tsv" in e_none, (rc5, rc_none, e_none)) - gr = genome(os.path.join(tmp, "running"), finished=False) - rc3, _, e3 = og(cmd, "--sample", "S1", "--genome-dir", gr, "--offline", off, "--rerun") - out["3. --rerun refuses while a run is going on"] = (rc3 == 3 and "going on" in e3, (rc3, e3)) - finally: - shutil.rmtree(tmp, ignore_errors=True) - return out - - -def fake_og(tmp, body): - """An executable that stands in for og: a planted mistake.""" - p = os.path.join(tmp, "og") - with open(p, "w") as f: - f.write("#!/bin/sh\n" + body + "\n") - os.chmod(p, 0o755) - return [p] - - -def lock_block(path): - """The ClinVar lock block of a script: from 'CVLOCK=' through its closing 'fi'.""" - x = re.search(r"^CVLOCK=.*?\n fi\nfi\n", open(path).read(), re.S | re.M) - return x.group(0) if x else "" - - -def lock_waits(block): - """block, run in bash while this process holds the lock exclusive: it must wait, then go on - once the lock is let go.""" - import time - tmp = tempfile.mkdtemp(prefix="recheck-cvlock.") - try: - g = os.path.join(tmp, "genome") - os.makedirs(os.path.join(g, "clinvar")) - lock = os.path.join(g, "clinvar", ".refresh.lock") - with open(lock, "a") as held: - fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) - p = subprocess.Popen(["bash", "-c", "set -euo pipefail\n" + block + "echo through\n"], - env=dict(os.environ, G=g, GENOME_DIR=g), stdout=subprocess.PIPE, text=True) - time.sleep(1.5) - waited = p.poll() is None - try: - o, _ = p.communicate(timeout=20) - except subprocess.TimeoutExpired: - p.kill() - o, _ = p.communicate() - return (bool(block) and waited and p.returncode == 0 and "Waiting" in o and o.rstrip().endswith("through"), - (waited, p.returncode, o)) - finally: - shutil.rmtree(tmp, ignore_errors=True) - - -def load_plugin(): - spec = importlib.util.spec_from_file_location("owngenome_live_recheck_test", os.path.join(REPO, "bin", "owngenome_live_recheck.py")) - p = importlib.util.module_from_spec(spec) - keep, sys.dont_write_bytecode = sys.dont_write_bytecode, True - try: - spec.loader.exec_module(p) - finally: - sys.dont_write_bytecode = keep - return p - - -def plugin_ok(p): - class App: - def __init__(self): - self.globs, self.hooks = [], {} - - def run_file(self, g): - self.globs.append(g) - - def state_hook(self, name, fn): - self.hooks[name] = fn - app = App() - p.register(app) - tmp = tempfile.mkdtemp(prefix="recheck-live.") - try: - run = os.path.join(tmp, "S1") - none = app.hooks["recheck"](run, {}, []) - os.makedirs(os.path.join(run, "recheck")) - json.dump({"checked_utc": "2026-10-06T00:00:00Z", "items": [ - {"item": "clinvar_file_date", "label": "ClinVar (step 06's file)", "class": "refresh-available", "steps": ["06", "24"]}, - {"item": "VEP_CACHE_RELEASE", "label": "Ensembl / VEP cache", "class": "no-newer-release", "steps": ["13"]}], - "rerun": {"finished_utc": "2026-10-05T10:00:00Z", "predicted_vs_redone": {"redone": ["06"]}}}, - open(os.path.join(run, "recheck", "state.json"), "w")) - got = app.hooks["recheck"](run, {}, []) - finally: - shutil.rmtree(tmp, ignore_errors=True) - ok = (app.globs == ["recheck/state.json"] and none == {} - and got.get("steps") == {"06": ["newer ClinVar available", "redone 2026-10-05"], "24": ["newer ClinVar available"]} - and got.get("banner") == "Database re-check of 2026-10-06T00:00:00Z: newer ClinVar available") - return ok, (app.globs, none, got) - - -def plugin_odd_rerun_ok(p): - """A rerun block that is not an object: the banner still comes, no badge from it.""" - tmp = tempfile.mkdtemp(prefix="recheck-live-odd.") - try: - run = os.path.join(tmp, "S1") - os.makedirs(os.path.join(run, "recheck")) - out = [] - for rr in ([1], {"finished_utc": "2026-10-05T10:00:00Z", "predicted_vs_redone": ["06"]}): - json.dump({"checked_utc": "2026-10-06T00:00:00Z", "items": [], "rerun": rr}, - open(os.path.join(run, "recheck", "state.json"), "w")) - try: - out.append(p.recheck_state(run, {}, [])) - except (AttributeError, TypeError) as e: - out.append(e) - finally: - shutil.rmtree(tmp, ignore_errors=True) - ok = all(isinstance(g, dict) and g.get("steps") == {} and "nothing newer" in g.get("banner", "") for g in out) - return ok, out - - -def plugin_stale_ok(p): - """A check older than the run's run_manifest.tsv describes an earlier run: no banner, no badge. - One written after the manifest is still shown.""" - tmp = tempfile.mkdtemp(prefix="recheck-live-stale.") - try: - run = os.path.join(tmp, "S1") - os.makedirs(os.path.join(run, "recheck")) - json.dump({"checked_utc": "2026-10-06T00:00:00Z", "items": [ - {"item": "clinvar_file_date", "label": "ClinVar", "class": "refresh-available", "steps": ["06"]}]}, - open(os.path.join(run, "recheck", "state.json"), "w")) - out = [] - for written in ("2026-10-07 09:00:00 UTC", "2026-10-05 09:00:00 UTC"): - with open(os.path.join(run, "run_manifest.tsv"), "w") as f: - f.write(f"run\twritten_utc\t{written}\n") - out.append(p.recheck_state(run, {}, [])) - finally: - shutil.rmtree(tmp, ignore_errors=True) - ok = out[0] == {} and out[1].get("steps") == {"06": ["newer ClinVar available"]} - return ok, out - - -def plugin_from_source(src, name): - p = os.path.join(REPO, "tmp") if os.path.isdir(os.path.join(REPO, "tmp")) else tempfile.gettempdir() - path = os.path.join(tempfile.mkdtemp(prefix="recheck-plugin.", dir=p), name + ".py") - open(path, "w").write(src) - spec = importlib.util.spec_from_file_location(name, path) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -def main(): - ogp = og_binary() - results = cli_checks([ogp]) - for name, (ok, detail) in results.items(): - check(name, ok, detail) - tmp = tempfile.mkdtemp(prefix="recheck-mutant.") - try: - # og that writes nothing and always exits 0: every check above must fail on it. - silent = cli_checks(fake_og(tmp, "exit 0")) - for name, (ok, _) in silent.items(): - check(f"fails on a planted mistake: {name} (an og that does nothing)", not ok, "passed on the mutant") - finally: - shutil.rmtree(tmp, ignore_errors=True) - - rc, o, _ = og([ogp], "--affected", "clinvar_file_date") - check("4. og recheck --affected names ClinVar's steps", rc == 0 and o.split() == CLINVAR_STEPS, (rc, o)) - # The shim runs the og beside it (bin/og). A copy of it beside an og that runs this og checks - # it on any machine, with or without a build in bin/.og. - tmp = tempfile.mkdtemp(prefix="recheck-shim.") - try: - shim = os.path.join(tmp, "owngenome_recheck.py") - shutil.copy(SHIM, shim) - with open(os.path.join(tmp, "og"), "w") as f: - f.write(f"#!/bin/sh\nexec '{ogp}' \"$@\"\n") - os.chmod(os.path.join(tmp, "og"), 0o755) - rc, o, e = og([sys.executable, shim], "--affected", "clinvar_file_date") - check("4. bin/owngenome_recheck.py runs og recheck and says it is deprecated", - rc == 0 and o.split() == CLINVAR_STEPS and "deprecated" in e, (rc, o, e)) - rc, _, _ = og([sys.executable, shim], "--sample", "S1", "--genome-dir", "/nonexistent") - check("4. bin/owngenome_recheck.py passes og recheck's exit code on", rc == 2, rc) - with open(os.path.join(tmp, "og"), "w") as f: - f.write("#!/bin/sh\nexit 0\n") - rc, o, _ = og([sys.executable, shim], "--affected", "clinvar_file_date") - check("fails on a planted mistake: bin/owngenome_recheck.py runs og recheck (an og that does nothing)", - o.split() != CLINVAR_STEPS, "passed on the mutant") - finally: - shutil.rmtree(tmp, ignore_errors=True) - - ok, detail = plugin_ok(load_plugin()) - check("5. the live-site plugin serves the check and gives each step its badges", ok, detail) - bad = load_plugin() - bad.ACT = dict(bad.ACT, **{"refresh-available": "newer {name}"}) - check("fails on a planted mistake: the live-site plugin serves the check and gives each step its badges", - not plugin_ok(bad)[0], "passed on the mutant") - ok, detail = plugin_odd_rerun_ok(load_plugin()) - check("5. the live-site plugin survives a rerun block that is not an object", ok, detail) - src = open(os.path.join(REPO, "bin", "owngenome_live_recheck.py")).read() - guard = "rr = rr if isinstance(rr, dict) else {}" - check("planted mistake is in the copy: the live-site plugin survives a rerun block that is not an object", guard in src) - unguarded = plugin_from_source(src.replace(guard, "rr = rr or {}"), "owngenome_live_recheck_unguarded") - check("fails on a planted mistake: the live-site plugin survives a rerun block that is not an object", - not plugin_odd_rerun_ok(unguarded)[0], "passed on the mutant") - ok, detail = plugin_stale_ok(load_plugin()) - check("5. the live-site plugin leaves out a check older than the run's manifest", ok, detail) - stale = "if utc14(manifest_written(run_dir)) > utc14(st.get(\"checked_utc\")):" - check("planted mistake is in the copy: the live-site plugin leaves out a check older than the run's manifest", - stale in src) - unfiltered = plugin_from_source(src.replace(stale, "if False:"), "owngenome_live_recheck_unfiltered") - check("fails on a planted mistake: the live-site plugin leaves out a check older than the run's manifest", - not plugin_stale_ok(unfiltered)[0], "passed on the mutant") - - for script in ("scripts/run-all.sh", "scripts/06-clinvar-screen.sh"): - block = lock_block(os.path.join(REPO, script)) - if not shutil.which("flock"): - check(f"6. the ClinVar lock block of {script} waits for a refresh (flock is not installed: " - f"{'FAIL in CI' if os.environ.get('CI') else 'not run here'})", not os.environ.get("CI")) - continue - ok, detail = lock_waits(block) - check(f"6. the ClinVar lock block of {script} waits for a refresh, then goes on", ok, detail) - planted = re.sub(r"(?m)^ flock -s -n 7 .*$", " :", block) - check(f"planted mistake is in the copy: the ClinVar lock block of {script}", planted != block) - check(f"fails on a planted mistake: the ClinVar lock block of {script} waits for a refresh (no flock call)", - not lock_waits(planted)[0], "passed on the mutant") - print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_prs_runopts.sh b/tests/test_prs_runopts.sh index 43c327c83..927499e31 100755 --- a/tests/test_prs_runopts.sh +++ b/tests/test_prs_runopts.sh @@ -14,9 +14,17 @@ # the images.config it wrote. MODULE defaults to the PRS module. # Without the block (the module before it existed) the test fails. # -# What the Groovy side hands the block (the run's docker.runOptions and -# task.containerOptions) is checked on real runs by the e2e case -# tests/e2e/prs-runopts-and-trace-1.sh. +# The Groovy side drops each user flag with its whole value first, by a +# pattern between markers in the module; the test applies that pattern +# before the block. What the Groovy side hands the block (the run's +# docker.runOptions and task.containerOptions) is checked on real runs by the +# e2e case tests/e2e/prs-runopts-and-trace-1.sh. +# +# It also checks the docker profile itself (nextflow.config), whose +# docker.runOptions every task gets: expanded by bash, as a task's +# .command.run expands it, its user is the caller's $(id -u):$(id -g), as +# with the bash steps' run_in; and the block hands it on with no word of +# that user left. The old --user root:root must fail the first check. set -euo pipefail REPO=$(cd "$(dirname "$0")/.." && pwd) @@ -56,13 +64,52 @@ then exit 1 fi +# The Groovy side first drops each user flag with its whole value: a value +# can hold spaces ($(id -u):$(id -g)), and the block splits the options into +# words, so the block alone drops only the value's first word. The pattern is +# the single-quoted Groovy string between the module's "user flag" markers, a +# Java regex. perl applies it here as the module's replaceAll(user_flag, ' ') +# does; the pattern uses only syntax the two read the same way. A module +# without the markers hands the options on as they are, as before it dropped +# them, so the cases below fail there. +# shellcheck disable=SC2016 # the dollar signs are awk's +if ! USER_FLAG=$(awk ' + /\/\/ >>> user flag/ { on = 1; next } + /\/\/ <<< user flag/ { on = 0 } + on && /^ *def user_flag = '\''.*'\'' *$/ { + s = $0; sub(/^ *def user_flag = '\''/, "", s); sub(/'\'' *$/, "", s) + out = "" + for (i = 1; i <= length(s); i++) { + c = substr(s, i, 1) + if (c == "\\") { + nxt = substr(s, i + 1, 1) + if (nxt != "\\") { print "unexpected Groovy escape \\" nxt " in user_flag" > "/dev/stderr"; exit 1 } + out = out nxt; i++ + } else out = out c + } + print out; n++ + } + END { if (n > 1) { print "more than one user_flag in " FILENAME > "/dev/stderr"; exit 1 } }' "$MODULE"); then + echo "FAIL: the user flag pattern of ${MODULE} could not be read" + exit 1 +fi +echo "user flag pattern: ${USER_FLAG:-none, the options pass as they are}" +# groovy_side OUTER: OUTER after the module's replaceAll(user_flag, ' '). +groovy_side() { + if [ -z "$USER_FLAG" ]; then printf '%s' "$1"; return; fi + # shellcheck disable=SC2016 # perl's own variable + printf '%s' "$1" | USER_FLAG=$USER_FLAG perl -0777 -pe 's/$ENV{USER_FLAG}/ /g' +} + # line OUTER MOUNT: the docker.runOptions line the block writes for those -# values, after the process block the task writes first (a file a glob would find). +# values, after the process block the task writes first (a file a glob would +# find). OUTER goes through the Groovy side first, as in the module. line() { - local d + local d outer d=$(mktemp -d "${WORK}/case.XXXXXX") + outer=$(groovy_side "$1") printf 'process {\n}\n' > "${d}/images.config" - (cd "$d" && OUTER_DOCKER=$1 MOUNT=$2 bash -euo pipefail "${WORK}/block.sh" > /dev/null) || { echo "the block failed"; return; } + (cd "$d" && OUTER_DOCKER=$outer MOUNT=$2 bash -euo pipefail "${WORK}/block.sh" > /dev/null) || { echo "the block failed"; return; } grep '^docker\.runOptions' "${d}/images.config" || echo "no docker.runOptions line" } @@ -83,7 +130,7 @@ expect() { } expect "no outer flags: the line of before" "" "" "$FIXED" -expect "the docker profile's --user root:root and --network none are dropped" \ +expect "a config's --user root:root and --network none are dropped" \ "--user root:root --network none" "" "$FIXED" expect "a low-priority runOptions reaches pgsc_calc after -u and --network none" \ "--user root:root --cpu-shares 256" "" "${FIXED} --cpu-shares 256" @@ -97,6 +144,65 @@ expect "a glob is passed as it is, not expanded to the files of the task folder" # A quote and a backslash, escaped for the single-quoted config string. expect "a quote and a backslash are escaped for the config" \ "--label 'a=b c' --env X=a\\b" "" "${FIXED} --label \\'a=b c\\' --env X=a\\\\b" +# A user flag whose value holds spaces is dropped whole, not word by word. +# shellcheck disable=SC2016 # the options' own text, expanded by the task +expect "a config's -u \$(id -u):\$(id -g), the common form, is dropped whole" \ + '-u $(id -u):$(id -g) -e HOME=/tmp' "" "${FIXED} -e HOME=/tmp" +# shellcheck disable=SC2016 +expect "--user=\$(...), a quoted --user value and a backquoted -u value are dropped whole" \ + '--user=$(id -u):$(id -g) --cpu-shares 256 --user "$(id -u):$(id -g)" -u `id -u`:`id -g` --label a=b' "" \ + "${FIXED} --cpu-shares 256 --label a=b" +# shellcheck disable=SC2016 +expect "the space-free -u \${UID}:\${GROUPS} passes clean" \ + '-u ${UID}:${GROUPS} -e HOME=/tmp' "" "${FIXED} -e HOME=/tmp" +expect "--userns is not a user flag and is kept" \ + "--userns host --cpu-shares 256" "" "${FIXED} --userns host --cpu-shares 256" +# The pattern must read a user value whole or not at all: it never stops in +# the middle of a word and leaves the rest (here `b"`) for pgsc_calc. +expect "a user value the pattern cannot read whole is left to the block, not cut in a word" \ + '-u "a\"b" --label c' "" "${FIXED} --label c" + +# --- The docker profile: every task runs as the caller --------------------------- +# runs_as_caller OPTIONS: OPTIONS, split and expanded by bash as a task's +# .command.run does, give the container the caller's ids. The last -u or +# --user wins, as in `docker run`. +runs_as_caller() { + local user='' a + local -a w=() + eval "w=($1)" + set -- ${w[@]+"${w[@]}"} + while [ $# -gt 0 ]; do + a=$1; shift + case "$a" in + -u|--user) user=${1-}; [ $# -eq 0 ] || shift ;; + -u?*) user=${a#-u} ;; + --user=*) user=${a#--user=} ;; + esac + done + [ "$user" = "$(id -u):$(id -g)" ] +} +# The docker profile's runOptions, the one docker.runOptions line of nextflow.config. +OPTS=$(sed -n "s/^ *docker\.runOptions *= *'\(.*\)'\$/\1/p" "${REPO}/nextflow.config") +if [ "$(printf '%s\n' "$OPTS" | grep -c .)" -ne 1 ]; then + echo "FAIL: want one docker.runOptions line in nextflow.config, got: ${OPTS:-none}" + FAILS=$((FAILS + 1)) +elif runs_as_caller "$OPTS"; then + echo "ok: the docker profile runs every task as the caller ($(id -u):$(id -g)): ${OPTS}" +else + echo "FAIL: the docker profile does not run tasks as the caller ($(id -u):$(id -g)): ${OPTS}" + FAILS=$((FAILS + 1)) +fi +# Controls: the profile of before, and the user flag dropped, are refused. +for bad in "--user root:root" "-e HOME=/tmp"; do + if runs_as_caller "$bad"; then + echo "FAIL: control: '${bad}' was read as running as the caller" + FAILS=$((FAILS + 1)) + else + echo "ok: control: '${bad}' is refused" + fi +done +# Through the block, pgsc_calc gets the profile's other flags and none of its user. +expect "the docker profile's own runOptions: its user is dropped, the rest kept" "$OPTS" "" "${FIXED} -e HOME=/tmp" # Control: a block that drops nothing must fail the second case, so the # comparison above can fail. @@ -104,6 +210,7 @@ expect "a quote and a backslash are escaped for the config" \ sed 's/skip=true ;;/out="$out $o" ;;/' "${WORK}/block.sh" > "${WORK}/broken.sh" mv "${WORK}/block.sh" "${WORK}/good.sh" mv "${WORK}/broken.sh" "${WORK}/block.sh" +USER_FLAG='' # the block alone: the Groovy side would drop the user first if [ "$(line "--user root:root --network none" "")" = "docker.runOptions = '${FIXED}'" ]; then echo "FAIL: control: a block that keeps --user root:root was not caught" FAILS=$((FAILS + 1)) diff --git a/tests/test_pypgx_genes.sh b/tests/test_pypgx_genes.sh index e5e8fb99a..b2790b5f1 100755 --- a/tests/test_pypgx_genes.sh +++ b/tests/test_pypgx_genes.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# og-leg: OG_REQUIRED # Step 32 (pypgx) takes its genes from assets/pgx_genes.tsv. This test checks: # 1. the table against pypgx's own gene table at the pinned tag: every bam # row is a gene pypgx marks SV=TRUE, and every SV=TRUE gene is a bam row @@ -12,10 +13,11 @@ # a BAM whose header lacks the gene's ALT contig and not for one that has # it, and not printed when the column says yes; # 4. the PYPGX module reads the same table and spells no gene list; -# 5. the reports: every bam gene has exactly one row in the PGx copy number -# block (its call, or why there is none), pypgx's, even beside a CNVpytor -# section, and a not-callable gene gets its not-assessable line (two -# planted summaries must be reported); +# 5. the reports (og summary collect, then og report render): every bam +# gene has exactly one row in the PGx copy number block (its call, or why +# there is none), pypgx's, even beside a CNVpytor section, and a +# not-callable gene gets its not-assessable line +# (two planted summaries must be reported); # 6. one owner per gene: no row of the depth engine's catalog # (bin/cn_loci.tsv) is a gene pypgx calls from the BAM (a planted UGT2B17 # row must be reported); @@ -245,82 +247,81 @@ if grep -qE '"(CYP[0-9A-Z]+ ){2,}' "$NF"; then bad "the PYPGX module spells a ge # --- 5. the reports ------------------------------------------------------------ # A summary with every gene of the table: GSTT1 not callable, CYP2E1 failed in # pypgx, CYP2D6 withheld by the depth check, every other bam gene with a -# copy-number call. Every bam gene must have a row in the PGx copy number block -# (its call, "not called: pypgx failed" or "not called: the CYP2D6 depth check -# failed"), -# and GSTT1 its not-assessable line. Two planted summaries must be reported: one -# where GSTT1 reads as called (no not-assessable line), and one where a bam -# gene's row says VCF (it drops out of the block). -python3 - "$REPO" "$CAT" <<'PY' || FAILS=$((FAILS + 1)) -import os, sys, tempfile -repo, cat = sys.argv[1], sys.argv[2] -sys.path.insert(0, os.path.join(repo, "bin")) -import collect_summary, render_report -rows = [l.rstrip("\n").split("\t") for l in open(cat) if l.strip() and not l.startswith("#")][1:] -bam = [g for g, mode, *_ in rows if mode == "bam"] -hdr = ["Gene", "Diplotype", "Phenotype", "CNV_call", "Source", "Status", "Defining_variants", "Min_GQ", - "Weakest_variant", "Defining_filter", "Quality"] - - -def summary(gstt1_status="not callable on this reference", vcf_source_for=None): - lines = [] - for g, mode, *_ in rows: - src = "VCF" if g == vcf_source_for else mode.upper() - if g == "GSTT1": - lines.append([g, "N/A", "N/A", "N/A", src, gstt1_status] + ["N/A"] * 5) - elif g == "CYP2E1": - lines.append([g, "FAILED", "N/A", "N/A", src, "failed"] + ["N/A"] * 5) - elif g == "CYP2D6": - # what step 32 writes when the depth check rejects CYP2D6 - lines.append([g, "Indeterminate", "Indeterminate (CYP2D6 depth check)", "N/A (CYP2D6 depth check)", - src, "no call (CYP2D6 depth check)"] + ["N/A"] * 5) - else: - lines.append([g, "*1/*1", "Normal Metabolizer", f"CNV-{g}" if mode == "bam" else "N/A", src, "called"] - + ["none", "N/A", "N/A", "N/A", "N/A"]) - return lines - - -def problems(lines): - with tempfile.TemporaryDirectory() as d: - os.makedirs(os.path.join(d, "pypgx")) - with open(os.path.join(d, "pypgx", "s1_pypgx_summary.tsv"), "w") as f: - f.write("\n".join("\t".join(r) for r in [hdr] + lines) + "\n") - data = collect_summary.collect("s1", d)["sections"]["pypgx"]["data"] - # The rows of the block (gene, text), with CNVpytor's section beside pypgx's: - # pypgx owns every bam gene, so each has one row, and it is pypgx's. - block = render_report.pgx_copy_number({"pypgx": {"state": "ok", "data": data}, - "cnvpytor": {"state": "ok", "data": {}}}) - cn = dict(block) - text = "\n".join(render_report.pypgx_lines(data)) - bad = [f"{g}: {n} rows in the copy-number block" for g in sorted({g for g, _ in block}) - if (n := sum(x == g for x, _ in block)) > 1] - for g in bam: - want = {"CYP2E1": "not called: pypgx failed", - "CYP2D6": "not called: the CYP2D6 depth check failed"}.get(g, f"CNV-{g}") + " (pypgx, read depth)" - if g == "GSTT1": - if "Not assessable on this reference: GSTT1" not in text: - bad.append("GSTT1 has no not-assessable line") - elif cn.get(g) != want: - bad.append(f"{g}: copy-number block says {cn.get(g)!r}, not {want!r}") - return bad - - -bad = problems(summary()) -for b in bad: - print("FAIL:", b) -if not bad: - print(f"ok: the reports show the copy-number call of {len(bam) - 3} bam genes, CYP2E1's failed call, " - "CYP2D6's depth-check no call and GSTT1's not-assessable line") -for name, planted in (("GSTT1 read as called", summary(gstt1_status="no call")), - ("SLC22A2's row says VCF", summary(vcf_source_for="SLC22A2"))): - got = problems(planted) - if got: - print(f"ok: control, {name} reported: {'; '.join(got)}") - else: - print(f"FAIL: control, {name} passed") - bad.append(name) -sys.exit(1 if bad else 0) -PY +# copy-number call. og summary collect reads it and og report render writes +# the text report, with a CNVpytor section beside pypgx's: every bam gene +# must have exactly one row in its PGx copy number block (its call, "not +# called: pypgx failed" or "not called: the CYP2D6 depth check failed"), and +# GSTT1 its not-assessable line. Two planted +# summaries must be reported: one where GSTT1 reads as called (no +# not-assessable line), and one where a bam gene's row says VCF (it drops out +# of the block). +OG_BIN=${OG:-${REPO}/bin/og} +# pypgx_table GSTT1_STATUS VCF_SOURCE_FOR: step 32's summary table for every gene. +pypgx_table() { + printf 'Gene\tDiplotype\tPhenotype\tCNV_call\tSource\tStatus\tDefining_variants\tMin_GQ\tWeakest_variant\tDefining_filter\tQuality\n' + rows "$CAT" | awk -F'\t' -v OFS='\t' -v gs="$1" -v vf="$2" '{ + g = $1; src = (g == vf) ? "VCF" : toupper($2); na5 = "N/A\tN/A\tN/A\tN/A\tN/A" + if (g == "GSTT1") print g, "N/A", "N/A", "N/A", src, gs, na5 + else if (g == "CYP2E1") print g, "FAILED", "N/A", "N/A", src, "failed", na5 + else if (g == "CYP2D6") print g, "Indeterminate", "Indeterminate (CYP2D6 depth check)", "N/A (CYP2D6 depth check)", src, "no call (CYP2D6 depth check)", na5 + else print g, "*1/*1", "Normal Metabolizer", ($2 == "bam" ? "CNV-" g : "N/A"), src, "called", "none", "N/A", "N/A", "N/A", "N/A" + }' +} +# report_problems NAME GSTT1_STATUS VCF_SOURCE_FOR: what is wrong with the reports of that summary, one per line. +report_problems() { + local d="${WORK}/report-$1" g want got n + mkdir -p "${d}/s1/pypgx" + pypgx_table "$2" "$3" > "${d}/s1/pypgx/s1_pypgx_summary.tsv" + "$OG_BIN" summary collect --sample s1 --sample-dir "${d}/s1" --out "${d}/collected.json" > /dev/null 2>&1 \ + || { echo "og summary collect failed"; return; } + # CNVpytor's section beside pypgx's: pypgx owns every bam gene, so each + # has one row, and it is pypgx's. + jq '.sections.cnvpytor = {"title": "Copy number (CNVpytor)", "step": "18", "state": "ok", "source": null, + "file_date": null, "note": null, + "data": {"total": 0, "deletions": 0, "duplications": 0, "significant": 0}}' "${d}/collected.json" > "${d}/summary.json" + "$OG_BIN" report render --summary "${d}/summary.json" -o "${d}/report.txt" > /dev/null 2>&1 \ + || { echo "og report render failed"; return; } + # The block's rows as GENE<TAB>TEXT. + sed -n '/^## PGx copy number$/,/^$/p' "${d}/report.txt" | sed -n 's/^ \([^ ]*\) *\(.*\)$/\1\t\2/p' > "${d}/block.tsv" + grep -qF 'could not be read' "${d}/report.txt" && echo "a report section could not be read" + # CNVpytor's section is written beside the PGx block, with its counts. + grep -qxF ' Total CNVs: 0 (0 deletions, 0 duplications)' "${d}/report.txt" \ + || echo "CNVpytor's section is not written" + cut -f1 "${d}/block.tsv" | sort | uniq -c | awk '$1 > 1 { print $2 ": " $1 " rows in the copy-number block" }' + while IFS= read -r g; do + if [ "$g" = GSTT1 ]; then + grep -qF 'Not assessable on this reference: GSTT1' "${d}/report.txt" || echo "GSTT1 has no not-assessable line" + continue + fi + case $g in + CYP2E1) want="not called: pypgx failed" ;; + CYP2D6) want="not called: the CYP2D6 depth check failed" ;; + *) want="CNV-${g}" ;; + esac + want="${want} (pypgx, read depth)" + got=$(awk -F'\t' -v g="$g" '$1 == g { print $2; exit }' "${d}/block.tsv") + [ "$got" = "$want" ] || echo "${g}: copy-number block says '${got}', not '${want}'" + done < <(rows "$CAT" | awk -F'\t' '$2 == "bam" { print $1 }') +} +if ! "$OG_BIN" version > /dev/null 2>&1; then + if [ -n "${OG_REQUIRED:-}" ]; then bad "the reports: og is not built (set OG or run scripts/setup.sh --og)"; else + echo "skip: the reports need og (set OG or run scripts/setup.sh --og)" + fi +else + got=$(report_problems shipped "not callable on this reference" "") + if [ -z "$got" ]; then + n=$(rows "$CAT" | awk -F'\t' '$2 == "bam"' | wc -l) + ok "the reports show the copy-number call of $((n - 3)) bam genes, CYP2E1's failed call, CYP2D6's depth-check no call and GSTT1's not-assessable line" + else + while IFS= read -r l; do bad "$l"; done <<< "$got" + fi + # Each control must be caught by its own diagnostic, on a report og wrote. + caught() { ! grep -qE '^og (summary collect|report render) failed$' <<< "$1" && grep -qF "$2" <<< "$1"; } + got=$(report_problems gstt1-called "no call" "") + if caught "$got" "GSTT1 has no not-assessable line"; then ok "control, GSTT1 read as called reported: $(tr '\n' ';' <<< "$got")"; else bad "control, GSTT1 read as called passed"; fi + got=$(report_problems slc22a2-vcf "not callable on this reference" SLC22A2) + if caught "$got" "SLC22A2: copy-number block says"; then ok "control, SLC22A2's row says VCF reported: $(tr '\n' ';' <<< "$got")"; else bad "control, SLC22A2's row says VCF passed"; fi +fi # --- 6. one owner per gene ------------------------------------------------------ # A gene pypgx calls from the BAM (a bam row) is pypgx's: the depth engine's diff --git a/tests/test_render_report.py b/tests/test_render_report.py deleted file mode 100644 index f782cf812..000000000 --- a/tests/test_render_report.py +++ /dev/null @@ -1,963 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""The PRS path around pgsc_calc (og prs format and og prs table, then -bin/collect_summary.py), and its card in bin/render_report.py, on synthetic -files shaped like pgsc_calc's own. - -Checks: - 1. og prs format (run when OG names an og binary; tests/parity pins it - against the Python it replaced) writes a harmonised file as pgsc_calc's - custom GRCh38 file, the folder the prs-table checks below read (written - by hand without OG): - chr_name/chr_position from hm_chr/hm_pos without "chr", the other allele - from other_allele or hm_inferOtherAllele (never "A/G"), rows without a - GRCh38 position dropped, rows off the autosomes (chrX, chrY) dropped - from both the score and the allele list, the label from - assets/pgs_scores.tsv; and it - refuses a GRCh37 file and a non-additive one; with --keep-x (the - sample's sex is known) the chrX rows stay in both and chrY still goes; - chrx_rows.tsv records per score how many chrX rows it has and whether - they were scored or left out; - 2. prs-table without a panel: the sum from aggregated_scores.txt.gz as - pgsc_calc wrote it but without ".0", matched and total counts from the - match summary, Percentile and Ancestry_Group NA, ChrX from - chrx_rows.tsv, Input last; a score pgsc_calc dropped (below its minimum - overlap) has no sum; - 3. prs-table with a panel: the percentile and group of the target sample - only (not the reference samples in the same file), and the ancestry table - with the population and the principal components; - 4. --zero-matches: every score unmatched, no sum; --below-threshold: no - sum, the matched count unknown and the rate from pgscatalog-match's log; - 4c. a sex-specific score, labelled from the real assets/pgs_scores.tsv - (PGS000662, prostate cancer, applies_to male): a sample declared female - keeps its sum and loses its percentile, Sex_Check "not applicable to - the declared sex", in the table and in both reports; declared male it - keeps its percentile; with no sex it keeps it too and both reports name - the score in a note. Red control: the same list without its applies_to - column gives the female sample a percentile, which the check rejects; - 4d. (og prs format, with OG) strand-ambiguous rows (A/T, C/G): counted and "dropped" by default - (pgsc_calc drops them), "kept" with --keep-ambiguous when every other - row agrees with the reference, and left out of the file ("dropped, - strand check failed") when a row agrees only after a strand flip; - --keep-ambiguous without --reference is refused; the Ambiguous column - of prs-table says it per score; - 4e. with a panel, a chrX row og prs format kept that pgsc_calc's match log did - not match (the panel lacks it) reads "left out (not matched in the - panel run)"; matched, it stays "scored"; partly matched, both counts are - named ("1 scored, 2 left out (not matched in the panel run)"); - 5. both reports: a percentile with its group when there is one, the - "Raw score only" line when there is none, and the not-assessed line about - percentiles only then; a panel run whose percentiles were withheld for a - variant-only input says so in both reports and in not-assessed, never - that no panel was installed; a score whose chrX rows were left out is named, - one whose chrX rows were scored is not; both captions, with a panel - and without, carry the ancestry-transfer caveat of docs/25-prs.md; - 6. the ClinVar card: a homozygous 3-star P/LP hit gets no green badge, - no hit gets a gray one, and both reports ask for every hit's population - frequency (heterozygous ones too) and say it is unknown without step 13; with step 06's cDNA read check table, both reports print - the flag beside the flagged hit and "clear" beside the other, the HTML - hits table gets its column and the card the flagged count; without the - table (a Nextflow folder) there is no such column; - 5b. the fine-grained ancestry section of step 26b in both reports, rendered - from the same summary as the PRS card: the groups with their bootstrap - ranges, a small regional group kept because its range is not small, the - painted shares, the picture and the "not a nationality" sentence; - 7. the telomere card says relative telomere content with its unit, and - no report in bin/ says "Telomere Length"; - 8. the mosaic card: a man with an estimated loss of Y shows it in both - reports with its 95% range, the depth ratio and the population-level - note; a woman shows the allele-fraction line too; a sample without the - estimate gets a card that says Not run, and no text section. - 9. the paralog card of step 35b is left out when the step did not run, and - a result older than a run in which step 35b failed is marked Stale. - 10. the blood group card (step 41): every system's row in the HTML table and - the text report, ABO and RhD on top, the "predicted, not typed" caption; - the card is absent when step 41 did not run; and neither the reports nor - og bloodtype say anything about transfusion or donation (the - table is information). - 11. the traits card (step 42): rust/og/tests/genome_traits_report.rs. - 12. the raw-data export (step 40): both reports show how many chip sites - got a genotype, the '--' reasons, the two files and the privacy note; - without the step's table there is no card and no text section. - 13. the couple card (step 39), from a table og couple screen wrote: - both reports name the partner, the shared gene with its chance for a - child and both partners' findings, and point to a genetic counsellor; - without the table there is no card and no text section. - - 14. the chapters (bin/render_report.py CHAPTERS), on DEMO-001, the invented - sample og dev demo-sample writes: no http(s) src or href other - than the loopback viewer and the links marked as opening a website; - the ClinVar section shows no green badge; each text chapter holds its - sections; the card titles are the golden list of the report before the - chapters; each caveat sits once, under its card title. Every detector - is run on a planted bad input as well, which it must catch. With a - previous run on record (summary new_since), a "New since <date>" badge - sits on the sections whose results changed and nowhere else (a badge on - every section is caught), and the card titles stay the golden list. - The frame's own rules (every SECTIONS key in one section, the - Content-Security-Policy and no script, unique ids and resolving - contents links, the chapter order, the badge of the quality block and - of a frame, the re-check line, the print rules and the header) moved to - og report render's cargo tests (rust/crates/og-render/tests/ - port_report_core.rs), with planted faults in place of the source - mutations. - -Run: python3 tests/test_render_report.py -""" -import glob -import gzip -import importlib.util -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -sys.path.insert(0, os.path.join(REPO, "tests", "schema")) -sys.path.insert(0, os.path.join(REPO, "tests")) -import collect_summary # noqa: E402 -import render_report # noqa: E402 -import validate # noqa: E402 -from og_binary import og_binary # noqa: E402 - -FAILS = 0 - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:600]}") - if not ok: - FAILS += 1 - - -PRS_NOT_ADJUSTED = "Polygenic score percentiles: no ancestry reference panel was used, so the scores are raw sums" -PRS_WITHHELD = ("Polygenic score percentiles: the input lists variants only, so the percentiles were withheld and " - "the scores are raw sums") - - -def haplogroup_explore(kind, hg): - """The explore links og summary collect writes for one Y ('y') or mitochondrial ('mt') haplogroup call.""" - with tempfile.TemporaryDirectory() as d: - if kind == "y": - put(f"{d}/y_haplogroup/S_y_haplogroup.txt", f"Sample_name\tHg\tValid_markers\tQC-score\nS\t{hg}\t9\t1\n") - key = "y_haplogroup" - else: - put(f"{d}/mito/S_haplogroup.txt", f"SampleID\tHaplogroup\nS\t{hg}\n") - key = "haplogroup" - return collect_summary.collect("S", d)["sections"][key]["data"]["explore"] - - -def put(path, text, gz=False): - os.makedirs(os.path.dirname(path), exist_ok=True) - with (gzip.open(path, "wt") if gz else open(path, "w")) as f: - f.write(text) - - -HM = ("#pgs_id=PGS000001\n#trait_reported=Something else\n#HmPOS_build=GRCh38\n" - "rsID\tchr_name\tchr_position\teffect_allele\tother_allele\teffect_weight\thm_chr\thm_pos\thm_inferOtherAllele\n" - "rs1\t1\t100\tA\tG\t0.5\t1\t1100\t\n" - "rs2\t1\t200\tc\t\t-0.25\tchr2\t2200\tT\n" - "rs3\t1\t300\tG\t\t1\t3\t3300\tA/C\n" - "rs4\t1\t400\tT\tC\t2\t\t\t\n" - "rs5\tX\t500\tA\tG\t1\tX\t51000000\t\n" - "rs6\tY\t600\tC\tT\t1\tchrY\t2800000\t\n") - - -# og prs format (PRS_PREPARE) writes the score folder prs-table reads. It runs -# only when OG names an og binary; the folders the prs-table checks read are -# written by hand as og writes them for these inputs, and with OG og must -# write the same. -OG = os.environ.get("OG") -if os.environ.get("OG_REQUIRED") and not OG: - sys.exit("FAIL: OG_REQUIRED is set but OG names no og binary") - - -def prs_table(argv): - """og prs table ARGV (the PRS_SUMMARY table, once collect_summary.py - prs-table): its exit code. It always runs; tests/og_binary.py finds og.""" - return subprocess.run([og_binary(), "prs", "table", *argv], capture_output=True).returncode - - -# The columns of og prs table without evaluations: Input last. -PRS_COLUMNS = ["Condition", "PGS_ID", "Score_SUM", "Variants_Matched", "Variants_Total", - "Matched_Pct", "Percentile", "Ancestry_Group", "ChrX", "Ambiguous", "Applies_To", - "Sex_Check", "Input"] - - -def og_format(args): - """og prs format ARGS: its exit code, or None without OG.""" - if not OG: - return None - return subprocess.run([OG, "prs", "format", *args], capture_output=True).returncode - - -def formatted(out, pid, trait, rows, nx=1, chrx="left out", applies="", amb="0\tdropped", evals=""): - """The score folder og prs format writes for one score.""" - put(f"{out}/{pid}.txt.gz", f"#pgs_id={pid}\n#pgs_name={pid}\n#trait_reported={trait}\n#genome_build=GRCh38\n" - "chr_name\tchr_position\teffect_allele\tother_allele\teffect_weight\n" + "".join(r + "\n" for r in rows), gz=True) - put(f"{out}/chrx_rows.tsv", f"pgs_id\tchrx_rows\tchrx\n{pid}\t{nx}\t{chrx}\n") - put(f"{out}/score_notes.tsv", f"pgs_id\tapplies_to\tambiguous_rows\tambiguous\n{pid}\t{applies}\t{amb}\n") - put(f"{out}/score_evaluations.tsv", "pgs_id\tppm_id\tancestry\teffect\n" + (f"{pid}\t{evals}\n" if evals else "")) - - -def folder_text(d): - """Each file of a score folder by name, gzip files read as text.""" - out = {} - for n in sorted(os.listdir(d)): - with (gzip.open if n.endswith(".gz") else open)(os.path.join(d, n), "rt") as f: - out[n] = f.read() - return out - - -def og_writes(desc, args, want_dir): - """With OG: og prs format ARGS must write WANT_DIR's files, into WANT_DIR + "_og".""" - if not OG: - return - rc = og_format(args + ["--out", want_dir + "_og"]) - got = folder_text(want_dir + "_og") if rc == 0 else rc - check(f"og prs format: {desc}", got == folder_text(want_dir), got) - - -def read_rows(path): - with open(path) as f: - lines = [l.rstrip("\n").split("\t") for l in f] - return lines[0], lines[1:] - - -def pgsc_results(d, adjusted): - """A pgsc_calc --outdir for sampleset 'sample': two scores, PGS000001 passed, PGS000002 was dropped. - The reference samples' rows come after the target's, as a reader that - forgot to skip them would take their values.""" - put(f"{d}/sample/match/sample_summary.csv", - "dataset,accession,ambiguous,is_multiallelic,match_flipped,duplicate_best_match,duplicate_ID,match_IDs,match_status,count,score_pass,match_rate\n" - "sample,PGS000001,false,false,false,false,false,NA,matched,2,true,0.67\n" - "sample,PGS000001,false,false,false,false,false,NA,unmatched,1,true,0.67\n" - "sample,PGS000002,false,false,false,false,false,NA,unmatched,5,false,0.0\n") - if not adjusted: - put(f"{d}/sample/score/aggregated_scores.txt.gz", - "sampleset\tFID\tIID\tPGS\tSUM\tDENOM\nsample\tHG\tHG\tPGS000001\t62.0\t6\n", gz=True) - return - put(f"{d}/sample/score/sample_pgs.txt.gz", - "sampleset\tFID\tIID\tPGS\tSUM\tZ_MostSimilarPop\tZ_norm1\tZ_norm2\tpercentile_MostSimilarPop\n" - "sample\tHG\tHG\tPGS000001\t1.25\t1.6\t1.5\t1.4\t94.44\n" - "reference\tR1\tR1\tPGS000001\t10.0\t0.1\t0.1\t0.1\t12.3\n", gz=True) - put(f"{d}/sample/score/sample_popsimilarity.txt.gz", - "sampleset\tFID\tIID\tPC1\tPC2\tPC10\tSuperPop\tUnrelated\tRF_P_AFR\tRF_P_EUR\tMostSimilarPop\tMostSimilarPop_LowConfidence\tREFERENCE\n" - "reference\tR1\tR1\t-1\t2\t0\tAFR\tTrue\t0.9\t0.1\tAFR\tFalse\tTrue\n" - "sample\tHG\tHG\t0.012\t-0.034\t0.5\t\t\t0.05\t0.95\tEUR\tFalse\tFalse\n", gz=True) - - -# The card titles of DEMO-001's HTML report before the chapters (check 14): -# the chapters move cards, they never add, drop or rename one. The CPIC gene card is there since the -# demo sample carries a CPIC table (og dev demo-sample, for the live demo's medicines widget). -GOLDEN_TITLES = sorted([ - "Quality Control", "Mosaic Loss of Chromosome Y or X", "Variant Calling", "ClinVar Screening", - "Carrier List (this sample)", "Couple Carrier Screen", "Pharmacogenomics", "pypgx Pharmacogenomics", - "PGx Copy Number", "CPIC Drug Recommendations", "Genes With a Non-Normal or Unresolved Phenotype (CPIC)", "CYP2D6 Across Callers", "HLA Typing", "Structural Variants", - "Structural Variants, Class 4 and 5 (AnnotSV)", "Cancer Predisposition", "Repeat Expansions", - "Runs of Homozygosity", "Mitochondrial Haplogroup", "Paralog Genes (Paraphase)", - "SMN1/SMN2 Copy Number (Parascopy)", "Y-Chromosome Haplogroup", "Y Placement on the Current Public Tree", "Y-STR Markers (HipSTR)", - "Raw-Data Export (23andMe v5 layout)", "Blood Groups (predicted)", "Archaic Ancestry (Neanderthal, Denisovan)", - "Lp(a): LPA KIV-2 Repeat Count", "MUC1 VNTR Frameshifts (VNtyper)", "Locityper Second Opinion at Hard Loci", "Viral DNA in the Unmapped Reads", - "Clonal Haematopoiesis (CHIP)", "Mosaic Chromosomal Alterations (MoChA)", "Mobile Element Insertions", - "Telomere content (relative)", "Mitochondrial Analysis", "Clinical Variant Filter", - "Variant Prioritization (slivar)", "Symptom-Driven Ranking (Exomiser)", "Polygenic Risk Scores", "Fine-Grained Ancestry", "Traits", - "Ancient Populations", "Genes This Genome Could Not Read Reliably", "ClinVar Hits (best-reviewed first, 3 of 3)", "Steps Not Run", - "Not Assessed by This Pipeline"]) - - -def external_problems(html): - """Every http(s) src or href that is not the loopback viewer and not - inside a link marked as opening a website.""" - marked = re.sub(r'<span class="offsite"><a href="[^"]*">.*?</a> <span class="offsite-note">\(opens a website[^)]*\)</span></span>', - "", html) - return [m for m in re.findall(r'(?:src|href)\s*=\s*["\']?(https?:[^"\' >]*)', marked, re.I) - if not m.startswith(render_report.VIEWER_URL)] - - -def frame_of(html, sid): - return html.split(f'id="{sid}">', 1)[-1].split("</section>", 1)[0] if f'id="{sid}">' in html else "" - - -def text_chapter_order(txt): - return re.findall(r"^# \d+\. (.+?): ", txt, re.M) - - -def chapter_checks(work): - """Check 14 (the docstring).""" - from og_binary import og_binary - g = f"{work}/demo" - os.makedirs(g) - subprocess.run([og_binary(), "dev", "demo-sample", "--genome-dir", g, "--records", "20000", "--repo", REPO], - check=True, stdout=subprocess.DEVNULL) - summ = collect_summary.collect("DEMO-001", f"{g}/DEMO-001") - txt, html = render_report.text_report(summ), render_report.html_report(summ) - - real_section_of = render_report.SECTION_OF - render_report.SECTION_OF = {k: v for k, v in real_section_of.items() if k != "telomere"} - try: - for name, fn in (("HTML", render_report.html_report), ("text", render_report.text_report)): - try: - fn(summ) - stopped = False - except KeyError: - stopped = True - check(f"chapters: a card for a key no section lists stops the {name} report", stopped) - finally: - render_report.SECTION_OF = real_section_of - - for kind, hg in (("y", "R-M269*(xR-L23)"), ("mt", "H1a+16189")): - rows = render_report.explore_html({"haplogroup": hg, "explore": haplogroup_explore(kind, hg)}) - marks = re.findall(r'<span class="offsite-note">\(([^)]*)\)</span>', "".join(rows)) - want = [render_report.OFFSITE_HAPLOGROUP] * (len(marks) - 1) + [render_report.OFFSITE] - check(f"chapters: {hg}: the FamilyTreeDNA and YFull links say the address carries the haplogroup, the docs link does not", - len(marks) >= 3 and marks == want, marks) - - check("chapters: no off-machine address but the marked links", external_problems(html) == [], external_problems(html)) - check("chapters: the marked links are there (FamilyTreeDNA, YFull, GitHub)", - all(f'<span class="offsite"><a href="{u}' in html for u in - ("https://discover.familytreedna.com/", "https://www.yfull.com/", "https://github.com/"))) - for bad in ('<a href="https://example.org/x">x</a>', '<img src="https://example.org/i.png">', - '<a class="offsite" href="https://example.org/y">y</a>'): - check(f"chapters: red control: {bad} is caught", external_problems(html + bad) != []) - - sids = [sid for _, _, _, secs in render_report.CHAPTERS for sid, _, _ in secs] - - cv = frame_of(html, "clinvar") - check("chapters: the ClinVar section has its two cards and no green badge", - "<h2>ClinVar Screening</h2>" in cv and "<h2>ClinVar Hits" in cv and "badge-green" not in cv, cv[:800]) - check("chapters: red control: the detector sees green where there is green (the QC sex badge)", - "badge-green" in frame_of(html, "qc")) - nohit = json.loads(json.dumps(summ)) - nohit["sections"]["clinvar"]["data"].update(count=0, hits=[]) - check("chapters: no ClinVar hit, still no green", "badge-green" not in frame_of(render_report.html_report(nohit), "clinvar")) - check("chapters: the CPIC table sits in the CPIC section", "CPIC Drug Recommendations" in frame_of(html, "cpic")) - - heads = {"QUALITY": "## Quality control", "CLINICAL": "## ClinVar Pathogenic Screen", - "CARRIER": "## Carrier List (this sample)", "MEDICINES": "## Pharmacogenomics (PharmCAT)", - "ANCESTRY": "## Mitochondrial Haplogroup", "TRAITS AND SCORES": "## Polygenic Risk Scores", - "OTHER AND EXPORTS": "## Raw-Data Export (23andMe v5 layout)"} - chunks = dict(zip(text_chapter_order(txt), re.split(r"^# \d+\. .*$", txt, flags=re.M)[1:])) - check("chapters: each text chapter holds its sections", all(h in chunks.get(c, "") for c, h in heads.items()), - {c: h in chunks.get(c, "") for c, h in heads.items()}) - - got = sorted(re.findall(r"<h2>([^<]*)</h2>", html)) - check("chapters: the card titles are the golden list", got == GOLDEN_TITLES, - sorted(set(got) ^ set(GOLDEN_TITLES))) - check("chapters: red control: a dropped card is caught", - sorted(re.findall(r"<h2>([^<]*)</h2>", html.replace("<h2>Traits</h2>", ""))) != GOLDEN_TITLES) - - # "New since" and the database re-check (bin/owngenome_recheck.py, og summary collect) - since = json.loads(json.dumps(summ)) - since["new_since"] = {"previous_utc": "2026-01-01T00:00:00Z", "sections": ["clinvar"]} - since["recheck"] = {"checked_utc": "2026-02-02T00:00:00Z", "affected_steps": ["06", "24"], "rerun": None, - "items": [{"item": "clinvar_file_date", "label": "ClinVar", "class": "refresh-available"}, - {"item": "VEP_CACHE_RELEASE", "label": "VEP", "class": "no-newer-release"}]} - h2, t2 = render_report.html_report(since), render_report.text_report(since) - badge = "New since 2026-01-01T00:00:00Z" - - def badge_problems(page, want_sids): - got = [sid for sid in sids if badge in frame_of(page, sid)] - return [] if got == want_sids and page.count(badge) == len(want_sids) else got - check("new since: the badge only on the section whose result changed", badge_problems(h2, ["clinvar"]) == [], - badge_problems(h2, ["clinvar"])) - check("new since: no badge without a previous run", "New since" not in html and "New since" not in txt) - every = json.loads(json.dumps(since)) - every["new_since"]["sections"] = [k for k, *_ in collect_summary.SECTIONS] - check("new since: red control: a badge on every section is caught", - badge_problems(render_report.html_report(every), ["clinvar"]) != []) - def text_with(sections, missing=None, mod=render_report): - x = json.loads(json.dumps(since)) - if missing: - x["sections"][missing]["state"] = "missing" - x["new_since"] = dict(since["new_since"], sections=sections) - return mod.html_report(x), mod.text_report(x) - - def under(txt, title): - i = txt.find(title) - return txt[i:i + 120] if i >= 0 else "" - marks = { - "new since: the text CYP2D6 block carries the badge when pypgx changed": - (["pypgx"], None, lambda h, t: f"[{badge}]" in under(t, "## CYP2D6 across callers")), - "new since: a section gone missing keeps its badge on its HTML card, the text leaves the section out": - (["roh"], "roh", lambda h, t: badge in frame_of(h, "roh") and "Not run" in frame_of(h, "roh") - and "## Runs of Homozygosity" not in t and badge not in t), - } - for name, (secs, miss, fn) in marks.items(): - check(name, fn(*text_with(secs, miss)), under(text_with(secs, miss)[1], "## ")) - rsrc = open(os.path.join(REPO, "bin", "render_report.py")).read() - planted = { - "new since: the text CYP2D6 block carries the badge when pypgx changed": - 'for k in ("pypgx", "cyrius")):', - } - for name, needle in planted.items(): - check(f"planted mistake is in the copy: {name}", needle in rsrc, needle) - mut = rsrc.replace(needle, "for k in ()):" if needle.startswith("for k") else '""') - mdir = tempfile.mkdtemp(prefix="render-mutant.", dir=work) - open(os.path.join(mdir, "render_report_mutant.py"), "w").write(mut) - spec = importlib.util.spec_from_file_location("render_report_mutant", os.path.join(mdir, "render_report_mutant.py")) - mm = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mm) - secs, miss, fn = marks[name] - check(f"fails on a planted mistake: {name}", not fn(*text_with(secs, miss, mod=mm)), "passed on the mutant") - check("new since: the card titles stay the golden list", sorted(re.findall(r"<h2>([^<]*)</h2>", h2)) == GOLDEN_TITLES, - sorted(set(re.findall(r"<h2>([^<]*)</h2>", h2)) ^ set(GOLDEN_TITLES))) - hist = f"{g}/DEMO-001/history/2026-01-01T000000Z" - os.makedirs(hist) - old = json.loads(json.dumps(summ)) - old["generated_utc"] = "2026-01-01T00:00:00Z" - if old["sections"]["clinvar"]["data"].get("hits"): - old["sections"]["clinvar"]["data"]["hits"][0]["significance"] = "Uncertain significance" - with open(f"{hist}/summary.json", "w") as f: - json.dump(old, f) - if os.path.isfile(f"{g}/DEMO-001/run_manifest.tsv"): - shutil.copy(f"{g}/DEMO-001/run_manifest.tsv", hist) - again = collect_summary.collect("DEMO-001", f"{g}/DEMO-001") - shutil.rmtree(f"{g}/DEMO-001/history") - check("new since: collect_summary compares with the newest history/ folder", - bool(summ["sections"]["clinvar"]["data"].get("hits")) and "new_since" not in summ - and again.get("new_since") == {"previous_utc": "2026-01-01T00:00:00Z", "sections": ["clinvar"]}, - again.get("new_since")) - check("new since: the text marks the ClinVar section once", t2.count(f"[{badge}]") == 1 - and f"## ClinVar Pathogenic Screen\n---\n [{badge}]" in t2, t2.count(f"[{badge}]")) - shown = [n for n in render_report.caveat_notes() if render_report.E(n) in html] - # PRS_ANCESTRY_NOTE is part of the PRS caption's sentence (prs_note), not a line of its own. - loose = [n[:40] for n in shown if n != render_report.PRS_ANCESTRY_NOTE - and html.count(render_report.E(n)) != html.count(f'<p class="caveat">{render_report.E(n)}</p>')] - check("chapters: every caveat shown is under a card title, never left in the body", len(shown) > 20 and not loose, - loose or len(shown)) - check("chapters: red control: a caveat left in a body is caught", - html.replace('<p class="caveat">' + render_report.E(render_report.EH_NOTE), "<p>" + render_report.E(render_report.EH_NOTE)) - .count('<p class="caveat">' + render_report.E(render_report.EH_NOTE)) == 0) - mo = html.split("<h2>Mosaic Loss of Chromosome Y or X</h2>", 1)[-1] - check("chapters: the caveat comes right after the title and the step line", - re.match(r'\s*<p class="sec-meta">Step 33 .*?</p>\s*<p class="caveat">' + re.escape(render_report.E(render_report.MOSAIC_NOTE)), - mo) is not None, mo[:400]) - - -def main(): - work = tempfile.mkdtemp() - try: - # 1. og prs format - put(f"{work}/in/PGS000001_hmPOS_GRCh38.txt.gz", HM, gz=True) - put(f"{work}/labels.tsv", "# comment\npgs_id\ttrait_reported\nPGS000001\tCoronary artery disease\n") - # GRCh38 positions without chr, the other allele from hm_inferOtherAllele - # (never A/C), unplaced and chrX/chrY rows dropped, labelled from the list. - rows1 = ["1\t1100\tA\tG\t0.5", "2\t2200\tC\tT\t-0.25", "3\t3300\tG\t\t1"] - formatted(f"{work}/pgs", "PGS000001", "Coronary artery disease", rows1) - if not OG: - print("[NOTE] OG is not set, so og prs format does not run here; tests/parity and its own tests pin it") - og_writes("the header pgsc_calc reads, the rows of the autosomes, chrX left out", - ["--scores", f"{work}/in", "--labels", f"{work}/labels.tsv", "--alleles", f"{work}/alleles.tsv"], f"{work}/pgs") - if OG: - with open(f"{work}/alleles.tsv") as f: - al = f.read().splitlines() - check("og prs format: every effect and other allele of the autosomes, chr-prefixed and sorted", - al == ["chr1\t1100\tA", "chr1\t1100\tG", "chr2\t2200\tC", "chr2\t2200\tT", "chr3\t3300\tG"], al) - # --keep-x: the sample's sex is known - formatted(f"{work}/pgsx", "PGS000001", "Coronary artery disease", rows1 + ["X\t51000000\tA\tG\t1"], chrx="scored") - og_writes("--keep-x: the chrX row stays, the chrY row still goes, chrx_rows.tsv says scored", - ["--scores", f"{work}/in", "--labels", f"{work}/labels.tsv", "--alleles", f"{work}/allelesx.tsv", "--keep-x"], - f"{work}/pgsx") - if OG: - with open(f"{work}/allelesx.tsv") as f: - alx = f.read().splitlines() - check("og prs format --keep-x: the chrX alleles are in the allele list, after the autosomes", - alx[-2:] == ["chrX\t51000000\tA", "chrX\t51000000\tG"] and not any(a.startswith("chrY") for a in alx), alx) - put(f"{work}/bad37/PGS000009.txt.gz", HM.replace("HmPOS_build=GRCh38", "HmPOS_build=GRCh37"), gz=True) - rc = og_format(["--scores", f"{work}/bad37", "--out", f"{work}/o37", "--alleles", f"{work}/a37"]) - check("og prs format refuses a file harmonised to GRCh37", rc == 1, rc) - put(f"{work}/badnonadd/PGS000010.txt.gz", - "#HmPOS_build=GRCh38\nhm_chr\thm_pos\teffect_allele\teffect_weight\tis_recessive\n1\t5\tA\t1\tTrue\n", gz=True) - rc = og_format(["--scores", f"{work}/badnonadd", "--out", f"{work}/onon", "--alleles", f"{work}/anon"]) - check("og prs format refuses a non-additive score", rc == 1, rc) - rc = og_format(["--scores", f"{work}/in", "--ids", "PGS000001,PGS000777", - "--out", f"{work}/omiss", "--alleles", f"{work}/amiss"]) - check("og prs format: an id of the list with no file is an error", rc == 1, rc) - - # A second formatted score, for the table - put(f"{work}/pgs/PGS000002.txt.gz", "#pgs_id=PGS000002\n#trait_reported=Trait two\n#genome_build=GRCh38\n" - "chr_name\tchr_position\teffect_allele\tother_allele\teffect_weight\n" + "1\t1\tA\t\t1\n" * 5, gz=True) - - # 2. prs-table, no panel - pgsc_results(f"{work}/raw", adjusted=False) - out = f"{work}/S_prs_summary.tsv" - rc = prs_table(["--sample", "S", "--results", f"{work}/raw", "--sampleset", "sample", - "--scores", f"{work}/pgs", "--input-kind", "gvcf", "--out", out]) - hdr, rows = read_rows(out) - check("prs-table exits 0", rc == 0, rc) - check("prs-table: the columns, Input last", hdr == PRS_COLUMNS and hdr[-1] == "Input", hdr) - check("prs-table: sum 62 (not 62.0), 2 of 3 matched, no percentile, its chrX row left out", - rows[0] == ["Coronary artery disease", "PGS000001", "62", "2", "3", "66.7", "NA", "NA", "1 left out", "0", "", "", "gvcf"], rows[0]) - check("prs-table: a score pgsc_calc dropped has no sum and its counts; no chrX entry for a score og prs format did not write", - rows[1] == ["Trait two", "PGS000002", "NA", "0", "5", "0.0", "NA", "NA", "", "", "", "", "gvcf"], rows[1]) - - # 3. prs-table with a panel - pgsc_results(f"{work}/adj", adjusted=True) - anc = f"{work}/S_ancestry.tsv" - rc = prs_table(["--sample", "S", "--results", f"{work}/adj", "--sampleset", "sample", - "--scores", f"{work}/pgs", "--input-kind", "gvcf", "--panel", "pgsc_1000G_v1", - "--ancestry-out", anc, "--out", out]) - hdr, rows = read_rows(out) - check("prs-table with a panel exits 0", rc == 0, rc) - check("prs-table: the target's sum, percentile and group, not the reference sample's", - rows[0][2:8] == ["1.25", "2", "3", "66.7", "94.4", "EUR"], rows[0]) - kv = dict(l.rstrip("\n").split("\t", 1) for l in open(anc)) - check("ancestry table: population, panel, probabilities and the PCs in order", - kv.get("population") == "EUR" and kv.get("reference_panel") == "pgsc_1000G_v1" - and kv.get("probability_EUR") == "0.95" and kv.get("PC1") == "0.012" and kv.get("PC2") == "-0.034" - and list(k for k in kv if k.startswith("PC")) == ["PC1", "PC2", "PC10"], kv) - - # 4. zero matches - rc = prs_table(["--sample", "S", "--results", f"{work}/none", "--sampleset", "sample", - "--scores", f"{work}/pgs", "--input-kind", "vcf", "--zero-matches", "--out", out]) - hdr, rows = read_rows(out) - check("prs-table --zero-matches: every score unmatched, Input vcf", - rc == 0 and [r[2:6] + r[12:] for r in rows] == [["NA", "0", "3", "0.0", "vcf"], ["NA", "0", "5", "0.0", "vcf"]], rows) - rc = prs_table(["--sample", "S", "--results", f"{work}/none", "--sampleset", "sample", - "--scores", f"{work}/pgs", "--input-kind", "vcf", "--out", out]) - check("prs-table: no pgsc_calc output and no --zero-matches is an error", rc == 1, rc) - - # 4b. every score under pgsc_calc's minimum overlap: no sum, matched unknown, the rate from the log - put(f"{work}/below.log", "ERROR Score PGS000001 fails minimum matching threshold (33.33% variants match)\n" - "ERROR pgscatalog.core.lib.pgsexceptions.ZeroMatchesError: All scores fail to meet match threshold 0.75\n") - rc = prs_table(["--sample", "S", "--results", f"{work}/none", "--sampleset", "sample", - "--scores", f"{work}/pgs", "--input-kind", "gvcf", "--below-threshold", f"{work}/below.log", - "--out", out]) - hdr, rows = read_rows(out) - check("prs-table --below-threshold: no sum, matched NA (not 0), the rate from the log, NA without one", - rc == 0 and [r[2:6] for r in rows] == [["NA", "NA", "3", "33.3"], ["NA", "NA", "5", "NA"]], rows) - - # 4c. sex-specific scores, from the real score list - assets = os.path.join(REPO, "assets", "pgs_scores.tsv") - put(f"{work}/sexin/PGS000662_hmPOS_GRCh38.txt.gz", HM.replace("PGS000001", "PGS000662"), gz=True) - - def sex_rows(labels, sex, tag, applies, evals): - pgs, res = f"{work}/sex_{tag}/pgs", f"{work}/sex_{tag}/res" - formatted(pgs, "PGS000662", "Prostate cancer", rows1, applies=applies, evals=evals) - og_writes(f"4c: PGS000662 labelled from {os.path.basename(labels)}", - ["--scores", f"{work}/sexin", "--labels", labels, "--alleles", f"{work}/sex_{tag}/alleles.tsv"], pgs) - pgsc_results(res, adjusted=True) - for f_ in glob.glob(f"{res}/sample/*/*"): - if f_.endswith(".gz"): - with gzip.open(f_, "rt") as g: - body = g.read() - put(f_, body.replace("PGS000001", "PGS000662"), gz=True) - else: - with open(f_) as g: - body = g.read() - put(f_, body.replace("PGS000001", "PGS000662")) - o = f"{work}/sex_{tag}/S_prs_summary.tsv" - rc2 = prs_table(["--sample", "S", "--results", res, "--sampleset", "sample", - "--scores", pgs, "--input-kind", "gvcf", "--panel", "pgsc_1000G_v1", - "--out", o] + (["--sex", sex] if sex else [])) - h, r = read_rows(o) - return rc2, dict(zip(h, r[0])), o - - male_only = ("male", "PPM022982;PPM015482\tEUR;AFR\tHR 1.72;OR 1.86") - rc2, female, f_out = sex_rows(assets, "female", "f", *male_only) - check("4c: prs-table exits 0 on PGS000662", rc2 == 0, rc2) - check("4c: declared female, PGS000662 (male): raw sum kept, no percentile, not applicable", - female.get("Score_SUM") == "1.25" and female.get("Percentile") == "NA" - and female.get("Applies_To") == "male" and female.get("Sex_Check") == "not applicable to the declared sex", - female) - _, male, m_out = sex_rows(assets, "male", "m", *male_only) - check("4c: declared male, PGS000662: the percentile stays, Sex_Check applies", - male.get("Percentile") == "94.4" and male.get("Sex_Check") == "applies", male) - _, nosex, n_out = sex_rows(assets, "", "n", *male_only) - check("4c: no sex declared, PGS000662: the percentile stays, Sex_Check says no sex was declared", - nosex.get("Percentile") == "94.4" and nosex.get("Sex_Check") == "no sex declared", nosex) - for tag, src in (("f", f_out), ("n", n_out)): - dd = f"{work}/sex_sample_{tag}/S" - os.makedirs(f"{dd}/prs", exist_ok=True) - shutil.copy(src, f"{dd}/prs/S_prs_summary.tsv") - put(f"{dd}/ancestry/S_ancestry.tsv", "key\tvalue\nsample\tS\nreference_panel\tpgsc_1000G_v1\npopulation\tEUR\n") - sm = collect_summary.collect("S", dd) - tx, ht = render_report.text_report(sm), render_report.html_report(sm) - if tag == "f": - check("4c: declared female: both reports print not applicable for PGS000662, no percentile", - "PGS000662 not applicable to the declared sex (male-specific score)" in tx - and "<td>not applicable to the declared sex (male-specific score)</td>" in ht - and "percentile 94.4" not in tx and "<td>94.4 (EUR)</td>" not in ht, tx) - else: - check("4c: no sex: both reports keep the percentile and name the score in a note", - "percentile 94.4 (EUR)" in tx and "<td>94.4 (EUR)</td>" in ht - and "No sex was declared, so these sex-specific scores are shown for any sex: PGS000662 (male)." in tx - and "PGS000662 (male)." in ht, tx) - # Red control: the score list without its applies_to column. - with open(assets, encoding="utf-8") as f_: - two = "".join("\t".join(l.rstrip("\n").split("\t")[:2]) + "\n" if not l.startswith("#") else l for l in f_) - put(f"{work}/two_columns.tsv", two) - _, ctl, _ = sex_rows(f"{work}/two_columns.tsv", "female", "ctl", "", "") - check("4c: control: without applies_to the female sample would get the percentile, which the check above rejects", - ctl.get("Percentile") == "94.4" and ctl.get("Sex_Check") == "", ctl) - - # 4d. strand-ambiguous rows against a small reference: chr1 is GAAGCCAT - put(f"{work}/ref/ref.fa", ">chr1\nGAAG\nCCAT\n") - put(f"{work}/ref/ref.fa.fai", "chr1\t8\t6\t4\t5\n") - amb = ("#pgs_id=PGS000003\n#HmPOS_build=GRCh38\nhm_chr\thm_pos\teffect_allele\tother_allele\teffect_weight\n" - "1\t2\tA\tG\t1\n1\t3\tA\tT\t1\n1\t4\tC\tG\t1\n1\t5\tT\tC\t1\n") - put(f"{work}/ambin/PGS000003.txt.gz", amb, gz=True) - # chr1:7 is A: T/C agrees only on the other strand (A/G) - put(f"{work}/flipin/PGS000003.txt.gz", amb + "1\t7\tT\tC\t1\n", gz=True) - - def amb_run(src, tag, extra): - out_ = f"{work}/amb_{tag}" - rc = og_format(["--scores", f"{work}/{src}", "--out", out_, "--alleles", f"{out_}.alleles"] + extra) - if rc: - return rc, None, None - with gzip.open(f"{out_}/PGS000003.txt.gz", "rt") as g: - body = [l.split("\t")[1] for l in g.read().splitlines()[5:]] - with open(f"{out_}/score_notes.tsv") as g: - notes = g.read().splitlines() - return rc, body, notes - - ref = ["--keep-ambiguous", "--reference", f"{work}/ref/ref.fa"] - if OG: - rc, body, notes = amb_run("ambin", "off", []) - check("4d: default: the 2 ambiguous rows are written and counted as dropped (pgsc_calc drops them)", - rc == 0 and body == ["2", "3", "4", "5"] and notes[1:] == ["PGS000003\t\t2\tdropped"], (rc, body, notes)) - rc, body, notes = amb_run("ambin", "on", ref) - check("4d: --keep-ambiguous, every other row agrees with the reference: kept", - rc == 0 and body == ["2", "3", "4", "5"] and notes[1:] == ["PGS000003\t\t2\tkept"], (rc, body, notes)) - rc, body, notes = amb_run("flipin", "flip", ref) - check("4d: --keep-ambiguous, a row that agrees only on the other strand: the ambiguous rows are left out", - rc == 0 and body == ["2", "5", "7"] and notes[1:] == ["PGS000003\t\t2\tdropped, strand check failed"], - (rc, body, notes)) - rc, _, _ = amb_run("ambin", "noref", ["--keep-ambiguous"]) - check("4d: --keep-ambiguous without --reference is refused", rc == 1, rc) - formatted(f"{work}/amb_off", "PGS000003", "PGS000003", - ["1\t2\tA\tG\t1", "1\t3\tA\tT\t1", "1\t4\tC\tG\t1", "1\t5\tT\tC\t1"], nx=0, amb="2\tdropped") - og_writes("4d: the folder of the default run", ["--scores", f"{work}/ambin", "--alleles", f"{work}/amb_off.alleles2"], - f"{work}/amb_off") - put(f"{work}/amb_res/sample/match/sample_summary.csv", - "dataset,accession,ambiguous,is_multiallelic,match_flipped,duplicate_best_match,duplicate_ID,match_IDs,match_status,count,score_pass,match_rate\n" - "sample,PGS000003,false,false,false,false,false,NA,matched,2,true,0.5\n" - "sample,PGS000003,true,false,false,false,false,NA,excluded,2,true,0.5\n") - put(f"{work}/amb_res/sample/score/aggregated_scores.txt.gz", - "sampleset\tFID\tIID\tPGS\tSUM\tDENOM\nsample\tHG\tHG\tPGS000003\t2\t2\n", gz=True) - rc = prs_table(["--sample", "S", "--results", f"{work}/amb_res", "--sampleset", "sample", - "--scores", f"{work}/amb_off", "--input-kind", "gvcf", "--out", f"{work}/amb.tsv"]) - h, r = read_rows(f"{work}/amb.tsv") - check("4d: prs-table: the Ambiguous column counts them per score", rc == 0 and dict(zip(h, r[0])).get("Ambiguous") == "2 dropped", r) - - # 4e. a chrX row and the panel's intersection: pgsc_calc's match log - formatted(f"{work}/pgsx_panel", "PGS000001", "Coronary artery disease", rows1 + ["X\t51000000\tA\tG\t1"], - chrx="scored") - pgsc_results(f"{work}/xpanel", adjusted=True) - log = "row_nr,accession,chr_name,chr_position,effect_allele,other_allele,match_status,dataset\n" - put(f"{work}/xpanel/sample/match/sample_log.csv.gz", log - + "0,PGS000001,1,1100,A,G,matched,sample\n1,PGS000001,X,51000000,A,G,excluded,sample\n", gz=True) - - def chrx_of(res): - prs_table(["--sample", "S", "--results", res, "--sampleset", "sample", - "--scores", f"{work}/pgsx_panel", "--input-kind", "gvcf", "--panel", "pgsc_1000G_v1", - "--out", f"{work}/xp.tsv"]) - h, r = read_rows(f"{work}/xp.tsv") - return dict(zip(h, r[0])).get("ChrX") - got = chrx_of(f"{work}/xpanel") - check("4e: panel run, the chrX row excluded by pgsc_calc: left out (not matched in the panel run)", - got == "1 left out (not matched in the panel run)", got) - put(f"{work}/xpanel/sample/match/sample_log.csv.gz", log - + "0,PGS000001,1,1100,A,G,matched,sample\n1,PGS000001,X,51000000,A,G,matched,sample\n", gz=True) - got = chrx_of(f"{work}/xpanel") - check("4e: panel run, the chrX row matched: scored", got == "1 scored", got) - # Three chrX rows, the log matched one: both counts are named. - put(f"{work}/pgsx_panel/chrx_rows.tsv", "pgs_id\tchrx_rows\tchrx\nPGS000001\t3\tscored\n") - got = chrx_of(f"{work}/xpanel") - check("4e: panel run, 1 of 3 chrX rows matched: 1 scored, 2 left out", - got == "1 scored, 2 left out (not matched in the panel run)", got) - - # 5. the reports - d = f"{work}/sample_raw/S" - put(f"{d}/prs/S_prs_summary.tsv", "\t".join(PRS_COLUMNS) + "\n" - "Coronary artery disease\tPGS000001\t62\t2\t3\t66.7\tNA\tNA\t8 left out\tgvcf\n" - "Trait two\tPGS000002\t1\t5\t5\t100.0\tNA\tNA\t3 scored\tgvcf\n") - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - check("no panel: the raw-score line in both reports", "Raw score only" in txt and "Raw score only" in html) - check("chrX left out: both reports name the score and its row count, not the scored one", - "Rows on chrX were left out, because the sample's sex was not given: PGS000001 (8)." in txt - and "Rows on chrX were left out, because the sample's sex was not given: PGS000001 (8)." in html - and "PGS000002 (3)" not in txt, txt) - check("no panel: the HTML row says raw score only", "<td>raw score only</td>" in html) - check("no panel: not-assessed names the missing percentiles", - PRS_NOT_ADJUSTED in summ["not_assessed"] and PRS_WITHHELD not in summ["not_assessed"]) - # A panel run whose input lists variants only: og prs table withheld - # every percentile and wrote why (Percentile_Note). Neither report - # may say that no panel was installed, nor tell the user to add one. - d = f"{work}/sample_withheld/S" - withheld = "percentile withheld: the input lists variants only" - put(f"{d}/prs/S_prs_summary.tsv", "\t".join(PRS_COLUMNS[:-1] + ["Percentile_Note", "Input"]) + "\n" - f"Coronary artery disease\tPGS000001\t62\t2\t3\t66.7\tNA\tEUR\t0\t0\t\t\t{withheld}\tvcf\n") - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - check("withheld: both reports say the percentile was withheld for a variant-only input", - "Percentile withheld: the input lists variants only" in txt - and "Percentile withheld: the input lists variants only" in html, txt) - check("withheld: no report says no panel was installed, nor asks for one", - "no ancestry reference panel was installed" not in txt + html - and "--ancestry-panel adds it" not in txt + html, txt) - check("withheld: not-assessed gives the true reason", - PRS_WITHHELD in summ["not_assessed"] and PRS_NOT_ADJUSTED not in summ["not_assessed"], - summ["not_assessed"]) - d = f"{work}/sample_adj/S" - put(f"{d}/prs/S_prs_summary.tsv", "\t".join(PRS_COLUMNS) + "\n" - "Coronary artery disease\tPGS000001\t1.25\t2\t3\t66.7\t94.4\tEUR\t0\tgvcf\n") - put(f"{d}/ancestry/S_ancestry.tsv", "key\tvalue\nsample\tS\nreference_panel\tpgsc_1000G_v1\npopulation\tEUR\n") - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - check("panel: the percentile with its group in the text report", "percentile 94.4 (EUR)" in txt, txt) - check("panel: the percentile with its group in the HTML report", "<td>94.4 (EUR)</td>" in html) - check("panel: the note names the group and the panel, no raw-score line, no chrX line", - "among the EUR samples of the pgsc_1000G_v1 reference panel" in txt and "Raw score only" not in txt - and "Raw score only" not in html and "Rows on chrX" not in txt, txt) - check("panel: no not-assessed line about percentiles", PRS_NOT_ADJUSTED not in summ["not_assessed"]) - caveat = ("Most PRS scores were developed in European-ancestry populations and predict less well for " - "other ancestries, even with an ancestry-adjusted percentile.") - check("panel: the ancestry-transfer caveat in both reports", caveat in txt and caveat in html, txt) - raw = collect_summary.collect("S", f"{work}/sample_raw/S") - check("no panel: the ancestry-transfer caveat in both reports", - caveat in render_report.text_report(raw) and caveat in render_report.html_report(raw)) - - # 5b. the fine-grained ancestry section (step 26b), in the same sample - # folder as the PRS card: both run in one report, and the PRS block's - # own local names must not break this one. - put(f"{d}/ancestry_fine/S_ancestry_fine.tsv", - "level\tcode\tgroup\tfraction\tse\tlow\thigh\n" - "continent\tWEA\tWest Eurasia (Europe, Middle East, North Africa)\t0.8200\t0.0300\t0.7600\t0.8800\n" - "continent\tCSA\tCentral and South Asia\t0.1500\t0.0300\t0.0900\t0.2100\n" - "continent\tAFR\tAfrica\t0.0300\t0.0100\t0.0100\t0.0500\n" - "region\tLEV\tLevant\t0.4600\t0.1000\t0.2600\t0.6600\n" - "region\tSAR\tSardinia\t0.0050\t0.0200\t0.0000\t0.0600\n" - "region\tOCE\tOceania\t0.0010\t0.0005\t0.0000\t0.0020\n" - "painting\tWEA\tWest Eurasia (Europe, Middle East, North Africa)\t0.6400\t.\t.\t.\n" - "painting\tAFR\tAfrica\t0.3600\t.\t.\t.\n") - with open(f"{d}/ancestry_fine/S_painting.png", "wb") as f_: - f_.write(b"\x89PNG\r\n\x1a\n" + b"0" * 64) - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - check("26b: both reports still render with the PRS card beside this one", - "percentile 94.4 (EUR)" in txt and "<td>94.4 (EUR)</td>" in html, txt) - check("26b: the text section has the group, its fraction and its bootstrap range", - "## Fine-Grained Ancestry (ADMIXTURE, RFMix)" in txt - and "West Eurasia (Europe, Middle East, North Africa) 82% (76% to 88%)" in txt, txt) - check("26b: a regional group of 0.5% whose range reaches 6% is shown, one of 0.1% is not", - "Sardinia" in txt and "Oceania" not in txt.split("## Fine-Grained")[1].split("##")[0], txt) - check("26b: the painted shares and the picture are in the HTML card", - "<h2>Fine-Grained Ancestry</h2>" in html and "West Eurasia (Europe, Middle East, North Africa) 64%" in html - and 'src="data:image/png;base64,' in html, html[:200]) - check("26b: both reports say a group name is not a nationality", - "not a nationality" in txt and "not a nationality" in html) - - # 6. ClinVar badge: one homozygous 3-star Pathogenic hit - d = f"{work}/clinvar_hom/S" - put(f"{d}/clinvar/S_clinvar_hits.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr1\t20\t2\tT\tG\t50\tPASS\tGENEINFO=GENEB:22;CLNSIG=Pathogenic;" - "CLNREVSTAT=reviewed_by_expert_panel\tGT\t1/1\n") - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - badge_line = next((l for l in html.splitlines() if "ClinVar matches" in l), "") - check("ClinVar: a homozygous 3-star P/LP hit gets no green badge", "badge-green" not in badge_line - and 'badge-yellow">1<' in badge_line, badge_line) - freq = "a heterozygous hit in a dominant gene as well as a homozygous one: a hit above 1% in gnomAD" - check("ClinVar: both reports ask for every hit's population frequency, not only a homozygous one's", - freq in txt and freq in html and "Look up a homozygous P/LP hit" not in txt + html, txt) - check("ClinVar: without step 13 both reports say the hit's frequency is unknown", - "frequency: gnomAD AF unknown" in txt and "<td>gnomAD AF unknown</td>" in html, txt) - put(f"{d}/clinvar/S_clinvar_hits.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n") - html0 = render_report.html_report(collect_summary.collect("S", d)) - check("ClinVar: no hit gets a gray badge, not a green all-clear", - 'ClinVar matches</span><span class="value"><span class="badge badge-gray">0<' in html0) - - # 6b. ClinVar: step 06's cDNA read check beside each hit - d = f"{work}/clinvar_cdna/S" - put(f"{d}/clinvar/S_clinvar_hits.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr1\t20\t2\tT\tG\t50\tPASS\tGENEINFO=GENEB:22;CLNSIG=Pathogenic;CLNREVSTAT=reviewed_by_expert_panel\tGT\t0/1\n" - "chr2\t30\t3\tC\tA\t50\tPASS\tGENEINFO=GENEC:33;CLNSIG=Pathogenic;CLNREVSTAT=reviewed_by_expert_panel\tGT\t0/1\n") - summ = collect_summary.collect("S", d) - html = render_report.html_report(summ) - check("cDNA check: no table, no column (a Nextflow folder)", "cDNA read check" not in html - and "cdna_flagged" not in summ["sections"]["clinvar"]["data"], html) - flag = "possible cDNA/transgene contamination: check the reads" - put(f"{d}/clinvar/S_clinvar_cdna_check.tsv", - "chrom\tpos\tref\talt\talt_reads\talt_clipped\tref_reads\tref_clipped\tcdna_check\tcdna_note\n" - f"chr1\t20\tT\tG\t9\t8\t11\t0\tflagged\t{flag}\n" - "chr2\t30\tC\tA\t12\t0\t10\t0\tclear\t\n") - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - shown = f"{flag} (8 of 9 alt reads and 0 of 11 ref reads clipped at an exon edge)" - check("cDNA check: the text report prints the flag under the flagged hit", - "chr1:20 T>G" in txt and f"reads: {shown}" in txt and "reads: clear" in txt, txt) - check("cDNA check: the HTML hits table has the column, the flag and clear", - "<th>cDNA read check</th>" in html and f"<td>{shown}</td>" in html and "<td>clear</td>" in html, html) - check("cDNA check: the card counts one flagged hit", - 'Flagged by the cDNA read check</span><span class="value"><span class="badge badge-red">1<' in html, html) - check("cDNA check: the summary validates against the schema", not validate.validate( - json.load(open(os.path.join(REPO, "tests", "schema", "summary.schema.json"))), - json.loads(json.dumps(summ)))) - - # 7. telomere card - d = f"{work}/telomere/S" - put(f"{d}/telomere/S/S_summary.tsv", "PID\tsample\tintratelomeric_reads\ttotal_reads_with_tel_gc\ttel_content\n" - "S\ttumor\t61873\t185204533\t334.08\n") - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - check("telomere: the text heading and the value with its unit", - "## Telomere content (relative, TelomereHunter)" in txt - and "Telomere content: 334.08 intratelomeric reads per million reads with 48-52% GC" in txt, txt) - check("telomere: the HTML card title and the unit", "<h2>Telomere content (relative)</h2>" in html - and "intratelomeric reads per million reads with 48-52% GC" in html and "not a telomere length" in html) - said = [p_ for p_ in glob.glob(os.path.join(REPO, "bin", "*.py")) if "Telomere Length" in open(p_).read()] - check("no file in bin/ says 'Telomere Length'", not said, said) - - # 8. mosaic loss of chrY or chrX - d = f"{work}/mosaic/S" - - def mosaic_table(rows): - put(f"{d}/qc/S_sample_qc.tsv", "key\tvalue\nsample\tS\ninferred_sex\tmale\n" - + "".join(f"mosaic_{k}\t{v}\n" for k, v in rows.items())) - summ_ = collect_summary.collect("S", d) - return render_report.text_report(summ_), render_report.html_report(summ_) - male = {"sex": "male", "chrom": "chrY", "depth_ratio": "0.351", "depth_ratio_expected": "0.5", - "loss_depth": "0.298", "loss_depth_low": "0.293", "loss_depth_high": "0.303", "floor": "0.100", - "call": "detected", "reason": "the low end of the 95% range is above 10%"} - txt, html = mosaic_table(male) - line = ("From depth: estimated share of the cells that lost the Y 29.8% (95% range 29.3% to 30.3%); " - "chrY to autosome depth ratio 0.351") - card_m = html.split("<h2>Mosaic Loss of Chromosome Y or X</h2>", 1)[-1].split("<h2>", 1)[0] - check("mosaic: the text report heading, call and estimate", - "## Mosaic loss of chromosome Y or X" in txt and "chrY: detected (the low end" in txt and line in txt, txt) - check("mosaic: the HTML card shows the same", 'badge-yellow">detected<' in card_m and line in card_m, card_m) - check("mosaic: both say population-level, not a diagnosis, and the 10% floor", - all("population-level association, not a diagnosis" in t and "Below about 10% of the cells" in t - for t in (txt, card_m)), card_m) - check("mosaic: a man has no allele-fraction line", "allele fractions at" not in txt, txt) - txt, html = mosaic_table(dict(male, sex="female", chrom="chrX", depth_ratio="0.849", - depth_ratio_expected="1.0", x_het_sites="600", loss_af="0.316", - loss_af_low="0.288", loss_af_high="0.340")) - check("mosaic: a woman shows one X and the allele-fraction line", - "of the cells that lost one X" in txt and "From chrX allele fractions at 600 heterozygous sites: 31.6% " - "(95% range 28.8% to 34.0%)" in txt and "31.6%" in html, txt) - txt, html = mosaic_table(dict(male, depth_ratio="", loss_depth="0.190", loss_depth_low="-0.002", - loss_depth_high="0.382", call="not_detected")) - card_m = html.split("<h2>Mosaic Loss of Chromosome Y or X</h2>", 1)[-1].split("<h2>", 1)[0] - check("mosaic: not detected says 'above 10%', and a missing depth ratio prints '?'", - "chrY: not detected above 10%" in txt and "below 10%" not in txt and "depth ratio ?," in txt - and 'badge-gray">not detected above 10%<' in card_m, txt) - put(f"{d}/qc/S_sample_qc.tsv", "key\tvalue\nsample\tS\ninferred_sex\tmale\n") - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - card_m = html.split("<h2>Mosaic Loss of Chromosome Y or X</h2>", 1)[-1].split("<h2>", 1)[0] - check("mosaic: no estimate, the card says Not run and the text has no section", - 'badge-gray">Not run<' in card_m and "## Mosaic loss" not in txt, card_m) - # 9. paralog card (step 35b): hidden when the step did not run; a - # result older than a run in which step 35b failed is marked Stale - check("paralogs: no card when step 35b did not run", "Paralog Genes (Paraphase)" not in html) - d = f"{work}/paralogs/S" - put(f"{d}/paralogs/S_paraphase_copy_number.tsv", - "region\tgene\tcopy_number\tstatus\tdetail\ttotal_cn\tregion_depth\tgenome_depth\n" - "pms2\tPMS2\t2\tcalled\t.\t4\t60\t30\n") - os.utime(f"{d}/paralogs/S_paraphase_copy_number.tsv", (1, 1)) - put(f"{d}/logs/run_status.tsv", "meta\tstarted_epoch\t1000\nstep\t35b\tfailed\n") - summ = collect_summary.collect("S", d) - html = render_report.html_report(summ) - card_p = html.split("<h2>Paralog Genes (Paraphase)</h2>", 1)[-1].split("<h2>", 1)[0] - check("paralogs: a result older than a run where step 35b failed is stale, with its PMS2 row", - summ["sections"]["paralogs"]["state"] == "stale" and 'badge-yellow">Stale<' in card_p - and "<td>PMS2</td><td>2</td>" in card_p, card_p) - # 12. raw-data export card - d = f"{work}/consumer/S" - os.makedirs(d) - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - check("raw-data export: no card and no text section without the step's table", - "Raw-Data Export" not in txt and "Raw-Data Export" not in html) - put(f"{d}/consumer_export/S_consumer_export.tsv", "key\tvalue\nsites\t200\ncalled\t150\nno_call\t50\n" - "from_reference_blocks\t120\nfile_23andme\tS_23andme_v5.zip\nfile_ancestrydna\tS_ancestrydna_layout.zip\n" - "no_call: no gVCF record\t30\nno_call: filtered call\t20\n") - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - want = ["Sites with a genotype: 150 of 200 (75.0%); '--' at 50", - "Homozygous reference from the gVCF's reference blocks: 120", - "consumer_export/S_23andme_v5.zip and consumer_export/S_ancestrydna_layout.zip", - "'--' because: no gVCF record 30; filtered call 20", "Uploading one to a service shares your genotypes"] - check("raw-data export: the text section has every line", "## Raw-Data Export (23andMe v5 layout)" in txt - and all(w_ in txt for w_ in want), txt) - check("raw-data export: the HTML card has every line", "<h2>Raw-Data Export (23andMe v5 layout)</h2>" in html - and all(render_report.E(w_) in html for w_ in want), html) - - # 10. blood groups - d = f"{work}/blood/S" - put(f"{d}/bloodtype/S_bloodtype.tsv", "system\tphenotype\tgenotype\tconfidence\tnote\n" - "ABO\tO\tO.01/O.01\thigh\t\nRhD\tD+\tRHD/RHD (RHD copies by read depth 1.78)\thigh\t\n" - "Duffy\tFy(a+b-)\tFY*A/FY*B GATA\thigh\tc.-67T>C\nHPA-1\tHPA-1(a+b+)\t1a/1b\thigh\t\n") - summ = collect_summary.collect("S", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - check("blood groups: the text section has every system", - "## Blood Groups (predicted from the genome)" in txt - and all(x in txt for x in ("ABO O ", "RhD D+ ", "Duffy Fy(a+b-) ", "HPA-1 HPA-1(a+b+) ")), txt) - check("blood groups: the HTML card has ABO, RhD and the table", - "<h2>Blood Groups (predicted)</h2>" in html and "<td>Fy(a+b-)</td>" in html - and "<td>FY*A/FY*B GATA</td>" in html and ">D+</span>" in html, html[-3000:]) - check("blood groups: both reports say it is predicted, not typed from blood", - txt.count("not typed from blood") == 1 and html.count("not typed from blood") == 1) - nothing = collect_summary.collect("S", f"{work}/telomere/S") - check("blood groups: no card without step 41", "Blood Groups" not in render_report.html_report(nothing)) - words = re.compile(r"transfus|donat|donor", re.I) - src = os.path.join(REPO, "rust", "crates", "og-bloodtype", "src") - said = [n for n, t in (("text report", txt), ("HTML report", html), - ("og bloodtype", "".join(open(os.path.join(src, f)).read() - for f in sorted(os.listdir(src)) if f.endswith(".rs")))) - if words.search(t)] - check("blood groups: nothing about transfusion or donation", not said, said) - check("control: the wording check finds the word", bool(words.search("for a transfusion"))) - - # 11. traits card: rust/og/tests/genome_traits_report.rs - - # 13. couple card (step 39) - with open(os.path.join(REPO, "assets", "carrier_genes.tsv")) as f: - g = next(x.split("\t") for x in f if x.startswith("NAGA\t")) - pos = (int(g[2]) + int(g[3])) // 2 - cv = ("##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr22\t%d\t1\tA\tG\t.\tPASS\tGENEINFO=NAGA:4668;CLNSIG=Pathogenic;" - "CLNREVSTAT=reviewed_by_expert_panel\tGT\t0/1\n") - c = f"{work}/couple" - put(f"{c}/ped.tsv", "sample\tsex\tfather\tmother\tpartner\nP1\tfemale\t.\t.\tP2\nP2\tmale\t.\t.\tP1\n") - put(f"{c}/P1.vcf", cv % pos) - put(f"{c}/P2.vcf", cv % (pos + 5)) - subprocess.run([og_binary(), "couple", "screen", "--pedigree", f"{c}/ped.tsv", "--genes", - os.path.join(REPO, "assets", "carrier_genes.tsv"), "--clinvar", f"P1={c}/P1.vcf", - "--clinvar", f"P2={c}/P2.vcf", "--out-dir", f"{c}/out"], check=True, stdout=subprocess.DEVNULL) - d = f"{c}/P2" - put(f"{d}/couple/P2_couple_carrier.tsv", open(f"{c}/out/P2_couple_carrier.tsv").read()) - summ = collect_summary.collect("P2", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - card_c = html.split("<h2>Couple Carrier Screen</h2>", 1)[-1].split("<h2>", 1)[0] - want = ("Partner: P1", "Shared: NAGA (Schindler disease): 1 in 4 per pregnancy", - "This sample: chr22:%d A>G het" % (pos + 5), "P1: chr22:%d A>G het, Pathogenic, 3 stars" % pos, - "genetic counsellor") - check("couple: the card names the partner, the shared gene, its chance, both findings and the counsellor", - all(w in card_c for w in want), card_c) - check("couple: the text report says the same", - all(w.replace(">", ">") in txt for w in want) and "## Couple Carrier Screen (step 39)" in txt, txt) - os.remove(f"{d}/couple/P2_couple_carrier.tsv") - summ = collect_summary.collect("P2", d) - txt, html = render_report.text_report(summ), render_report.html_report(summ) - check("couple: without the table there is no card and no text section", - "Couple Carrier Screen" not in html and "Couple Carrier Screen" not in txt) - - # 14. chapters - chapter_checks(work) - finally: - shutil.rmtree(work) - print("\nRESULT:", "ALL PASS" if FAILS == 0 else f"{FAILS} FAILED") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/workflows/reporting.nf b/workflows/reporting.nf index e25530dc3..323679d69 100644 --- a/workflows/reporting.nf +++ b/workflows/reporting.nf @@ -3,7 +3,7 @@ REPORTING — HTML Report Generation & MultiQC Aggregation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Produces two kinds of report: - 1. Per-sample consolidated HTML report, rendered by bin/render_report.py + 1. Per-sample consolidated HTML report, rendered by og report render as the bash report is (QC, sample identity and contamination, variant counts, ClinVar, PGx, CPIC, CPSR, clinical filter, slivar, ROH, mito haplogroup)
    not applicable to the declared sex (male-specific score)94.4 (EUR)94.4 (EUR)raw score only94.4 (EUR)94.4 (EUR)gnomAD AF unknowncDNA read check{shown}clearPMS22Fy(a+b-)FY*A/FY*B GATADEL 20:20000000-2045000043G9ig_tr_locus