From a8f3b4e701e3006e6b9798ce1938242858fe39f8 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:36:45 +0100 Subject: [PATCH 1/4] feat(report): write the ancestry, haplogroup and PRS sections in Rust og report render now writes the fine-grained ancestry, mitochondrial haplogroup, Y chromosome (Yleaf, the second placement, Y-STR), archaic, ancient and polygenic score sections, byte for byte as bin/render_report.py does. A section lays out its tables and pictures in a Raw line, escaping each text it puts in. The report-render parity cases gain 14 summaries made from case c-full (every state, a planted escape in the haplogroup links, the painting and its legend, the raw and sex-specific scores, numbers where the summary writes text), and the oracle filter drops the charts, which og report charts will write. Bead: D4 Bead: D5 Bead: D8 --- rust/crates/og-render/src/chapters.rs | 15 +- rust/crates/og-render/src/html.rs | 1 + rust/crates/og-render/src/lib.rs | 6 +- rust/crates/og-render/src/model.rs | 10 +- .../crates/og-render/src/sections/ancestry.rs | 176 +++++++++- .../og-render/src/sections/ancestry_fine.rs | 286 +++++++++++++++- rust/crates/og-render/src/sections/ancient.rs | 132 ++++++- rust/crates/og-render/src/sections/archaic.rs | 91 ++++- .../og-render/src/sections/haplogroups.rs | 201 ++++++++++- rust/crates/og-render/src/sections/prs.rs | 324 +++++++++++++++++- rust/crates/og-render/src/sections/y_tree.rs | 142 +++++++- rust/crates/og-render/src/sections/ystr.rs | 102 +++++- tests/parity/cases.tsv | 16 + tests/parity/gen/report-corpus.sh | 80 ++++- 14 files changed, 1555 insertions(+), 27 deletions(-) diff --git a/rust/crates/og-render/src/chapters.rs b/rust/crates/og-render/src/chapters.rs index 92ace64e9..5367904e9 100644 --- a/rust/crates/og-render/src/chapters.rs +++ b/rust/crates/og-render/src/chapters.rs @@ -207,21 +207,24 @@ pub fn writer(sid: &str) -> Option<(TextFn, HtmlFn)> { "locityper" => None, - "ancestry-fine" => None, + "ancestry-fine" => Some((sections::ancestry_fine::text, sections::ancestry_fine::html)), - "mt-haplogroup" => None, + "mt-haplogroup" => Some((sections::haplogroups::text, sections::haplogroups::html)), - "y-chromosome" => None, + "y-chromosome" => Some(( + sections::haplogroups::y_chromosome_text, + sections::haplogroups::y_chromosome_html, + )), - "archaic" => None, + "archaic" => Some((sections::archaic::text, sections::archaic::html)), - "ancient" => None, + "ancient" => Some((sections::ancient::text, sections::ancient::html)), "roh" => None, "traits-panel" => None, - "prs" => None, + "prs" => Some((sections::prs::text, sections::prs::html)), "bloodtype" => None, diff --git a/rust/crates/og-render/src/html.rs b/rust/crates/og-render/src/html.rs index 43e75f02b..34883da28 100644 --- a/rust/crates/og-render/src/html.rs +++ b/rust/crates/og-render/src/html.rs @@ -70,6 +70,7 @@ fn line(out: &mut Vec, l: &Line) { out.extend(items.iter().map(|t| format!("
  • {}
  • ", e(t)))); out.push(" ".to_owned()); } + Line::Raw(t) => out.push(t.clone()), } } diff --git a/rust/crates/og-render/src/lib.rs b/rust/crates/og-render/src/lib.rs index 839f2be52..83e46ab58 100644 --- a/rust/crates/og-render/src/lib.rs +++ b/rust/crates/og-render/src/lib.rs @@ -10,9 +10,9 @@ //! [`css`] the page's style. //! //! Ported so far: the frame (chapters, contents, section frames, the run -//! facts, what the report leaves out) and the quality-control and variant -//! sections. A section not ported yet is left out of both reports; the index -//! and charts verbs say they are not ported and exit 2. +//! facts, what the report leaves out) and the sections +//! [`chapters::writer`] names. A section not ported yet is left out of both +//! reports; the index and charts verbs say they are not ported and exit 2. pub mod chapters; pub mod charts; diff --git a/rust/crates/og-render/src/model.rs b/rust/crates/og-render/src/model.rs index 54ea06364..71093922f 100644 --- a/rust/crates/og-render/src/model.rs +++ b/rust/crates/og-render/src/model.rs @@ -3,8 +3,10 @@ //! `==`), and the document the section modules build from it, which the //! text and HTML writers lay out. //! -//! A section module never writes markup. It returns cards of typed lines; -//! every text in them is escaped once, by the HTML writer. +//! A section module returns cards of typed lines; every text in them is +//! escaped once, by the HTML writer. The one exception is [`Line::Raw`]: +//! a table or picture the section lays out itself, escaping each text it +//! puts in. use std::fmt; @@ -325,6 +327,10 @@ pub enum Line { Para(String), /// A plain bulleted list. List(Vec), + /// Markup a section built itself, each text in it escaped with + /// [`crate::escape::e`]: a table, a picture or a link whose layout the + /// report fixes byte for byte. + Raw(String), } /// One card: a title, the caveats shown under it, the notes saying its diff --git a/rust/crates/og-render/src/sections/ancestry.rs b/rust/crates/og-render/src/sections/ancestry.rs index f882faadb..f091cf9e9 100644 --- a/rust/crates/og-render/src/sections/ancestry.rs +++ b/rust/crates/og-render/src/sections/ancestry.rs @@ -1,2 +1,174 @@ -//! The report section `ancestry`. Empty until its unit of the 1.0.0 plan -//! lands. +//! What the ancestry chapter's sections share: reading a row of the +//! summary the way the Python report did (`r["key"]`, `f"{v:<35}"`, a list +//! walked with `for`), the percentages of the fine-grained ancestry, and the +//! links where a haplogroup can be explored further (steps 12 and 37). +//! +//! Used by the fine-grained ancestry, haplogroup, Y chromosome, archaic, +//! ancient and polygenic score sections. + +use pycompat::fmt::{fixed, ljust, rjust}; +use pycompat::json::Value; + +use crate::escape::e; +use crate::html::offsite_link; +use crate::model::{Line, Malformed, get, need, num, py, truthy}; +use crate::notes::EXPLORE_NOTE; + +/// Every explore address but the docs one carries the haplogroup in it. +pub const EXPLORE_DOCS: &str = "https://github.com/GeiserX/OwnGenome/blob/main/docs/"; + +/// The mark of an address that leaves this computer with the haplogroup in it. +pub const OFFSITE_HAPLOGROUP: &str = "opens a website; the address carries the haplogroup"; + +/// `str(v)` of an optional value: `None` when it is absent. +pub fn py_or(v: Option<&Value>, default: &str) -> Result { + v.map_or(Ok(default.to_owned()), py) +} + +/// `d.get(key, default)` as text: the default only when the key is absent. +pub fn get_text(d: &Value, key: &str, default: &str, path: &str) -> Result { + py_or(get(d, key, path)?, default) +} + +/// `str(d[key])`. +pub fn need_text(d: &Value, key: &str, path: &str) -> Result { + py(need(d, key, path)?) +} + +/// The items `for x in v` walks: a list's items, a string's characters, a +/// dict's keys. `None` and numbers cannot be walked. +pub fn items(v: &Value, path: &str) -> Result, Malformed> { + match v { + Value::Array(a) => Ok(a.clone()), + Value::Str(s) => Ok(s.chars().map(|c| Value::Str(c.to_string())).collect()), + Value::Object(o) => Ok(o.keys().map(|k| Value::Str(k.clone())).collect()), + _ => Err(Malformed(format!("{path} is not a list"))), + } +} + +/// `d.get(key, [])` walked with `for`: nothing when the key is absent. +pub fn list(d: &Value, key: &str, path: &str) -> Result, Malformed> { + match get(d, key, path)? { + None => Ok(Vec::new()), + Some(v) => items(v, &format!("{path}.{key}")), + } +} + +/// `len(v)`. +pub fn py_len(v: &Value, path: &str) -> Result { + match v { + Value::Array(a) => Ok(a.len()), + Value::Str(s) => Ok(s.chars().count()), + Value::Object(o) => Ok(o.len()), + _ => Err(Malformed(format!("{path} has no length"))), + } +} + +/// Which side `f"{v:N}"` (right) pads. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Align { + Left, + Right, +} + +/// `f"{v:N}"`: a string or a number padded to `width` +/// characters. A bool formats as the number it is, as in Python; `None`, a +/// list or a dict cannot take a width. +pub fn align(v: &Value, width: usize, side: Align, path: &str) -> Result { + let text = match v { + Value::Str(s) => s.clone(), + Value::Int(i) => i.to_string(), + Value::Float(_) => py(v)?, + Value::Bool(b) => u8::from(*b).to_string(), + _ => return Err(Malformed(format!("{path} cannot be padded"))), + }; + Ok(match side { + Align::Left => ljust(&text, width), + Align::Right => rjust(&text, width), + }) +} + +/// `f"{d[key]:N}"`. +pub fn need_align( + d: &Value, + key: &str, + width: usize, + side: Align, + path: &str, +) -> Result { + align(need(d, key, path)?, width, side, &format!("{path}.{key}")) +} + +/// A number as Python compares it with `>=`; anything else is an error. +pub fn number(v: &Value, path: &str) -> Result { + num(Some(v)).ok_or_else(|| Malformed(format!("{path} is not a number"))) +} + +/// A fraction as a whole percentage (`95%`), `.` for none. +pub fn share(v: Option<&Value>, path: &str) -> Result { + match v { + None | Some(Value::Null) => Ok(".".to_owned()), + Some(x) => Ok(format!("{}%", fixed(100.0 * number(x, path)?, 0))), + } +} + +/// A stat line whose value is markup built here. +pub fn stat_raw(label: &str, value: &str) -> Line { + Line::Raw(format!( + "
    {}{value}
    ", + e(label) + )) +} + +/// One explore entry of a haplogroup section: its name and address. +fn explore(d: &Value, path: &str) -> Result, Malformed> { + let at = format!("{path}.explore[]"); + list(d, "explore", path)? + .iter() + .map(|x| Ok((need_text(x, "name", &at)?, need(x, "url", &at)?.clone()))) + .collect() +} + +/// The text report's explore lines: `name: url` for each link, then the +/// note that nothing is uploaded; nothing when there is no link. +pub fn explore_lines(d: &Value, path: &str) -> Result, Malformed> { + let mut out = Vec::new(); + for (name, url) in explore(d, path)? { + out.push(format!("{name}: {}", py(&url)?)); + } + if !out.is_empty() { + out.push(EXPLORE_NOTE.to_owned()); + } + Ok(out) +} + +/// The HTML report's explore lines: one stat per link, each marked as +/// leaving this computer, and whether the card shows the explore caveat. +/// The address is escaped, so a branch name cannot open a tag or end the +/// attribute. +pub fn explore_html(d: &Value, path: &str) -> Result<(Vec, bool), Malformed> { + let mut out = Vec::new(); + for (name, url) in explore(d, path)? { + let Value::Str(url) = url else { + return Err(Malformed(format!("{path}.explore[].url is not a text"))); + }; + let why = if url.starts_with(EXPLORE_DOCS) { + crate::html::OFFSITE + } else { + OFFSITE_HAPLOGROUP + }; + out.push(stat_raw(&name, &offsite_link(&url, &url, why))); + } + let note = !out.is_empty(); + Ok((out, note)) +} + +/// `d.get(key, ".") not in (".", "")`: a value worth a line. +pub fn shown<'a>(d: &'a Value, key: &str, path: &str) -> Result, Malformed> { + Ok(get(d, key, path)?.filter(|v| !matches!(v, Value::Str(s) if s.is_empty() || s == "."))) +} + +/// `bool(d.get(key))`. +pub fn has(d: &Value, key: &str, path: &str) -> Result { + Ok(get(d, key, path)?.is_some_and(truthy)) +} diff --git a/rust/crates/og-render/src/sections/ancestry_fine.rs b/rust/crates/og-render/src/sections/ancestry_fine.rs index d87e67915..67b9c2134 100644 --- a/rust/crates/og-render/src/sections/ancestry_fine.rs +++ b/rust/crates/og-render/src/sections/ancestry_fine.rs @@ -1,2 +1,284 @@ -//! The report section `ancestry_fine`. Empty until its unit of the 1.0.0 -//! plan lands. +//! Fine-grained ancestry (step 26b): the continental and regional groups +//! with their bootstrap ranges, the groups the model leaves out, the share +//! of the genome each group paints, and the chromosome painting with its +//! legend. + +use pycompat::json::Value; + +use crate::escape::e; +use crate::model::{ + Card, HtmlPart, Inline, Line, Malformed, State, Summary, get, need, py, truthy, +}; +use crate::notes::ANCESTRY_FINE_NOTE; +use crate::sections::ancestry::{get_text, has, list, need_text, number, share}; +use crate::sections::haplogroups::shows_body; +use crate::{html, text}; + +const PATH: &str = "sections.ancestry_fine.data"; + +/// The two levels the model reports, with their headings. +pub const LEVELS: [(&str, &str); 2] = [ + ("continent", "Continental groups"), + ("region", "Regional groups"), +]; + +/// The groups of one level worth a line: every continental group of 1% or +/// more, a regional one of 1% or more or whose bootstrap range reaches 5%. +pub fn rows(d: &Value, level: &str) -> Result, Malformed> { + let at = format!("{PATH}.{level}[]"); + let mut out = Vec::new(); + for r in list(d, level, PATH)? { + let f = need(&r, "fraction", &at)?; + if matches!(f, Value::Null) { + continue; + } + let keep = number(f, &format!("{at}.fraction"))? >= 0.01 + || (level == "region" + && match get(&r, "high", &at)?.filter(|v| truthy(v)) { + Some(h) => number(h, &format!("{at}.high"))? >= 0.05, + None => false, + }); + if keep { + out.push(r); + } + } + Ok(out) +} + +/// ` (93% to 97%)`: the bootstrap range of a row, when it has one. +pub fn spread(r: &Value, at: &str) -> Result { + match get(r, "low", at)? { + None | Some(Value::Null) => Ok(String::new()), + Some(low) => Ok(format!( + " ({} to {})", + share(Some(low), at)?, + share(Some(need(r, "high", at)?), at)? + )), + } +} + +/// `(.*) \((.*) group\)` matched against the whole name: the group and its +/// kind (`Oceania (continental group)`), or the name alone. +fn name_and_kind(group: &str) -> (&str, &str) { + let Some(inner) = group.strip_suffix(" group)") else { + return (group, ""); + }; + match inner.rfind(" (") { + Some(i) if !inner.contains('\n') => (&inner[..i], &inner[i + 2..]), + _ => (group, ""), + } +} + +/// The groups the model leaves out, as one sentence, or "". +pub fn left_out(d: &Value) -> Result { + if !has(d, "left_out", PATH)? { + return Ok(String::new()); + } + let at = format!("{PATH}.left_out[]"); + // One row per level ("Oceania (continental group)"); a group both + // levels leave out reads as one name. + let mut kinds: Vec<(String, Vec)> = Vec::new(); + for r in list(d, "left_out", PATH)? { + let Value::Str(group) = need(&r, "group", &at)? else { + return Err(Malformed(format!("{at}.group is not a text"))); + }; + let (name, kind) = name_and_kind(group); + let i = match kinds.iter().position(|(n, _)| n == name) { + Some(i) => i, + None => { + kinds.push((name.to_owned(), Vec::new())); + kinds.len() - 1 + } + }; + if !kind.is_empty() && !kinds[i].1.iter().any(|k| k == kind) { + kinds[i].1.push(kind.to_owned()); + } + } + let names: Vec = kinds + .iter() + .map(|(n, k)| { + if k.is_empty() { + n.clone() + } else { + format!("{n} ({} group)", k.join(" and ")) + } + }) + .collect(); + Ok(format!( + "Left out of the model: {}. The panel has fewer than {} unrelated reference people for \ + it, so its ancestry is spread over the groups shown.", + names.join(", "), + get_text(d, "min_references", "10", PATH)? + )) +} + +/// The painted shares, or why there are none, and the chromosomes the +/// painting failed on. +pub fn painted(d: &Value) -> Result { + let codes = list(d, "not_painted", PATH)? + .iter() + .map( + |r| match need(r, "code", &format!("{PATH}.not_painted[]"))? { + Value::Str(c) => Ok(c.clone()), + _ => Err(Malformed(format!( + "{PATH}.not_painted[].code is not a text" + ))), + }, + ) + .collect::, _>>()?; + let failed = codes.join(", "); + let mut text = if has(d, "painting", PATH)? { + let at = format!("{PATH}.painting[]"); + list(d, "painting", PATH)? + .iter() + .map(|r| { + Ok(format!( + "{} {}", + need_text(r, "group", &at)?, + share(Some(need(r, "fraction", &at)?), &at)? + )) + }) + .collect::, Malformed>>()? + .join(", ") + } else if failed.is_empty() { + "none (no chromosome long enough)".to_owned() + } else { + "none".to_owned() + }; + if !failed.is_empty() { + text += &format!("; not painted: RFMix failed on {failed}"); + } + Ok(text) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head( + s, + "ancestry_fine", + Some("Fine-Grained Ancestry (ADMIXTURE, RFMix)"), + )? + else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = s.section("ancestry_fine").data; + for (level, title) in LEVELS { + let rs = rows(d, level)?; + if rs.is_empty() { + continue; + } + w.push(format!(" {title} (fraction, bootstrap range):")); + let at = format!("{PATH}.{level}[]"); + for r in &rs { + let group = crate::sections::ancestry::need_align( + r, + "group", + 50, + crate::sections::ancestry::Align::Left, + &at, + )?; + let frac = share(Some(need(r, "fraction", &at)?), &at)?; + w.push(format!( + " {group} {}{}", + pycompat::fmt::rjust(&frac, 5), + spread(r, &at)? + )); + } + } + let lo = left_out(d)?; + if !lo.is_empty() { + w.push(format!(" {lo}")); + } + let lead = if has(d, "painting", PATH)? { + " Chromosome painting, share of the painted genome: " + } else { + " Chromosome painting: " + }; + w.push(format!("{lead}{}", painted(d)?)); + w.push(format!(" {ANCESTRY_FINE_NOTE}")); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, when step 26b has a result: one table per +/// level, the painted shares, and the painting with its legend. +pub fn html(s: &Summary) -> Result { + let sec = s.section("ancestry_fine"); + if sec.state == State::Missing { + return Ok(HtmlPart::default()); + } + let mut c = Card { + title: "Fine-Grained Ancestry".to_owned(), + full: true, + ..Card::default() + }; + if shows_body(&sec) { + let d = sec.data; + for (level, title) in LEVELS { + let at = format!("{PATH}.{level}[]"); + let mut rows_html = String::new(); + for r in rows(d, level)? { + let range = spread(&r, &at)?; + let range = range.trim_matches([' ', '(', ')']); + rows_html += &format!( + "{}{}{}\n", + e(&need_text(&r, "group", &at)?), + e(&share(Some(need(&r, "fraction", &at)?), &at)?), + e(if range.is_empty() { "." } else { range }) + ); + } + if !rows_html.is_empty() { + c.body.extend([ + Line::Raw(format!("

    {}

    ", e(title))), + Line::Raw(" ".to_owned()), + Line::Raw( + " " + .to_owned(), + ), + Line::Raw(rows_html.trim_end_matches('\n').to_owned()), + Line::Raw("
    GroupFractionBootstrap range
    ".to_owned()), + ]); + } + } + let lo = left_out(d)?; + if !lo.is_empty() { + c.body.push(Line::Para(lo)); + } + c.body.push(Line::Stat( + "Painted genome".to_owned(), + Inline::Text(painted(d)?), + )); + if let Some(png) = get(d, "painting_png", PATH)?.filter(|v| truthy(v)) { + c.body.push(Line::Raw(format!( + " \"Chromosome", + e(&py(png)?) + ))); + let at = format!("{PATH}.legend[]"); + let keys = list(d, "legend", PATH)? + .iter() + .map(|x| { + Ok(format!( + "■ {}", + e(&need_text(x, "colour", &at)?), + e(&need_text(x, "group", &at)?) + )) + }) + .collect::, Malformed>>()?; + c.body.push(Line::Raw(format!( + "

    {} ■ not painted

    ", + keys.join(" ") + ))); + } + c.caveats.push(ANCESTRY_FINE_NOTE); + } + Ok(HtmlPart { + cards: vec![html::card(&sec, c)?], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/ancient.rs b/rust/crates/og-render/src/sections/ancient.rs index 0bcd3409a..d6000869a 100644 --- a/rust/crates/og-render/src/sections/ancient.rs +++ b/rust/crates/og-render/src/sections/ancient.rs @@ -1,2 +1,130 @@ -//! The report section `ancient`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Ancient populations (step 44): where the genome falls on a PCA of +//! present-day West Eurasians, and the ancient groups of the AADR nearest +//! to it there, with the plot. + +use pycompat::json::Value; + +use crate::escape::e; +use crate::model::{Card, HtmlPart, Line, Malformed, State, Summary, get, py, truthy}; +use crate::notes::ANCIENT_NOTE; +use crate::sections::ancestry::{Align, get_text, need_align, need_text}; +use crate::{html, text}; + +const PATH: &str = "sections.ancient.data"; +const ROW: &str = "sections.ancient.data.nearest[]"; + +/// `(d.get("nearest") or [])[:10]`: the ten nearest groups. +fn nearest(d: &Value) -> Result, Malformed> { + match get(d, "nearest", PATH)?.filter(|v| truthy(v)) { + None => Ok(Vec::new()), + Some(Value::Array(a)) => Ok(a.iter().take(10).collect()), + Some(_) => Err(Malformed(format!("{PATH}.nearest is not a list"))), + } +} + +/// Step 44's lines, the same in both reports: what the PCA was built from, +/// then the nearest groups. +pub fn lines(d: &Value) -> Result, Malformed> { + let g = |k: &str| get_text(d, k, ".", PATH); + let mut out = vec![ + format!( + "AADR {}: {} of {} sites used; axes from {} present-day West Eurasians, {} ancient \ + people in {} groups projected.", + g("release")?, + g("sites_used")?, + g("panel_sites")?, + g("present_day_people")?, + g("ancient_people")?, + g("ancient_groups")? + ), + "Nearest ancient groups:".to_owned(), + ]; + for r in nearest(d)? { + let n = |k: &str| need_text(r, k, ROW); + out.push(format!( + " {}. {} ({}, {}; {} people)", + need_align(r, "rank", 2, Align::Right, ROW)?, + n("group")?, + n("dates")?, + n("country")?, + n("n")? + )); + } + Ok(out) +} + +/// The text report's block, with the plot's file name. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head( + s, + "ancient", + Some("Ancient Populations (AADR PCA, step 44)"), + )? + else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + w.extend( + lines(s.section("ancient").data)? + .iter() + .map(|l| format!(" {l}")), + ); + w.push(format!( + " {ANCIENT_NOTE} Plot: ancient/{}_ancient_pca.svg", + py(s.need("sample")?)? + )); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's card: the nearest groups as a table, and the plot as +/// an image (shown, never run as part of the page). +pub fn html(s: &Summary) -> Result { + let sec = s.section("ancient"); + if sec.state == State::Missing { + return Ok(HtmlPart::default()); + } + let mut c = Card { + title: "Ancient Populations".to_owned(), + full: true, + ..Card::default() + }; + if crate::sections::haplogroups::shows_body(&sec) { + let d = sec.data; + let first = lines(d)?.swap_remove(0); + let mut rows = String::new(); + for r in nearest(d)? { + rows += ""; + for k in ["rank", "group", "dates", "country", "locality", "n"] { + rows += &format!("{}", e(&need_text(r, k, ROW)?)); + } + rows += "\n"; + } + c.body = vec![ + Line::Para(first), + Line::Raw(" ".to_owned()), + Line::Raw( + " \ + " + .to_owned(), + ), + Line::Raw(rows.trim_end_matches('\n').to_owned()), + Line::Raw("
    #Ancient groupDatesCountryPlacePeople
    ".to_owned()), + ]; + if let Some(svg) = get(d, "plot_svg", PATH)?.filter(|v| truthy(v)) { + c.body.push(Line::Raw(format!( + " \"PC1", + e(&py(svg)?) + ))); + } + c.caveats.push(ANCIENT_NOTE); + } + Ok(HtmlPart { + cards: vec![html::card(&sec, c)?], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/archaic.rs b/rust/crates/og-render/src/sections/archaic.rs index 4295ec431..42cc364a2 100644 --- a/rust/crates/og-render/src/sections/archaic.rs +++ b/rust/crates/og-render/src/sections/archaic.rs @@ -1,2 +1,89 @@ -//! The report section `archaic`. Empty until its unit of the 1.0.0 plan -//! lands. +//! Archaic ancestry (step 43, IBDmix): the share of the analysed autosomes +//! in Neanderthal and Denisovan segments, and the longest segments. + +use pycompat::fmt::fixed; +use pycompat::json::Value; + +use crate::model::{Card, Inline, Line, Malformed, State, Summary, need, py}; +use crate::notes::ARCHAIC_NOTE; +use crate::sections::ancestry::{get_text, has, items, need_text, number}; +use crate::{html, text}; + +const PATH: &str = "sections.archaic.data"; + +/// A length in base pairs as megabases (`50.0 Mb`). +fn mb(d: &Value, key: &str) -> Result { + let bp = number(need(d, key, PATH)?, &format!("{PATH}.{key}"))?; + Ok(format!("{} Mb", fixed(bp / 1e6, 1))) +} + +/// Step 43's lines, the same in both reports. +pub fn lines(d: &Value) -> Result, Malformed> { + let n = |k: &str| need_text(d, k, PATH); + let mut out = Vec::new(); + for (name, key) in [("Neanderthal", "neanderthal"), ("Denisovan", "denisovan")] { + out.push(format!( + "{name}: {}% ({} in {} segments)", + n(&format!("{key}_percent"))?, + mb(d, &format!("{key}_bp"))?, + n(&format!("{key}_segments"))? + )); + } + out.push(format!( + "Analysed: {} of the autosomes; modern frequencies: 1000 Genomes {}", + mb(d, "analysed_bp")?, + get_text(d, "population", ".", PATH)? + )); + if has(d, "longest", PATH)? { + out.push("Longest segments:".to_owned()); + for x in items(need(d, "longest", PATH)?, &format!("{PATH}.longest"))? { + out.push(format!(" {}", py(&x)?)); + } + } + Ok(out) +} + +/// The text report's block. It ends without a blank line, as the Python +/// report's did. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "archaic", Some("Archaic Ancestry (IBDmix, step 43)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let d = s.section("archaic").data; + w.extend(lines(d)?.iter().map(|l| format!(" {l}"))); + w.push(format!(" {ARCHAIC_NOTE}")); + } + Ok(w) +} + +/// The HTML report's card, when step 43 has a result. +pub fn html(s: &Summary) -> Result { + let sec = s.section("archaic"); + if sec.state == State::Missing { + return Ok(crate::model::HtmlPart::default()); + } + let mut c = Card { + title: "Archaic Ancestry (Neanderthal, Denisovan)".to_owned(), + ..Card::default() + }; + if sec.state.ran() { + let d = sec.data; + for (label, key) in [ + ("Neanderthal", "neanderthal_percent"), + ("Denisovan", "denisovan_percent"), + ] { + c.body.push(Line::Stat( + label.to_owned(), + Inline::Text(format!("{}%", need_text(d, key, PATH)?)), + )); + } + c.body.extend(lines(d)?.into_iter().map(Line::Para)); + c.caveats.push(ARCHAIC_NOTE); + } + Ok(crate::model::HtmlPart { + cards: vec![html::card(&sec, c)?], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/haplogroups.rs b/rust/crates/og-render/src/sections/haplogroups.rs index 8f28f8f32..94fdad73e 100644 --- a/rust/crates/og-render/src/sections/haplogroups.rs +++ b/rust/crates/og-render/src/sections/haplogroups.rs @@ -1,2 +1,199 @@ -//! The report section `haplogroups`. Empty until its unit of the 1.0.0 plan -//! lands. +//! The haplogroups: the mitochondrial one with haplocheck's contamination +//! verdict (step 12), and the Y chromosome section, which shows Yleaf's +//! haplogroup (step 37), the second placement on the current public tree +//! ([`crate::sections::y_tree`]) and the Y-STR markers +//! ([`crate::sections::ystr`]). Each haplogroup carries the links where it +//! can be explored further. + +use pycompat::json::Value; + +use crate::model::{Card, Colour, HtmlPart, Inline, Line, Malformed, Section, State, Summary, py}; +use crate::notes::EXPLORE_NOTE; +use crate::sections::ancestry::{explore_html, explore_lines, get_text, need_text}; +use crate::sections::{y_tree, ystr}; +use crate::{html, text}; + +/// The Status badge of a section that ran: Stale for an earlier run's +/// result, else Complete. +pub fn done_badge(sec: &Section<'_>) -> Inline { + if sec.state == State::Stale { + Inline::Badge("Stale".to_owned(), Colour::Yellow) + } else { + Inline::Badge("Complete".to_owned(), Colour::Green) + } +} + +/// Whether a card's lines are shown: the card says Not run, Failed or +/// Unreadable in their place otherwise. +pub fn shows_body(sec: &Section<'_>) -> bool { + !matches!( + sec.state, + State::Missing | State::Failed | State::Unreadable + ) +} + +fn path(sec: &Section<'_>) -> String { + format!("sections.{}.data", sec.key) +} + +/// haplocheck's verdict: whether the reads hold two mtDNA haplogroups. +pub fn contamination_text(sec: &Section<'_>) -> Result { + let Some(status) = sec.get("contamination_status")? else { + return Ok("not checked (needs step 20's Mutect2 chrM calls)".to_owned()); + }; + let word = match status { + Value::Str(s) => match s.as_str() { + "YES" => "YES, two mtDNA haplogroups in the reads (another person's DNA?)".to_owned(), + "NO" => "no".to_owned(), + "ND" => "not determined".to_owned(), + _ => s.clone(), + }, + Value::Array(_) | Value::Object(_) => { + return Err(Malformed(format!( + "{}.contamination_status is not a value", + path(sec) + ))); + } + v => py(v)?, + }; + let level = get_text(sec.data, "contamination_level", ".", &path(sec))?; + Ok(format!("{word} (level {level})")) +} + +/// The text report's mitochondrial block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "haplogroup", Some("Mitochondrial Haplogroup"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let sec = s.section("haplogroup"); + let p = path(&sec); + w.push(format!( + " Haplogroup: {}", + need_text(sec.data, "haplogroup", &p)? + )); + w.push(format!( + " Contamination (haplocheck): {}", + contamination_text(&sec)? + )); + w.extend( + explore_lines(sec.data, &p)? + .iter() + .map(|l| format!(" {l}")), + ); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's mitochondrial card. It is always there, so a run +/// without step 12 says so. +pub fn html(s: &Summary) -> Result { + let sec = s.section("haplogroup"); + let mut c = Card { + title: "Mitochondrial Haplogroup".to_owned(), + ..Card::default() + }; + if shows_body(&sec) { + let p = path(&sec); + c.body = vec![ + Line::Stat("Status".to_owned(), done_badge(&sec)), + Line::Stat( + "Haplogroup".to_owned(), + Inline::Text(get_text(sec.data, "haplogroup", ".", &p)?), + ), + Line::Stat( + "Contamination (haplocheck)".to_owned(), + Inline::Text(contamination_text(&sec)?), + ), + ]; + let (links, note) = explore_html(sec.data, &p)?; + c.body.extend(links); + if note { + c.caveats.push(EXPLORE_NOTE); + } + } + Ok(HtmlPart { + cards: vec![html::card(&sec, c)?], + files: Vec::new(), + }) +} + +/// The text report's block of Yleaf's haplogroup. +fn y_text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "y_haplogroup", Some("Y-Chromosome Haplogroup (Yleaf)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if head.ran { + let sec = s.section("y_haplogroup"); + let p = path(&sec); + let n = |k: &str| need_text(sec.data, k, &p); + w.push(format!( + " Haplogroup: {} ({} markers, QC-score {})", + n("haplogroup")?, + n("valid_markers")?, + n("qc_score")? + )); + w.extend( + explore_lines(sec.data, &p)? + .iter() + .map(|l| format!(" {l}")), + ); + w.push(String::new()); + } + Ok(w) +} + +/// The HTML report's card of Yleaf's haplogroup, when step 37 has a result. +fn y_card(s: &Summary) -> Result, Malformed> { + let sec = s.section("y_haplogroup"); + if sec.state == State::Missing { + return Ok(None); + } + let mut c = Card { + title: "Y-Chromosome Haplogroup".to_owned(), + ..Card::default() + }; + if shows_body(&sec) { + let p = path(&sec); + for (label, key) in [ + ("Haplogroup", "haplogroup"), + ("Markers", "valid_markers"), + ("QC-score", "qc_score"), + ] { + c.body.push(Line::Stat( + label.to_owned(), + Inline::Text(get_text(sec.data, key, ".", &p)?), + )); + } + let (links, note) = explore_html(sec.data, &p)?; + c.body.extend(links); + if note { + c.caveats.push(EXPLORE_NOTE); + } + } + Ok(Some(html::card(&sec, c)?)) +} + +/// The text report's Y chromosome blocks: Yleaf, the second placement, +/// the Y-STR markers. +pub fn y_chromosome_text(s: &Summary) -> Result, Malformed> { + let mut w = y_text(s)?; + w.extend(y_tree::text(s)?); + w.extend(ystr::text(s)?); + Ok(w) +} + +/// The HTML report's Y chromosome cards, each when its step has a result. +pub fn y_chromosome_html(s: &Summary) -> Result { + let cards = [y_card(s)?, y_tree::card(s)?, ystr::card(s)?] + .into_iter() + .flatten() + .collect(); + Ok(HtmlPart { + cards, + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/prs.rs b/rust/crates/og-render/src/sections/prs.rs index 3d832baa8..c9faba465 100644 --- a/rust/crates/og-render/src/sections/prs.rs +++ b/rust/crates/og-render/src/sections/prs.rs @@ -1 +1,323 @@ -//! The report section `prs`. Empty until its unit of the 1.0.0 plan lands. +//! Polygenic risk scores (step 25, pgsc_calc): each score with the variants +//! it matched, its percentile among the most similar reference group when +//! the ancestry panel is installed, the effect a published evaluation +//! measured in that group, and the line that says what the numbers can be +//! compared with: the ancestry caveat, the chrX rows left out and the +//! sex-specific scores when no sex was declared. +//! +//! The scores are shown as the summary lists them. + +use pycompat::json::Value; + +use crate::escape::e; +use crate::model::{Card, HtmlPart, Line, Malformed, Summary, get, is_str, need, py, truthy}; +use crate::notes::PRS_ANCESTRY_NOTE; +use crate::sections::ancestry::{Align, items, need_align, need_text, py_or}; +use crate::{html, text}; + +const PATH: &str = "sections.prs.data"; +const ROW: &str = "sections.prs.data.scores[]"; + +/// What a score of the other sex says in place of a percentile. +pub const NOT_APPLICABLE: &str = "not applicable to the declared sex"; + +/// Part of the line under the scores, so not a caveat of its own. +pub const EFFECT_TEXT: &str = "Effect: the odds ratio (OR) or hazard ratio (HR) a published \ + evaluation of the score measured in people of the same reference group, in that \ + evaluation's unit (its PGS Catalog PPM page says which). It describes the score in that \ + group, not your own risk, and is shown only for a group the score was evaluated in."; + +fn field<'a>(r: &'a Value, key: &str) -> Result, Malformed> { + get(r, key, ROW) +} + +/// `r.get(key)` is true. +fn on<'a>(r: &'a Value, key: &str) -> Result, Malformed> { + Ok(field(r, key)?.filter(|v| truthy(v))) +} + +fn other_sex(r: &Value) -> Result { + Ok(is_str(field(r, "sex_check")?, NOT_APPLICABLE)) +} + +/// The scores of the summary. +fn scores(d: &Value) -> Result, Malformed> { + items(need(d, "scores", PATH)?, &format!("{PATH}.scores")) +} + +/// The percentile cell: the percentile with its group, the note for a score +/// of the other sex, or "" when there is none. +pub fn percentile(r: &Value) -> Result { + if other_sex(r)? { + let sex = py_or(on(r, "applies_to")?, "sex")?; + return Ok(format!("{NOT_APPLICABLE} ({sex}-specific score)")); + } + match on(r, "percentile")? { + Some(p) => Ok(format!( + "{} ({})", + py(p)?, + py_or(on(r, "group")?, "group unknown")? + )), + None => Ok(String::new()), + } +} + +/// `", ".join(v)`: the items of a list of texts, or a text's characters. +fn join(v: &Value, at: &str) -> Result { + let parts = items(v, at)? + .into_iter() + .map(|x| match x { + Value::Str(s) => Ok(s), + _ => Err(Malformed(format!("{at} holds an item that is not a text"))), + }) + .collect::, _>>()?; + Ok(parts.join(", ")) +} + +/// The effect cell: the odds or hazard ratio the PGS Catalog records for +/// the score in the sample's group, a line saying the score has none for +/// that group, or "" when the sample has no group, the score does not apply +/// to the declared sex, or the table names no evaluations. The bool says +/// whether the cell is the ratio itself. +pub fn effect(r: &Value) -> Result<(String, bool), Malformed> { + let evaluated = match field(r, "evaluated_in")? { + None | Some(Value::Null) => return Ok((String::new(), false)), + Some(v) => v, + }; + if on(r, "percentile")?.is_some() && on(r, "group")?.is_some() && !other_sex(r)? { + if let Some(x) = on(r, "effect")? { + return Ok((py(x)?, true)); + } + let group = py(need(r, "group", ROW)?)?; + let place = if truthy(evaluated) { + format!( + "; evaluated in {}", + join(evaluated, &format!("{ROW}.evaluated_in"))? + ) + } else { + String::new() + }; + return Ok((format!("no published OR or HR for {group}{place}"), false)); + } + Ok((String::new(), false)) +} + +/// The chrX rows left out of the scores because no sex was given. +fn chrx_note(rs: &[Value]) -> Result { + let mut left = Vec::new(); + for r in rs { + let Some(c) = on(r, "chrx")? else { continue }; + let Value::Str(c) = c else { + return Err(Malformed(format!("{ROW}.chrx is not a text"))); + }; + if c.ends_with("left out") { + let first = pycompat::text::split(c.as_str(), None, None) + .first() + .copied() + .unwrap_or_default() + .to_owned(); + left.push(format!("{} ({first})", need_text(r, "pgs_id", ROW)?)); + } + } + if left.is_empty() { + return Ok(String::new()); + } + Ok(format!( + " Rows on chrX were left out, because the sample's sex was not given: {}. Give the sex \ + to score them (docs/25-prs.md).", + left.join(", ") + )) +} + +/// The sex-specific scores shown for any sex, when no sex was declared. +fn sex_note(rs: &[Value]) -> Result { + let mut named = Vec::new(); + for r in rs { + if is_str(field(r, "sex_check")?, "no sex declared") + && let Some(a) = on(r, "applies_to")? + { + named.push(format!("{} ({})", need_text(r, "pgs_id", ROW)?, py(a)?)); + } + } + if named.is_empty() { + return Ok(String::new()); + } + Ok(format!( + " No sex was declared, so these sex-specific scores are shown for any sex: {}. Give the \ + sex to mark a score of the other sex not applicable (docs/25-prs.md).", + named.join(", ") + )) +} + +/// Whether the summary's table names evaluations (its Evaluated_In column). +fn names_evaluations(rs: &[Value]) -> Result { + for r in rs { + if !matches!(field(r, "evaluated_in")?, None | Some(Value::Null)) { + return Ok(true); + } + } + Ok(false) +} + +/// The line under the scores, in both reports: what the numbers can be +/// compared with. +pub fn note(d: &Value) -> Result { + let rs = scores(d)?; + let tail = chrx_note(&rs)? + &sex_note(&rs)?; + if !get(d, "adjusted", PATH)?.is_some_and(truthy) { + return Ok(format!( + "Raw score only: no ancestry reference panel was installed, so there is no percentile \ + and the sum cannot be compared with anyone (scripts/setup.sh --ancestry-panel adds \ + it). {PRS_ANCESTRY_NOTE} See docs/25-prs.md.{tail}" + )); + } + let empty = Value::Object(indexmap::IndexMap::new()); + let anc = get(d, "ancestry", PATH)? + .filter(|v| truthy(v)) + .unwrap_or(&empty); + let at = format!("{PATH}.ancestry"); + let mut group = match get(anc, "population", &at)?.filter(|v| truthy(v)) { + Some(p) => py(p)?, + None => { + let mut g = String::new(); + for r in &rs { + if let Some(x) = on(r, "group")? { + g = py(x)?; + break; + } + } + g + } + }; + if group.is_empty() { + group = "most similar".to_owned(); + } + let panel = match get(anc, "panel", &at)?.filter(|v| truthy(v)) { + Some(p) => format!(" of the {} reference panel", py(p)?), + None => " of the reference panel".to_owned(), + }; + let low = if is_str(get(anc, "low_confidence", &at)?, "True") { + " The ancestry match is low-confidence: read the percentile with care." + } else { + "" + }; + let eff = if names_evaluations(&rs)? { + format!(" {EFFECT_TEXT}") + } else { + String::new() + }; + Ok(format!( + "Percentile: where the score falls among the {group} samples{panel}, the group whose \ + genetic ancestry is most similar to this sample's (pgsc_calc).{low} A percentile is not \ + a risk.{eff} {PRS_ANCESTRY_NOTE} See docs/25-prs.md.{tail}" + )) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "prs", Some("Polygenic Risk Scores"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = s.section("prs").data; + for r in scores(d)? { + let condition = need_align(&r, "condition", 35, Align::Left, ROW)?; + let n = |k: &str| need_text(&r, k, ROW); + let line = format!( + " {condition} {} ({}/{} variants matched) {}", + n("score")?, + n("matched")?, + n("total")?, + n("pgs_id")? + ); + let pct = percentile(&r)?; + let pct = if other_sex(&r)? { + format!(" {pct}") + } else if pct.is_empty() { + String::new() + } else { + format!(" percentile {pct}") + }; + let (e, is_ratio) = effect(&r)?; + let eff = if e.is_empty() { + String::new() + } else if is_ratio { + format!(" effect {e}") + } else { + format!(" {e}") + }; + w.push(format!("{line}{pct}{eff}")); + } + w.push(format!(" {}", note(d)?)); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, when step 25 has a result: one row per score, +/// with the effect column when the table names evaluations. +pub fn html(s: &Summary) -> Result { + let sec = s.section("prs"); + if !sec.state.ran() { + return Ok(HtmlPart::default()); + } + let d = sec.data; + let rs = scores(d)?; + let adjusted = get(d, "adjusted", PATH)?.is_some_and(truthy); + let show_effect = adjusted && names_evaluations(&rs)?; + let mut rows = String::new(); + for x in &rs { + let n = |k: &str| need_text(x, k, ROW).map(|t| e(&t).into_owned()); + let pct = percentile(x)?; + rows += &format!( + "{}{}{}{}/{}{}", + n("condition")?, + n("pgs_id")?, + n("score")?, + n("matched")?, + n("total")?, + e(if pct.is_empty() { + "raw score only" + } else { + &pct + }) + ); + if show_effect { + rows += &format!("{}", e(&effect(x)?.0)); + } + rows += "\n"; + } + let effect_th = if show_effect { + "Effect in your group" + } else { + "" + }; + let card = Card { + title: format!( + "Polygenic Risk Scores{}", + if adjusted { + "" + } else { + " (raw, not percentiles)" + } + ), + full: true, + body: vec![ + Line::Raw(" ".to_owned()), + Line::Raw(format!( + " \ + {effect_th}" + )), + Line::Raw(rows.trim_end_matches('\n').to_owned()), + Line::Raw("
    ConditionPGSScoreVariants matchedPercentile
    ".to_owned()), + Line::Para(note(d)?), + ], + ..Card::default() + }; + Ok(HtmlPart { + cards: vec![html::card(&sec, card)?], + files: Vec::new(), + }) +} diff --git a/rust/crates/og-render/src/sections/y_tree.rs b/rust/crates/og-render/src/sections/y_tree.rs index 892584a1c..da59d9f0e 100644 --- a/rust/crates/og-render/src/sections/y_tree.rs +++ b/rust/crates/og-render/src/sections/y_tree.rs @@ -1,2 +1,140 @@ -//! The report section `y_tree`. Empty until its unit of the 1.0.0 plan -//! lands. +//! The second Y placement (step 37, `og y tree-place`): the branch on a +//! public tree release newer than Yleaf's, the SNP counts on the path to +//! it, and one row per level of the walk. + +use crate::escape::e; +use crate::model::{Card, Inline, Line, Malformed, State, Summary}; +use crate::notes::{EXPLORE_NOTE, Y_TREE_NOTE}; +use crate::sections::ancestry::{ + Align, explore_html, get_text, list, need_align, need_text, py_or, shown, +}; +use crate::{html, text}; + +const PATH: &str = "sections.y_tree.data"; +const ROW: &str = "sections.y_tree.data.path[]"; + +/// The two lists of ancestral calls a reader should see, with their labels. +const ANCESTRAL: [(&str, &str); 2] = [ + ( + "Ancestral inside the terminal branch", + "ancestral_in_terminal", + ), + ( + "Ancestral on the path, outvoted by the SNPs below it", + "outvoted_ancestral", + ), +]; + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "y_tree", Some("Y Placement on the Current Public Tree"))? + else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = s.section("y_tree").data; + let n = |k: &str| need_text(d, k, PATH); + w.push(format!( + " Branch: {} (level {}, {})", + n("haplogroup")?, + n("level")?, + n("tree")? + )); + w.push(format!( + " Path SNPs: {} derived, {} ancestral, {} no call; {} SNPs called at depth {} or more", + n("path_derived")?, + n("path_ancestral")?, + n("path_no_call")?, + n("snps_called")?, + n("min_depth")? + )); + for (label, key) in ANCESTRAL { + if shown(d, key, PATH)?.is_some() { + w.push(format!(" {label}: {}", n(key)?)); + } + } + for x in list(d, "path", PATH)? { + let a = |k: &str, width: usize, side: Align| need_align(&x, k, width, side, ROW); + w.push(format!( + " {} {} {}D {}A {}N", + a("Level", 3, Align::Right)?, + a("Branch", 22, Align::Left)?, + a("Derived", 3, Align::Right)?, + a("Ancestral", 3, Align::Right)?, + a("No_call", 4, Align::Right)? + )); + } + w.push(format!(" {Y_TREE_NOTE}")); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, when step 37 placed the sample on the tree. +pub fn card(s: &Summary) -> Result, Malformed> { + let sec = s.section("y_tree"); + if sec.state == State::Missing { + return Ok(None); + } + let mut c = Card { + title: "Y Placement on the Current Public Tree".to_owned(), + full: true, + ..Card::default() + }; + if sec.state.ran() { + let d = sec.data; + let g = |k: &str| get_text(d, k, ".", PATH); + for (label, key) in [ + ("Branch", "haplogroup"), + ("Level", "level"), + ("Tree", "tree"), + ] { + c.body + .push(Line::Stat(label.to_owned(), Inline::Text(g(key)?))); + } + c.body.push(Line::Para(format!( + "Path SNPs: {} derived, {} ancestral, {} no call; {} SNPs called at depth {} or more", + g("path_derived")?, + g("path_ancestral")?, + g("path_no_call")?, + g("snps_called")?, + g("min_depth")? + ))); + for (label, key) in ANCESTRAL { + if let Some(v) = shown(d, key, PATH)? { + c.body + .push(Line::Para(format!("{label}: {}", py_or(Some(v), "")?))); + } + } + let mut rows = String::new(); + for x in list(d, "path", PATH)? { + rows += ""; + for k in [ + "Level", + "Branch", + "Derived", + "Ancestral", + "No_call", + "Ancestral_SNPs", + ] { + rows += &format!("{}", e(&need_text(&x, k, ROW)?)); + } + rows += ""; + } + if !rows.is_empty() { + c.body.push(Line::Raw(format!( + " \ + {rows}
    LevelBranchDerivedAncestralNo callAncestral SNPs
    " + ))); + } + let (links, note) = explore_html(d, PATH)?; + c.body.extend(links); + if note { + c.caveats.push(EXPLORE_NOTE); + } + c.caveats.push(Y_TREE_NOTE); + } + Ok(Some(html::card(&sec, c)?)) +} diff --git a/rust/crates/og-render/src/sections/ystr.rs b/rust/crates/og-render/src/sections/ystr.rs index 7e7922e39..91019f76c 100644 --- a/rust/crates/og-render/src/sections/ystr.rs +++ b/rust/crates/og-render/src/sections/ystr.rs @@ -1 +1,101 @@ -//! The report section `ystr`. Empty until its unit of the 1.0.0 plan lands. +//! The Y-STR markers (step 45, HipSTR): the repeat count of each marker +//! called, in FamilyTreeDNA's Y-111 order and panels, so a count lines up +//! with FamilyTreeDNA results and surname-project tables. + +use pycompat::json::Value; + +use crate::escape::e; +use crate::model::{Card, Line, Malformed, State, Summary, need, truthy}; +use crate::notes::YSTR_NOTE; +use crate::sections::ancestry::{Align, items, need_align, need_text, py_len}; +use crate::{html, text}; + +const PATH: &str = "sections.y_str.data"; +const ROW: &str = "sections.y_str.data.markers[]"; + +/// How many markers were called, and with what confidence. +pub fn counts(d: &Value) -> Result { + let n = |k: &str| need_text(d, k, PATH); + let high = n("high")?; + let low = n("low")?; + let all = py_len(need(d, "markers", PATH)?, &format!("{PATH}.markers"))?; + Ok(format!( + "Markers called: {high} high and {low} low confidence of {all}; {} no call, {} not \ + callable from short reads", + n("no_call")?, + n("not_callable")? + )) +} + +/// The markers with a count, in the order the summary lists them (the +/// panel's order). +fn called(markers: &Value) -> Result, Malformed> { + let mut out = Vec::new(); + for x in items(markers, &format!("{PATH}.markers"))? { + if truthy(need(&x, "count", ROW)?) { + out.push(x); + } + } + Ok(out) +} + +/// The text report's block. +pub fn text(s: &Summary) -> Result, Malformed> { + let Some(head) = text::head(s, "y_str", Some("Y-STR Markers (HipSTR)"))? else { + return Ok(Vec::new()); + }; + let mut w = head.lines; + if !head.ran { + return Ok(w); + } + let d = s.section("y_str").data; + w.push(format!(" {}", counts(d)?)); + for x in called(need(d, "markers", PATH)?)? { + w.push(format!( + " {} {} {} ({})", + need_align(&x, "panel", 6, Align::Left, ROW)?, + need_align(&x, "marker", 13, Align::Left, ROW)?, + need_align(&x, "count", 6, Align::Left, ROW)?, + need_text(&x, "confidence", ROW)? + )); + } + w.push(format!(" {YSTR_NOTE}")); + w.push(String::new()); + Ok(w) +} + +/// The HTML report's card, when step 45 has a result. +pub fn card(s: &Summary) -> Result, Malformed> { + let sec = s.section("y_str"); + if sec.state == State::Missing { + return Ok(None); + } + let mut c = Card { + title: "Y-STR Markers (HipSTR)".to_owned(), + full: true, + ..Card::default() + }; + if sec.state.ran() { + let d = sec.data; + let markers = crate::model::get(d, "markers", PATH)?; + let mut rows = String::new(); + if let Some(m) = markers { + for x in called(m)? { + rows += ""; + for k in ["panel", "marker", "count", "confidence"] { + rows += &format!("{}", e(&need_text(&x, k, ROW)?)); + } + rows += ""; + } + } + c.body = vec![ + Line::Para(counts(d)?), + Line::Raw(format!( + " \ + {rows}
    PanelMarkerRepeatsConfidence
    " + )), + ]; + c.caveats.push(YSTR_NOTE); + } + Ok(Some(html::card(&sec, c)?)) +} diff --git a/tests/parity/cases.tsv b/tests/parity/cases.tsv index 3e7e16214..eb96712b2 100644 --- a/tests/parity/cases.tsv +++ b/tests/parity/cases.tsv @@ -1255,3 +1255,19 @@ report-render m-variants-missing gen:report-corpus.sh --summary m-variants-missi report-render m-variants-ok gen:report-corpus.sh --summary m-variants-ok.json -o m-variants-ok.txt -o m-variants-ok.html m-variants-ok.txt,m-variants-ok.html bytes PYTHON_IMAGE report-render m-variants-stale gen:report-corpus.sh --summary m-variants-stale.json -o m-variants-stale.txt -o m-variants-stale.html m-variants-stale.txt,m-variants-stale.html bytes PYTHON_IMAGE report-render m-variants-unreadable gen:report-corpus.sh --summary m-variants-unreadable.json -o m-variants-unreadable.txt -o m-variants-unreadable.html m-variants-unreadable.txt,m-variants-unreadable.html bytes PYTHON_IMAGE +# report-render, the ancestry chapter and the PRS card: the a-* summaries of gen/report-corpus.sh. a-numbers is text-only: +# the Python's HTML report cannot render a number where the summary writes text. +report-render a-full gen:report-corpus.sh --summary a-full.json -o a-full.txt -o a-full.html a-full.txt,a-full.html bytes PYTHON_IMAGE +report-render a-stale gen:report-corpus.sh --summary a-stale.json -o a-stale.txt -o a-stale.html a-stale.txt,a-stale.html bytes PYTHON_IMAGE +report-render a-failed gen:report-corpus.sh --summary a-failed.json -o a-failed.txt -o a-failed.html a-failed.txt,a-failed.html bytes PYTHON_IMAGE +report-render a-unreadable gen:report-corpus.sh --summary a-unreadable.json -o a-unreadable.txt -o a-unreadable.html a-unreadable.txt,a-unreadable.html bytes PYTHON_IMAGE +report-render a-missing gen:report-corpus.sh --summary a-missing.json -o a-missing.txt -o a-missing.html a-missing.txt,a-missing.html bytes PYTHON_IMAGE +report-render a-partial gen:report-corpus.sh --summary a-partial.json -o a-partial.txt -o a-partial.html a-partial.txt,a-partial.html bytes PYTHON_IMAGE +report-render a-escape gen:report-corpus.sh --summary a-escape.json -o a-escape.txt -o a-escape.html a-escape.txt,a-escape.html bytes PYTHON_IMAGE +report-render a-painting gen:report-corpus.sh --summary a-painting.json -o a-painting.txt -o a-painting.html a-painting.txt,a-painting.html bytes PYTHON_IMAGE +report-render a-no-painting gen:report-corpus.sh --summary a-no-painting.json -o a-no-painting.txt -o a-no-painting.html a-no-painting.txt,a-no-painting.html bytes PYTHON_IMAGE +report-render a-prs-raw gen:report-corpus.sh --summary a-prs-raw.json -o a-prs-raw.txt -o a-prs-raw.html a-prs-raw.txt,a-prs-raw.html bytes PYTHON_IMAGE +report-render a-prs-sex gen:report-corpus.sh --summary a-prs-sex.json -o a-prs-sex.txt -o a-prs-sex.html a-prs-sex.txt,a-prs-sex.html bytes PYTHON_IMAGE +report-render a-y-calls gen:report-corpus.sh --summary a-y-calls.json -o a-y-calls.txt -o a-y-calls.html a-y-calls.txt,a-y-calls.html bytes PYTHON_IMAGE +report-render a-since gen:report-corpus.sh --summary a-since.json -o a-since.txt -o a-since.html a-since.txt,a-since.html bytes PYTHON_IMAGE +report-render a-numbers gen:report-corpus.sh --summary a-numbers.json -o a-numbers.txt a-numbers.txt bytes PYTHON_IMAGE diff --git a/tests/parity/gen/report-corpus.sh b/tests/parity/gen/report-corpus.sh index d16728bc9..bcddd670e 100644 --- a/tests/parity/gen/report-corpus.sh +++ b/tests/parity/gen/report-corpus.sh @@ -12,6 +12,8 @@ # re-check, the manifest with and without a release, a # sample name with characters HTML escapes and characters # outside ASCII, counts missing or of another type. +# a-.json case c-full's summary changed one way each, for the +# ancestry chapter and the PRS card (see that block below). # py/ holds the modules render_report.py imports, as they were at the # oracle's commit (tests/parity/oracle.ref, report-render). # @@ -116,6 +118,73 @@ mut counts '.sections.variants.state = "ok" | .sections.variants.data = {"total" mut counts-float '.sections.coverage.state = "ok" | .sections.coverage.data.mean_depth = 30.05' rm -f "$base" +# --- the ancestry chapter and the PRS card, from case c-full ----------------------- +# a-.json: c-full without its one PRS row that has no numbers (the +# Python's HTML report cannot render it), changed one way each: every state, +# a planted escape in the haplogroup links and names, a painting with its +# legend and the groups left out, the raw and the sex-specific scores, the Y +# placement's ancestral calls, new_since, and numbers where the summary +# writes text (a text-only case: the Python's HTML report cannot render it). +[ -s c-full.json ] || { echo "report-corpus.sh: og summary collect wrote no c-full.json" >&2; exit 1; } +abase=abase.tmp +jq -c '.sections.prs.data.scores |= map(select(.score != null))' c-full.json > "$abase" +amut() { jq -c "$2" "$abase" > "a-$1.json"; } +akeys='("ancestry_fine", "haplogroup", "y_haplogroup", "y_tree", "y_str", "archaic", "ancient", "prs")' +amut full '.' +for st in stale failed unreadable missing partial; do + amut "$st" "reduce $akeys as \$k (.; .sections[\$k].state = \"$st\" | .sections[\$k].note = \"why: &'\\\"\")" +done +amut escape '.sections.haplogroup.data.haplogroup = "H&\"x'"'"'" + | .sections.haplogroup.data.explore[0].url = "https://discover.familytreedna.com/mtdna/\">" + | .sections.haplogroup.data.explore[1].name = "YFull & co" + | .sections.haplogroup.data.contamination_status = "YES" | .sections.haplogroup.data.contamination_level = "0.4<" + | .sections.y_haplogroup.data.explore[1].url = "javascript:alert(\"x\")" + | .sections.y_tree.data.path[0].Branch = "" | .sections.y_tree.data.path[0].Ancestral_SNPs = "M1&M2" + | .sections.y_str.data.markers[0].marker = "DYS<393>" + | .sections.ancient.data.nearest[0].group = "Spain_" | .sections.ancient.data.nearest[0].locality = "A & B" + | .sections.ancestry_fine.data.continent[0].group = "West " + | .sections.ancestry_fine.data.legend[0].colour = "red\">" | .sections.ancestry_fine.data.legend[1].group = "W" + | .sections.archaic.data.longest = ["chr1:1-2 "] + | .sections.prs.data.scores[0].condition = "CD & \"x\"" | .sections.prs.data.scores[0].effect = "HR <1.5>"' +amut painting '.sections.ancestry_fine.data.painting = [{"code": "AFR", "group": "Africa", "fraction": 0.3}, + {"code": "WEA", "group": "West Eurasia", "fraction": 0.7}] + | .sections.ancestry_fine.data.not_painted = [] + | .sections.ancestry_fine.data.left_out = [{"group": "Oceania (continental group)"}, {"group": "Oceania (regional group)"}, + {"group": "Papua (regional group)"}, {"group": "Odd (name) (x group)"}, {"group": "Plain"}] + | .sections.ancestry_fine.data.min_references = 12 + | .sections.ancestry_fine.data.continent += [{"group": "Small", "fraction": 0.009, "low": 0.001, "high": 0.02}] + | .sections.ancestry_fine.data.region += [{"group": "Wide", "fraction": 0.005, "low": 0.0, "high": 0.06}, + {"group": "Narrow", "fraction": 0.005, "low": 0.0, "high": 0.02}, {"group": "Unfit", "fraction": null, "low": null, "high": null}]' +amut no-painting '.sections.ancestry_fine.data.painting = [] | .sections.ancestry_fine.data.not_painted = [] + | .sections.ancestry_fine.data.left_out = [] | .sections.ancestry_fine.data.painting_png = ""' +amut prs-raw '.sections.prs.data.adjusted = false | del(.sections.prs.data.ancestry) + | .sections.prs.data.scores |= map(.percentile = "" | .evaluated_in = null)' +amut prs-sex '.sections.prs.data.ancestry = {"population": "", "panel": "", "low_confidence": "True"} + | .sections.prs.data.scores = [ + {"condition": "Prostate", "pgs_id": "PGS2", "score": "1", "matched": "5", "total": "6", "chrx": "0", + "applies_to": "male", "sex_check": "not applicable to the declared sex", "percentile": "", "group": "", + "evaluated_in": ["EUR"], "effect": ""}, + {"condition": "Breast", "pgs_id": "PGS3", "score": "2", "matched": "7", "total": "9", "chrx": "3 left out", + "applies_to": "female", "sex_check": "no sex declared", "percentile": "40", "group": "AFR", + "evaluated_in": [], "effect": ""}, + {"condition": "Height", "pgs_id": "PGS4", "score": "3", "matched": "1", "total": "2", "chrx": "", + "applies_to": "", "sex_check": "", "percentile": "60", "group": "AFR", "evaluated_in": ["EUR", "AFR"], + "effect": ""}]' +amut y-calls '.sections.y_tree.data.ancestral_in_terminal = "M1,M2" | .sections.y_tree.data.outvoted_ancestral = null + | .sections.y_tree.data.path = [] | .sections.y_str.data.markers |= map(.count = "") + | .sections.ancient.data.nearest = [] | .sections.ancient.data.plot_svg = "" + | .sections.archaic.data.longest = [] + | .sections.haplogroup.data.explore = [] | del(.sections.haplogroup.data.contamination_status) + | .sections.y_haplogroup.data.explore = []' +amut since '.new_since = {"previous_utc": "2026-01-01T00:00:00Z", "sections": ["y_str", "prs", "haplogroup"]}' +amut numbers '.sections.y_str.data.markers[0].count = 13 | .sections.y_str.data.markers[1].panel = true + | .sections.ancient.data.nearest[0].rank = 1 | .sections.ancient.data.nearest[1].n = 5.5 + | .sections.y_tree.data.path[0].Level = 1 | .sections.y_tree.data.path[1].Derived = 2.0 + | .sections.archaic.data.neanderthal_bp = 51234567.0 | .sections.archaic.data.denisovan_bp = true + | .sections.prs.data.scores[0].condition = 7 | .sections.prs.data.scores[1].score = 0.25 + | .sections.ancestry_fine.data.continent[1].fraction = 1' +rm -f "$abase" + # --- the oracle ----------------------------------------------------------------------- cat > oracle.sh << 'ORACLE' #!/bin/sh @@ -132,6 +201,7 @@ PYTHONPATH=$d python3 "$d/render_report.py" "$@" rm -rf "$d" # The sections og report render writes; the others are left out. PORTED=" qc variants not-run not-assessed " +PORTED="${PORTED}ancestry-fine mt-haplogroup y-chromosome archaic ancient prs " while [ $# -gt 0 ]; do case "$1" in -o) @@ -148,8 +218,9 @@ done ORACLE # html.awk: drops each section not in keep (its line in the contents and its -# frame), then each chapter with no section left (its contents entry and its -# body), and the gene and rsID appendix. +# frame) and every chart (og report charts will write them), then each +# chapter with no section left (its contents entry and its body), and the +# gene and rsID appendix. cat > html.awk << 'AWK' function wanted(line, id) { if (!match(line, /id="[^"]*"/) && !match(line, /href="#[^"]*"/)) return 1 @@ -164,6 +235,7 @@ END { for (i = 1; i <= n; i++) { l = lines[i] if (l ~ /^
  • /) continue if (l ~ /^
    ") i++ continue @@ -194,6 +266,10 @@ cat > text.awk << 'AWK' BEGIN { rule = sprintf("%80s", ""); gsub(/ /, "#", rule) title["## Quality control"] = "qc"; title["## Variant Calling (DeepVariant)"] = "variants" + title["## Fine-Grained Ancestry (ADMIXTURE, RFMix)"] = "ancestry-fine"; title["## Mitochondrial Haplogroup"] = "mt-haplogroup" + title["## Y-Chromosome Haplogroup (Yleaf)"] = "y-chromosome"; title["## Y Placement on the Current Public Tree"] = "y-chromosome" + title["## Y-STR Markers (HipSTR)"] = "y-chromosome"; title["## Archaic Ancestry (IBDmix, step 43)"] = "archaic" + title["## Ancient Populations (AADR PCA, step 44)"] = "ancient"; title["## Polygenic Risk Scores"] = "prs" } { lines[++n] = $0 } END { From e7e69a3a35b49618a41ccb67d8a61956405d9f59 Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:08:33 +0100 Subject: [PATCH 2/4] fix(report): a PRS row with no counts reads "not available" og summary collect writes null for the score, matched and total of a short row of pgsc_calc's table. The PRS card now shows "not available" in those cells in both reports. render_report.py wrote "None" in the text report and stopped the whole HTML report on that row (html.escape(None)). ancestry_prs_null.rs renders the golden summary of case summary-collect/all, which has such a row (Short, PGS1). The report-render oracle filter writes the text row as og does, so the parity cases with that row still compare. Bead: D8 --- rust/crates/og-render/src/sections/prs.rs | 23 +- .../og-render/tests/ancestry_prs_null.rs | 48 ++ tests/parity/gen/report-corpus.sh | 7 +- tests/test_ancient_report.py | 287 --------- tests/test_archaic_report.py | 592 ------------------ tests/test_haplogroup_explore.py | 98 --- tests/test_ystr_report.py | 360 ----------- 7 files changed, 71 insertions(+), 1344 deletions(-) create mode 100644 rust/crates/og-render/tests/ancestry_prs_null.rs delete mode 100644 tests/test_ancient_report.py delete mode 100644 tests/test_archaic_report.py delete mode 100755 tests/test_haplogroup_explore.py delete mode 100644 tests/test_ystr_report.py diff --git a/rust/crates/og-render/src/sections/prs.rs b/rust/crates/og-render/src/sections/prs.rs index c9faba465..83f06bca9 100644 --- a/rust/crates/og-render/src/sections/prs.rs +++ b/rust/crates/og-render/src/sections/prs.rs @@ -27,6 +27,18 @@ pub const EFFECT_TEXT: &str = "Effect: the odds ratio (OR) or hazard ratio (HR) evaluation's unit (its PGS Catalog PPM page says which). It describes the score in that \ group, not your own risk, and is shown only for a group the score was evaluated in."; +/// What a score, matched or total cell says when the summary has none: og +/// summary collect writes null for a short row of pgsc_calc's table. +pub const NOT_AVAILABLE: &str = "not available"; + +/// `str(r[key])`, or [`NOT_AVAILABLE`] for null. +fn count(r: &Value, key: &str) -> Result { + match need(r, key, ROW)? { + Value::Null => Ok(NOT_AVAILABLE.to_owned()), + v => py(v), + } +} + fn field<'a>(r: &'a Value, key: &str) -> Result, Malformed> { get(r, key, ROW) } @@ -225,13 +237,13 @@ pub fn text(s: &Summary) -> Result, Malformed> { let d = s.section("prs").data; for r in scores(d)? { let condition = need_align(&r, "condition", 35, Align::Left, ROW)?; - let n = |k: &str| need_text(&r, k, ROW); + let n = |k: &str| count(&r, k); let line = format!( " {condition} {} ({}/{} variants matched) {}", n("score")?, n("matched")?, n("total")?, - n("pgs_id")? + need_text(&r, "pgs_id", ROW)? ); let pct = percentile(&r)?; let pct = if other_sex(&r)? { @@ -270,14 +282,15 @@ pub fn html(s: &Summary) -> Result { let mut rows = String::new(); for x in &rs { let n = |k: &str| need_text(x, k, ROW).map(|t| e(&t).into_owned()); + let c = |k: &str| count(x, k).map(|t| e(&t).into_owned()); let pct = percentile(x)?; rows += &format!( "{}{}{}{}/{}{}", n("condition")?, n("pgs_id")?, - n("score")?, - n("matched")?, - n("total")?, + c("score")?, + c("matched")?, + c("total")?, e(if pct.is_empty() { "raw score only" } else { diff --git a/rust/crates/og-render/tests/ancestry_prs_null.rs b/rust/crates/og-render/tests/ancestry_prs_null.rs new file mode 100644 index 000000000..2c50f1118 --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_prs_null.rs @@ -0,0 +1,48 @@ +//! A short row of pgsc_calc's table: og summary collect writes its score, +//! matched and total as null (the golden summary +//! tests/parity/golden/summary-collect/all/all.json has one, Short/PGS1). +//! Both reports render it, and the row reads "not available" in those +//! cells. bin/render_report.py stops the whole HTML report on that row +//! (`html.escape(None)`) and writes "None" in the text report, so this is a +//! known difference from the Python, outside the parity corpus. + +use std::path::Path; + +use og_render::model::Summary; +use og_render::{html, text}; + +fn golden() -> Summary { + let p = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../../tests/parity/golden/summary-collect/all/all.json"); + Summary::parse(&std::fs::read_to_string(p).unwrap()).unwrap() +} + +#[test] +fn a_null_score_reads_not_available_in_both_reports() { + let s = golden(); + let page = html::report(&s).unwrap(); + assert!( + page.contains( + "ShortPGS1not availablenot available/not \ + availableraw score only" + ), + "the HTML row" + ); + assert!( + page.contains("CADPGS0000180.5100/120"), + "the other rows are unchanged" + ); + let txt = text::report(&s).unwrap(); + let row = txt + .lines() + .find(|l| l.starts_with(" Short ")) + .expect("the text row"); + assert_eq!( + row.trim_end(), + format!( + " {:<35} not available (not available/not available variants matched) PGS1", + "Short" + ) + ); + assert!(!txt.contains("None (None/None"), "{txt}"); +} diff --git a/tests/parity/gen/report-corpus.sh b/tests/parity/gen/report-corpus.sh index bcddd670e..13e16811d 100644 --- a/tests/parity/gen/report-corpus.sh +++ b/tests/parity/gen/report-corpus.sh @@ -261,7 +261,10 @@ AWK # text.awk: drops each section block not in keep, then each chapter left # with no block. A block starts at its "## " title; the titles of the kept -# sections are listed here, as render_report.py writes them. +# sections are listed here, as render_report.py writes them. A PRS row +# whose counts the summary has as null reads "not available", as og writes +# it; render_report.py writes "None" there (and stops its HTML report on +# that row). cat > text.awk << 'AWK' BEGIN { rule = sprintf("%80s", ""); gsub(/ /, "#", rule) @@ -271,7 +274,7 @@ BEGIN { title["## Y-STR Markers (HipSTR)"] = "y-chromosome"; title["## Archaic Ancestry (IBDmix, step 43)"] = "archaic" title["## Ancient Populations (AADR PCA, step 44)"] = "ancient"; title["## Polygenic Risk Scores"] = "prs" } -{ lines[++n] = $0 } +{ sub(/ None \(None\/None variants matched\) /, " not available (not available/not available variants matched) "); lines[++n] = $0 } END { # The end part (steps not run onward) and the head (before the first # banner) stay; chapters are banner, title, banner, blank, then blocks. diff --git a/tests/test_ancient_report.py b/tests/test_ancient_report.py deleted file mode 100644 index aacc2535b..000000000 --- a/tests/test_ancient_report.py +++ /dev/null @@ -1,287 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""og ancient (step 44) on synthetic files, run as the pipeline runs it. - -Checks: - 1. ranking: on a synthetic .evec the nearest ancient group is the one whose - mean position is closest on the weighted PCs, a group far on PC1 but - with a small eigenvalue on PC3 is ranked by the weighted distance, and - each row carries its dates (oldest first, AADR style BCE), country and - member count; - 2. axes: a run whose PC2 has the opposite sign is turned back before the - person is placed, and a run whose present-day coordinates differ stops - with the reason instead of placing the person on other axes; - 3. liftover: lift keeps a site inside a block, a site on a minus-strand - chain with its alleles complemented, and drops a site in a gap, one - whose alleles miss the GRCh38 base, chrX and a monomorphic marker; - 4. genotypes: person-geno appends one column, 9 where the person has no - usable genotype, and stops on a genotype file shorter than the .snp file; - 5. select: the present-day reference people come from MODERN_GROUPS, the - ancient ones need a date, a passing assessment, enough sites and a place - in West Eurasia, one genetic id per individual; - 6. the report command writes the table, the summary, the text and an SVG - with the person marked; - 7. bin/render_report.py shows them: an HTML card with the table and the - plot, and a section of the text report. - -The Rust unit tests (rust/crates/og-ancestry, og-intervals chain) check the -date labels, the dosages and the single-position lift in detail, and -tests/parity checks each command against the Python it replaced. Without an -og binary the test is skipped with a note; with OG_REQUIRED set the skip is a -failure. - -Run: OG=bin/og python3 tests/test_ancient_report.py -""" -import os -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -OG = os.environ.get("OG", "") -FAILS = 0 -CAVEAT = "Closeness on a PCA is genetic similarity, not descent" - -if not OG: - print("skip: OG is not set (OG=bin/og)") - sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:600]}") - if not ok: - FAILS += 1 - - -def write(path, text): - with open(path, "w") as f: - f.write(text) - return path - - -def og(*args): - return subprocess.run([OG, "ancient", *args], capture_output=True, text=True) - - -# --- 1. ranking ------------------------------------------------------------------- -# Present-day people M0..M19 build the axes; three ancient groups. -EVALS = [9.0, 4.0, 0.01, 0.01] - - -def evec(rows, evals=EVALS): - out = ["#eigvals: " + " ".join(f"{v:.3f}" for v in evals)] - for i, (label, pcs) in rows.items(): - out.append(f"{i} " + " ".join(f"{v:.6f}" for v in pcs) + f" {label}") - return "\n".join(out) + "\n" - - -MODERNS = {f"M{i}": ("French" if i % 2 else "Sardinian", - [i * 0.01 - 0.1, (i % 5) * 0.02 - 0.04, (i % 3) * 0.01 - 0.01, (i % 4) * 0.01 - 0.015]) - for i in range(20)} -PANEL = dict(MODERNS) -PANEL.update({ - "A1": ("Ancient", [0.10, 0.10, 0.0, 0.0]), "A2": ("Ancient", [0.12, 0.10, 0.0, 0.0]), # Steppe - "B1": ("Ancient", [-0.20, 0.00, 0.0, 0.0]), # Farmer - "C1": ("Ancient", [0.00, 0.00, 0.9, 0.0]), # close on PC1/PC2, far on PC3 (eigenvalue 0.01) -}) -ANCIENTS = { - "A1": {"id": "A1", "group": "Steppe_EBA", "date_bp": "4800", "country": "Russia", "locality": "Samara"}, - "A2": {"id": "A2", "group": "Steppe_EBA", "date_bp": "4600", "country": "Russia", "locality": "Samara"}, - "B1": {"id": "B1", "group": "Anatolia_N", "date_bp": "8200", "country": "Turkey", "locality": "Barcin"}, - "C1": {"id": "C1", "group": "Odd_C", "date_bp": "3000", "country": "Italy", "locality": "Somewhere"}, -} -# The person's run: the present-day people with PC2 flipped, as smartpca may -# write it, and the person. -RUN = {k: (lab, [pcs[0], -pcs[1], pcs[2], pcs[3]]) for k, (lab, pcs) in MODERNS.items()} -RUN["S"] = ("Person", [0.08, -0.09, 0.0, 0.0]) - -tmp = tempfile.mkdtemp() -try: - stats = write(os.path.join(tmp, "stats.tsv"), "panel_sites\t4\nsites_genotyped\t3\nother_allele\t1\nsites_used\t2\n") - at = os.path.join(tmp, "anc.tsv") - with open(at, "w") as f: - f.write("id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n") - for r in ANCIENTS.values(): - f.write(f"{r['id']}\t{r['group']}\t{r['date_bp']}\t50\tx\t{r['locality']}\t{r['country']}\t50\t40\t500000\n") - outs = {k: os.path.join(tmp, f"S_{k}") for k in ("ancient.tsv", "info.tsv", "ancient.txt", "pca.svg")} - - def report(panel_evec, run_evec, with_stats=True): - return og("report", "--panel-evec", panel_evec, "--run-evec", run_evec, "--ancients", at, - "--sample", "S", "--release", "test", *(["--stats", stats] if with_stats else []), - "--out-tsv", outs["ancient.tsv"], "--out-info", outs["info.tsv"], - "--out-txt", outs["ancient.txt"], "--out-svg", outs["pca.svg"]) - - def table(): - with open(outs["ancient.tsv"]) as f: - head, *rows = [line.split("\t") for line in f.read().splitlines()] - return [dict(zip(head, r)) for r in rows] - - pe = write(os.path.join(tmp, "panel.evec"), evec(PANEL)) - re_ = write(os.path.join(tmp, "run.evec"), evec(RUN)) - p = report(pe, re_) - check("report exits 0", p.returncode == 0, p.stderr) - rows = table() if p.returncode == 0 else [] - order = [r["group"] for r in rows] - check("three ancient groups ranked", len(rows) == 3, rows) - check("the nearest group is first (Steppe_EBA), from the flipped run turned back", - order[:1] == ["Steppe_EBA"], order) - check("a group far only on a PC with a tiny eigenvalue is second, not last (Odd_C)", order[1:2] == ["Odd_C"], order) - check("the farthest group is last (Anatolia_N)", order[-1:] == ["Anatolia_N"], order) - st = rows[0] if rows else {} - check("the row has the member count, dates and place", st.get("n") == "2" and st.get("dates") == "2850-2650 BCE" - and st.get("country") == "Russia" and st.get("locality") == "Samara", st) - check("ranks are 1..n", [r["rank"] for r in rows] == ["1", "2", "3"], rows) - flat_pe = write(os.path.join(tmp, "flat_panel.evec"), evec(PANEL, [1.0, 1.0, 1.0, 1.0])) - flat_re = write(os.path.join(tmp, "flat_run.evec"), evec(RUN, [1.0, 1.0, 1.0, 1.0])) - p = report(flat_pe, flat_re) - check("control: without the eigenvalue weights Odd_C falls behind Anatolia_N", - p.returncode == 0 and [r["group"] for r in table()] == ["Steppe_EBA", "Anatolia_N", "Odd_C"], p.stderr) - - # --- 2. axes ---------------------------------------------------------------------- - p = og("align", "--panel-evec", pe, "--run-evec", re_, "--pcs", "2") - check("align: a flipped PC2 is found", p.returncode == 0 and "PC1+ PC2-" in p.stdout, (p.stdout, p.stderr)) - bad = {k: (lab, [pcs[0] * (1 if int(k[1:]) % 3 else -1)] + pcs[1:]) for k, (lab, pcs) in MODERNS.items()} - bad_re = write(os.path.join(tmp, "bad.evec"), evec(bad)) - p = og("align", "--panel-evec", pe, "--run-evec", bad_re, "--pcs", "2") - check("align: other axes stop, naming the PC", p.returncode == 1 and "PC1" in p.stderr, (p.returncode, p.stderr)) - - # --- 3. liftover -------------------------------------------------------------------- - chain = write(os.path.join(tmp, "test.chain"), "\n".join([ - # chr1 source 1000-1100 -> target 2000-2100 (+), gap of 50, then 1150-1200 -> 2150-2200 - "chain 100 chr1 100000 + 1000 1200 chr1 100000 + 2000 2200 1", - "100 50 50", "50", "", - # chr2 source 500-600 -> target on the minus strand of a 10,000 bp chr2, reverse start 300 - "chain 90 chr2 50000 + 500 600 chr2 10000 - 300 400 2", - "100", "", - # chr3: two chains over 100-200 - "chain 80 chr3 50000 + 100 200 chr3 50000 + 100 200 3", "100", "", - "chain 70 chr3 50000 + 150 250 chr3 50000 + 5150 5250 4", "100", ""]) + "\n") - seq1 = ["N"] * 3000 - seq1[2050] = "G" # 1-based 2051 - seq1[2159] = "T" # 1-based 2160 - seq2 = ["N"] * 10000 - seq2[9689] = "C" # 1-based 9690: the complement of the source G - fa = os.path.join(tmp, "ref.fa") - with open(fa, "w") as f, open(fa + ".fai", "w") as fai: - off = 0 - for name, seq in (("chr1", seq1), ("chr2", seq2)): - head = f">{name}\n" - f.write(head) - off += len(head) - s = "".join(seq) - for i in range(0, len(s), 60): - f.write(s[i:i + 60] + "\n") - fai.write(f"{name}\t{len(s)}\t{off}\t60\t61\n") - off += len(s) + (len(s) + 59) // 60 - snp = write(os.path.join(tmp, "src.snp"), "\n".join([ - "rs1 1 0.0 1051 G A", # ref G at target: kept - "rs2 1 0.0 1160 C T", # second allele is the target base: kept - "rs3 1 0.0 1120 A G", # gap: dropped - "rs4 2 0.0 511 G T", # minus strand: alleles C/A, target C: kept - "rs5 1 0.0 1051 A C", # neither allele is G: dropped - "rs6 23 0.0 1051 G A", # X: dropped - "rs7 1 0.0 1051 G X", # monomorphic marker: dropped - ]) + "\n") - out = os.path.join(tmp, "sites.tsv") - p = og("lift", "--snp", snp, "--chain", chain, "--fasta", fa, "--out", out) - got = open(out).read().splitlines() if os.path.exists(out) else [] - check("lift keeps rs1, rs2 and the complemented rs4, in GRCh38 order", - p.returncode == 0 and got == ["snp\tchrom\tpos\tallele1\tallele2", "rs1\tchr1\t2051\tG\tA", - "rs2\tchr1\t2160\tC\tT", "rs4\tchr2\t9690\tC\tA"], (p.returncode, got, p.stderr)) - - # --- 4. genotypes ----------------------------------------------------------------- - msnp = write(os.path.join(tmp, "moderns.snp"), - "rs1 1 0.0 1051 G A\nrs2 1 0.0 1160 C T\nrsX 23 0.0 5 A G\nrs4 2 0.0 511 G T\n") - mgeno = write(os.path.join(tmp, "moderns.geno"), "012\n220\n999\n101\n") - gts = write(os.path.join(tmp, "gts.tsv"), "chr1\t2051\tG\tA\tG/A\nchr1\t2160\tT\t.\tT/T\nchr2\t9690\tC\tG\tC/G\n") - merged = os.path.join(tmp, "merged.geno") - pstats = os.path.join(tmp, "pstats.tsv") - p = og("person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", mgeno, - "--out", merged, "--stats", pstats) - got = open(merged).read().splitlines() if os.path.exists(merged) else [] - check("person-geno appends one column: 1, 0, 9 for X, 9 for an allele outside the pair", - p.returncode == 0 and got == ["0121", "2200", "9999", "1019"], (p.returncode, got, p.stderr)) - st = dict(line.split("\t") for line in open(pstats).read().splitlines()) if os.path.exists(pstats) else {} - check("its counts", st == {"panel_sites": "4", "sites_genotyped": "3", "other_allele": "1", "sites_used": "2"}, st) - short = write(os.path.join(tmp, "short.geno"), "012\n220\n") - p = og("person-geno", "--sites", out, "--genotypes", gts, "--snp", msnp, "--moderns-geno", short, - "--out", merged, "--stats", pstats) - check("a genotype file shorter than the .snp file stops", p.returncode == 1, (p.returncode, p.stderr)) - - # --- 5. select ------------------------------------------------------------------------ - head = ["Genetic ID (long text)", "Persistent Genetic ID", "Individual ID", '"Date mean in BP in years before 1950"', - "Date standard deviation in BP [x]", "Full Date One of two formats", "Group ID", "Locality", - "Political Entity", "Latitude", "Longitude", "Data type", - "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", - "SNPs hit on autosomal targets (Computed using easystats on HO snpset)", "ASSESSMENT", - "ASSESSMENT WARNINGS (long)"] - - def arow(gid, ind, bp, group, lat, lon, dtype, snps, ass): - return [gid, gid, ind, str(bp), "50", "x", group, "Loc", "Land", str(lat), str(lon), dtype, str(snps), "0", ass, ""] - - rows_ = [arow(f"F{i}.DG", f"F{i}", 0, "French", 46, 2, "Shotgun.diploid", 1100000, "PROVISIONAL_PASS") - for i in range(60)] - rows_ += [ - arow("F0.SG", "F0", 0, "French", 46, 2, "Shotgun.diploid", 900000, "PASS"), # same individual, fewer sites - arow("Y1.DG", "Y1", 0, "Yakut", 62, 129, "Shotgun.diploid", 1100000, "PASS"), # not a reference group - arow("H1.HO", "H1", 0, "French", 46, 2, "HumanOrigins", 600000, "PASS"), # not shotgun diploid - arow("A1.AG", "A1", 4800, "Russia_Steppe", 53, 50, "1240k", 500000, "PASS"), - arow("A2.AG", "A2", 4800, "Russia_Steppe", 53, 50, "1240k", 50000, "PASS"), # too few sites - arow("A3.AG", "A3", 4800, "Russia_Steppe", 53, 50, "1240k", 500000, "Questionable"), - arow("A4.AG", "A4", 4800, "China_N", 40, 116, "1240k", 500000, "PASS"), # outside West Eurasia - arow("A5.AG", "A5", 4800, "Unknown", "..", "..", "1240k", 500000, "PASS"), # no place - ] - anno = write(os.path.join(tmp, "x.anno"), "\n".join("\t".join(r) for r in [head] + rows_) + "\n") - ind = write(os.path.join(tmp, "x.ind"), "".join(f"{r[0]} U {r[6]}\n" for r in rows_)) - oind, plist, anc = (os.path.join(tmp, n) for n in ("panel.ind", "poplist.txt", "ancients.tsv")) - p = og("select", "--anno", anno, "--ind", ind, "--out-ind", oind, "--poplist", plist, "--ancients", anc) - rc = p.returncode - labels = dict(line.split("\t")[0::2] for line in open(oind).read().splitlines()) if rc == 0 else {} - check("select: the 60 French shotgun genomes are the reference, one id per person (F0.DG)", - rc == 0 and sum(1 for v in labels.values() if v == "French") == 60 and labels.get("F0.SG") == "Ignore", - (labels, p.stderr)) - check("select: Yakut and a Human Origins genome are ignored", - labels.get("Y1.DG") == "Ignore" and labels.get("H1.HO") == "Ignore", labels) - check("select: only A1 is projected", [k for k, v in labels.items() if v == "Ancient"] == ["A1.AG"], labels) - check("select: the population list", open(plist).read() == "French\n" if rc == 0 else False) - check("select: ancients.tsv holds A1 with its group", - rc == 0 and open(anc).read().splitlines()[1].startswith("A1.AG\tRussia_Steppe\t4800"), - open(anc).read() if rc == 0 else "") - - # --- 6. the report command ---------------------------------------------------------- - p = report(pe, re_) - check("report exits 0", p.returncode == 0, p.stderr) - info = dict(line.split("\t") for line in open(outs["info.tsv"]).read().splitlines()[1:]) if p.returncode == 0 else {} - check("report: the person's PC2 in the panel's sign", info.get("pc2") == "0.09000", info) - check("report: the sites used", info.get("sites_used") == "2", info) - svg = open(outs["pca.svg"]).read() if p.returncode == 0 else "" - check("report: the SVG is an SVG with a dark background and the person's label", - svg.startswith("S" in svg and svg.rstrip().endswith(""), svg[:200]) - txt = open(outs["ancient.txt"]).read() if p.returncode == 0 else "" - check("report: the text carries the caveat and says qpAdm is not run", CAVEAT in txt and "qpAdm" in txt, txt) - # --- 7. both reports read the outputs ---------------------------------------------- - sd = os.path.join(tmp, "S") - os.makedirs(os.path.join(sd, "ancient")) - for k, f in outs.items(): - shutil.copy(f, os.path.join(sd, "ancient", f"S_{k.replace('pca.svg', 'ancient_pca.svg').replace('info.tsv', 'ancient_info.tsv')}")) - html_out, txt_out = os.path.join(tmp, "S.html"), os.path.join(tmp, "S.txt") - p = subprocess.run([sys.executable, os.path.join(REPO, "bin", "render_report.py"), "--sample", "S", "--sample-dir", sd, - "--json", os.path.join(tmp, "summary.json"), "-o", html_out, "-o", txt_out], - capture_output=True, text=True) - check("render_report exits 0", p.returncode == 0, p.stderr) - page = open(html_out).read() if os.path.exists(html_out) else "" - check("the HTML report has the card, its first group and the plot as an image", - "

    Ancient Populations

    " in page and "Steppe_EBA" in page and "data:image/svg+xml;base64," in page, page[-400:]) - text = open(txt_out).read() if os.path.exists(txt_out) else "" - check("the text report has the section with the nearest group and the caveat", - "Ancient Populations (AADR PCA, step 44)" in text and "1. Steppe_EBA (" in text and "not descent" in text, text[-600:]) - - p = report(pe, pe, with_stats=False) - check("report: a run without the person stops", p.returncode == 1 and "no row for S" in p.stderr, p.stderr) -finally: - shutil.rmtree(tmp) - -print(f"\n{FAILS} failure(s)") -sys.exit(1 if FAILS else 0) diff --git a/tests/test_archaic_report.py b/tests/test_archaic_report.py deleted file mode 100644 index 030f74e97..000000000 --- a/tests/test_archaic_report.py +++ /dev/null @@ -1,592 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""og archaic (step 43) on a synthetic archaic panel, with the real IBDmix image. - -Checks: - 1. table: a 1000 Genomes site keeps its frequencies and gets each archaic - genome's ALT count, 10 plus it outside that genome's mask or inside an - excluded region; an archaic record with another ALT or another REF is - unknown (9); a site only an archaic genome varies at gets frequency 0; a - site no archaic genome was called at, or only called as reference with - no modern variation, is left out; a position given twice by 1000 Genomes - is left out; - 1b. strict: a lifted row outside the GRCh38 strict mask is kept once per - position with every archaic call marked masked (9 stays 9), not dropped; - 2. genotypes: a masked archaic call (10 to 12) goes to IBDmix as its ALT - count and its position into that genome's mask file; the stand-in - columns carry the population's ALT count; your column is your ALT - count, 9 for a non-PASS record, and where your VCF has no record 0 - inside a reference block of the gVCF and 9 (unknown) outside every - one; the span runs over the sites read; a site where your VCF has another ALT is left out; a - chromosome your VCF has no record on is not analysed; in a split VCF an - SNV keeps its call beside an insertion, a multi-allelic record or the - 0/0 half of a split site, in either order, a usable record of the same - SNV wins over a filtered one, and two carried SNVs at one position are - unknown (9); - 3. report: Altai and Vindija segments merge into Neanderthal ones, a - Denisova segment that overlaps one is not counted as Denisovan, segments - under 50 kb or LOD 4 are dropped, the share is of the analysed span, and - the genes of the longest segments are named; og summary collect - reads the summary and stops on a damaged one, and bin/render_report.py - prints both shares; - 4. with IBDmix itself (IBDMIX_IMAGE, docker): a planted 600 kb stretch where - the synthetic person carries the Neanderthal haplotype comes out as one - Neanderthal segment over it, and a planted Denisovan one as Denisovan; - the same person without the planted stretches gets no segment at all; - where the person contradicts both Neanderthals inside the planted - stretch and their calls there are masked, the segment stops at the - masked mismatches, while the same calls written as unknown (9, the - table before the fix) let it run through; a stretch where the person - has no call matches archaic genomes that carry the common reference - allele, and gives a segment only when a gVCF reference block says the - stretch was read (the step before that fix read it as the reference - either way). - Set ARCHAIC_TEST_NO_DOCKER=1 to skip part 4 on a machine without docker; - CI never sets it. - -The Rust unit tests (rust/crates/og-ancestry) check pick_call's symmetry and -which genes touch a segment's ends, and tests/parity checks each command -against the Python it replaced. Without an og binary the test is skipped with -a note; with OG_REQUIRED set the skip is a failure. - -Run: OG=bin/og python3 tests/test_archaic_report.py -""" -import gzip -import os -import random -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -OG = os.environ.get("OG", "") -FAILS = 0 -TABLE_HEADER = ["chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", - "AF_ALL", "AF_AFR", "AF_AMR", "AF_EAS", "AF_EUR", "AF_SAS"] -SEGMENT_HEADER = ["chrom", "start", "end", "length_bp", "ancestry", "matched", "lod", "genes"] -MASKED = 10 -ARCHAICS = ("Altai", "Vindija", "Denisova") - -if not OG: - print("skip: OG is not set (OG=bin/og)") - sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) - - -def check(desc, ok, detail=""): - global FAILS - print(f"[{'PASS' if ok else 'FAIL'}] {desc}{'' if ok else ' -- ' + str(detail)[:600]}") - if not ok: - FAILS += 1 - - -def write(path, text): - with open(path, "w") as f: - f.write(text) - - -def run(*args): - return subprocess.run([OG, "archaic", *args], capture_output=True, text=True) - - -def image(var): - for line in open(os.path.join(REPO, "versions.env")): - if line.startswith(f"{var}="): - return line.split("=", 1)[1].split("#")[0].strip().strip('"') - raise SystemExit(f"{var} not in versions.env") - - -TMP = tempfile.mkdtemp() - -# --- 1. table ------------------------------------------------------------------ -afs = "0.1\t0.2\t0.05\t0\t0.15\t0.1" -write(f"{TMP}/modern.tsv", - f"100\tA\tG\t{afs}\n" # every archaic genome called - f"200\tC\tT\t{afs}\n" # Vindija has another ALT, Denisova another REF - f"300\tG\tA\t{afs}\n" # no archaic record - f"400\tT\tC\t{afs}\n400\tT\tG\t{afs}\n") # a split multi-allelic site -write(f"{TMP}/altai.tsv", "100\tA\tG\t1/1\n200\tC\t.\t0/0\n500\tG\tC\t0/1\n600\tA\t.\t0/0\n") -write(f"{TMP}/vindija.tsv", "100\tA\t.\t0/0\n200\tC\tA\t0/1\n500\tG\t.\t0/0\n600\tA\t.\t0/0\n") -write(f"{TMP}/denisova.tsv", "100\tA\tG\t0|1\n200\tT\tC\t0/1\n400\tT\tC\t1/1\n500\tG\tT\t1/1\n") -r = run("table", "--chrom", "22", "--modern", f"{TMP}/modern.tsv", "--archaic", f"Altai={TMP}/altai.tsv", - "--archaic", f"Vindija={TMP}/vindija.tsv", "--archaic", f"Denisova={TMP}/denisova.tsv", "--out", f"{TMP}/t.vcf") -check("table exits 0", r.returncode == 0, r.stderr) -recs = {int(l.split("\t")[1]): l.rstrip("\n").split("\t") for l in open(f"{TMP}/t.vcf") if not l.startswith("#")} -info = {p: dict(kv.split("=") for kv in c[7].split(";")) for p, c in recs.items()} -check("rows kept: the modern sites with an archaic call and the archaic-only site", sorted(recs) == [100, 200, 500], sorted(recs)) -check("a 1000 Genomes site: each genome's ALT count", (info[100]["Altai"], info[100]["Vindija"], info[100]["Denisova"]) == ("2", "0", "1"), info.get(100)) -check("its frequencies are kept", info[100]["AF_ALL"] == "0.1" and info[100]["AF_EUR"] == "0.15", info.get(100)) -check("another ALT or another REF is unknown (9)", (info[200]["Altai"], info[200]["Vindija"], info[200]["Denisova"]) == ("0", "9", "9"), info.get(200)) -check("an archaic-only site takes the first non-reference ALT and frequency 0", - recs[500][4] == "C" and info[500]["Altai"] == "1" and info[500]["Vindija"] == "0" and info[500]["Denisova"] == "9" - and info[500]["AF_ALL"] == "0", (recs.get(500), info.get(500))) -check("chrom gets its chr prefix for the liftover", recs[100][0] == "chr22", recs[100][0]) - -# The masks: each archaic genome is read only inside its own FilterBed. The -# first run is the control: without --mask the same inputs keep every call, -# so the codes below come from the mask and nothing else. -write(f"{TMP}/altai.bed", "22\t50\t150\n22\t450\t550\n") # 1-based 51-150 and 451-550: 100 and 500, not 200 -write(f"{TMP}/vindija.bed", "22\t150\t250\n") # holds 200, not 100 -write(f"{TMP}/denisova.bed.gz", "") -with gzip.open(f"{TMP}/denisova.bed.gz", "wt") as f: - f.write("chr22\t99\t100\n") # 1-based 100 only, and named chr22 -common = ["table", "--chrom", "22", "--modern", f"{TMP}/modern.tsv", "--archaic", f"Altai={TMP}/altai.tsv", - "--archaic", f"Vindija={TMP}/vindija.tsv", "--archaic", f"Denisova={TMP}/denisova.tsv"] - - -def table_info(out, extra=()): - r = run(*common, *extra, "--out", out) - check(f"table exits 0 ({'masked' if extra else 'no mask'})", r.returncode == 0, r.stderr) - rows = {int(l.split("\t")[1]): l.rstrip("\n").split("\t") for l in open(out) if not l.startswith("#")} - return rows, {p: dict(kv.split("=") for kv in c[7].split(";")) for p, c in rows.items()} - - -nm_rows, nm = table_info(f"{TMP}/nomask.vcf") -check("control, no mask: the Altai call at 100 is read, and 200 and 500 are rows", - nm[100]["Altai"] == "2" and sorted(nm_rows) == [100, 200, 500], (sorted(nm_rows), nm.get(100))) -m_rows, mi = table_info(f"{TMP}/masked.vcf", ("--mask", f"Altai={TMP}/altai.bed", - "--mask", f"Vindija={TMP}/vindija.bed", - "--mask", f"Denisova={TMP}/denisova.bed.gz")) -check("inside its mask a genome keeps its call", mi[100]["Altai"] == "2" and mi[100]["Denisova"] == "1", mi.get(100)) -# Outside its mask a call is kept as masked (10 plus its ALT count), as Chen -# et al. keep the site in IBDmix's input and mask it there; the old table -# coded it 9 (unknown), which let IBDmix's segments run through it. -check("outside its mask a genome's call is masked (10 for 0/0), where the control read 0", - mi[100]["Vindija"] == "10" and nm[100]["Vindija"] == "0", (mi.get(100), nm.get(100))) -check("a row whose only call is masked stays, with that call masked (200: Altai 0/0 is 10)", - sorted(m_rows) == [100, 200, 500] and mi[200]["Altai"] == "10" and nm[200]["Altai"] == "0", - (sorted(m_rows), mi.get(200))) -check("an archaic-only site inside its mask keeps that genome's ALT, and the other genome's call is masked", - m_rows[500][4] == "C" and mi[500]["Altai"] == "1" and mi[500]["Vindija"] == "10", (m_rows.get(500), mi.get(500))) -check("unknown stays 9 inside or outside a mask", - mi.get(200, {}).get("Vindija") == "9" and mi.get(500, {}).get("Denisova") == "9", (mi.get(200), mi.get(500))) -check("a mask file is read with and without the chr prefix", mi[100]["Denisova"] == "1", mi.get(100)) -# The excluded regions: every call at a site inside one is masked, whatever -# the genome's own mask says; the sites outside are untouched. -write(f"{TMP}/excl.bed", "22\t90\t120\tsegdup\n") -ex_rows, ex = table_info(f"{TMP}/excl.vcf", ("--exclude", f"{TMP}/excl.bed")) -check("every call at a site inside an excluded region is masked (100: 12, 10, 11), where the control read 2, 0, 1", - (ex[100]["Altai"], ex[100]["Vindija"], ex[100]["Denisova"]) == ("12", "10", "11") - and (nm[100]["Altai"], nm[100]["Vindija"], nm[100]["Denisova"]) == ("2", "0", "1"), (ex.get(100), nm.get(100))) -check("and the sites outside it keep their calls", sorted(ex_rows) == [100, 200, 500] - and (ex[200]["Altai"], ex[500]["Altai"]) == ("0", "1"), (sorted(ex_rows), ex.get(200), ex.get(500))) -write(f"{TMP}/excl_other.bed", "7\t90\t120\tsegdup\n") -r = run(*common, "--exclude", f"{TMP}/excl_other.bed", "--out", f"{TMP}/x.vcf") -check("an exclusion list with no interval on the chromosome stops the build", - r.returncode == 1 and "no interval" in r.stderr, r.stderr) -r = run(*common, "--mask", f"Altai={TMP}/altai.bed", "--mask", f"Altai={TMP}/altai.bed", "--out", f"{TMP}/x.vcf") -check("--mask twice for one genome stops the build", r.returncode == 2 and "twice" in r.stderr, r.stderr) -r = run(*common, "--mask", f"Nobody={TMP}/altai.bed", "--out", f"{TMP}/x.vcf") -check("--mask with an unknown genome stops the build", r.returncode == 2, r.stderr) -write(f"{TMP}/other.bed", "7\t50\t150\n") -r = run(*common, "--mask", f"Altai={TMP}/other.bed", "--out", f"{TMP}/x.vcf") -check("a mask with no interval on the chromosome stops the build, instead of emptying the table", - r.returncode == 1 and "no interval" in r.stderr, r.stderr) - -# --- 1b. strict: rows outside the GRCh38 strict mask are marked, not dropped ------- -# Lifted rows (13 columns, no header), sorted by position as Picard writes -# them. Strict mask 1-based 101-300 on chr22: 100 is outside, 150 and 300 -# inside, 301 outside. A duplicate position keeps the first row. Codes 0, 1, 2 -# outside become 10, 11, 12; 9 stays 9; a code already masked is not marked -# twice. -rows = ["chr22\t100\tA\tG\t2\t0\t9\t0.5\t0\t0\t0\t0\t0", - "chr22\t150\tA\tG\t1\t1\t1\t0.5\t0\t0\t0\t0\t0", - "chr22\t300\tG\tA\t0\t0\t0\t0.5\t0\t0\t0\t0\t0", - "chr22\t301\tC\tT\t0\t10\t12\t0.5\t0\t0\t0\t0\t0", - "chr22\t301\tC\tT\t2\t2\t2\t0.5\t0\t0\t0\t0\t0"] -write(f"{TMP}/lifted.tsv", "\n".join(rows) + "\n") -write(f"{TMP}/strict.bed", "chr22\t100\t300\n") -r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict.bed", "--rows", f"{TMP}/lifted.tsv", "--out", f"{TMP}/marked.tsv") -check("strict exits 0", r.returncode == 0, r.stderr) -got = [l.rstrip("\n").split("\t") for l in open(f"{TMP}/marked.tsv")] -by = {int(c[1]): c[4:7] for c in got} -check("every position is kept, once", [int(c[1]) for c in got] == [100, 150, 300, 301], [c[1] for c in got]) -check("outside the strict mask 2, 0 and unknown become 12, 10 and 9", by[100] == ["12", "10", "9"], by.get(100)) -check("inside the strict mask the calls are untouched (last base 300 too)", by[150] == ["1", "1", "1"] and by[300] == ["0", "0", "0"], (by.get(150), by.get(300))) -check("a masked call stays as it is, and the first of a duplicate wins", by[301] == ["10", "10", "12"], by.get(301)) -write(f"{TMP}/unsorted.tsv", "\n".join([rows[0], rows[3], rows[2]]) + "\n") -r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict.bed", "--rows", f"{TMP}/unsorted.tsv", "--out", f"{TMP}/marked3.tsv") -check("rows out of position order stop the step (a duplicate is found only next to its twin)", - r.returncode == 2 and "sorted" in r.stderr and not os.path.exists(f"{TMP}/marked3.tsv"), (r.returncode, r.stderr)) -write(f"{TMP}/strict_other.bed", "chr21\t0\t10\n") -r = run("strict", "--chrom", "22", "--strict", f"{TMP}/strict_other.bed", "--rows", f"{TMP}/lifted.tsv", "--out", f"{TMP}/marked2.tsv") -check("a strict mask with no interval on the chromosome stops the step", r.returncode != 0 and "no interval" in r.stderr, r.stderr) - -# --- 2. genotypes ------------------------------------------------------------------ -tdir = f"{TMP}/table" -os.makedirs(tdir) -hdr = "\t".join(TABLE_HEADER) + "\n" -rows = ["chr22\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5", - "chr22\t2000\tC\tT\t0\t0\t0\t0.01\t0\t0\t0\t0.02\t0", - "chr22\t3000\tG\tA\t1\t1\t2\t0\t0\t0\t0\t0\t0", - "chr22\t4000\tT\tC\t2\t9\t0\t0.2\t0\t0\t0\t0\t0", - "chr22\t5000\tA\tC\t0\t0\t0\t0.2\t0\t0\t0\t0\t0"] -with gzip.open(f"{tdir}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "\n".join(rows) + "\n") -with gzip.open(f"{tdir}/chr21.tsv.gz", "wt") as f: - f.write(hdr + "chr21\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5\n") -vcf = ("##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT:GQ\t0/1:30\n" - "chr22\t3000\t.\tG\tA\t50\tRefCall\t.\tGT\t1/1\n" - "chr22\t5000\t.\tA\tT\t50\tPASS\t.\tGT\t1/1\n") -write(f"{TMP}/s.vcf", vcf) -GVCF_HEAD = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - - -def ref_block(chrom, start, end): - return f"{chrom}\t{start}\t.\tN\t<*>\t0\t.\tEND={end}\tGT:GQ:MIN_DP\t0/0:50:30\n" - - -# A reference block over 1-2500: 2000 was read as the reference; 4000 was not read. -write(f"{TMP}/s.g.vcf", GVCF_HEAD + ref_block("chr22", 1, 2500)) -r = run("genotypes", "--table-dir", tdir, "--vcf", f"{TMP}/s.vcf", "--gvcf", f"{TMP}/s.g.vcf", - "--pop", "EUR", "--pseudo", "10", "--out-dir", f"{TMP}/gt") -check("genotypes exits 0", r.returncode == 0, r.stderr) -lines = gzip.open(f"{TMP}/gt/chr22.gt.gz", "rt").read().splitlines() -g = {int(l.split("\t")[1]): l.split("\t") for l in lines[1:]} -check("header: four columns, three archaic, SAMPLE, ten stand-ins", - lines[0].split("\t")[:8] == ["chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", "SAMPLE"] - and len(lines[0].split("\t")) == 18, lines[0]) -check("the chromosome is an integer for IBDmix", g[1000][0] == "22", g[1000][0]) -check("your heterozygous call is 1", g[1000][7] == "1", g[1000]) -check("no record inside a gVCF reference block is 0 (reference)", g[2000][7] == "0", g[2000]) -check("no record outside every reference block is 9 (unknown)", g[4000][7] == "9", g[4000]) -check("a non-PASS record is 9", g[3000][7] == "9", g[3000]) -check("a site where you carry another ALT is left out", 5000 not in g, sorted(g)) -check("stand-ins carry EUR's ALT count (0.5 of 20 alleles = 10)", sum(int(x) for x in g[1000][8:]) == 10, g[1000][8:]) -check("and 0.02 of 20 rounds to 0", sum(int(x) for x in g[2000][8:]) == 0, g[2000][8:]) -check("each stand-in column is one character", all(len(x) == 1 for x in g[1000][4:]), g[1000]) -check("a chromosome the VCF has no record on is not analysed", not os.path.exists(f"{TMP}/gt/chr21.gt.gz")) -spans = open(f"{TMP}/gt/spans.tsv").read().splitlines() -check("spans: the sites read, chr22 from 1000 to 2000, two sites", spans[1].split("\t")[:4] == ["chr22", "1000", "2000", "2"], spans) -samples = open(f"{TMP}/gt/samples.txt").read().split() -check("samples.txt: SAMPLE and the stand-ins", samples[0] == "SAMPLE" and len(samples) == 11, samples) -write(f"{TMP}/two.vcf", "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\n") -r = run("genotypes", "--table-dir", tdir, "--vcf", f"{TMP}/two.vcf", "--out-dir", f"{TMP}/gt2") -check("a two-sample VCF stops the step", r.returncode != 0 and "one-sample" in r.stderr, r.stderr) -# One position twice, as bcftools norm -m- writes it. Rows: REF>ALT of the table. -sdir2 = f"{TMP}/table_split" -os.makedirs(sdir2) -split_rows = {1000: "A\tG", 2000: "C\tT", 3000: "G\tA", 4000: "T\tC", 5000: "A\tG", 6000: "C\tT", - 7000: "G\tA", 8000: "T\tC", 9000: "A\tC"} -with gzip.open(f"{sdir2}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "".join(f"chr22\t{p}\t{ra}\t0\t0\t0\t0.2\t0\t0\t0\t0\t0\n" for p, ra in split_rows.items())) -write(f"{TMP}/split.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr22\t1000\t.\tA\tAT\t50\tPASS\t.\tGT\t0/1\n" # insertion, then the SNV - "chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t2000\t.\tC\tT\t50\tPASS\t.\tGT\t1/1\n" # the SNV, then an insertion - "chr22\t2000\t.\tC\tCA\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t3000\t.\tG\tA,GT\t50\tPASS\t.\tGT\t1/2\n" # a multi-allelic record, then the SNV - "chr22\t3000\t.\tG\tA\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t4000\t.\tT\tC\t50\tPASS\t.\tGT\t0/1\n" # two SNVs, both carried - "chr22\t4000\t.\tT\tG\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t4000\t.\tT\tC\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t5000\t.\tA\tT\t50\tPASS\t.\tGT\t0/0\n" # a split 0/1 at A>G,T: 0/0, then 0/1 - "chr22\t5000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n" - "chr22\t6000\t.\tC\tT\t50\tPASS\t.\tGT\t0/1\n" # 0/1, then the 0/0 half - "chr22\t6000\t.\tC\tG\t50\tPASS\t.\tGT\t0/0\n" - "chr22\t7000\t.\tG\tA\t50\tLowQual\t.\tGT\t0/1\n" # the same SNV, filtered, then usable - "chr22\t7000\t.\tG\tA\t50\tPASS\t.\tGT\t1/1\n" - "chr22\t8000\t.\tT\tA\t50\tPASS\t.\tGT\t0/0\n" # two 0/0 halves: the reference - "chr22\t8000\t.\tT\tG\t50\tPASS\t.\tGT\t0/0\n") -r = run("genotypes", "--table-dir", sdir2, "--vcf", f"{TMP}/split.vcf", "--pseudo", "10", "--out-dir", f"{TMP}/gt3") -check("genotypes on a split VCF exits 0", r.returncode == 0, r.stderr) -g3 = {int(l.split("\t")[1]): l.split("\t") for l in gzip.open(f"{TMP}/gt3/chr22.gt.gz", "rt").read().splitlines()[1:]} -code3 = lambda p: g3.get(p, ["left out"] * 8)[7] -check("an insertion before the SNV at one position keeps the SNV call", code3(1000) == "1", g3.get(1000)) -check("an insertion after the SNV at one position keeps the SNV call", code3(2000) == "2", g3.get(2000)) -check("a multi-allelic record beside the SNV keeps the SNV call", code3(3000) == "1", g3.get(3000)) -check("two carried SNVs at one position are unknown (9), also when a third record follows", code3(4000) == "9", g3.get(4000)) -check("a 0/0 half of a split site before the carried SNV keeps the SNV call", code3(5000) == "1", g3.get(5000)) -check("a 0/0 half after the carried SNV keeps the SNV call", code3(6000) == "1", g3.get(6000)) -check("a usable record of the same SNV wins over a filtered one", code3(7000) == "2", g3.get(7000)) -check("two 0/0 halves are the reference (0)", code3(8000) == "0", g3.get(8000)) - -# Masked calls: IBDmix gets the call itself and a mask per genome of the -# positions where it is masked, the runs of written rows closed by the next -# unmasked one. Codes per row: Altai, Vindija, Denisova. -mdir = f"{TMP}/table_masked" -os.makedirs(mdir) -mrows = {100: ("A\tG", "12\t0\t9"), 200: ("C\tT", "10\t11\t0"), 300: ("G\tA", "2\t0\t0"), - 400: ("T\tC", "10\t0\t0"), 500: ("A\tC", "0\t0\t0"), 600: ("C\tT", "11\t0\t0")} -with gzip.open(f"{mdir}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "".join(f"chr22\t{p}\t{ra}\t{c}\t0.2\t0\t0\t0\t0\t0\n" for p, (ra, c) in mrows.items())) -write(f"{TMP}/m.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" - "chr22\t500\t.\tA\tT\t50\tPASS\t.\tGT\t0/1\n") # another ALT: row 500 is left out -write(f"{TMP}/m.g.vcf", GVCF_HEAD + ref_block("chr22", 1, 1000)) -r = run("genotypes", "--table-dir", mdir, "--vcf", f"{TMP}/m.vcf", "--gvcf", f"{TMP}/m.g.vcf", "--pseudo", "10", - "--out-dir", f"{TMP}/gtm") -check("genotypes on a table with masked calls exits 0", r.returncode == 0, r.stderr) -gm = {int(l.split("\t")[1]): l.split("\t")[4:7] for l in gzip.open(f"{TMP}/gtm/chr22.gt.gz", "rt").read().splitlines()[1:]} -check("IBDmix gets each masked call as its ALT count (12, 10, 11 are 2, 0, 1)", - gm.get(100) == ["2", "0", "9"] and gm.get(200) == ["0", "1", "0"] and gm.get(600) == ["1", "0", "0"], gm) -masks = {g: open(f"{TMP}/gtm/chr22.{g}.mask.bed").read() if os.path.exists(f"{TMP}/gtm/chr22.{g}.mask.bed") else None - for g in ARCHAICS} -check("Altai's mask: 100-200, then 400-600 (the left-out row 500 does not close it)", - masks["Altai"] == "22\t99\t200\n22\t399\t600\n", masks["Altai"]) -check("Vindija's mask: 200 alone", masks["Vindija"] == "22\t199\t200\n", masks["Vindija"]) -check("Denisova's mask is written and empty", masks["Denisova"] == "", masks["Denisova"]) -check("an unmasked table writes empty masks", open(f"{TMP}/gt/chr22.Altai.mask.bed").read() == "") -bdir = f"{TMP}/table_bad" -os.makedirs(bdir) -with gzip.open(f"{bdir}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "chr22\t100\tA\tG\t7\t0\t0\t0.2\t0\t0\t0\t0\t0\n") -r = run("genotypes", "--table-dir", bdir, "--vcf", f"{TMP}/m.vcf", "--out-dir", f"{TMP}/gtb") -check("a code the table never holds (7) stops the step", r.returncode == 1 and "code '7'" in r.stderr, r.stderr) - -# A streaming join must refuse backwards positions, including records past -# the table's end and records whose genotype it would otherwise ignore. -vcf_header = "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" -for label, records in ( - ("within-table", [(3000, "G", "A", "GT"), (1000, "A", "G", "GT")]), - ("past-table", [(9000, "A", "G", "GT"), (8000, "A", "G", "GT")]), - ("ignored-call", [(3000, "G", "A", "GT"), (1000, "A", ".", "GQ")])): - bad_vcf = f"{TMP}/{label}.vcf" - write(bad_vcf, vcf_header + "".join( - f"chr22\t{p}\t.\t{ref}\t{alt}\t.\tPASS\t.\t{fmt}\t0/1\n" - for p, ref, alt, fmt in records)) - out = f"{TMP}/order-{label}" - os.makedirs(out) - protected = ["chr22.gt.gz", *(f"chr22.{g}.mask.bed" for g in ARCHAICS)] - for name in protected: - write(f"{out}/{name}", "previous output\n") - r = run("genotypes", "--table-dir", tdir, "--vcf", bad_vcf, "--out-dir", out) - check(f"unsorted VCF {label} exits 2", r.returncode == 2 and "not sorted" in r.stderr, r.stderr) - check(f"unsorted VCF {label} keeps prior genotype and masks", - all(open(f"{out}/{name}", "rb").read() == b"previous output\n" for name in protected)) - -unordered_table = f"{TMP}/table_unordered" -os.makedirs(unordered_table) -with gzip.open(f"{unordered_table}/chr22.tsv.gz", "wt") as f: - f.write(hdr + "\n".join([rows[1], rows[0], *rows[2:]]) + "\n") -r = run("genotypes", "--table-dir", unordered_table, "--vcf", f"{TMP}/s.vcf", - "--out-dir", f"{TMP}/gt_unordered") -check("an unsorted table exits 2", r.returncode == 2 and "not sorted" in r.stderr, r.stderr) -check("an unsorted table leaves no partial genotype or mask", - not any(name.endswith((".gt.gz", ".mask.bed")) for name in os.listdir(f"{TMP}/gt_unordered"))) - -# --- 3. report --------------------------------------------------------------------- -idir = f"{TMP}/ibd" -os.makedirs(idir) -head = "ID\tchrom\tstart\tend\tslod\n" -write(f"{idir}/chr22.Altai.txt", head + "SAMPLE\t22\t1000000\t1300000\t40\nP001\t22\t5000000\t9000000\t99\n" - "SAMPLE\t22\t2000000\t2040000\t30\nSAMPLE\t22\t3000000\t3100000\t3.5\n") -write(f"{idir}/chr22.Vindija.txt", head + "SAMPLE\t22\t1200000\t1500000\t50\n") -write(f"{idir}/chr22.Denisova.txt", head + "SAMPLE\t22\t1400000\t1450000\t12\nSAMPLE\t22\t6000000\t6200000\t20\n") -write(f"{TMP}/spans.tsv", "chrom\tfirst\tlast\tsites\nchr22\t1\t10000000\t5000\n") -write(f"{TMP}/genes.gtf", - 'chr22\tHAVANA\tgene\t1100000\t1110000\t.\t+\t.\tgene_id "G1"; gene_type "protein_coding"; gene_name "GENEA";\n' - 'chr22\tHAVANA\tgene\t1120000\t1130000\t.\t+\t.\tgene_id "G2"; gene_type "lncRNA"; gene_name "LNC1";\n' - 'chr22\tHAVANA\tgene\t6100000\t6300000\t.\t-\t.\tgene_id "G3"; gene_type "protein_coding"; gene_name "GENEB";\n') -r = run("report", "--sample", "S1", "--ibdmix-dir", idir, "--spans", f"{TMP}/spans.tsv", "--genes", f"{TMP}/genes.gtf", - "--out-dir", f"{TMP}/out") -check("report exits 0", r.returncode == 0, r.stderr) -segs = [l.split("\t") for l in open(f"{TMP}/out/S1_archaic_segments.tsv").read().splitlines()] -check("segments header", segs[0] == SEGMENT_HEADER, segs[0]) -body = segs[1:] -check("two segments: the merged Neanderthal one and the Denisovan one", len(body) == 2, body) -check("Altai 1.0-1.3 and Vindija 1.2-1.5 Mb merge into Neanderthal 1.0-1.5 Mb", - body and body[0][:6] == ["chr22", "1000000", "1500000", "500000", "Neanderthal", "Altai,Vindija,Denisova"], body[:1]) -check("the merged segment keeps the higher LOD", body and body[0][6] == "50.0", body[:1]) -check("a Denisova stretch inside a Neanderthal one is not Denisovan; one apart is", len(body) > 1 and body[1][:5] == ["chr22", "6000000", "6200000", "200000", "Denisovan"], body) -check("protein-coding genes only", body and body[0][7] == "GENEA", body[:1]) -check("a stand-in's segment is not yours, nor one under 50 kb or LOD 4", all(r[1] not in ("5000000", "2000000", "3000000") for r in body), body) -summ = dict(l.split("\t", 1) for l in open(f"{TMP}/out/S1_archaic.tsv").read().splitlines()[1:]) -check("Neanderthal share of the 10 Mb span: 5.00%", summ.get("neanderthal_percent") == "5.00", summ) -check("Denisovan share: 2.00%", summ.get("denisovan_percent") == "2.00", summ) -check("segment counts", summ.get("neanderthal_segments") == "1" and summ.get("denisovan_segments") == "1", summ) -check("the longest segment is listed first with its genes", summ.get("longest_1", "").startswith("chr22:1000000-1500000 Neanderthal") - and "GENEA" in summ.get("longest_1", ""), summ.get("longest_1")) -txt = open(f"{TMP}/out/S1_archaic.txt").read() -check("the text names the caveat", "An estimate, not a measurement." in txt, txt[-300:]) -r = run("report", "--sample", "S1", "--ibdmix-dir", f"{TMP}/nothing", "--spans", f"{TMP}/spans.tsv", "--out-dir", f"{TMP}/out2") -check("a missing IBDmix output stops the report", r.returncode != 0 and "no IBDmix output" in r.stderr, r.stderr) - -# --- 3b. the reports read it ------------------------------------------------------ -import collect_summary as cs # noqa: E402 -import render_report as rr # noqa: E402 - - -def archaic(d): - """The archaic section og summary collect writes for folder D.""" - return cs.collect("S1", d)["sections"]["archaic"] - - -sdir = f"{TMP}/sample" -os.makedirs(f"{sdir}/archaic") -shutil.copy(f"{TMP}/out/S1_archaic.tsv", f"{sdir}/archaic/S1_archaic.tsv") -sec = archaic(sdir) -data = sec["data"] -check("og summary collect reads the summary", sec["state"] == "ok" and data.get("neanderthal_percent") == "5.00" - and data.get("denisovan_segments") == 1 and len(data.get("longest", [])) == 2, sec) -check("the archaic section is in SECTIONS as step 43", any(x[0] == "archaic" and x[2] == "43" for x in cs.SECTIONS) - and sec["step"] == "43") -lines = rr.archaic_lines(data) -check("the report lines give both shares", lines[0].startswith("Neanderthal: 5.00% (0.5 Mb in 1 segments)") - and lines[1].startswith("Denisovan: 2.00%"), lines) -write(f"{sdir}/archaic/S1_archaic.tsv", "key\tvalue\nneanderthal_percent\tlots\n") -check("a damaged summary is unreadable, not a share", archaic(sdir)["state"] == "unreadable", archaic(sdir)) -write(f"{sdir}/archaic/S1_archaic.tsv", "key\tvalue\nneanderthal_segments\n") -check("a row cut after its key is unreadable, not a crash", archaic(sdir)["state"] == "unreadable", archaic(sdir)) -nowhere = archaic(f"{TMP}/nowhere") -check("no file: the section is missing", (nowhere["state"], nowhere["data"]) == ("missing", {}), nowhere) - -# --- 4. the real IBDmix ------------------------------------------------------------- -NEA = (2000000, 2600000) -DEN = (3200000, 3600000) - - -MID = (2200000, 2400000) - - -def synthetic(d, plant, mid=None): - """A 4 Mb chr22 table and one person. A third of the sites carry an allele - the Neanderthals share and modern people rarely do, a third one only the - Denisovan carries, a third are common. The person draws both copies from - the frequencies, except inside the planted stretches, where one copy - carries the archaic allele. - - mid: the middle of the planted Neanderthal stretch (MID) holds sites the - person contradicts (0/0 where both Neanderthals are 1/1), and the two - Neanderthals' calls there are outside their masks. "masked" writes them - as this table does (10 plus the call); "unknown" as the table before the - fix did (9).""" - rnd = random.Random(11) - os.makedirs(f"{d}/table") - rows, calls = [], [] - for i, pos in enumerate(range(1000000, 5000000, 1000)): - kind = i % 3 - if kind == 0: - f, al, vi, de = rnd.uniform(0.01, 0.04), 2, 2, 0 - elif kind == 1: - f, al, vi, de = rnd.uniform(0.01, 0.04), 0, 0, 2 - else: - f = rnd.uniform(0.1, 0.9) - al = vi = de = sum(rnd.random() < f for _ in range(2)) - ta, tv = al, vi # the codes the table writes - g = [rnd.random() < f, rnd.random() < f] - if plant and NEA[0] <= pos < NEA[1]: - g[0] = al == 2 - if mid and MID[0] <= pos < MID[1]: - if kind == 0: - g = [False, False] - ta, tv = (al + MASKED, vi + MASKED) if mid == "masked" else (9, 9) - rows.append(f"chr22\t{pos}\tA\tG\t{ta}\t{tv}\t{de}\t" + "\t".join([f"{f:.4f}"] * 6)) - if plant and DEN[0] <= pos < DEN[1]: - g[0] = de == 2 - if any(g): - calls.append(f"chr22\t{pos}\t.\tA\tG\t50\tPASS\t.\tGT\t{'1/1' if all(g) else '0/1'}") - with gzip.open(f"{d}/table/chr22.tsv.gz", "wt") as f: - f.write("\t".join(TABLE_HEADER) + "\n" + "\n".join(rows) + "\n") - write(f"{d}/s.vcf", "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n" + "\n".join(calls) + "\n") - - -def ibdmix_run(d, gvcf=None): - if gvcf is None: - gvcf = GVCF_HEAD + ref_block("chr22", 1, 6000000) # every site read - write(f"{d}/s.g.vcf", gvcf) - r = run("genotypes", "--table-dir", f"{d}/table", "--vcf", f"{d}/s.vcf", "--gvcf", f"{d}/s.g.vcf", - "--out-dir", f"{d}/gt") - if r.returncode: - return r - os.makedirs(f"{d}/ibd") - # The flags of scripts/43-archaic.sh. - sh = ("set -e; gzip -dc /w/gt/chr22.gt.gz > /w/gt/chr22.gt; for a in Altai Vindija Denisova; do " - "ibdmix -g /w/gt/chr22.gt -s /w/gt/samples.txt -n $a -r /w/gt/chr22.$a.mask.bed -o /w/ibd/chr22.$a.txt " - "-d 4 -m 1 -a 0.01 -e 0.002 -c 2; done") - r = subprocess.run(["docker", "run", "--rm", "--network", "none", "-u", f"{os.getuid()}:{os.getgid()}", - "-v", f"{d}:/w", image("IBDMIX_IMAGE"), "sh", "-c", sh], capture_output=True, text=True) - if r.returncode: - return r - return run("report", "--sample", "S", "--ibdmix-dir", f"{d}/ibd", "--spans", f"{d}/gt/spans.tsv", "--out-dir", f"{d}/out") - - -if os.environ.get("ARCHAIC_TEST_NO_DOCKER") == "1": - print("[SKIP] part 4: ARCHAIC_TEST_NO_DOCKER=1") -else: - for plant in (True, False): - d = f"{TMP}/real_{plant}" - os.makedirs(d) - os.chmod(d, 0o777) - synthetic(d, plant) - r = ibdmix_run(d) - check(f"IBDmix chain exits 0 ({'planted' if plant else 'control'})", r.returncode == 0, r.stderr + r.stdout) - if r.returncode: - continue - body = [l.split("\t") for l in open(f"{d}/out/S_archaic_segments.tsv").read().splitlines()[1:]] - print(f" segments ({'planted' if plant else 'control'}): {[(b[1], b[2], b[4]) for b in body]}") - if plant: - nea = [b for b in body if b[4] == "Neanderthal"] - den = [b for b in body if b[4] == "Denisovan"] - cover = lambda segs, lo, hi: sum(max(0, min(int(b[2]), hi) - max(int(b[1]), lo)) for b in segs) - check("the planted Neanderthal stretch: one Neanderthal segment covering 90% of it", - len(nea) == 1 and cover(nea, *NEA) >= 0.9 * (NEA[1] - NEA[0]), nea) - check("and no more than 100 kb beyond it", nea and int(nea[0][3]) <= NEA[1] - NEA[0] + 100000, nea) - check("the planted Denisovan stretch: one Denisovan segment covering 90% of it", - len(den) == 1 and cover(den, *DEN) >= 0.9 * (DEN[1] - DEN[0]), den) - else: - check("without the planted stretches: no segment", body == [], body) - - # The masked middle. Chen et al. keep a masked site in IBDmix's input and - # mask it there, so a homozygous mismatch still counts against a segment; - # the table before the fix wrote such calls as unknown (9), and a segment - # ran through them unopposed. "unknown" is that control: the planted - # segment crosses the contradicted middle. "masked" is this table: it - # must not. - cover_mid = {} - for mid in ("unknown", "masked"): - d = f"{TMP}/real_mid_{mid}" - os.makedirs(d) - os.chmod(d, 0o777) - synthetic(d, True, mid) - r = ibdmix_run(d) - check(f"IBDmix chain exits 0 (middle {mid})", r.returncode == 0, r.stderr + r.stdout) - if r.returncode: - continue - body = [l.split("\t") for l in open(f"{d}/out/S_archaic_segments.tsv").read().splitlines()[1:]] - print(f" segments (middle {mid}): {[(b[1], b[2], b[4]) for b in body]}") - cover_mid[mid] = sum(max(0, min(int(b[2]), MID[1]) - max(int(b[1]), MID[0])) - for b in body if b[4] == "Neanderthal") - span = MID[1] - MID[0] - check("control, calls written as unknown (9): the Neanderthal segment runs through the contradicted middle", - cover_mid.get("unknown", 0) >= 0.9 * span, cover_mid) - check("calls written as masked: IBDmix's mask keeps the mismatches, and no segment covers the middle", - "masked" in cover_mid and cover_mid["masked"] <= 0.1 * span, cover_mid) - - # A stretch with no call (lin-01 of the 1.0.0 review). 150 sites where - # all three archaic genomes carry the reference allele and modern people - # carry the ALT at 0.9, and a person with one call far away: read as the - # reference, the stretch matches the archaic genomes. Control: with a - # reference block over it (the person was read and matches the - # reference) IBDmix calls a segment there. With no block over it, - # nothing was read, and no segment may come out. - gap_lo, gap_hi = 200000, 349000 - gap_segment = {} - for covered in (True, False): - d = f"{TMP}/real_gap_{covered}" - os.makedirs(f"{d}/table") - os.chmod(d, 0o777) - with gzip.open(f"{d}/table/chr22.tsv.gz", "wt") as f: - f.write("\t".join(TABLE_HEADER) + "\n" + "".join( - f"chr22\t{p}\tC\tT\t0\t0\t0\t" + "\t".join(["0.9"] * 6) + "\n" - for p in range(gap_lo, gap_hi + 1, 1000))) - write(f"{d}/s.vcf", GVCF_HEAD + "chr22\t10000000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n") - # Without the block the person has one read site left, at the start. - block = ref_block("chr22", 1, 400000) if covered else ref_block("chr22", 1, gap_lo) - r = ibdmix_run(d, GVCF_HEAD + block) - check(f"IBDmix chain exits 0 (stretch {'read' if covered else 'with no call'})", r.returncode == 0, - r.stderr + r.stdout) - if r.returncode: - continue - body = [l.split("\t") for l in open(f"{d}/out/S_archaic_segments.tsv").read().splitlines()[1:]] - print(f" segments (stretch {'read' if covered else 'with no call'}): {[(b[1], b[2], b[4]) for b in body]}") - gap_segment[covered] = sum(int(b[3]) for b in body if b[4] == "Neanderthal") - check("control: a stretch read as the reference gives a Neanderthal segment of 100 kb or more", - gap_segment.get(True, 0) >= 100000, gap_segment) - check("a stretch with no call and no reference block gives no segment (it scored 149 kb when read as the reference)", - gap_segment.get(False, 1) == 0, gap_segment) - -shutil.rmtree(TMP, ignore_errors=True) -print(f"{FAILS} check(s) failed" if FAILS else "all checks passed") -sys.exit(1 if FAILS else 0) diff --git a/tests/test_haplogroup_explore.py b/tests/test_haplogroup_explore.py deleted file mode 100755 index 635392884..000000000 --- a/tests/test_haplogroup_explore.py +++ /dev/null @@ -1,98 +0,0 @@ -#!/usr/bin/env python3 -"""Both reports point to where a haplogroup can be explored further (steps 12 and 37). - -Each case writes the haplogroup files of one made-up sample S into a bash -output folder, collects the summary and renders both reports: - - 1. a male sample (Yleaf table with R-M269, haplogrep3 with H5a7): the Y and - the mtDNA links are built from those names, in the text and the HTML - report, with the note that nothing is uploaded - 2. Yleaf's paragroup form R-M269*(xR-L23): the links use R-M269 - 3. a female sample (no Yleaf table): no Y links; the mtDNA links stay - 4. no call (Hg NA, haplogroup '.'): no links at all - 5. a branch name with characters a URL cannot carry raw is escaped - -The branch names are the fixture's and public examples, not anyone's result. - -Run: python3 tests/test_haplogroup_explore.py -""" -import os -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, os.path.join(REPO, "bin")) -import collect_summary # noqa: E402 -import render_report # noqa: E402 - -FAILS = [] -Y_HEAD = "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\tQC-1\tQC-2\tQC-3\n" - - -def check(desc, ok, detail=""): - print(f"{'PASS' if ok else 'FAIL'} {desc}" + ("" if ok else f": {detail}")) - if not ok: - FAILS.append(desc) - - -def put(path, text): - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w") as f: - f.write(text) - - -def render(d): - summ = collect_summary.collect("S", d) - return summ, render_report.text_report(summ), render_report.html_report(summ) - - -def sample(tmp, name, y_hg=None, mt_hg="H5a7"): - d = os.path.join(tmp, name) - put(f"{d}/mito/S_haplogroup.txt", f'"SampleID"\t"Haplogroup"\t"Rank"\n"S"\t"{mt_hg}"\t"0.98"\n') - if y_hg is not None: - put(f"{d}/y_haplogroup/S_y_haplogroup.txt", Y_HEAD + f"S_sorted\t{y_hg}\tM269\t100\t42\t0.97\t1\t1\t0.97\n") - return d - - -def main(): - with tempfile.TemporaryDirectory() as tmp: - # 1. male - summ, txt, html = render(sample(tmp, "male", y_hg="R-M269")) - for url in ("https://discover.familytreedna.com/y-dna/R-M269", "https://www.yfull.com/tree/R-M269/", - "https://discover.familytreedna.com/mtdna/H5a7", "https://www.yfull.com/mtree/H5a7/"): - check(f"male: {url} in the text report", url in txt, txt) - check(f"male: {url} as a link in the HTML report", f'
    ' in html, "") - check("male: the docs section for the routes is linked", - "docs/37-y-haplogroup.md#explore-your-branch-further" in txt - and "docs/12-mito-haplogroup.md#explore-your-branch-further" in txt, txt) - check("male: both reports say the pipeline uploads nothing", - "Nothing is uploaded by this pipeline" in txt and "Nothing is uploaded by this pipeline" in html, txt) - - # 2. paragroup - summ, txt, html = render(sample(tmp, "para", y_hg="R-M269*(xR-L23)")) - check("paragroup: the link is built from R-M269", - "https://www.yfull.com/tree/R-M269/" in txt and "y-dna/R-M269*" not in txt and "%2A" not in txt, txt) - - # 3. female: no Yleaf table, so no Y section and no Y link - summ, txt, html = render(sample(tmp, "female")) - check("female: no Y link", "y-dna/" not in txt and "yfull.com/tree/" not in txt - and "y-dna/" not in html and "yfull.com/tree/" not in html, txt) - check("female: the mtDNA links stay", "https://discover.familytreedna.com/mtdna/H5a7" in txt - and "https://www.yfull.com/mtree/H5a7/" in html, txt) - - # 4. no call - summ, txt, html = render(sample(tmp, "nocall", y_hg="NA", mt_hg="")) - check("no call: no explore links", "discover.familytreedna.com" not in txt and "yfull.com" not in txt - and "discover.familytreedna.com" not in html, txt) - check("no call: the Y section still says insufficient markers", "insufficient markers" in txt, txt) - - # 5. escaping (a made-up PhyloTree-style name) - summ, txt, html = render(sample(tmp, "esc", mt_hg="L3e2b+152")) - check("escaped: '+' is encoded in the mtDNA links", "mtdna/L3e2b%2B152" in txt and "mtree/L3e2b%2B152/" in txt, txt) - - print(f"\n{len(FAILS)} failed" if FAILS else "\nall passed") - return 1 if FAILS else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_ystr_report.py b/tests/test_ystr_report.py deleted file mode 100644 index d8608e342..000000000 --- a/tests/test_ystr_report.py +++ /dev/null @@ -1,360 +0,0 @@ -#!/usr/bin/env python3 -# og-leg: OG_REQUIRED -"""og y str-bed and og y str-report: Y-STR repeat counts from HipSTR's calls (step 45). - -Each case writes a synthetic HipSTR VCF (one sample, only the records it -plants), runs og y str-report on assets/ystr_panel.tsv, and checks the table: - - 1. the panel: 111 markers in FamilyTreeDNA's order (the Y-12, Y-25 and - Y-37 heads checked by name), every multi-copy marker kind multicopy, - the BED holds every region of the single and parts markers once, and - one region over the whole locus for each span marker - 2. counts: reference copies plus GB / motif length; a partial repeat as - N.r; a negative GB; the sum of a marker's parts (DYS449); a span - marker's parts' copies plus its one GB (DYS510, DYS389II at 250 bp); - an offset from a panel that gives one - 3. confidence: high; low for a posterior under 0.9, for over 15 % of - reads with stutter or with a flank indel, and for under 10 reads; no - call for a region without a record or without a genotype, for two - lengths on the haploid Y, and for a marker whose other part is missing; - a region without a record carries the reason HipSTR's log gives for it - 4. not callable: multi-copy markers (DYS464, DYF395S1), markers the - marker set has no region for (DYS439), and a region longer than the - reads can span (DYS452 at 150 bp reads, callable at 250 bp; the spans - of DYS389II and DYS448) - 5. the CSV: one row per marker in the panel's order, the count and the - confidence - 6. a damaged input: a VCF with two samples, a panel out of order - 7. the reports: bin/collect_summary.py reads the table (counts per - confidence) and refuses one with an unknown confidence; the HTML card of - bin/render_report.py lists the called markers only, with the note, and - the HTML report shows an unreadable table as Unreadable - -Without OG (the og binary) the test is skipped with a note; with -OG_REQUIRED set too, it fails (Guard's rust job sets both). - -Run: OG=bin/og python3 tests/test_ystr_report.py -""" -import atexit -import csv -import gzip -import os -import shutil -import subprocess -import sys -import tempfile - -REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -OG = os.environ.get("OG", "") -PANEL = os.path.join(REPO, "assets", "ystr_panel.tsv") -sys.path.insert(0, os.path.join(REPO, "bin")) -if not OG: - print("skip: OG is not set (OG=bin/og)") - sys.exit(1 if os.environ.get("OG_REQUIRED") else 0) - -FAILS = [] - - -def check(name, got, want): - if got == want: - print(f"[PASS] {name}") - else: - print(f"[FAIL] {name}: got {got!r}, want {want!r}") - FAILS.append(name) - - -def read_panel(path): - """The panel's rows as dicts, each region as (chrom, start, end, period, copies, name); a span - marker has one region over its whole locus, its parts' copies rounded half up and summed.""" - with open(path, encoding="utf-8") as f: - lines = [line.rstrip("\n").split("\t") for line in f if line.strip() and not line.startswith("#")] - rows = [dict(zip(lines[0], c)) for c in lines[1:]] - for r in rows: - r["regions"] = [] if r["regions"] == "." else [ - (where.split(":")[0], int(where.split(":")[1].split("-")[0]), int(where.split(":")[1].split("-")[1]), - int(period), float(copies), name) - for where, period, copies, name in (x.split("/", 3) for x in r["regions"].split(";"))] - if r["kind"] == "span": # one region over the whole locus, named after the marker - regs = r["regions"] - r["regions"] = [(regs[0][0], regs[0][1], regs[-1][2], regs[0][3], - float(sum(int(x[4] + 0.5) for x in regs)), r["marker"])] - return rows - - -TMP = tempfile.mkdtemp(prefix="ystr-test-") -atexit.register(shutil.rmtree, TMP, ignore_errors=True) -FORMAT = "GT:GB:Q:PQ:DP:DSNP:PSNP:PDP:GLDIFF:DSTUTTER:DFLANKINDEL" - - -def vcf(records, samples=("S1",), gz=True): - """records: {HipSTR name: (GT, GB, Q, DP, DSTUTTER, DFLANKINDEL)}.""" - panel = read_panel(PANEL) - where = {n: (c, s) for r in panel for c, s, _e, _p, _k, n in r["regions"]} - lines = ["##fileformat=VCFv4.1", "##source=HipSTR", - "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + "\t".join(samples)] - for name, (gt, gb, q, dp, st, fl) in records.items(): - c, s = where.get(name, ("chrY", 1)) - cell = f"{gt}:{gb}:{q}:{q}:{dp}:0:0|0:{dp}|0:.:{st}:{fl}" - lines.append("\t".join([c, str(s - 1), name, "A", ".", ".", "PASS", "PERIOD=4", FORMAT] - + [cell] * len(samples))) - p = os.path.join(TMP, f"calls{len(os.listdir(TMP))}.vcf" + (".gz" if gz else "")) - data = ("\n".join(lines) + "\n").encode() - with (gzip.open(p, "wb") if gz else open(p, "wb")) as f: - f.write(data) - return p - - -def run(records=None, panel=PANEL, read_length=None, samples=("S1",), want_rc=0, log=None): - v = vcf(records or {}, samples) - tsv, out_csv = v + ".tsv", v + ".csv" - cmd = [OG, "y", "str-report", "--panel", panel, "--vcf", v, "--tsv", tsv, "--csv", out_csv] - if read_length: - cmd += ["--read-length", str(read_length)] - if log is not None: - lp = v + ".log" - with open(lp, "w") as f: - f.write(log) - cmd += ["--log", lp] - p = subprocess.run(cmd, capture_output=True, text=True) - if p.returncode != want_rc: - print(p.stdout, p.stderr) - check(f"exit code {want_rc} ({' '.join(cmd[2:3])})", p.returncode, want_rc) - if want_rc: - return p - with open(tsv, encoding="utf-8") as f: - rows = list(csv.DictReader(f, delimiter="\t")) - with open(out_csv, encoding="utf-8") as f: - crow = list(csv.reader(f)) - return {r["marker"]: r for r in rows}, rows, crow, p.stdout - - -GOOD = ("0", "0", "0.99", "20", "0", "0") - - -def good(gb="0", dp="20"): - return ("0" if gb == "0" else "1", gb, "0.99", dp, "0", "0") - - -# --- 1. the panel -------------------------------------------------------------------- -panel = read_panel(PANEL) -names = [r["marker"] for r in panel] -check("1 111 markers", len(panel), 111) -check("1 Y-12 in FamilyTreeDNA's order", names[:12], - ["DYS393", "DYS390", "DYS19", "DYS391", "DYS385a", "DYS385b", "DYS426", "DYS388", "DYS439", - "DYS389I", "DYS392", "DYS389II"]) -check("1 Y-25 (13-25)", names[12:25], - ["DYS458", "DYS459a", "DYS459b", "DYS455", "DYS454", "DYS447", "DYS437", "DYS448", "DYS449", - "DYS464a", "DYS464b", "DYS464c", "DYS464d"]) -check("1 Y-37 (26-37)", names[25:37], - ["DYS460", "Y-GATA-H4", "YCAIIa", "YCAIIb", "DYS456", "DYS607", "DYS576", "DYS570", "CDYa", "CDYb", - "DYS442", "DYS438"]) -check("1 the last marker is DYS435", names[-1], "DYS435") -check("1 panel sizes", [sum(1 for r in panel if r["panel"] == p) for p in ("Y-12", "Y-25", "Y-37", "Y-67", "Y-111")], - [12, 13, 12, 30, 44]) -multi = {r["marker"] for r in panel if r["kind"] == "multicopy"} -check("1 the multi-copy markers", multi, - {"DYS385a", "DYS385b", "DYS459a", "DYS459b", "DYS464a", "DYS464b", "DYS464c", "DYS464d", "YCAIIa", "YCAIIb", - "CDYa", "CDYb", "DYF395S1a", "DYF395S1b", "DYS413a", "DYS413b"}) -bed = os.path.join(TMP, "regions.bed") -check("1 bed exits 0", subprocess.run([OG, "y", "str-bed", "--panel", PANEL, "--out", bed]).returncode, 0) -with open(bed) as f: - bl = [line.rstrip("\n").split("\t") for line in f] -want_regions = {n for r in panel for *_x, n in r["regions"]} -check("1 the BED has every region once", sorted(x[5] for x in bl), sorted(want_regions)) -check("1 the BED has six columns, 1-based start below the end", all(len(x) == 6 and 0 < int(x[1]) < int(x[2]) for x in bl), - True) -check("1 the BED is sorted", [int(x[1]) for x in bl], sorted(int(x[1]) for x in bl)) -check("1 DYS393's region", [x for x in bl if x[5] == "DYS393"], [["chrY", "3263111", "3263158", "4", "12", "DYS393"]]) -spans = {r["marker"] for r in panel if r["kind"] == "span"} -check("1 the span markers", spans, {"DYS389II", "DYS448", "DYS450", "DYS504", "DYS510", "DYS532"}) -check("1 the BED: 86 regions (76 single, 6 spans, the 4 parts of DYS449 and DYS552)", len(bl), 86) -check("1 DYS450 is one region from its first part's start to its last part's end, 9 + 5 copies", - [x for x in bl if x[5].startswith("DYS450")], [["chrY", "8258259", "8258341", "5", "14", "DYS450"]]) -check("1 DYS510's three parts are one region, 17 + 5 + 3 copies (4.5 halves up)", - [x for x in bl if x[5].startswith("DYS510")], [["chrY", "15188021", "15188138", "4", "25", "DYS510"]]) -check("1 no part of a span marker is in the BED", [x[5] for x in bl if x[5].split(".")[0] in spans and "." in x[5]], []) -check("1 DYS449 keeps its two parts", sorted(x[5] for x in bl if x[5].startswith("DYS449")), ["DYS449.1", "DYS449.2"]) - -# --- 2. counts ------------------------------------------------------------------------- -by, rows, crow, out = run({ - "DYS393": good("4"), # one repeat longer than the reference's 12 - "DYS390": good("-8"), # two shorter than 24 - "DYS391": good("2"), # a partial repeat: 11.2 - "DYS392": good("-1"), # three-base motif, one base short: 12.2 - "DYS449.1": good("4"), "DYS449.2": good("-4"), # 15 (15.25 rounded) + 1, plus 21 - 1 - "DYS510": good("4"), # one region: 17 + 5 (4.5 halves up) + 3, plus 1 - "DYS389II": good("4"), "DYS448": good("6"), # spans longer than 150 bp reads can cover - "DYS19/DYS394": GOOD, -}) -check("2 a reference call keeps the reference copies (DYS19: 16)", by["DYS19"]["count"], "16") -check("2 GB +4 at a 4-base motif: one repeat more (DYS393 13)", by["DYS393"]["count"], "13") -check("2 GB -8: two repeats fewer (DYS390 22)", by["DYS390"]["count"], "22") -check("2 GB +2: a partial repeat (DYS391 11.2)", by["DYS391"]["count"], "11.2") -check("2 GB -1 at a 3-base motif (DYS392 12.2)", by["DYS392"]["count"], "12.2") -check("2 DYS449 is the sum of its two parts (16 + 20)", by["DYS449"]["count"], "36") -check("2 DYS510: its parts' copies (17 + 5 + 3, 4.5 halves up) plus its one GB", by["DYS510"]["count"], "26") -check("2 DYS510 reads one region: one read count, no part names in the note", - (by["DYS510"]["confidence"], by["DYS510"]["reads"], by["DYS510"]["note"]), ("high", "20", ".")) -for m, bp in (("DYS389II", 164), ("DYS448", 156)): - check(f"2 {m}: the span is {bp} bp, not callable from 150 bp reads", - (by[m]["confidence"], by[m]["count"], f"a {bp} bp repeat" in by[m]["note"]), ("not callable", "", True)) -by250 = run({"DYS389II": good("4")}, read_length=250)[0] -check("2 DYS389II at 250 bp reads: 17 + 12 copies, plus 1", by250["DYS389II"]["count"], "30") -check("2 the DYS389II note says the count covers DYS389I", "covers DYS389I" in by250["DYS389II"]["note"], True) -check("2 the summary line counts the confidences", out.strip().startswith("Y-STR markers: "), True) -off = os.path.join(TMP, "offset_panel.tsv") -with open(PANEL) as f, open(off, "w") as g: - for line in f: - c = line.rstrip("\n").split("\t") - if len(c) == 8 and c[2] == "DYS393": - c[6] = "-1" - g.write("\t".join(c) + "\n") -oby = run({"DYS393": good("4")}, panel=off)[0] -check("2 a panel offset is added (13 - 1)", oby["DYS393"]["count"], "12") - -# --- 3. confidence --------------------------------------------------------------------- -by = run({ - "DYS393": GOOD, - "DYS390": ("0", "0", "0.6", "20", "0", "0"), # posterior below 0.9 - "DYS391": ("0", "0", "0.99", "20", "4", "0"), # 20 % stutter - "DYS426": ("0", "0", "0.99", "20", "0", "4"), # 20 % flank indels - "DYS388": ("0", "0", "0.99", "6", "0", "0"), # six reads - "DYS392": (".", ".", ".", "0", "0", "0"), # no genotype - "DYS458": ("0/1", "0/4", "0.99", "20", "0", "0"), # two lengths - "DYS449.1": GOOD, # DYS449.2 missing - "DYS456": ("0", "0", "0.99", "20", "3", "3"), # exactly 15 %: still high -})[0] -check("3 a clean call is high", (by["DYS393"]["confidence"], by["DYS393"]["reads"], by["DYS393"]["q"]), - ("high", "20", "0.99")) -check("3 posterior 0.6: low", by["DYS390"]["confidence"], "low") -check("3 and the note says why", "posterior 0.6 below 0.9" in by["DYS390"]["note"], True) -check("3 a low call keeps its count", by["DYS390"]["count"], "24") -check("3 4 of 20 reads with stutter: low", (by["DYS391"]["confidence"], by["DYS391"]["stutter_reads"]), ("low", "4")) -check("3 4 of 20 reads with a flank indel: low", by["DYS426"]["confidence"], "low") -check("3 6 reads: low", (by["DYS388"]["confidence"], "6 reads, fewer than 10" in by["DYS388"]["note"]), ("low", True)) -check("3 3 of 20 (15 %) is not above the limit: high", by["DYS456"]["confidence"], "high") -check("3 no genotype: no call, no count", (by["DYS392"]["confidence"], by["DYS392"]["count"]), ("no call", "")) -check("3 two lengths on the Y: no call", (by["DYS458"]["confidence"], "two lengths" in by["DYS458"]["note"]), - ("no call", True)) -check("3 a missing part: no call", (by["DYS449"]["confidence"], "DYS449.2" in by["DYS449"]["note"]), - ("no call", True)) -check("3 a region without a record: no call", by["DYS19"]["confidence"], "no call") -check("3 without HipSTR's log, the note points to it", "see HipSTR's log" in by["DYS19"]["note"], True) -# HipSTR's log: a "Processing region" line per region (0-based start), then why it stopped. -LOG = """Processing region chrY 9684379 9684443 -Aligning reads to each candidate haplotype -Aborting genotyping of the locus as the sequence downstream of the repeat is too repetitive for accurate genotyping -\tFlanking sequence = AAACACTATATATATATAACACTATATATATAATAC -Processing region chrY 18041621 18041651 -Skipping locus with too few reads: TOTAL=3, MIN=5 -Processing region chrY 3263110 3263158 -Locus timing: -Processing region chrY 8258258 8258341 -Skipping locus with too few informative reads for stutter training: TOTAL=2, MIN=5 -Processing region chrY 12500447 12500611 -Skipping region as the reference allele length exceeds the threshold (164 vs 150) -""" -by = run({"DYS390": GOOD}, log=LOG, read_length=250)[0] -check("3 the log's reason for DYS19 (its flank)", by["DYS19"]["note"], - "HipSTR wrote no record for DYS19/DYS394; its log: Aborting genotyping of the locus as the sequence " - "downstream of the repeat is too repetitive for accurate genotyping") -check("3 the log's reason for DYS425 (too few reads)", by["DYS425"]["note"], - "HipSTR wrote no record for DYS425; its log: Skipping locus with too few reads: TOTAL=3, MIN=5") -check("3 a span's reason, found by its one region (DYS450)", - "too few informative reads for stutter training" in by["DYS450"]["note"], True) -check("3 a span longer than --max-str-len (DYS389II at 250 bp reads)", - "exceeds the threshold (164 vs 150)" in by["DYS389II"]["note"], True) -check("3 a region the log names without a reason: the note says so", by["DYS393"]["note"], - "HipSTR wrote no record for DYS393, and its log gives no reason") -check("3 a region the log does not name: the note says so", by["DYS391"]["note"], - "HipSTR wrote no record for DYS391, and its log gives no reason") -check("3 a called marker carries no log line", by["DYS390"]["note"], ".") - -# --- 4. not callable --------------------------------------------------------------------- -by = run({"DYS452": GOOD})[0] -for m in ("DYS464a", "DYS464d", "DYF395S1a", "DYS385b", "CDYa"): - check(f"4 {m}: not callable, multi-copy", (by[m]["confidence"], "multi-copy" in by[m]["note"], by[m]["count"]), - ("not callable", True, "")) -check("4 DYS439 (no region in the marker set): not callable", (by["DYS439"]["confidence"], "no region" in by["DYS439"]["note"]), - ("not callable", True)) -check("4 DYS452 (155 bp) at 150 bp reads: not callable even with a call", - (by["DYS452"]["confidence"], by["DYS452"]["count"]), ("not callable", "")) -by = run({"DYS452": GOOD}, read_length=250)[0] -check("4 DYS452 at 250 bp reads: called", (by["DYS452"]["confidence"], by["DYS452"]["count"]), ("high", "31")) - -# --- 5. the CSV ---------------------------------------------------------------------------- -_by, rows, crow, _o = run({"DYS393": good("4")}) -check("5 CSV header", crow[0], ["marker", "value", "confidence"]) -check("5 CSV: a row per marker in the panel's order", [r[0] for r in crow[1:]], names) -check("5 CSV: DYS393's row", crow[1], ["DYS393", "13", "high"]) -check("5 CSV: a multi-copy row has no value", crow[5], ["DYS385a", "", "not callable"]) -check("5 TSV: rows in the panel's order", [r["marker"] for r in rows], names) - -# --- 6. damaged input -------------------------------------------------------------------- -p = run({"DYS393": GOOD}, samples=("S1", "S2"), want_rc=1) -check("6 two samples: the error says so", "2 samples" in p.stderr, True) -bad = os.path.join(TMP, "bad_panel.tsv") -with open(PANEL) as f, open(bad, "w") as g: - lines = f.readlines() - head = [x for x in lines if x.startswith("#") or x.startswith("order\t")] - body = [x for x in lines if x not in head] - g.writelines(head + [body[1], body[0]] + body[2:]) -p = run({}, panel=bad, want_rc=1) -check("6 a panel out of order stops", "order column" in p.stderr, True) - -# --- 7. the reports ----------------------------------------------------------------------- -import collect_summary as cs # noqa: E402 -import render_report as rr # noqa: E402 - -class Unreadable(Exception): - """A section og summary collect could not read (its state is unreadable).""" - - -def sec(d, s, key): - """(source path, data) of section KEY of the summary og summary collect writes - for folder D: (None, {}) without its file; Unreadable when it cannot be read.""" - x = cs.collect(s, d)["sections"][key] - if x["state"] == "unreadable": - raise Unreadable(x["note"]) - return (os.path.join(d, x["source"]) if x["source"] else None), x["data"] - - - -sd = os.path.join(TMP, "sample") -os.makedirs(os.path.join(sd, "y_str")) -v = vcf({"DYS393": good("4"), "DYS390": ("0", "0", "0.6", "20", "0", "0")}) -subprocess.run([OG, "y", "str-report", "--panel", PANEL, "--vcf", v, "--tsv", - os.path.join(sd, "y_str", "S1_y_str.tsv"), "--csv", os.path.join(sd, "y_str", "S1_y_str.csv")], - check=True, capture_output=True) -path, data = sec(sd, "S1", "y_str") -check("7 collect_summary finds the table", os.path.basename(path or ""), "S1_y_str.tsv") -check("7 counts per confidence", (data["high"], data["low"], data["not_callable"], data["no_call"]), - (1, 1, sum(1 for r in panel if r["kind"] in ("multicopy", "absent")) + 3, # + DYS452, DYS389II, DYS448: too long - 111 - 2 - sum(1 for r in panel if r["kind"] in ("multicopy", "absent")) - 3)) -check("7 the markers keep their order", [x["marker"] for x in data["markers"]], names) -check("7 a sample without the step has no section", sec(TMP, "nobody", "y_str"), (None, {})) -html = "\n".join(rr.ystr_body(data)) -check("7 the card lists DYS393 at 13", "DYS39313high" in html, True) -check("7 the card lists the low call too", "DYS39024low" in html, True) -check("7 and leaves out markers without a count", "DYS464a" in html, False) -check("7 the card carries the convention note", "fixed number of repeats" in html, True) -with open(os.path.join(sd, "y_str", "S1_y_str.tsv")) as f: - t = f.read().replace("\thigh\t", "\tgreat\t", 1) -with open(os.path.join(sd, "y_str", "S1_y_str.tsv"), "w") as f: - f.write(t) -try: - sec(sd, "S1", "y_str") - check("7 an unknown confidence is unreadable", "read", "Unreadable") -except Unreadable: - check("7 an unknown confidence is unreadable", "Unreadable", "Unreadable") -try: - page = rr.html_report(cs.collect("S1", sd)) -except (KeyError, TypeError) as e: - page = f"{type(e).__name__}: {e}" -check("7 the HTML report renders an unreadable Y-STR table as such", - "

    Y-STR Markers (HipSTR)

    " in page and "Unreadable" in page, True) - -if FAILS: - print(f"{len(FAILS)} check(s) failed") - sys.exit(1) -print("all checks passed") From 09ba83bbc9d07c96786d96d8a6e6b44098c60bef Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:08:34 +0100 Subject: [PATCH 3/4] test(report): move the ancestry chapter's tests from Python to cargo The tests of the Y-STR, haplogroup explore links, ancient and archaic steps and the fine-grained ancestry reports now run in cargo test, in og-render/tests/ancestry_*.rs: each runs the step's own verb, then og summary collect and both reports, as the Python tests did through bin/og and render_report.py. tests/test_ancient_report.py, test_archaic_report.py, test_haplogroup_explore.py and test_ystr_report.py are deleted; test_ancestry_fine_report.py keeps the og ancestry-fine checks and drops its render part, so remnants.tsv no longer lists the four as importers. The archaic test's IBDmix runs need docker and an amd64 image: they compile on x86_64 only and run in CI (or with OG_DOCKER_TESTS=1). Their fixture is drawn with Python's Mersenne Twister, seeded as before, so it is the same data; a test checks the generator against Python's first draws. A planted haplogroup address that closes the link's attribute and opens a script tag stays escaped in the HTML report. port_report_core's chapter order test now expects the mitochondrial card, which the report shows even when step 12 did not run. Bead: D1 Bead: D5 Bead: D14 --- rust/Cargo.lock | 3 + rust/crates/og-render/Cargo.toml | 7 + .../og-render/tests/ancestry_ancient.rs | 718 ++++++++++ .../og-render/tests/ancestry_archaic.rs | 1275 +++++++++++++++++ rust/crates/og-render/tests/ancestry_fine.rs | 117 ++ .../og-render/tests/ancestry_haplogroups.rs | 169 +++ rust/crates/og-render/tests/ancestry_ystr.rs | 811 +++++++++++ .../og-render/tests/port_report_core.rs | 10 +- tests/parity/remnants.tsv | 2 +- tests/test_ancestry_fine_report.py | 20 +- 10 files changed, 3111 insertions(+), 21 deletions(-) create mode 100644 rust/crates/og-render/tests/ancestry_ancient.rs create mode 100644 rust/crates/og-render/tests/ancestry_archaic.rs create mode 100644 rust/crates/og-render/tests/ancestry_fine.rs create mode 100644 rust/crates/og-render/tests/ancestry_haplogroups.rs create mode 100644 rust/crates/og-render/tests/ancestry_ystr.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index ec8e92fcc..960526ee3 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -848,8 +848,11 @@ dependencies = [ name = "og-render" version = "0.1.0" dependencies = [ + "flate2", "indexmap", + "og-ancestry", "og-io", + "og-reports", "og-summary", "pycompat", ] diff --git a/rust/crates/og-render/Cargo.toml b/rust/crates/og-render/Cargo.toml index 5112b56a7..08c642327 100644 --- a/rust/crates/og-render/Cargo.toml +++ b/rust/crates/og-render/Cargo.toml @@ -20,5 +20,12 @@ og-io.workspace = true og-summary.workspace = true indexmap.workspace = true +# The ancestry chapter's tests (tests/ancestry_*.rs) run each step's own +# verb, then collect and render its result, as the run does. +[dev-dependencies] +og-ancestry.workspace = true +og-reports.workspace = true +flate2.workspace = true + [lints] workspace = true diff --git a/rust/crates/og-render/tests/ancestry_ancient.rs b/rust/crates/og-render/tests/ancestry_ancient.rs new file mode 100644 index 000000000..97973c0ee --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_ancient.rs @@ -0,0 +1,718 @@ +//! `og ancient` (step 44) on synthetic files, run as the pipeline runs it, +//! then the summary and both reports. It replaces +//! `tests/test_ancient_report.py`. +//! +//! 1. ranking: on a synthetic .evec the nearest ancient group is the one +//! whose mean position is closest on the weighted PCs, a group far on PC1 +//! but with a small eigenvalue on PC3 is ranked by the weighted distance, +//! and each row carries its dates (oldest first, AADR style BCE), country +//! and member count; +//! 2. axes: a run whose PC2 has the opposite sign is turned back before the +//! person is placed, and a run whose present-day coordinates differ +//! stops with the reason instead of placing the person on other axes; +//! 3. liftover: lift keeps a site inside a block, a site on a minus-strand +//! chain with its alleles complemented, and drops a site in a gap, one +//! whose alleles miss the GRCh38 base, chrX and a monomorphic marker; +//! 4. genotypes: person-geno appends one column, 9 where the person has no +//! usable genotype, and stops on a genotype file shorter than the .snp +//! file; +//! 5. select: the present-day reference people come from MODERN_GROUPS, the +//! ancient ones need a date, a passing assessment, enough sites and a +//! place in West Eurasia, one genetic id per individual; +//! 6. the report command writes the table, the summary, the text and an SVG +//! with the person marked; +//! 7. both reports show them: an HTML card with the table and the plot, and +//! a section of the text report. +//! +//! The unit tests in og-ancestry and og-intervals check the date labels, +//! the dosages and the single-position lift in detail. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use og_ancestry::ancient::cli; +use og_render::model::Summary; +use og_render::{html, text}; +use pycompat::json::{Dump, dumps}; + +const CAVEAT: &str = "Closeness on a PCA is genetic similarity, not descent"; +const EVALS: [f64; 4] = [9.0, 4.0, 0.01, 0.01]; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn tmp(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-ancestry-ancient-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn write(d: &Path, name: &str, text: &str) -> String { + let p = d.join(name); + std::fs::write(&p, text).unwrap(); + p.display().to_string() +} + +fn read(p: &str) -> String { + std::fs::read_to_string(p).unwrap_or_default() +} + +fn og(args: &[&str]) -> (u8, String, String) { + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&args, &mut out, &mut err); + ( + code, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + ) +} + +/// One person of an .evec: id, label, four PCs. +type Pc = (String, &'static str, [f64; 4]); + +fn evec(rows: &[Pc], evals: [f64; 4]) -> String { + let mut out = vec![format!( + "#eigvals: {}", + evals + .iter() + .map(|v| format!("{v:.3}")) + .collect::>() + .join(" ") + )]; + for (i, label, pcs) in rows { + out.push(format!( + "{i} {} {label}", + pcs.iter() + .map(|v| format!("{v:.6}")) + .collect::>() + .join(" ") + )); + } + out.join("\n") + "\n" +} + +/// Present-day people M0..M19: they build the axes. +fn moderns() -> Vec { + (0..20) + .map(|i| { + let f = f64::from(i); + ( + format!("M{i}"), + if i % 2 == 1 { "French" } else { "Sardinian" }, + [ + f * 0.01 - 0.1, + f64::from(i % 5) * 0.02 - 0.04, + f64::from(i % 3) * 0.01 - 0.01, + f64::from(i % 4) * 0.01 - 0.015, + ], + ) + }) + .collect() +} + +/// The panel: the present-day people and three ancient groups. +fn panel() -> Vec { + let mut p = moderns(); + p.extend([ + ("A1".to_owned(), "Ancient", [0.10, 0.10, 0.0, 0.0]), // Steppe + ("A2".to_owned(), "Ancient", [0.12, 0.10, 0.0, 0.0]), + ("B1".to_owned(), "Ancient", [-0.20, 0.00, 0.0, 0.0]), // Farmer + // Close on PC1 and PC2, far on PC3 (eigenvalue 0.01). + ("C1".to_owned(), "Ancient", [0.00, 0.00, 0.9, 0.0]), + ]); + p +} + +/// The person's run: the present-day people with PC2 flipped, as smartpca +/// may write it, and the person. +fn run_rows() -> Vec { + let mut r: Vec = moderns() + .into_iter() + .map(|(k, l, p)| (k, l, [p[0], -p[1], p[2], p[3]])) + .collect(); + r.push(("S".to_owned(), "Person", [0.08, -0.09, 0.0, 0.0])); + r +} + +/// The files of one report run. +struct Report { + tsv: String, + info: String, + txt: String, + svg: String, + ancients: String, + stats: String, +} + +impl Report { + fn new(d: &Path) -> Report { + let mut anc = "id\tgroup\tdate_bp\tdate_sd\tfull_date\tlocality\tcountry\tlat\tlon\tsnps\n" + .to_owned(); + for (id, group, bp, country, locality) in [ + ("A1", "Steppe_EBA", "4800", "Russia", "Samara"), + ("A2", "Steppe_EBA", "4600", "Russia", "Samara"), + ("B1", "Anatolia_N", "8200", "Turkey", "Barcin"), + ("C1", "Odd_C", "3000", "Italy", "Somewhere"), + ] { + anc += &format!("{id}\t{group}\t{bp}\t50\tx\t{locality}\t{country}\t50\t40\t500000\n"); + } + let o = |k: &str| d.join(format!("S_{k}")).display().to_string(); + Report { + tsv: o("ancient.tsv"), + info: o("info.tsv"), + txt: o("ancient.txt"), + svg: o("pca.svg"), + ancients: write(d, "anc.tsv", &anc), + stats: write( + d, + "stats.tsv", + "panel_sites\t4\nsites_genotyped\t3\nother_allele\t1\nsites_used\t2\n", + ), + } + } + + fn run(&self, panel_evec: &str, run_evec: &str, with_stats: bool) -> (u8, String, String) { + let mut a = vec![ + "report", + "--panel-evec", + panel_evec, + "--run-evec", + run_evec, + "--ancients", + &self.ancients, + "--sample", + "S", + "--release", + "test", + ]; + if with_stats { + a.extend(["--stats", &self.stats]); + } + a.extend([ + "--out-tsv", + &self.tsv, + "--out-info", + &self.info, + "--out-txt", + &self.txt, + "--out-svg", + &self.svg, + ]); + og(&a) + } + + fn table(&self) -> Vec> { + let t = read(&self.tsv); + let mut lines = t.lines(); + let head: Vec<&str> = lines.next().unwrap_or_default().split('\t').collect(); + lines + .map(|l| { + head.iter() + .zip(l.split('\t')) + .map(|(h, v)| ((*h).to_owned(), v.to_owned())) + .collect() + }) + .collect() + } + + fn groups(&self) -> Vec { + self.table() + .into_iter() + .map(|r| r["group"].clone()) + .collect() + } +} + +#[test] +fn the_nearest_group_on_the_weighted_pcs_comes_first() { + let d = tmp("rank"); + let r = Report::new(&d); + let pe = write(&d, "panel.evec", &evec(&panel(), EVALS)); + let re = write(&d, "run.evec", &evec(&run_rows(), EVALS)); + let (code, _, err) = r.run(&pe, &re, true); + assert_eq!(code, 0, "{err}"); + assert_eq!( + r.groups(), + ["Steppe_EBA", "Odd_C", "Anatolia_N"], + "Steppe_EBA first from the flipped run turned back; Odd_C, far only on a PC with a tiny \ + eigenvalue, second" + ); + let rows = r.table(); + let st = &rows[0]; + assert_eq!( + [&st["n"], &st["dates"], &st["country"], &st["locality"]], + ["2", "2850-2650 BCE", "Russia", "Samara"] + ); + let ranks: Vec<&str> = rows.iter().map(|x| x["rank"].as_str()).collect(); + assert_eq!(ranks, ["1", "2", "3"]); + + // Control: without the eigenvalue weights Odd_C falls behind Anatolia_N. + let fe = write(&d, "flat_panel.evec", &evec(&panel(), [1.0; 4])); + let fr = write(&d, "flat_run.evec", &evec(&run_rows(), [1.0; 4])); + let (code, _, err) = r.run(&fe, &fr, true); + assert_eq!(code, 0, "{err}"); + assert_eq!(r.groups(), ["Steppe_EBA", "Anatolia_N", "Odd_C"]); +} + +#[test] +fn a_flipped_axis_is_turned_back_and_other_axes_stop() { + let d = tmp("align"); + let pe = write(&d, "panel.evec", &evec(&panel(), EVALS)); + let re = write(&d, "run.evec", &evec(&run_rows(), EVALS)); + let (code, out, err) = og(&[ + "align", + "--panel-evec", + &pe, + "--run-evec", + &re, + "--pcs", + "2", + ]); + assert_eq!(code, 0, "{err}"); + assert!(out.contains("PC1+ PC2-"), "{out}"); + let bad: Vec = moderns() + .into_iter() + .map(|(k, l, p)| { + let n: u32 = k[1..].parse().unwrap(); + let s = if n.is_multiple_of(3) { -1.0 } else { 1.0 }; + (k, l, [p[0] * s, p[1], p[2], p[3]]) + }) + .collect(); + let be = write(&d, "bad.evec", &evec(&bad, EVALS)); + let (code, _, err) = og(&[ + "align", + "--panel-evec", + &pe, + "--run-evec", + &be, + "--pcs", + "2", + ]); + assert_eq!(code, 1, "{err}"); + assert!(err.contains("PC1"), "{err}"); +} + +#[test] +fn lift_keeps_sites_in_blocks_and_complements_the_minus_strand() { + let d = tmp("lift"); + let chain = write( + &d, + "test.chain", + &([ + // chr1 source 1000-1100 -> target 2000-2100 (+), gap of 50, then + // 1150-1200 -> 2150-2200. + "chain 100 chr1 100000 + 1000 1200 chr1 100000 + 2000 2200 1", + "100 50 50", + "50", + "", + // chr2 source 500-600 -> target on the minus strand of a 10,000 bp + // chr2, reverse start 300. + "chain 90 chr2 50000 + 500 600 chr2 10000 - 300 400 2", + "100", + "", + // chr3: two chains over 100-200. + "chain 80 chr3 50000 + 100 200 chr3 50000 + 100 200 3", + "100", + "", + "chain 70 chr3 50000 + 150 250 chr3 50000 + 5150 5250 4", + "100", + "", + ] + .join("\n") + + "\n"), + ); + let mut seq1 = vec![b'N'; 3000]; + seq1[2050] = b'G'; // 1-based 2051 + seq1[2159] = b'T'; // 1-based 2160 + let mut seq2 = vec![b'N'; 10000]; + seq2[9689] = b'C'; // 1-based 9690: the complement of the source G + let (mut fa, mut fai, mut off) = (String::new(), String::new(), 0usize); + for (name, seq) in [("chr1", &seq1), ("chr2", &seq2)] { + let head = format!(">{name}\n"); + fa += &head; + off += head.len(); + for c in seq.chunks(60) { + fa += std::str::from_utf8(c).unwrap(); + fa.push('\n'); + } + fai += &format!("{name}\t{}\t{off}\t60\t61\n", seq.len()); + off += seq.len() + seq.len().div_ceil(60); + } + let fa = write(&d, "ref.fa", &fa); + write(&d, "ref.fa.fai", &fai); + let snp = write( + &d, + "src.snp", + &([ + "rs1 1 0.0 1051 G A", // ref G at target: kept + "rs2 1 0.0 1160 C T", // second allele is the target base: kept + "rs3 1 0.0 1120 A G", // gap: dropped + "rs4 2 0.0 511 G T", // minus strand: alleles C/A, target C: kept + "rs5 1 0.0 1051 A C", // neither allele is G: dropped + "rs6 23 0.0 1051 G A", // X: dropped + "rs7 1 0.0 1051 G X", // monomorphic marker: dropped + ] + .join("\n") + + "\n"), + ); + let out = d.join("sites.tsv").display().to_string(); + let (code, _, err) = og(&[ + "lift", "--snp", &snp, "--chain", &chain, "--fasta", &fa, "--out", &out, + ]); + assert_eq!(code, 0, "{err}"); + assert_eq!( + read(&out).lines().collect::>(), + [ + "snp\tchrom\tpos\tallele1\tallele2", + "rs1\tchr1\t2051\tG\tA", + "rs2\tchr1\t2160\tC\tT", + "rs4\tchr2\t9690\tC\tA" + ] + ); + + // person-geno on those sites. + let msnp = write( + &d, + "moderns.snp", + "rs1 1 0.0 1051 G A\nrs2 1 0.0 1160 C T\nrsX 23 0.0 5 A G\nrs4 2 0.0 511 G T\n", + ); + let mgeno = write(&d, "moderns.geno", "012\n220\n999\n101\n"); + let gts = write( + &d, + "gts.tsv", + "chr1\t2051\tG\tA\tG/A\nchr1\t2160\tT\t.\tT/T\nchr2\t9690\tC\tG\tC/G\n", + ); + let merged = d.join("merged.geno").display().to_string(); + let pstats = d.join("pstats.tsv").display().to_string(); + let geno = |m: &str| { + og(&[ + "person-geno", + "--sites", + &out, + "--genotypes", + >s, + "--snp", + &msnp, + "--moderns-geno", + m, + "--out", + &merged, + "--stats", + &pstats, + ]) + }; + let (code, _, err) = geno(&mgeno); + assert_eq!(code, 0, "{err}"); + assert_eq!( + read(&merged).lines().collect::>(), + ["0121", "2200", "9999", "1019"], + "one column: 1, 0, 9 for X, 9 for an allele outside the pair" + ); + let st: BTreeMap = read(&pstats) + .lines() + .map(|l| { + let (k, v) = l.split_once('\t').unwrap(); + (k.to_owned(), v.to_owned()) + }) + .collect(); + let want: BTreeMap = [ + ("panel_sites", "4"), + ("sites_genotyped", "3"), + ("other_allele", "1"), + ("sites_used", "2"), + ] + .map(|(k, v)| (k.to_owned(), v.to_owned())) + .into(); + assert_eq!(st, want); + let short = write(&d, "short.geno", "012\n220\n"); + let (code, _, err) = geno(&short); + assert_eq!( + code, 1, + "a genotype file shorter than the .snp file stops: {err}" + ); +} + +#[test] +fn select_takes_reference_and_ancient_people_by_the_rules() { + let d = tmp("select"); + let head = [ + "Genetic ID (long text)", + "Persistent Genetic ID", + "Individual ID", + "\"Date mean in BP in years before 1950\"", + "Date standard deviation in BP [x]", + "Full Date One of two formats", + "Group ID", + "Locality", + "Political Entity", + "Latitude", + "Longitude", + "Data type", + "SNPs hit on autosomal targets (Computed using easystats on 1240k snpset)", + "SNPs hit on autosomal targets (Computed using easystats on HO snpset)", + "ASSESSMENT", + "ASSESSMENT WARNINGS (long)", + ] + .map(str::to_owned) + .to_vec(); + #[allow(clippy::too_many_arguments)] + fn arow( + gid: &str, + ind: &str, + bp: u32, + group: &str, + lat: &str, + lon: &str, + dtype: &str, + snps: u32, + ass: &str, + ) -> Vec { + [ + gid, + gid, + ind, + &bp.to_string(), + "50", + "x", + group, + "Loc", + "Land", + lat, + lon, + dtype, + &snps.to_string(), + "0", + ass, + "", + ] + .map(str::to_owned) + .to_vec() + } + let mut rows: Vec> = (0..60) + .map(|i| { + arow( + &format!("F{i}.DG"), + &format!("F{i}"), + 0, + "French", + "46", + "2", + "Shotgun.diploid", + 1_100_000, + "PROVISIONAL_PASS", + ) + }) + .collect(); + rows.extend([ + // The same individual with fewer sites. + arow( + "F0.SG", + "F0", + 0, + "French", + "46", + "2", + "Shotgun.diploid", + 900_000, + "PASS", + ), + // Not a reference group. + arow( + "Y1.DG", + "Y1", + 0, + "Yakut", + "62", + "129", + "Shotgun.diploid", + 1_100_000, + "PASS", + ), + // Not shotgun diploid. + arow( + "H1.HO", + "H1", + 0, + "French", + "46", + "2", + "HumanOrigins", + 600_000, + "PASS", + ), + arow( + "A1.AG", + "A1", + 4800, + "Russia_Steppe", + "53", + "50", + "1240k", + 500_000, + "PASS", + ), + // Too few sites. + arow( + "A2.AG", + "A2", + 4800, + "Russia_Steppe", + "53", + "50", + "1240k", + 50_000, + "PASS", + ), + arow( + "A3.AG", + "A3", + 4800, + "Russia_Steppe", + "53", + "50", + "1240k", + 500_000, + "Questionable", + ), + // Outside West Eurasia. + arow( + "A4.AG", "A4", 4800, "China_N", "40", "116", "1240k", 500_000, "PASS", + ), + // No place. + arow( + "A5.AG", "A5", 4800, "Unknown", "..", "..", "1240k", 500_000, "PASS", + ), + ]); + let anno_text: String = std::iter::once(&head) + .chain(rows.iter()) + .map(|r| r.join("\t") + "\n") + .collect(); + let anno = write(&d, "x.anno", &anno_text); + let ind_text: String = rows + .iter() + .map(|r| format!("{} U {}\n", r[0], r[6])) + .collect(); + let ind = write(&d, "x.ind", &ind_text); + let o = |n: &str| d.join(n).display().to_string(); + let (oind, plist, anc) = (o("panel.ind"), o("poplist.txt"), o("ancients.tsv")); + let (code, _, err) = og(&[ + "select", + "--anno", + &anno, + "--ind", + &ind, + "--out-ind", + &oind, + "--poplist", + &plist, + "--ancients", + &anc, + ]); + assert_eq!(code, 0, "{err}"); + let labels: Vec<(String, String)> = read(&oind) + .lines() + .map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + (c[0].to_owned(), c[2].to_owned()) + }) + .collect(); + let label = |k: &str| labels.iter().find(|(i, _)| i == k).map(|(_, v)| v.as_str()); + assert_eq!( + labels.iter().filter(|(_, v)| v == "French").count(), + 60, + "the 60 French shotgun genomes are the reference" + ); + assert_eq!(label("F0.SG"), Some("Ignore"), "one id per person (F0.DG)"); + assert_eq!(label("Y1.DG"), Some("Ignore")); + assert_eq!(label("H1.HO"), Some("Ignore")); + let projected: Vec<&str> = labels + .iter() + .filter(|(_, v)| v == "Ancient") + .map(|(k, _)| k.as_str()) + .collect(); + assert_eq!(projected, ["A1.AG"]); + assert_eq!(read(&plist), "French\n"); + assert!( + read(&anc) + .lines() + .nth(1) + .unwrap() + .starts_with("A1.AG\tRussia_Steppe\t4800") + ); +} + +/// The summary og summary collect writes for S, as text. +fn collect(sd: &Path) -> String { + let input = og_summary::collect::Input { + sample: "S".to_owned(), + sample_dir: sd.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + dumps( + &og_summary::collect::post::collect(&input).unwrap(), + &Dump::default(), + ) + .unwrap() +} + +#[test] +fn the_report_writes_its_files_and_both_reports_show_them() { + let d = tmp("report"); + let r = Report::new(&d); + let pe = write(&d, "panel.evec", &evec(&panel(), EVALS)); + let re = write(&d, "run.evec", &evec(&run_rows(), EVALS)); + let (code, _, err) = r.run(&pe, &re, true); + assert_eq!(code, 0, "{err}"); + let info: BTreeMap = read(&r.info) + .lines() + .skip(1) + .map(|l| { + let (k, v) = l.split_once('\t').unwrap(); + (k.to_owned(), v.to_owned()) + }) + .collect(); + assert_eq!( + info["pc2"], "0.09000", + "the person's PC2 in the panel's sign" + ); + assert_eq!(info["sites_used"], "2"); + let svg = read(&r.svg); + assert!(svg.starts_with("S"), "the person's label"); + assert!(svg.trim_end().ends_with("")); + let txt = read(&r.txt); + assert!(txt.contains(CAVEAT) && txt.contains("qpAdm"), "{txt}"); + + // Both reports read the outputs. + let sd = d.join("S"); + std::fs::create_dir_all(sd.join("ancient")).unwrap(); + for (from, to) in [ + (&r.tsv, "S_ancient.tsv"), + (&r.info, "S_ancient_info.tsv"), + (&r.txt, "S_ancient.txt"), + (&r.svg, "S_ancient_pca.svg"), + ] { + std::fs::copy(from, sd.join("ancient").join(to)).unwrap(); + } + let s = Summary::parse(&collect(&sd)).unwrap(); + let page = html::report(&s).unwrap(); + assert!(page.contains("

    Ancient Populations

    ")); + assert!(page.contains("1Steppe_EBA2850-2650 BCERussia")); + assert!(page.contains("data:image/svg+xml;base64,"), "the plot"); + let report = text::report(&s).unwrap(); + assert!(report.contains("Ancient Populations (AADR PCA, step 44)")); + assert!(report.contains("1. Steppe_EBA ("), "{report}"); + assert!(report.contains("not descent")); + assert!(report.contains("Plot: ancient/S_ancient_pca.svg")); + + let (code, _, err) = r.run(&pe, &pe, false); + assert_eq!(code, 1, "a run without the person stops"); + assert!(err.contains("no row for S"), "{err}"); +} diff --git a/rust/crates/og-render/tests/ancestry_archaic.rs b/rust/crates/og-render/tests/ancestry_archaic.rs new file mode 100644 index 000000000..6763b71ae --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_archaic.rs @@ -0,0 +1,1275 @@ +//! `og archaic` (step 43) on a synthetic archaic panel, then the summary and +//! the report lines; on amd64 in CI also with the real IBDmix image. It +//! replaces `tests/test_archaic_report.py`. +//! +//! 1. table: a 1000 Genomes site keeps its frequencies and gets each archaic +//! genome's ALT count, 10 plus it outside that genome's mask or inside an +//! excluded region; an archaic record with another ALT or another REF is +//! unknown (9); a site only an archaic genome varies at gets frequency 0; +//! a site no archaic genome was called at is left out; a position given +//! twice by 1000 Genomes is left out; +//! 2. strict: a lifted row outside the GRCh38 strict mask is kept once per +//! position with every archaic call marked masked (9 stays 9); +//! 3. genotypes: a masked archaic call goes to IBDmix as its ALT count and +//! its position into that genome's mask file; the stand-in columns carry +//! the population's ALT count; your column is your ALT count, 9 for a +//! non-PASS record, and where your VCF has no record 0 inside a reference +//! block of the gVCF and 9 outside every one; a site where your VCF has +//! another ALT is left out; a chromosome your VCF has no record on is not +//! analysed; the split-VCF rules; unsorted input stops and keeps the +//! previous outputs; +//! 4. report: Altai and Vindija segments merge into Neanderthal ones, a +//! Denisova segment that overlaps one is not Denisovan, segments under +//! 50 kb or LOD 4 are dropped, the share is of the analysed span, and the +//! genes of the longest segments are named; og summary collect reads the +//! summary and marks a damaged one unreadable, and the report lines give +//! both shares; +//! 5. with IBDmix itself (`IBDMIX_IMAGE`, docker; amd64 only, run in CI or +//! with `OG_DOCKER_TESTS=1`): a planted Neanderthal and a planted +//! Denisovan stretch come out as one segment each, the same person +//! without them gets none; masked mismatches stop a segment where the +//! same calls written as unknown let it run through; a stretch with no +//! call gives a segment only when a gVCF reference block says it was read. + +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use flate2::Compression; +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use og_ancestry::archaic::cli; +use pycompat::json::Value; + +const TABLE_HEADER: &str = "chrom\tpos\tref\talt\tAltai\tVindija\tDenisova\tAF_ALL\tAF_AFR\tAF_AMR\tAF_EAS\tAF_EUR\tAF_SAS"; +const VCF_HEAD: &str = + "##fileformat=VCFv4.2\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tS\n"; +const ARCHAICS: [&str; 3] = ["Altai", "Vindija", "Denisova"]; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn tmp(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-ancestry-archaic-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn p(d: &Path, name: &str) -> String { + d.join(name).display().to_string() +} + +fn write(path: &str, text: &str) { + std::fs::write(path, text).unwrap(); +} + +fn gzip(path: &str, text: &str) { + let mut z = GzEncoder::new(std::fs::File::create(path).unwrap(), Compression::default()); + z.write_all(text.as_bytes()).unwrap(); + z.finish().unwrap(); +} + +fn gunzip(path: &str) -> String { + let mut s = String::new(); + GzDecoder::new(std::fs::File::open(path).unwrap()) + .read_to_string(&mut s) + .unwrap(); + s +} + +fn og(args: &[&str]) -> (u8, String) { + let args: Vec = args.iter().map(|a| (*a).to_owned()).collect(); + let (mut out, mut err) = (Vec::new(), Vec::new()); + let code = cli(&args, &mut out, &mut err); + (code, String::from_utf8(err).unwrap()) +} + +fn ref_block(chrom: &str, start: u32, end: u32) -> String { + format!("{chrom}\t{start}\t.\tN\t<*>\t0\t.\tEND={end}\tGT:GQ:MIN_DP\t0/0:50:30\n") +} + +/// The records of a table VCF by position, and each one's INFO. +type Rows = BTreeMap>; +type Info = BTreeMap>; + +fn vcf_rows(path: &str) -> (Rows, Info) { + let rows: Rows = std::fs::read_to_string(path) + .unwrap() + .lines() + .filter(|l| !l.starts_with('#')) + .map(|l| { + let c: Vec = l.split('\t').map(str::to_owned).collect(); + (c[1].parse().unwrap(), c) + }) + .collect(); + let info = rows + .iter() + .map(|(k, c)| { + ( + *k, + c[7].split(';') + .map(|kv| { + let (a, b) = kv.split_once('=').unwrap(); + (a.to_owned(), b.to_owned()) + }) + .collect(), + ) + }) + .collect(); + (rows, info) +} + +fn codes(i: &Info, pos: u32) -> [&str; 3] { + ARCHAICS.map(|g| i[&pos][g].as_str()) +} + +/// The inputs of part 1 in `d`. +fn table_inputs(d: &Path) -> Vec { + let afs = "0.1\t0.2\t0.05\t0\t0.15\t0.1"; + write( + &p(d, "modern.tsv"), + &format!( + // every archaic genome called; Vindija has another ALT and + // Denisova another REF; no archaic record; a split multi-allelic + // site. + "100\tA\tG\t{afs}\n200\tC\tT\t{afs}\n300\tG\tA\t{afs}\n400\tT\tC\t{afs}\n400\tT\tG\t{afs}\n" + ), + ); + write( + &p(d, "altai.tsv"), + "100\tA\tG\t1/1\n200\tC\t.\t0/0\n500\tG\tC\t0/1\n600\tA\t.\t0/0\n", + ); + write( + &p(d, "vindija.tsv"), + "100\tA\t.\t0/0\n200\tC\tA\t0/1\n500\tG\t.\t0/0\n600\tA\t.\t0/0\n", + ); + write( + &p(d, "denisova.tsv"), + "100\tA\tG\t0|1\n200\tT\tC\t0/1\n400\tT\tC\t1/1\n500\tG\tT\t1/1\n", + ); + [ + "table".to_owned(), + "--chrom".to_owned(), + "22".to_owned(), + "--modern".to_owned(), + p(d, "modern.tsv"), + "--archaic".to_owned(), + format!("Altai={}", p(d, "altai.tsv")), + "--archaic".to_owned(), + format!("Vindija={}", p(d, "vindija.tsv")), + "--archaic".to_owned(), + format!("Denisova={}", p(d, "denisova.tsv")), + ] + .to_vec() +} + +fn table(common: &[String], extra: &[&str], out: &str) -> (u8, String) { + let mut a: Vec<&str> = common.iter().map(String::as_str).collect(); + a.extend(extra); + a.extend(["--out", out]); + og(&a) +} + +#[test] +fn the_table_codes_each_archaic_call() { + let d = tmp("table"); + let common = table_inputs(&d); + let (code, err) = table(&common, &[], &p(&d, "t.vcf")); + assert_eq!(code, 0, "{err}"); + let (recs, info) = vcf_rows(&p(&d, "t.vcf")); + assert_eq!( + recs.keys().copied().collect::>(), + [100, 200, 500], + "the modern sites with an archaic call and the archaic-only site" + ); + assert_eq!( + codes(&info, 100), + ["2", "0", "1"], + "each genome's ALT count" + ); + assert_eq!( + [&info[&100]["AF_ALL"], &info[&100]["AF_EUR"]], + ["0.1", "0.15"] + ); + assert_eq!( + codes(&info, 200), + ["0", "9", "9"], + "another ALT or another REF is unknown" + ); + assert_eq!(recs[&500][4], "C", "the first non-reference ALT"); + assert_eq!(codes(&info, 500), ["1", "0", "9"]); + assert_eq!(info[&500]["AF_ALL"], "0"); + assert_eq!(recs[&100][0], "chr22", "the chr prefix for the liftover"); +} + +#[test] +fn a_call_outside_its_mask_or_inside_an_excluded_region_is_masked() { + let d = tmp("mask"); + let common = table_inputs(&d); + // 1-based 51-150 and 451-550: 100 and 500, not 200. + write(&p(&d, "altai.bed"), "22\t50\t150\n22\t450\t550\n"); + write(&p(&d, "vindija.bed"), "22\t150\t250\n"); // holds 200, not 100 + gzip(&p(&d, "denisova.bed.gz"), "chr22\t99\t100\n"); // 100 only, named chr22 + + // Control: without --mask the same inputs keep every call. + let (code, err) = table(&common, &[], &p(&d, "nomask.vcf")); + assert_eq!(code, 0, "{err}"); + let (nm_rows, nm) = vcf_rows(&p(&d, "nomask.vcf")); + assert_eq!(nm[&100]["Altai"], "2"); + assert_eq!(nm_rows.keys().copied().collect::>(), [100, 200, 500]); + + let masks = [ + format!("Altai={}", p(&d, "altai.bed")), + format!("Vindija={}", p(&d, "vindija.bed")), + format!("Denisova={}", p(&d, "denisova.bed.gz")), + ]; + let (code, err) = table( + &common, + &[ + "--mask", &masks[0], "--mask", &masks[1], "--mask", &masks[2], + ], + &p(&d, "masked.vcf"), + ); + assert_eq!(code, 0, "{err}"); + let (m_rows, mi) = vcf_rows(&p(&d, "masked.vcf")); + assert_eq!( + [&mi[&100]["Altai"], &mi[&100]["Denisova"]], + ["2", "1"], + "inside its mask a genome keeps its call; a mask is read with and without chr" + ); + assert_eq!( + [&mi[&100]["Vindija"], &nm[&100]["Vindija"]], + ["10", "0"], + "outside its mask a call is masked (10 for 0/0)" + ); + assert_eq!(m_rows.keys().copied().collect::>(), [100, 200, 500]); + assert_eq!( + mi[&200]["Altai"], "10", + "a row whose only call is masked stays" + ); + assert_eq!(m_rows[&500][4], "C"); + assert_eq!([&mi[&500]["Altai"], &mi[&500]["Vindija"]], ["1", "10"]); + assert_eq!( + [&mi[&200]["Vindija"], &mi[&500]["Denisova"]], + ["9", "9"], + "unknown stays 9" + ); + + // The excluded regions mask every call at a site inside one. + write(&p(&d, "excl.bed"), "22\t90\t120\tsegdup\n"); + let excl = p(&d, "excl.bed"); + let (code, err) = table(&common, &["--exclude", &excl], &p(&d, "excl.vcf")); + assert_eq!(code, 0, "{err}"); + let (ex_rows, ex) = vcf_rows(&p(&d, "excl.vcf")); + assert_eq!(codes(&ex, 100), ["12", "10", "11"]); + assert_eq!(codes(&nm, 100), ["2", "0", "1"]); + assert_eq!(ex_rows.keys().copied().collect::>(), [100, 200, 500]); + assert_eq!([&ex[&200]["Altai"], &ex[&500]["Altai"]], ["0", "1"]); + + write(&p(&d, "excl_other.bed"), "7\t90\t120\tsegdup\n"); + let other = p(&d, "excl_other.bed"); + let (code, err) = table(&common, &["--exclude", &other], &p(&d, "x.vcf")); + assert!(code == 1 && err.contains("no interval"), "{code} {err}"); + let (code, err) = table( + &common, + &["--mask", &masks[0], "--mask", &masks[0]], + &p(&d, "x.vcf"), + ); + assert!(code == 2 && err.contains("twice"), "{code} {err}"); + let nobody = format!("Nobody={}", p(&d, "altai.bed")); + let (code, err) = table(&common, &["--mask", &nobody], &p(&d, "x.vcf")); + assert_eq!(code, 2, "{err}"); + write(&p(&d, "other.bed"), "7\t50\t150\n"); + let elsewhere = format!("Altai={}", p(&d, "other.bed")); + let (code, err) = table(&common, &["--mask", &elsewhere], &p(&d, "x.vcf")); + assert!( + code == 1 && err.contains("no interval"), + "a mask with no interval on the chromosome stops: {code} {err}" + ); +} + +#[test] +fn strict_marks_rows_outside_the_mask_and_keeps_each_position_once() { + let d = tmp("strict"); + // Strict mask 1-based 101-300 on chr22. + let rows = [ + "chr22\t100\tA\tG\t2\t0\t9\t0.5\t0\t0\t0\t0\t0", + "chr22\t150\tA\tG\t1\t1\t1\t0.5\t0\t0\t0\t0\t0", + "chr22\t300\tG\tA\t0\t0\t0\t0.5\t0\t0\t0\t0\t0", + "chr22\t301\tC\tT\t0\t10\t12\t0.5\t0\t0\t0\t0\t0", + "chr22\t301\tC\tT\t2\t2\t2\t0.5\t0\t0\t0\t0\t0", + ]; + write(&p(&d, "lifted.tsv"), &(rows.join("\n") + "\n")); + write(&p(&d, "strict.bed"), "chr22\t100\t300\n"); + let strict = |mask: &str, rows: &str, out: &str| { + og(&[ + "strict", "--chrom", "22", "--strict", mask, "--rows", rows, "--out", out, + ]) + }; + let (code, err) = strict( + &p(&d, "strict.bed"), + &p(&d, "lifted.tsv"), + &p(&d, "marked.tsv"), + ); + assert_eq!(code, 0, "{err}"); + let got: Vec> = std::fs::read_to_string(p(&d, "marked.tsv")) + .unwrap() + .lines() + .map(|l| l.split('\t').map(str::to_owned).collect()) + .collect(); + assert_eq!( + got.iter().map(|c| c[1].as_str()).collect::>(), + ["100", "150", "300", "301"] + ); + let by = |pos: &str| -> Vec<&str> { + got.iter().find(|c| c[1] == pos).unwrap()[4..7] + .iter() + .map(String::as_str) + .collect() + }; + assert_eq!(by("100"), ["12", "10", "9"], "outside: 2, 0, unknown"); + assert_eq!(by("150"), ["1", "1", "1"], "inside: untouched"); + assert_eq!(by("300"), ["0", "0", "0"], "the last base inside too"); + assert_eq!( + by("301"), + ["10", "10", "12"], + "a masked call stays, the first of a duplicate wins" + ); + write( + &p(&d, "unsorted.tsv"), + &([rows[0], rows[3], rows[2]].join("\n") + "\n"), + ); + let (code, err) = strict( + &p(&d, "strict.bed"), + &p(&d, "unsorted.tsv"), + &p(&d, "marked3.tsv"), + ); + assert!(code == 2 && err.contains("sorted"), "{code} {err}"); + assert!(!d.join("marked3.tsv").exists()); + write(&p(&d, "strict_other.bed"), "chr21\t0\t10\n"); + let (code, err) = strict( + &p(&d, "strict_other.bed"), + &p(&d, "lifted.tsv"), + &p(&d, "marked2.tsv"), + ); + assert!(code != 0 && err.contains("no interval"), "{code} {err}"); +} + +/// A genotype file by position. +fn gt_rows(path: &str) -> BTreeMap> { + gunzip(path) + .lines() + .skip(1) + .map(|l| { + let c: Vec = l.split('\t').map(str::to_owned).collect(); + (c[1].parse().unwrap(), c) + }) + .collect() +} + +const ROWS: [&str; 5] = [ + "chr22\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5", + "chr22\t2000\tC\tT\t0\t0\t0\t0.01\t0\t0\t0\t0.02\t0", + "chr22\t3000\tG\tA\t1\t1\t2\t0\t0\t0\t0\t0\t0", + "chr22\t4000\tT\tC\t2\t9\t0\t0.2\t0\t0\t0\t0\t0", + "chr22\t5000\tA\tC\t0\t0\t0\t0.2\t0\t0\t0\t0\t0", +]; + +/// The table of part 2 and the person's VCF. +fn genotype_inputs(d: &Path) -> String { + let tdir = p(d, "table"); + std::fs::create_dir_all(&tdir).unwrap(); + gzip( + &format!("{tdir}/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{}\n", ROWS.join("\n")), + ); + gzip( + &format!("{tdir}/chr21.tsv.gz"), + &format!("{TABLE_HEADER}\nchr21\t1000\tA\tG\t2\t2\t0\t0.5\t0.5\t0.5\t0.5\t0.5\t0.5\n"), + ); + write( + &p(d, "s.vcf"), + &format!( + "{VCF_HEAD}chr22\t1000\t.\tA\tG\t50\tPASS\t.\tGT:GQ\t0/1:30\n\ + chr22\t3000\t.\tG\tA\t50\tRefCall\t.\tGT\t1/1\n\ + chr22\t5000\t.\tA\tT\t50\tPASS\t.\tGT\t1/1\n" + ), + ); + tdir +} + +#[test] +fn genotypes_code_your_calls_and_the_stand_ins() { + let d = tmp("genotypes"); + let tdir = genotype_inputs(&d); + // A reference block over 1-2500: 2000 was read; 4000 was not. + write( + &p(&d, "s.g.vcf"), + &format!("{VCF_HEAD}{}", ref_block("chr22", 1, 2500)), + ); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &tdir, + "--vcf", + &p(&d, "s.vcf"), + "--gvcf", + &p(&d, "s.g.vcf"), + "--pop", + "EUR", + "--pseudo", + "10", + "--out-dir", + &p(&d, "gt"), + ]); + assert_eq!(code, 0, "{err}"); + let text = gunzip(&p(&d, "gt/chr22.gt.gz")); + let head: Vec<&str> = text.lines().next().unwrap().split('\t').collect(); + assert_eq!( + head[..8], + [ + "chrom", "pos", "ref", "alt", "Altai", "Vindija", "Denisova", "SAMPLE" + ] + ); + assert_eq!(head.len(), 18, "ten stand-ins"); + let g = gt_rows(&p(&d, "gt/chr22.gt.gz")); + assert_eq!(g[&1000][0], "22", "an integer chromosome for IBDmix"); + assert_eq!(g[&1000][7], "1", "a heterozygous call"); + assert_eq!(g[&2000][7], "0", "no record inside a reference block"); + assert_eq!(g[&4000][7], "9", "no record outside every block"); + assert_eq!(g[&3000][7], "9", "a non-PASS record"); + assert!(!g.contains_key(&5000), "another ALT is left out"); + let sum = |pos: u32| -> u32 { g[&pos][8..].iter().map(|x| x.parse::().unwrap()).sum() }; + assert_eq!(sum(1000), 10, "0.5 of 20 alleles"); + assert_eq!(sum(2000), 0, "0.02 of 20 rounds to 0"); + assert!(g[&1000][4..].iter().all(|x| x.len() == 1)); + assert!( + !d.join("gt/chr21.gt.gz").exists(), + "a chromosome the VCF has no record on is not analysed" + ); + let spans = std::fs::read_to_string(p(&d, "gt/spans.tsv")).unwrap(); + let span: Vec<&str> = spans.lines().nth(1).unwrap().split('\t').collect(); + assert_eq!(span[..4], ["chr22", "1000", "2000", "2"]); + let samples = std::fs::read_to_string(p(&d, "gt/samples.txt")).unwrap(); + let samples: Vec<&str> = samples.split_whitespace().collect(); + assert_eq!((samples[0], samples.len()), ("SAMPLE", 11)); + assert_eq!( + std::fs::read_to_string(p(&d, "gt/chr22.Altai.mask.bed")).unwrap(), + "", + "an unmasked table writes empty masks" + ); + + write( + &p(&d, "two.vcf"), + "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\tA\tB\n", + ); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &tdir, + "--vcf", + &p(&d, "two.vcf"), + "--out-dir", + &p(&d, "gt2"), + ]); + assert!(code != 0 && err.contains("one-sample"), "{code} {err}"); +} + +#[test] +fn a_split_vcf_keeps_each_snv_call() { + let d = tmp("split"); + let sdir = p(&d, "table_split"); + std::fs::create_dir_all(&sdir).unwrap(); + let rows: String = [ + (1000, "A\tG"), + (2000, "C\tT"), + (3000, "G\tA"), + (4000, "T\tC"), + (5000, "A\tG"), + (6000, "C\tT"), + (7000, "G\tA"), + (8000, "T\tC"), + (9000, "A\tC"), + ] + .iter() + .map(|(pos, ra)| format!("chr22\t{pos}\t{ra}\t0\t0\t0\t0.2\t0\t0\t0\t0\t0\n")) + .collect(); + gzip( + &format!("{sdir}/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{rows}"), + ); + let recs = [ + "1000\tA\tAT\tPASS\t0/1", // an insertion, then the SNV + "1000\tA\tG\tPASS\t0/1", + "2000\tC\tT\tPASS\t1/1", // the SNV, then an insertion + "2000\tC\tCA\tPASS\t0/1", + "3000\tG\tA,GT\tPASS\t1/2", // a multi-allelic record, then the SNV + "3000\tG\tA\tPASS\t0/1", + "4000\tT\tC\tPASS\t0/1", // two SNVs, both carried + "4000\tT\tG\tPASS\t0/1", + "4000\tT\tC\tPASS\t0/1", + "5000\tA\tT\tPASS\t0/0", // a split 0/1 at A>G,T: 0/0, then 0/1 + "5000\tA\tG\tPASS\t0/1", + "6000\tC\tT\tPASS\t0/1", // 0/1, then the 0/0 half + "6000\tC\tG\tPASS\t0/0", + "7000\tG\tA\tLowQual\t0/1", // filtered, then usable + "7000\tG\tA\tPASS\t1/1", + "8000\tT\tA\tPASS\t0/0", // two 0/0 halves: the reference + "8000\tT\tG\tPASS\t0/0", + ]; + let body: String = recs + .iter() + .map(|r| { + let c: Vec<&str> = r.split('\t').collect(); + format!( + "chr22\t{}\t.\t{}\t{}\t50\t{}\t.\tGT\t{}\n", + c[0], c[1], c[2], c[3], c[4] + ) + }) + .collect(); + write(&p(&d, "split.vcf"), &format!("{VCF_HEAD}{body}")); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &sdir, + "--vcf", + &p(&d, "split.vcf"), + "--pseudo", + "10", + "--out-dir", + &p(&d, "gt3"), + ]); + assert_eq!(code, 0, "{err}"); + let g = gt_rows(&p(&d, "gt3/chr22.gt.gz")); + let code = |pos: u32| g.get(&pos).map_or("left out", |c| c[7].as_str()); + assert_eq!(code(1000), "1", "an insertion before the SNV"); + assert_eq!(code(2000), "2", "an insertion after the SNV"); + assert_eq!(code(3000), "1", "a multi-allelic record beside the SNV"); + assert_eq!(code(4000), "9", "two carried SNVs at one position"); + assert_eq!(code(5000), "1", "a 0/0 half before the carried SNV"); + assert_eq!(code(6000), "1", "a 0/0 half after the carried SNV"); + assert_eq!(code(7000), "2", "a usable record wins over a filtered one"); + assert_eq!(code(8000), "0", "two 0/0 halves are the reference"); +} + +#[test] +fn masked_calls_go_to_ibdmix_with_a_mask_per_genome() { + let d = tmp("masked"); + let mdir = p(&d, "table_masked"); + std::fs::create_dir_all(&mdir).unwrap(); + let rows: String = [ + (100, "A\tG", "12\t0\t9"), + (200, "C\tT", "10\t11\t0"), + (300, "G\tA", "2\t0\t0"), + (400, "T\tC", "10\t0\t0"), + (500, "A\tC", "0\t0\t0"), + (600, "C\tT", "11\t0\t0"), + ] + .iter() + .map(|(pos, ra, c)| format!("chr22\t{pos}\t{ra}\t{c}\t0.2\t0\t0\t0\t0\t0\n")) + .collect(); + gzip( + &format!("{mdir}/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{rows}"), + ); + // Another ALT: row 500 is left out. + write( + &p(&d, "m.vcf"), + &format!("{VCF_HEAD}chr22\t500\t.\tA\tT\t50\tPASS\t.\tGT\t0/1\n"), + ); + write( + &p(&d, "m.g.vcf"), + &format!("{VCF_HEAD}{}", ref_block("chr22", 1, 1000)), + ); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &mdir, + "--vcf", + &p(&d, "m.vcf"), + "--gvcf", + &p(&d, "m.g.vcf"), + "--pseudo", + "10", + "--out-dir", + &p(&d, "gtm"), + ]); + assert_eq!(code, 0, "{err}"); + let g = gt_rows(&p(&d, "gtm/chr22.gt.gz")); + let calls = |pos: u32| g[&pos][4..7].to_vec(); + assert_eq!(calls(100), ["2", "0", "9"], "12 is 2"); + assert_eq!(calls(200), ["0", "1", "0"], "10 and 11 are 0 and 1"); + assert_eq!(calls(600), ["1", "0", "0"]); + let mask = + |a: &str| std::fs::read_to_string(p(&d, &format!("gtm/chr22.{a}.mask.bed"))).unwrap(); + assert_eq!( + mask("Altai"), + "22\t99\t200\n22\t399\t600\n", + "the left-out row 500 does not close the run" + ); + assert_eq!(mask("Vindija"), "22\t199\t200\n"); + assert_eq!(mask("Denisova"), "", "written and empty"); + + let bdir = p(&d, "table_bad"); + std::fs::create_dir_all(&bdir).unwrap(); + gzip( + &format!("{bdir}/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\nchr22\t100\tA\tG\t7\t0\t0\t0.2\t0\t0\t0\t0\t0\n"), + ); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &bdir, + "--vcf", + &p(&d, "m.vcf"), + "--out-dir", + &p(&d, "gtb"), + ]); + assert!(code == 1 && err.contains("code '7'"), "{code} {err}"); +} + +#[test] +fn unsorted_input_stops_and_keeps_the_previous_outputs() { + let d = tmp("order"); + let tdir = genotype_inputs(&d); + for (label, records) in [ + ( + "within-table", + [(3000, "G", "A", "GT"), (1000, "A", "G", "GT")], + ), + ( + "past-table", + [(9000, "A", "G", "GT"), (8000, "A", "G", "GT")], + ), + ( + "ignored-call", + [(3000, "G", "A", "GT"), (1000, "A", ".", "GQ")], + ), + ] { + let bad = p(&d, &format!("{label}.vcf")); + let body: String = records + .iter() + .map(|(pos, r, a, f)| format!("chr22\t{pos}\t.\t{r}\t{a}\t.\tPASS\t.\t{f}\t0/1\n")) + .collect(); + write(&bad, &format!("{VCF_HEAD}{body}")); + let out = p(&d, &format!("order-{label}")); + std::fs::create_dir_all(&out).unwrap(); + let protected: Vec = std::iter::once("chr22.gt.gz".to_owned()) + .chain(ARCHAICS.iter().map(|g| format!("chr22.{g}.mask.bed"))) + .collect(); + for name in &protected { + write(&format!("{out}/{name}"), "previous output\n"); + } + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &tdir, + "--vcf", + &bad, + "--out-dir", + &out, + ]); + assert!( + code == 2 && err.contains("not sorted"), + "{label}: {code} {err}" + ); + for name in &protected { + assert_eq!( + std::fs::read(format!("{out}/{name}")).unwrap(), + b"previous output\n", + "{label} keeps {name}" + ); + } + } + let udir = p(&d, "table_unordered"); + std::fs::create_dir_all(&udir).unwrap(); + gzip( + &format!("{udir}/chr22.tsv.gz"), + &format!( + "{TABLE_HEADER}\n{}\n", + [ROWS[1], ROWS[0], ROWS[2], ROWS[3], ROWS[4]].join("\n") + ), + ); + let out = p(&d, "gt_unordered"); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &udir, + "--vcf", + &p(&d, "s.vcf"), + "--out-dir", + &out, + ]); + assert!(code == 2 && err.contains("not sorted"), "{code} {err}"); + let left: Vec = std::fs::read_dir(&out) + .map(|r| { + r.map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect() + }) + .unwrap_or_default(); + assert!( + !left + .iter() + .any(|n| n.ends_with(".gt.gz") || n.ends_with(".mask.bed")), + "no partial genotype or mask: {left:?}" + ); +} + +/// The archaic section og summary collect writes for S1 in `dir`. +fn archaic_section(dir: &Path) -> Value { + let input = og_summary::collect::Input { + sample: "S1".to_owned(), + sample_dir: dir.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + let v = og_summary::collect::post::collect(&input).unwrap(); + let get = |v: &Value, k: &str| pycompat::value::get(v, k).unwrap().unwrap().clone(); + get(&get(&v, "sections"), "archaic") +} + +fn field<'a>(v: &'a Value, k: &str) -> &'a Value { + pycompat::value::get(v, k).unwrap().unwrap() +} + +#[test] +fn the_report_merges_segments_and_both_reports_read_it() { + let d = tmp("report"); + let idir = p(&d, "ibd"); + std::fs::create_dir_all(&idir).unwrap(); + let head = "ID\tchrom\tstart\tend\tslod\n"; + write( + &format!("{idir}/chr22.Altai.txt"), + &format!( + "{head}SAMPLE\t22\t1000000\t1300000\t40\nP001\t22\t5000000\t9000000\t99\n\ + SAMPLE\t22\t2000000\t2040000\t30\nSAMPLE\t22\t3000000\t3100000\t3.5\n" + ), + ); + write( + &format!("{idir}/chr22.Vindija.txt"), + &format!("{head}SAMPLE\t22\t1200000\t1500000\t50\n"), + ); + write( + &format!("{idir}/chr22.Denisova.txt"), + &format!("{head}SAMPLE\t22\t1400000\t1450000\t12\nSAMPLE\t22\t6000000\t6200000\t20\n"), + ); + write( + &p(&d, "spans.tsv"), + "chrom\tfirst\tlast\tsites\nchr22\t1\t10000000\t5000\n", + ); + write( + &p(&d, "genes.gtf"), + "chr22\tHAVANA\tgene\t1100000\t1110000\t.\t+\t.\tgene_id \"G1\"; gene_type \"protein_coding\"; gene_name \"GENEA\";\n\ + chr22\tHAVANA\tgene\t1120000\t1130000\t.\t+\t.\tgene_id \"G2\"; gene_type \"lncRNA\"; gene_name \"LNC1\";\n\ + chr22\tHAVANA\tgene\t6100000\t6300000\t.\t-\t.\tgene_id \"G3\"; gene_type \"protein_coding\"; gene_name \"GENEB\";\n", + ); + let (code, err) = og(&[ + "report", + "--sample", + "S1", + "--ibdmix-dir", + &idir, + "--spans", + &p(&d, "spans.tsv"), + "--genes", + &p(&d, "genes.gtf"), + "--out-dir", + &p(&d, "out"), + ]); + assert_eq!(code, 0, "{err}"); + let segs = std::fs::read_to_string(p(&d, "out/S1_archaic_segments.tsv")).unwrap(); + let segs: Vec> = segs.lines().map(|l| l.split('\t').collect()).collect(); + assert_eq!( + segs[0], + [ + "chrom", + "start", + "end", + "length_bp", + "ancestry", + "matched", + "lod", + "genes" + ] + ); + let body = &segs[1..]; + assert_eq!(body.len(), 2, "{body:?}"); + assert_eq!( + body[0][..6], + [ + "chr22", + "1000000", + "1500000", + "500000", + "Neanderthal", + "Altai,Vindija,Denisova" + ], + "Altai 1.0-1.3 and Vindija 1.2-1.5 Mb merge" + ); + assert_eq!(body[0][6], "50.0", "the higher LOD"); + assert_eq!( + body[1][..5], + ["chr22", "6000000", "6200000", "200000", "Denisovan"], + "a Denisova stretch inside a Neanderthal one is not Denisovan; one apart is" + ); + assert_eq!(body[0][7], "GENEA", "protein-coding genes only"); + let summ: BTreeMap = std::fs::read_to_string(p(&d, "out/S1_archaic.tsv")) + .unwrap() + .lines() + .skip(1) + .map(|l| { + let (k, v) = l.split_once('\t').unwrap(); + (k.to_owned(), v.to_owned()) + }) + .collect(); + assert_eq!(summ["neanderthal_percent"], "5.00"); + assert_eq!(summ["denisovan_percent"], "2.00"); + assert_eq!( + [&summ["neanderthal_segments"], &summ["denisovan_segments"]], + ["1", "1"] + ); + assert!(summ["longest_1"].starts_with("chr22:1000000-1500000 Neanderthal")); + assert!(summ["longest_1"].contains("GENEA")); + let txt = std::fs::read_to_string(p(&d, "out/S1_archaic.txt")).unwrap(); + assert!(txt.contains("An estimate, not a measurement."), "{txt}"); + let (code, err) = og(&[ + "report", + "--sample", + "S1", + "--ibdmix-dir", + &p(&d, "nothing"), + "--spans", + &p(&d, "spans.tsv"), + "--out-dir", + &p(&d, "out2"), + ]); + assert!( + code != 0 && err.contains("no IBDmix output"), + "{code} {err}" + ); + + // The summary and the report lines read it. + let sd = d.join("sample"); + std::fs::create_dir_all(sd.join("archaic")).unwrap(); + let tsv = sd.join("archaic/S1_archaic.tsv"); + std::fs::copy(p(&d, "out/S1_archaic.tsv"), &tsv).unwrap(); + let sec = archaic_section(&sd); + assert_eq!(field(&sec, "state"), &Value::Str("ok".into())); + assert_eq!(field(&sec, "step"), &Value::Str("43".into())); + let data = field(&sec, "data"); + assert_eq!( + field(data, "neanderthal_percent"), + &Value::Str("5.00".into()) + ); + assert_eq!(field(data, "denisovan_segments"), &Value::Int(1)); + let Value::Array(longest) = field(data, "longest") else { + panic!("longest is not a list") + }; + assert_eq!(longest.len(), 2); + let lines = og_render::sections::archaic::lines(data).unwrap(); + assert!( + lines[0].starts_with("Neanderthal: 5.00% (0.5 Mb in 1 segments)"), + "{lines:?}" + ); + assert!(lines[1].starts_with("Denisovan: 2.00%"), "{lines:?}"); + for damaged in [ + "key\tvalue\nneanderthal_percent\tlots\n", + "key\tvalue\nneanderthal_segments\n", + ] { + std::fs::write(&tsv, damaged).unwrap(); + assert_eq!( + field(&archaic_section(&sd), "state"), + &Value::Str("unreadable".into()), + "{damaged:?}" + ); + } + let nowhere = archaic_section(&d.join("nowhere")); + assert_eq!(field(&nowhere, "state"), &Value::Str("missing".into())); + assert_eq!(field(&nowhere, "data"), &Value::Object(Default::default())); +} + +/// Python's `random.Random(seed)` for a small seed: MT19937 seeded by +/// `init_by_array([seed])`, and `random()` from two draws. The IBDmix +/// fixture below is the one the Python test drew. +struct PyRandom { + mt: [u32; 624], + i: usize, +} + +impl PyRandom { + fn new(seed: u32) -> PyRandom { + let mut mt = [0u32; 624]; + mt[0] = 19_650_218; + for i in 1..624 { + mt[i] = 1_812_433_253u32 + .wrapping_mul(mt[i - 1] ^ (mt[i - 1] >> 30)) + .wrapping_add(i as u32); + } + let key = [seed]; + let (mut i, mut j) = (1usize, 0usize); + for _ in 0..624 { + mt[i] = (mt[i] ^ (mt[i - 1] ^ (mt[i - 1] >> 30)).wrapping_mul(1_664_525)) + .wrapping_add(key[j]) + .wrapping_add(j as u32); + i += 1; + j += 1; + if i >= 624 { + mt[0] = mt[623]; + i = 1; + } + if j >= key.len() { + j = 0; + } + } + for _ in 0..623 { + mt[i] = (mt[i] ^ (mt[i - 1] ^ (mt[i - 1] >> 30)).wrapping_mul(1_566_083_941)) + .wrapping_sub(i as u32); + i += 1; + if i >= 624 { + mt[0] = mt[623]; + i = 1; + } + } + mt[0] = 0x8000_0000; + PyRandom { mt, i: 624 } + } + + fn next_u32(&mut self) -> u32 { + if self.i >= 624 { + for k in 0..624 { + let y = (self.mt[k] & 0x8000_0000) | (self.mt[(k + 1) % 624] & 0x7fff_ffff); + let mut v = self.mt[(k + 397) % 624] ^ (y >> 1); + if y & 1 == 1 { + v ^= 0x9908_b0df; + } + self.mt[k] = v; + } + self.i = 0; + } + let mut y = self.mt[self.i]; + self.i += 1; + y ^= y >> 11; + y ^= (y << 7) & 0x9d2c_5680; + y ^= (y << 15) & 0xefc6_0000; + y ^ (y >> 18) + } + + fn random(&mut self) -> f64 { + let a = f64::from(self.next_u32() >> 5); + let b = f64::from(self.next_u32() >> 6); + (a * 67_108_864.0 + b) / 9_007_199_254_740_992.0 + } + + fn uniform(&mut self, a: f64, b: f64) -> f64 { + a + (b - a) * self.random() + } +} + +#[test] +fn the_fixture_generator_draws_what_python_drew() { + // Python 3: random.Random(11), three random() then uniform(0.01, 0.04). + let mut r = PyRandom::new(11); + let got = [r.random(), r.random(), r.random(), r.uniform(0.01, 0.04)]; + assert_eq!( + got, + [ + 0.452_379_553_509_818_6, + 0.559_772_386_080_496, + 0.924_210_584_023_729_4, + 0.023_969_502_102_993_2 + ] + ); +} + +#[cfg(target_arch = "x86_64")] +mod ibdmix { + //! Part 4: the chain with IBDmix itself. + + use super::*; + use std::process::Command; + + const NEA: (u32, u32) = (2_000_000, 2_600_000); + const DEN: (u32, u32) = (3_200_000, 3_600_000); + const MID: (u32, u32) = (2_200_000, 2_400_000); + const MASKED: u32 = 10; + + /// Whether the docker tests run here: always in CI. + fn docker_tests(test: &str) -> bool { + let on = std::env::var_os("CI").is_some_and(|v| !v.is_empty()) + || std::env::var("OG_DOCKER_TESTS").as_deref() == Ok("1"); + if !on { + eprintln!( + "skip: {test} runs IBDmix in docker; set OG_DOCKER_TESTS=1 (CI always runs it)" + ); + } + on + } + + fn image() -> String { + let text = std::fs::read_to_string(repo().join("versions.env")).unwrap(); + let line = text + .lines() + .find(|l| l.starts_with("IBDMIX_IMAGE=")) + .expect("IBDMIX_IMAGE in versions.env"); + let v = &line["IBDMIX_IMAGE=".len()..]; + v.split('#') + .next() + .unwrap() + .trim() + .trim_matches('"') + .to_owned() + } + + fn id(flag: &str) -> String { + let o = Command::new("id").arg(flag).output().unwrap(); + String::from_utf8(o.stdout).unwrap().trim().to_owned() + } + + fn open_dir(name: &str) -> PathBuf { + let d = tmp(name); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&d, std::fs::Permissions::from_mode(0o777)).unwrap(); + d + } + + /// A 4 Mb chr22 table and one person, drawn as the Python test drew + /// them. A third of the sites carry an allele the Neanderthals share + /// and modern people rarely do, a third one only the Denisovan carries, + /// a third are common. Inside a planted stretch one copy carries the + /// archaic allele. `mid`: the middle of the planted Neanderthal stretch + /// holds sites the person contradicts, with the Neanderthals' calls + /// written masked (10 plus the call) or unknown (9). + fn synthetic(d: &Path, plant: bool, mid: Option<&str>) { + let mut rnd = PyRandom::new(11); + std::fs::create_dir_all(d.join("table")).unwrap(); + let (mut rows, mut calls) = (Vec::new(), Vec::new()); + for (i, pos) in (1_000_000u32..5_000_000).step_by(1000).enumerate() { + let (f, al, vi, de) = match i % 3 { + 0 => (rnd.uniform(0.01, 0.04), 2, 2, 0), + 1 => (rnd.uniform(0.01, 0.04), 0, 0, 2), + _ => { + let f = rnd.uniform(0.1, 0.9); + let n = u32::from(rnd.random() < f) + u32::from(rnd.random() < f); + (f, n, n, n) + } + }; + let (mut ta, mut tv) = (al, vi); + let mut g = [rnd.random() < f, rnd.random() < f]; + if plant && NEA.0 <= pos && pos < NEA.1 { + g[0] = al == 2; + } + if let Some(m) = mid + && MID.0 <= pos + && pos < MID.1 + { + if i.is_multiple_of(3) { + g = [false, false]; + } + (ta, tv) = if m == "masked" { + (al + MASKED, vi + MASKED) + } else { + (9, 9) + }; + } + let fs = format!("{f:.4}"); + rows.push(format!( + "chr22\t{pos}\tA\tG\t{ta}\t{tv}\t{de}\t{}", + [fs.as_str(); 6].join("\t") + )); + if plant && DEN.0 <= pos && pos < DEN.1 { + g[0] = de == 2; + } + if g.iter().any(|x| *x) { + let gt = if g.iter().all(|x| *x) { "1/1" } else { "0/1" }; + calls.push(format!("chr22\t{pos}\t.\tA\tG\t50\tPASS\t.\tGT\t{gt}")); + } + } + gzip( + &p(d, "table/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{}\n", rows.join("\n")), + ); + write(&p(d, "s.vcf"), &format!("{VCF_HEAD}{}\n", calls.join("\n"))); + } + + /// genotypes, IBDmix with the flags of scripts/43-archaic.sh, report; + /// the segments as (start, end, length, ancestry). + fn chain(d: &Path, gvcf: Option) -> Vec<(u32, u32, u32, String)> { + let gvcf = + gvcf.unwrap_or_else(|| format!("{VCF_HEAD}{}", ref_block("chr22", 1, 6_000_000))); + write(&p(d, "s.g.vcf"), &gvcf); + let (code, err) = og(&[ + "genotypes", + "--table-dir", + &p(d, "table"), + "--vcf", + &p(d, "s.vcf"), + "--gvcf", + &p(d, "s.g.vcf"), + "--out-dir", + &p(d, "gt"), + ]); + assert_eq!(code, 0, "{err}"); + std::fs::create_dir_all(d.join("ibd")).unwrap(); + let sh = "set -e; gzip -dc /w/gt/chr22.gt.gz > /w/gt/chr22.gt; for a in Altai Vindija Denisova; do \ + ibdmix -g /w/gt/chr22.gt -s /w/gt/samples.txt -n $a -r /w/gt/chr22.$a.mask.bed -o /w/ibd/chr22.$a.txt \ + -d 4 -m 1 -a 0.01 -e 0.002 -c 2; done"; + let o = Command::new("docker") + .args(["run", "--rm", "--network", "none", "-u"]) + .arg(format!("{}:{}", id("-u"), id("-g"))) + .arg("-v") + .arg(format!("{}:/w", d.display())) + .arg(image()) + .args(["sh", "-c", sh]) + .output() + .unwrap_or_else(|e| panic!("docker: {e}")); + assert!( + o.status.success(), + "IBDmix: {}{}", + String::from_utf8_lossy(&o.stderr), + String::from_utf8_lossy(&o.stdout) + ); + let (code, err) = og(&[ + "report", + "--sample", + "S", + "--ibdmix-dir", + &p(d, "ibd"), + "--spans", + &p(d, "gt/spans.tsv"), + "--out-dir", + &p(d, "out"), + ]); + assert_eq!(code, 0, "{err}"); + let body: Vec<(u32, u32, u32, String)> = + std::fs::read_to_string(p(d, "out/S_archaic_segments.tsv")) + .unwrap() + .lines() + .skip(1) + .map(|l| { + let c: Vec<&str> = l.split('\t').collect(); + ( + c[1].parse().unwrap(), + c[2].parse().unwrap(), + c[3].parse().unwrap(), + c[4].to_owned(), + ) + }) + .collect(); + eprintln!("segments in {}: {body:?}", d.display()); + body + } + + fn cover(segs: &[(u32, u32, u32, String)], kind: &str, (lo, hi): (u32, u32)) -> u32 { + segs.iter() + .filter(|s| s.3 == kind) + .map(|s| s.1.min(hi).saturating_sub(s.0.max(lo))) + .sum() + } + + #[test] + fn planted_stretches_come_out_as_one_segment_each() { + if !docker_tests("planted_stretches_come_out_as_one_segment_each") { + return; + } + let d = open_dir("real_planted"); + synthetic(&d, true, None); + let body = chain(&d, None); + let nea: Vec<_> = body + .iter() + .filter(|s| s.3 == "Neanderthal") + .cloned() + .collect(); + let den: Vec<_> = body + .iter() + .filter(|s| s.3 == "Denisovan") + .cloned() + .collect(); + assert_eq!(nea.len(), 1, "{body:?}"); + assert!( + cover(&nea, "Neanderthal", NEA) * 10 >= 9 * (NEA.1 - NEA.0), + "{nea:?}" + ); + assert!( + nea[0].2 <= NEA.1 - NEA.0 + 100_000, + "no more than 100 kb beyond it" + ); + assert_eq!(den.len(), 1, "{body:?}"); + assert!( + cover(&den, "Denisovan", DEN) * 10 >= 9 * (DEN.1 - DEN.0), + "{den:?}" + ); + + let d = open_dir("real_control"); + synthetic(&d, false, None); + assert!(chain(&d, None).is_empty(), "without the planted stretches"); + } + + #[test] + fn masked_mismatches_stop_a_segment_that_unknown_calls_let_through() { + if !docker_tests("masked_mismatches_stop_a_segment_that_unknown_calls_let_through") { + return; + } + let span = MID.1 - MID.0; + let d = open_dir("real_mid_unknown"); + synthetic(&d, true, Some("unknown")); + let unknown = cover(&chain(&d, None), "Neanderthal", MID); + assert!( + unknown * 10 >= 9 * span, + "control: written as unknown the segment runs through ({unknown})" + ); + let d = open_dir("real_mid_masked"); + synthetic(&d, true, Some("masked")); + let masked = cover(&chain(&d, None), "Neanderthal", MID); + assert!( + masked * 10 <= span, + "written as masked no segment covers the middle ({masked})" + ); + } + + #[test] + fn a_stretch_with_no_call_needs_a_reference_block() { + if !docker_tests("a_stretch_with_no_call_needs_a_reference_block") { + return; + } + let (lo, hi) = (200_000u32, 349_000u32); + let mut got = Vec::new(); + for covered in [true, false] { + let d = open_dir(&format!("real_gap_{covered}")); + std::fs::create_dir_all(d.join("table")).unwrap(); + let rows: String = (lo..=hi) + .step_by(1000) + .map(|pos| format!("chr22\t{pos}\tC\tT\t0\t0\t0\t{}\n", ["0.9"; 6].join("\t"))) + .collect(); + gzip( + &p(&d, "table/chr22.tsv.gz"), + &format!("{TABLE_HEADER}\n{rows}"), + ); + write( + &p(&d, "s.vcf"), + &format!("{VCF_HEAD}chr22\t10000000\t.\tA\tG\t50\tPASS\t.\tGT\t0/1\n"), + ); + // Without the block the person has one read site left, at the start. + let block = if covered { + ref_block("chr22", 1, 400_000) + } else { + ref_block("chr22", 1, lo) + }; + let body = chain(&d, Some(format!("{VCF_HEAD}{block}"))); + got.push( + body.iter() + .filter(|s| s.3 == "Neanderthal") + .map(|s| s.2) + .sum::(), + ); + } + assert!( + got[0] >= 100_000, + "control: a stretch read as the reference gives a segment ({got:?})" + ); + assert_eq!(got[1], 0, "no call and no reference block: no segment"); + } +} diff --git a/rust/crates/og-render/tests/ancestry_fine.rs b/rust/crates/og-render/tests/ancestry_fine.rs new file mode 100644 index 000000000..be90b2238 --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_fine.rs @@ -0,0 +1,117 @@ +//! Both reports of step 26b's table (`og ancestry-fine report` writes it): +//! the groups with their bootstrap range, the group the model leaves out, +//! the chromosome RFMix failed on, the painted shares and the painting. +//! It replaces the render part of `tests/test_ancestry_fine_report.py`; +//! og-ancestry's own tests check the table the report command writes. + +use std::path::{Path, PathBuf}; + +use og_render::model::Summary; +use og_render::{html, text}; +use pycompat::json::{Dump, dumps}; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +/// The table `og ancestry-fine report` wrote for the Python test's inputs. +const TABLE: &str = "level\tcode\tgroup\tfraction\tse\tlow\thigh +continent\tWEA\tWest Eurasia (Europe, Middle East, North Africa)\t0.9000\t0.0332\t0.8025\t0.8975 +continent\tEAS\tEast Asia\t0.0500\t0.0000\t0.0500\t0.0500 +continent\tAFR\tAfrica\t0.0500\t0.0332\t0.0525\t0.1475 +region\tIBE\tIberia\t0.6000\t.\t.\t. +region\tITA\tItaly\t0.4000\t.\t.\t. +painting\tWEA\tWest Eurasia (Europe, Middle East, North Africa)\t0.7000\t.\t.\t. +painting\tAFR\tAfrica\t0.3000\t.\t.\t. +left_out\tOCE\tOceania (continental group)\t.\t.\t.\t. +left_out\tOCE\tOceania (regional group)\t.\t.\t.\t. +not_painted\tchr19\tRFMix failed\t.\t.\t.\t. +"; + +fn sample(tag: &str, painting: bool) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-ancestry-fine-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(d.join("ancestry_fine")).unwrap(); + std::fs::write(d.join("ancestry_fine/S_ancestry_fine.tsv"), TABLE).unwrap(); + if painting { + std::fs::write( + d.join("ancestry_fine/S_painting.png"), + b"\x89PNG\r\n\x1a\nnot a real picture", + ) + .unwrap(); + } + d +} + +fn reports(d: &Path) -> (String, String) { + let input = og_summary::collect::Input { + sample: "S".to_owned(), + sample_dir: d.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + let summary = dumps( + &og_summary::collect::post::collect(&input).unwrap(), + &Dump::default(), + ) + .unwrap(); + let s = Summary::parse(&summary).unwrap(); + (text::report(&s).unwrap(), html::report(&s).unwrap()) +} + +#[test] +fn both_reports_name_the_group_left_out_and_the_chromosome_not_painted() { + let (txt, page) = reports(&sample("full", true)); + for doc in [&txt, &page] { + assert!( + doc.contains( + "Left out of the model: Oceania (continental and regional group). The panel has \ + fewer than 10" + ), + "{doc}" + ); + assert!(doc.contains("not painted: RFMix failed on chr19"), "{doc}"); + } + assert!(txt.contains("Fine-Grained Ancestry (ADMIXTURE, RFMix)")); + assert!( + txt.contains(" West Eurasia (Europe, Middle East, North Africa) 90% (80% to 90%)"), + "{txt}" + ); + assert!( + txt.contains( + " Chromosome painting, share of the painted genome: West Eurasia (Europe, Middle \ + East, North Africa) 70%, Africa 30%; not painted: RFMix failed on chr19" + ), + "{txt}" + ); + assert!(page.contains("

    Fine-Grained Ancestry

    ")); + assert!(page.contains( + "West Eurasia (Europe, Middle East, North Africa)90%80% to \ + 90%" + )); + assert!( + page.contains("Iberia60%."), + "a level without bootstrap replicates has no range" + ); + assert!( + page.contains("src=\"data:image/png;base64,iVBORw0KGg"), + "the painting" + ); + assert!( + page.contains("■ not painted

    "), + "its legend" + ); +} + +#[test] +fn without_a_picture_the_card_has_no_image() { + let (_, page) = reports(&sample("nopng", false)); + assert!(page.contains("

    Fine-Grained Ancestry

    ")); + assert!(!page.contains("data:image/png;base64,")); + assert!( + !page.contains("not painted

    "), + "no legend without the painting" + ); +} diff --git a/rust/crates/og-render/tests/ancestry_haplogroups.rs b/rust/crates/og-render/tests/ancestry_haplogroups.rs new file mode 100644 index 000000000..b8499ae0a --- /dev/null +++ b/rust/crates/og-render/tests/ancestry_haplogroups.rs @@ -0,0 +1,169 @@ +//! Both reports point to where a haplogroup can be explored further (steps +//! 12 and 37). It replaces `tests/test_haplogroup_explore.py`. +//! +//! Each case writes the haplogroup files of one made-up sample S into an +//! output folder, collects the summary and renders both reports: +//! +//! 1. a male sample (Yleaf table with R-M269, haplogrep3 with H5a7): the Y +//! and the mtDNA links are built from those names, in the text and the +//! HTML report, with the note that nothing is uploaded; +//! 2. Yleaf's paragroup form R-M269*(xR-L23): the links use R-M269; +//! 3. a female sample (no Yleaf table): no Y links; the mtDNA links stay; +//! 4. no call (Hg NA, haplogroup ''): no links at all; +//! 5. a branch name with characters a URL cannot carry raw is encoded; +//! 6. an address that would close the link's attribute and open a tag is +//! escaped in the HTML report, so the page never runs it. +//! +//! The branch names are the fixture's and public examples, not anyone's +//! result. + +use std::path::{Path, PathBuf}; + +use og_render::model::Summary; +use og_render::{html, text}; +use pycompat::json::{Dump, dumps}; + +const Y_HEAD: &str = + "Sample_name\tHg\tHg_marker\tTotal_reads\tValid_markers\tQC-score\tQC-1\tQC-2\tQC-3\n"; + +fn repo() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..") +} + +fn put(path: &Path, text: &str) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, text).unwrap(); +} + +/// A sample folder with haplogrep3's call and, for a male, Yleaf's table. +fn sample(name: &str, y_hg: Option<&str>, mt_hg: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("og-ancestry-hg-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + put( + &d.join("mito/S_haplogroup.txt"), + &format!("\"SampleID\"\t\"Haplogroup\"\t\"Rank\"\n\"S\"\t\"{mt_hg}\"\t\"0.98\"\n"), + ); + if let Some(hg) = y_hg { + put( + &d.join("y_haplogroup/S_y_haplogroup.txt"), + &format!("{Y_HEAD}S_sorted\t{hg}\tM269\t100\t42\t0.97\t1\t1\t0.97\n"), + ); + } + d +} + +/// The summary og summary collect writes for S, as text. +fn collect(d: &Path) -> String { + let input = og_summary::collect::Input { + sample: "S".to_owned(), + sample_dir: d.display().to_string(), + declared_sex: None, + assets: repo().join("assets").display().to_string(), + group: og_summary::collect::Group::All, + tables: repo().join("bin").display().to_string(), + }; + dumps( + &og_summary::collect::post::collect(&input).unwrap(), + &Dump::default(), + ) + .unwrap() +} + +/// Both reports of a summary. +fn render(summary: &str) -> (String, String) { + let s = Summary::parse(summary).unwrap(); + (text::report(&s).unwrap(), html::report(&s).unwrap()) +} + +fn reports(d: &Path) -> (String, String) { + render(&collect(d)) +} + +#[test] +fn a_male_sample_gets_the_y_and_the_mtdna_links() { + let (txt, page) = reports(&sample("male", Some("R-M269"), "H5a7")); + for url in [ + "https://discover.familytreedna.com/y-dna/R-M269", + "https://www.yfull.com/tree/R-M269/", + "https://discover.familytreedna.com/mtdna/H5a7", + "https://www.yfull.com/mtree/H5a7/", + ] { + assert!(txt.contains(url), "{url} in the text report:\n{txt}"); + assert!( + page.contains(&format!("
    ")), + "{url} as a link in the HTML report" + ); + } + assert!(txt.contains("docs/37-y-haplogroup.md#explore-your-branch-further")); + assert!(txt.contains("docs/12-mito-haplogroup.md#explore-your-branch-further")); + assert!(txt.contains("Nothing is uploaded by this pipeline")); + assert!(page.contains("Nothing is uploaded by this pipeline")); +} + +#[test] +fn a_paragroup_links_its_named_branch() { + let (txt, _) = reports(&sample("para", Some("R-M269*(xR-L23)"), "H5a7")); + assert!(txt.contains("https://www.yfull.com/tree/R-M269/"), "{txt}"); + assert!(!txt.contains("y-dna/R-M269*"), "{txt}"); + assert!(!txt.contains("%2A"), "{txt}"); +} + +#[test] +fn a_female_sample_has_no_y_link_and_keeps_the_mtdna_ones() { + let (txt, page) = reports(&sample("female", None, "H5a7")); + for t in [&txt, &page] { + assert!(!t.contains("y-dna/"), "{t}"); + assert!(!t.contains("yfull.com/tree/"), "{t}"); + } + assert!(txt.contains("https://discover.familytreedna.com/mtdna/H5a7")); + assert!(page.contains("https://www.yfull.com/mtree/H5a7/")); +} + +#[test] +fn no_call_has_no_link() { + let (txt, page) = reports(&sample("nocall", Some("NA"), "")); + assert!(!txt.contains("discover.familytreedna.com"), "{txt}"); + assert!(!txt.contains("yfull.com"), "{txt}"); + assert!(!page.contains("discover.familytreedna.com")); + assert!( + txt.contains("insufficient markers"), + "the Y section still says why:\n{txt}" + ); +} + +#[test] +fn a_plus_in_the_branch_is_encoded() { + let (txt, _) = reports(&sample("esc", None, "L3e2b+152")); + assert!(txt.contains("mtdna/L3e2b%2B152"), "{txt}"); + assert!(txt.contains("mtree/L3e2b%2B152/"), "{txt}"); +} + +#[test] +fn a_planted_address_cannot_open_a_tag_in_the_html_report() { + let summary = collect(&sample("planted", Some("R-M269"), "H5a7")); + let planted = r#"https://x.example/\"><"), "it ended the attribute"); + assert_eq!( + page.matches(""><script>alert(1)</script>") + .count(), + 4, + "escaped in the address and the text of both links" + ); + assert!( + txt.contains("https://x.example/\">