diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 0000000..128eae6
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1,37 @@
+# Working on strain
+
+Build and test: `pip install -e . && pytest`
+
+## Changing the `--ui live` dashboard
+
+Unit tests check strings in the frame. They can't tell you whether the layout reads well, so
+look at the dashboard before you commit any change to `strain/dashboard.py`, `strain/console.py`
+or `strain/keys.py`:
+
+1. `pip install -e '.[ui-snapshot]'` (adds `pyte` and `pillow`).
+2. Configure a copy of the fixture as a git repository with a coverage build (strain refuses a folder
+ without `CMakeCache.txt`), using the flags from `configure_project` in `tests/test_end_to_end.py`:
+ ```sh
+ demo=build/ui-demo && rm -rf $demo && mkdir -p $demo && cp -r tests/fixtures/sample_project $demo/project
+ git -C $demo/project init -q && git -C $demo/project add . && git -C $demo/project -c user.name=strain -c user.email=strain@example.com commit -qm fixture
+ cmake -S $demo/project -B $demo/build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+ "-DCMAKE_CXX_FLAGS=--coverage -fsanitize=address,undefined -fno-omit-frame-pointer -D_GLIBCXX_ASSERTIONS -ftrivial-auto-var-init=pattern" \
+ "-DCMAKE_EXE_LINKER_FLAGS=--coverage -fsanitize=address,undefined"
+ cmake --build $demo/build
+ python tools/ui_snapshot.py $demo/project --size 120x36 --keys down,tab,q --out ui-snapshots \
+ -- --build-dir $PWD/$demo/build --report-dir $PWD/$demo/report
+ ```
+ - This needs the fixture's toolchain (CMake, a C++ compiler, gcov) and takes about a minute.
+ - Arguments after `--` go to `strain run`.
+ - Also try a small size such as `--size 80x24` to check truncation and panel sizing.
+3. Read the `ui-snapshots/*.txt` grids to check alignment, and view the `*.png` frames to check
+ colours and reverse video. `pyte` drops SGR 2 (faint), so `DIM` text looks like normal text in the PNGs.
+4. Check these points, which have regressed before:
+ - The header totals match the survivors view, so each mutant is counted once across tests.
+ - The progress bar and the title name the same test as the queue.
+ - Each survivor detail row is padded to `DETAIL_LABEL_WIDTH`.
+ - No panel leaves large blank areas while another panel is truncated.
+5. Attach before and after frames, or the relevant `.txt` excerpt, to the pull request.
+ Don't commit `ui-snapshots/`.
+
+Keep iterations short: render the dashboard once, fix what you see, and render it once more to confirm.
diff --git a/README.md b/README.md
index 2be2183..44b9ca9 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,10 @@
strain changes your code in small ways (`<` becomes `<=`, `&&` becomes `||`) and checks whether your tests notice. Every mutant that survives points at a test that runs the code but doesn't really check it.
+
+
+*`strain run --ui live` on `tests/fixtures/sample_project` after the run, with the second survivor selected.*
+
> **Status:** design stage, not yet usable.
> Design doc:
diff --git a/docs/images/dashboard-80x24.png b/docs/images/dashboard-80x24.png
new file mode 100644
index 0000000..68bb7b5
Binary files /dev/null and b/docs/images/dashboard-80x24.png differ
diff --git a/pyproject.toml b/pyproject.toml
index 5c2d856..7f5858f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,6 +12,7 @@ dependencies = []
[project.optional-dependencies]
tree-sitter = ["tree-sitter>=0.26.0", "tree-sitter-cpp>=0.23.4"]
+ui-snapshot = ["pyte>=0.8", "pillow>=10"]
[project.scripts]
strain = "strain.cli:main"
diff --git a/strain/console.py b/strain/console.py
index bd1be5f..43f0002 100644
--- a/strain/console.py
+++ b/strain/console.py
@@ -12,6 +12,7 @@
RESET = "\033[0m"
BOLD = "\033[1m"
DIM = "\033[2m"
+REVERSE = "\033[7m"
RED = "\033[31m"
GREEN = "\033[32m"
YELLOW = "\033[33m"
diff --git a/strain/dashboard.py b/strain/dashboard.py
index c0ba3dd..ea2b54e 100644
--- a/strain/dashboard.py
+++ b/strain/dashboard.py
@@ -18,18 +18,21 @@
GREEN,
RED,
RESET,
+ REVERSE,
STATE_STYLES,
STATE_WIDTH,
+ YELLOW,
Console,
UiMode,
is_terminal,
paint,
plural,
+ score_style,
)
from .coverage import LineCoverage
from .keys import KeyHandler, KeyReader, stdin_key_reader
from .mutators import Mutant, apply_mutant, describe_mutant
-from .report import action_items, hint
+from .report import FileScore, action_items, file_scores, format_score, hint
from .results import CombinedResult, MutantResult, MutantState, combine, mutation_score
ALTERNATE_SCREEN = "\033[?1049h\033[?25l"
@@ -40,6 +43,11 @@
ELLIPSIS = "…"
TAB_WIDTH = 4
SNIPPET_CONTEXT = 3
+SNIPPET_SEPARATOR = "│ "
+CHANGE_STYLES = {"-": RED, "+": GREEN}
+PROGRESS_BAR_WIDTH = 20
+# The per-file scores take at most this share of the survivors view.
+FILES_PANEL_SHARE = 4
BUILD_LOG_LINES = 200
# Builds can print thousands of lines; redrawing for each one would cost more than the build.
BUILD_OUTPUT_REFRESH_SECONDS = 0.05
@@ -53,6 +61,7 @@
DETAIL_LABEL_WIDTH = 10
# How often the finished run checks whether the user pressed q.
QUIT_POLL_SECONDS = 0.1
+EQUIVALENT_HINT = f"equivalent? list it under [ignore] mutants in {CONFIG_FILE}"
TerminalSize = Callable[[], os.terminal_size]
KeyReaderFactory = Callable[[KeyHandler], KeyReader | None]
@@ -79,6 +88,16 @@ def fit(text: str, width: int) -> str:
return f"{''.join(kept)}{ELLIPSIS}{reset}"
+def fit_keeping_end(text: str, end: str, width: int) -> str:
+ """`text` then `end` in exactly `width` visible characters, truncating `text` before `end` loses anything."""
+ if not end:
+ return fit(text, width)
+ end_width = visible_length(end)
+ if end_width >= width:
+ return fit(end, width)
+ return fit(text, width - end_width) + end
+
+
def code_snippet(source: str, mutant: Mutant, context: int = SNIPPET_CONTEXT) -> list[str]:
"""The lines around `mutant`, with its line shown before (`-`) and after (`+`) the mutation."""
lines = source.splitlines()
@@ -99,6 +118,17 @@ def numbered(marker: str, number: int, text: str) -> str:
return snippet
+def token_offset(source: str, mutant: Mutant) -> int:
+ """Where the mutated token starts in the tab-expanded text of its line."""
+ text = source.splitlines()[mutant.line - 1]
+ return len(text[: mutant.column - 1].expandtabs(TAB_WIDTH))
+
+
+def progress_bar(done: int, total: int, width: int = PROGRESS_BAR_WIDTH) -> str:
+ filled = 0 if total <= 0 else min(width, done * width // total)
+ return "█" * filled + "░" * (width - filled)
+
+
def window_start(focus: int, count: int, rows: int) -> int:
"""The first of `rows` consecutive items out of `count` that keeps `focus` in the middle."""
return max(0, min(focus - rows // 2, count - rows))
@@ -152,10 +182,19 @@ def layout(body_rows: int, snippet_rows: int) -> Layout:
return Layout(snippet, queue, findings, rest - queue - findings)
-def survivor_layout(body_rows: int, survivors: int) -> tuple[int, int]:
- """Rows for the survivor list and the details of the selected survivor, titles included."""
- survivor_list = min(body_rows, max(1, survivors) + 1, max(2, body_rows // 3))
- return survivor_list, body_rows - survivor_list
+@dataclass(frozen=True, slots=True)
+class SurvivorLayout:
+ survivor_list: int
+ details: int
+ files: int
+
+
+def survivor_layout(body_rows: int, survivors: int, files: int) -> SurvivorLayout:
+ """Rows for the survivor list, the details of the selected survivor and the per-file scores, titles included."""
+ file_rows = 0 if files == 0 else min(files + 1, body_rows // FILES_PANEL_SHARE)
+ rest = body_rows - file_rows
+ survivor_list = min(rest, max(1, survivors) + 1, max(2, rest // 3))
+ return SurvivorLayout(survivor_list, rest - survivor_list, file_rows)
class Dashboard:
@@ -169,20 +208,25 @@ def __init__(self, color: bool, interactive: bool = False) -> None:
self.contexts: dict[str, MutantContext] = {}
self._combined: list[CombinedResult] | None = None
self.test = ""
+ self.test_name = ""
self.queue: list[QueueEntry] = []
self.current: int | None = None
self.mutant_title = "waiting for the first mutant"
+ self.mutant: Mutant | None = None
+ self.mutant_offset = 0
self.snippet: list[str] = []
self.build_command = ""
self.build_log: collections.deque[str] = collections.deque(maxlen=BUILD_LOG_LINES)
self.status = ""
- self.finished: list[MutantState] = []
def paint(self, text: str, style: str) -> str:
return paint(text, style, self.color)
def set_test(self, name: str, number: int, total: int) -> None:
+ """Starts a test with an empty queue, so the title never pairs it with the previous test's mutants."""
self.test = f"Test {number}/{total}: {name}"
+ self.test_name = name
+ self.set_plan([])
def set_plan(self, descriptions: Sequence[str]) -> None:
self.queue = [QueueEntry(description) for description in descriptions]
@@ -192,12 +236,12 @@ def start_mutant(self, index: int, source: str, mutant: Mutant) -> None:
self.current = index
self.queue[index].state = RUNNING
self.mutant_title = describe_mutant(mutant)
+ self.mutant = mutant
+ self.mutant_offset = token_offset(source, mutant)
self.snippet = code_snippet(source, mutant)
def finish_mutant(self, index: int, state: str) -> None:
self.queue[index].state = state
- if isinstance(state, MutantState):
- self.finished.append(state)
def record_result(self, result: MutantResult, source: str, line: LineCoverage | None) -> None:
self.results.append(result)
@@ -227,7 +271,11 @@ def press(self, key: str) -> None:
self.selected = max(0, min(self.selected_survivor(survivors) + steps[key], len(survivors) - 1))
def finish_run(self) -> None:
+ """Ends the run with every mutant in the queue, in its state combined over all tests."""
self.run_done = True
+ self.test_name = "all tests"
+ self.queue = [QueueEntry(describe_mutant(entry.mutant), entry.state) for entry in self.combined()]
+ self.current = None
if self.survivors():
self.view = SURVIVORS_VIEW
@@ -249,32 +297,60 @@ def key_hints(self) -> str:
hints.append("q quit")
return self.paint(" ".join(hints), DIM)
- def title(self) -> str:
+ def title(self, width: int) -> str:
+ """The run's position on the left and the key hints on the right, so a narrow terminal never hides `q quit`."""
parts = [self.paint("strain live", BOLD + CYAN), self.test]
if self.run_done:
parts.append("finished")
elif self.current is not None:
parts.append(f"mutant {self.current + 1}/{len(self.queue)}")
- parts.append(self.key_hints())
- return " · ".join(part for part in parts if part)
+ return fit_keeping_end(" · ".join(part for part in parts if part) + " ", self.key_hints(), width)
+
+ def progress(self) -> str:
+ done = sum(entry.state not in (PENDING, RUNNING) for entry in self.queue)
+ label = " ".join(part for part in (self.test_name, f"{done}/{len(self.queue)}") if part)
+ return f"{self.paint(progress_bar(done, len(self.queue)), CYAN)} {label}"
+
+ def score(self, score: float | None) -> str:
+ percent = None if score is None else score * 100
+ return self.paint(format_score(score), score_style(percent))
def tally(self) -> str:
+ """The progress of the current test, the score, then the totals over every test, counting each mutant once.
+
+ The score comes before the counts, so a narrow terminal cuts the counts first.
+ """
+ states = [entry.state for entry in self.combined()]
counts = [
- self.paint(f"{state} {self.finished.count(state)}", STATE_STYLES[state])
+ self.paint(f"{state} {states.count(state)}", STATE_STYLES[state])
for state in MutantState
- if state in self.finished
+ if state in states
]
- score = mutation_score(self.finished)
- score_text = "n/a" if score is None else f"{score * 100:.1f}%"
- return " ".join([*counts, f"score {score_text}"])
+ return " ".join([self.progress(), f"score {self.score(mutation_score(states))}", *counts])
def rule(self, title: str, width: int) -> str:
- heading = f"─ {title} "
- return self.paint(fit(heading + "─" * max(0, width - len(heading)), width), BOLD + CYAN)
-
- def snippet_lines(self, snippet: Sequence[str]) -> list[str]:
- styles = {"-": RED, "+": GREEN}
- return [self.paint(line, styles.get(line[0], "")) for line in snippet]
+ filler = "─" * max(0, width - len(f"─ {title} "))
+ return fit(f"{self.paint('─', DIM)} {self.paint(title, BOLD + CYAN)} {self.paint(filler, DIM)}", width)
+
+ def snippet_line(self, line: str, mutant: Mutant | None, offset: int) -> str:
+ """`line` with a dim gutter and, on a changed line, the mutated token highlighted."""
+ gutter, separator, text = line.partition(SNIPPET_SEPARATOR)
+ style = CHANGE_STYLES.get(line[:1], "")
+ if not style:
+ return self.paint(gutter + separator, DIM) + text
+ prefix = self.paint(gutter + separator, style)
+ if mutant is None:
+ return prefix + self.paint(text, style)
+ end = offset + len(mutant.original if line.startswith("-") else mutant.replacement)
+ return (
+ prefix
+ + self.paint(text[:offset], style)
+ + self.paint(text[offset:end], BOLD + REVERSE + style)
+ + self.paint(text[end:], style)
+ )
+
+ def snippet_lines(self, snippet: Sequence[str], mutant: Mutant | None, offset: int) -> list[str]:
+ return [self.snippet_line(line, mutant, offset) for line in snippet]
def queue_line(self, index: int, entry: QueueEntry) -> str:
marker = "▶" if index == self.current else " "
@@ -306,7 +382,7 @@ def findings_lines(self, rows: int) -> list[str]:
items = action_items(self.combined(), limit=None)
if not items:
return [self.paint(" no findings yet", DIM)]
- lines = [f" {item}" for item in items]
+ lines = [f"{self.paint(' •', YELLOW)} {item}" for item in items]
if len(lines) > rows:
return [*lines[: rows - 1], self.paint(f" … {len(lines) - rows + 1} more; tab shows the survivors", DIM)]
return lines
@@ -320,32 +396,63 @@ def survivor_list_lines(self, rows: int) -> list[str]:
selected = self.selected_survivor(survivors)
start = window_start(selected, len(survivors), rows)
return [
- f"{'▶' if index == selected else ' '} {describe_mutant(survivors[index].mutant)}"
+ self.survivor_list_line(survivors[index], index == selected)
for index in range(start, min(len(survivors), start + rows))
]
+ def survivor_list_line(self, entry: CombinedResult, selected: bool) -> str:
+ kind = "" if entry.survivor is None else f" {self.paint(str(entry.survivor), DIM)}"
+ description = describe_mutant(entry.mutant)
+ if selected:
+ return f"{self.paint('▶', BOLD + CYAN)} {self.paint(description, BOLD)}{kind}"
+ return f" {description}{kind}"
+
+ def file_line(self, row: FileScore, width: int) -> str:
+ percent = None if row.score is None else row.score * 100
+ survived_style = STATE_STYLES[MutantState.SURVIVED] if row.survived else DIM
+ return (
+ f" {row.file:<{width}} {self.paint(f'{format_score(row.score):>6}', score_style(percent))}"
+ f" {self.paint(progress_bar(row.detected, row.detected + row.survived), CYAN)}"
+ f" {plural(row.mutants, 'mutant')} {self.paint(f'{row.survived} survived', survived_style)}"
+ )
+
+ def file_lines(self, rows: int) -> list[str]:
+ scores = file_scores(self.combined())
+ if rows <= 0 or not scores:
+ return []
+ width = max(len(row.file) for row in scores)
+ return [self.file_line(row, width) for row in scores[:rows]]
+
def detail(self, label: str, value: str) -> str:
return f"{self.paint(f'{label:<{DETAIL_LABEL_WIDTH}}', DIM)} {value}"
- def survivor_details(self, entry: CombinedResult) -> list[str]:
+ def survivor_facts(self, entry: CombinedResult) -> list[str]:
mutant = entry.mutant
context = self.contexts[mutant.id]
kind = "" if entry.survivor is None else f"{entry.survivor} "
return [
- f"id {mutant.id} {self.paint(f'equivalent? list it under [ignore] mutants in {CONFIG_FILE}', DIM)}",
+ self.detail("id", f"{mutant.id} {self.paint(EQUIVALENT_HINT, DIM)}"),
self.detail("function", entry.function or "unknown"),
self.detail("coverage", execution_summary(mutant.line, context.executions)),
self.detail(str(MutantState.SURVIVED), f"{kind}under {', '.join(entry.covered_by)}"),
self.detail("hint", hint(entry)),
"",
- *self.snippet_lines(code_snippet(context.source, mutant)),
]
+ def survivor_details(self, entry: CombinedResult, rows: int) -> list[str]:
+ """The facts about `entry`, then as much of its source as fits in `rows`."""
+ mutant = entry.mutant
+ source = self.contexts[mutant.id].source
+ facts = self.survivor_facts(entry)
+ context = max(SNIPPET_CONTEXT, (rows - len(facts) - len(CHANGE_STYLES)) // 2)
+ snippet = self.snippet_lines(code_snippet(source, mutant, context), mutant, token_offset(source, mutant))
+ return [*facts, *snippet][:rows]
+
def survivor_detail_lines(self, rows: int) -> list[str]:
survivors = self.survivors()
if rows <= 0 or not survivors:
return []
- return self.survivor_details(survivors[self.selected_survivor(survivors)])[:rows]
+ return self.survivor_details(survivors[self.selected_survivor(survivors)], rows)
def build_log_lines(self, rows: int) -> list[str]:
if rows <= 0:
@@ -365,7 +472,7 @@ def run_panels(self, body_rows: int, width: int) -> list[str]:
*self.panel(
f"Mutant: {self.mutant_title}",
rows.snippet,
- lambda count: self.snippet_lines(self.snippet)[:count],
+ lambda count: self.snippet_lines(self.snippet, self.mutant, self.mutant_offset)[:count],
width,
),
*self.panel("Mutants", rows.queue, self.queue_lines, width),
@@ -375,17 +482,18 @@ def run_panels(self, body_rows: int, width: int) -> list[str]:
def survivor_panels(self, body_rows: int, width: int) -> list[str]:
survivors = self.survivors()
- list_rows, detail_rows = survivor_layout(body_rows, len(survivors))
+ rows = survivor_layout(body_rows, len(survivors), len(file_scores(self.combined())))
detail_title = "Survivor"
if survivors:
detail_title = f"Survivor: {describe_mutant(survivors[self.selected_survivor(survivors)].mutant)}"
return [
- *self.panel(f"Survivors ({len(survivors)})", list_rows, self.survivor_list_lines, width),
- *self.panel(detail_title, detail_rows, self.survivor_detail_lines, width),
+ *self.panel(f"Survivors ({len(survivors)})", rows.survivor_list, self.survivor_list_lines, width),
+ *self.panel(detail_title, rows.details, self.survivor_detail_lines, width),
+ *self.panel("Files", rows.files, self.file_lines, width),
]
def render(self, width: int, height: int) -> list[str]:
- header = [self.title(), self.tally()]
+ header = [self.title(width), self.tally()]
footer = [self.paint(self.status, DIM)]
body_rows = max(0, height - len(header) - len(footer))
panels = self.run_panels if self.view == RUN_VIEW else self.survivor_panels
diff --git a/strain/html_report.py b/strain/html_report.py
new file mode 100644
index 0000000..892b445
--- /dev/null
+++ b/strain/html_report.py
@@ -0,0 +1,170 @@
+"""`index.html`: one self-contained page with the score, a table of files and the source annotated with its mutants."""
+
+from collections.abc import Iterable, Mapping, Sequence
+from html import escape
+
+from .console import HIGH_SCORE, LOW_SCORE
+from .report import TAB_WIDTH, FileScore, file_scores, format_score, hint
+from .results import DETECTED_STATES, CombinedResult, MutantState, mutation_score
+
+KILLED_CLASS = "killed"
+SURVIVED_CLASS = "survived"
+UNSCORED_CLASS = "unscored"
+STYLE = """
+body { font-family: system-ui, sans-serif; margin: 2rem auto; max-width: 75rem; color: #1f2328; padding: 0 1rem; }
+h1 { margin-bottom: 0.25rem; }
+.score { font-size: 2.5rem; font-weight: bold; }
+.high { color: #1a7f37; } .medium { color: #9a6700; } .low { color: #cf222e; } .none { color: #656d76; }
+table { border-collapse: collapse; }
+.files td, .files th { padding: 0.25rem 0.75rem; text-align: right; border-bottom: 1px solid #d0d7de; }
+.files td:first-child, .files th:first-child { text-align: left; }
+.source { width: 100%; font-family: ui-monospace, monospace; font-size: 0.85rem; }
+.source pre { margin: 0; white-space: pre-wrap; }
+.number { color: #656d76; text-align: right; padding-right: 0.75rem; user-select: none; vertical-align: top; }
+.number a { color: inherit; text-decoration: none; }
+.line.killed { background: #dafbe1; } .line.survived { background: #ffebe9; } .line.unscored { background: #f6f8fa; }
+.mutants ul { margin: 0.25rem 0 0.5rem; padding-left: 1.25rem; font-family: system-ui, sans-serif; }
+.state { font-weight: bold; } .killed .state { color: #1a7f37; } .survived .state { color: #cf222e; }
+.unscored .state { color: #656d76; }
+.detail { color: #656d76; }
+:target { outline: 2px solid #0969da; }
+"""
+
+
+def _score_class(score: float | None) -> str:
+ if score is None:
+ return "none"
+ if score * 100 >= HIGH_SCORE:
+ return "high"
+ return "medium" if score * 100 >= LOW_SCORE else "low"
+
+
+def _state_class(state: MutantState) -> str:
+ if state is MutantState.SURVIVED:
+ return SURVIVED_CLASS
+ return KILLED_CLASS if state in DETECTED_STATES else UNSCORED_CLASS
+
+
+def _line_class(entries: Sequence[CombinedResult]) -> str:
+ classes = {_state_class(entry.state) for entry in entries}
+ return next(name for name in (SURVIVED_CLASS, KILLED_CLASS, UNSCORED_CLASS) if name in classes)
+
+
+def _anchor(file: str, line: int) -> str:
+ return escape(f"{file}:{line}")
+
+
+def _change(entry: CombinedResult) -> str:
+ mutant = entry.mutant
+ return f"{escape(mutant.original)} → {escape(mutant.replacement)}"
+
+
+def _explanation(entry: CombinedResult) -> str:
+ if entry.state in DETECTED_STATES:
+ return f"killed by {escape(', '.join(entry.killed_by))}"
+ if entry.state is MutantState.SURVIVED:
+ return f"{escape(hint(entry))}; covered by {escape(', '.join(entry.covered_by))}"
+ return ""
+
+
+def _mutant_item(entry: CombinedResult) -> str:
+ mutant = entry.mutant
+ survivor = "" if entry.survivor is None else f" ({escape(entry.survivor)})"
+ return (
+ f'
{escape(entry.state)}{survivor} '
+ f'{_change(entry)} column {mutant.column} [{escape(mutant.operator)}] id {mutant.id}'
+ f" {_explanation(entry)}"
+ )
+
+
+def _source_rows(file: str, source: str, entries: Iterable[CombinedResult]) -> list[str]:
+ by_line: dict[int, list[CombinedResult]] = {}
+ for entry in entries:
+ by_line.setdefault(entry.mutant.line, []).append(entry)
+ rows: list[str] = []
+ for number, text in enumerate(source.splitlines(), start=1):
+ mutants = by_line.get(number, [])
+ line_class = f"line {_line_class(mutants)}" if mutants else "line"
+ rows.append(
+ f'| '
+ f'{number} | '
+ f"{escape(text.expandtabs(TAB_WIDTH))} |
"
+ )
+ if mutants:
+ items = "".join(_mutant_item(entry) for entry in mutants)
+ rows.append(f' | |
')
+ return rows
+
+
+def _file_row(row: FileScore) -> str:
+ return (
+ f'| {escape(row.file)} | {row.mutants} | '
+ f"{row.detected} | {row.survived} | "
+ f'{format_score(row.score)} |
'
+ )
+
+
+def _files_table(combined: Sequence[CombinedResult]) -> list[str]:
+ return [
+ "Files
",
+ '| File | Mutants | Detected | Survived | Score |
',
+ *(_file_row(row) for row in file_scores(combined)),
+ "
",
+ ]
+
+
+def _survivor_list(combined: Sequence[CombinedResult]) -> list[str]:
+ survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED]
+ if not survivors:
+ return []
+ links = (
+ f''
+ f"{escape(entry.mutant.file)}:{entry.mutant.line}:{entry.mutant.column} {_change(entry)}: "
+ f"{escape(hint(entry))}"
+ for entry in survivors
+ )
+ return [f"Survivors ({len(survivors)})
", ""]
+
+
+def _file_sections(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> list[str]:
+ sections: list[str] = []
+ for row in file_scores(combined):
+ entries = [entry for entry in combined if entry.mutant.file == row.file]
+ sections += [
+ f'{escape(row.file)}
'
+ f"{format_score(row.score)}
",
+ '',
+ *_source_rows(row.file, sources[row.file], entries),
+ "
",
+ ]
+ return sections
+
+
+def _counts(combined: Sequence[CombinedResult]) -> str:
+ states = [entry.state for entry in combined]
+ return ", ".join(f"{escape(state)} {states.count(state)}" for state in MutantState if state in states)
+
+
+def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> str:
+ score = mutation_score(entry.state for entry in combined)
+ body = (
+ [
+ f"{len(combined)} mutants: {_counts(combined)}
",
+ *_files_table(combined),
+ *_survivor_list(combined),
+ *_file_sections(combined, sources),
+ ]
+ if combined
+ else ["No mutants were generated.
"]
+ )
+ page = [
+ "",
+ '',
+ f"strain mutation report: {format_score(score)}",
+ f"",
+ "strain mutation report
",
+ f'{format_score(score)}
',
+ *body,
+ "",
+ ]
+ return "\n".join(page) + "\n"
diff --git a/strain/report.py b/strain/report.py
index ed54e6e..89c735b 100644
--- a/strain/report.py
+++ b/strain/report.py
@@ -2,11 +2,21 @@
import json
from collections.abc import Iterable, Mapping, Sequence
+from dataclasses import dataclass
from pathlib import Path
from types import TracebackType
from typing import Any, Self, TextIO
-from .results import CombinedResult, MutantResult, MutantState, SurvivorKind, SCORED_STATES, mutation_score
+from .mutators import Mutant
+from .results import (
+ DETECTED_STATES,
+ CombinedResult,
+ MutantResult,
+ MutantState,
+ SurvivorKind,
+ SCORED_STATES,
+ mutation_score,
+)
MUTATION_TESTING_ELEMENTS_STATUS = {
MutantState.KILLED: "Killed",
@@ -18,16 +28,18 @@
MutantState.IGNORED: "Ignored",
}
NOT_REACHED_HINT = "add a test input that takes the other branch"
+TAB_WIDTH = 4
+EXCERPT_GUTTER = 5
-def _format_score(score: float | None) -> str:
+def format_score(score: float | None) -> str:
return "n/a" if score is None else f"{score * 100:.1f}%"
def summary_line(test: str, results: Sequence[MutantResult], width: int) -> str:
states = [result.state for result in results]
return (
- f"{test:<{width}} score {_format_score(mutation_score(states))}"
+ f"{test:<{width}} score {format_score(mutation_score(states))}"
f" killed {states.count(MutantState.KILLED)}"
f" sanitizer {states.count(MutantState.KILLED_BY_SANITIZER)}"
f" timeout {states.count(MutantState.TIMEOUT)}"
@@ -51,6 +63,49 @@ def compiler_warning(entry: CombinedResult, source_root: Path) -> str:
)
+def source_excerpt(mutant: Mutant, source: str) -> list[str]:
+ """The mutated line with a caret under the operator and the replacement below it, as GCC shows a fix-it."""
+ lines = source.splitlines()
+ if not 1 <= mutant.line <= len(lines):
+ return []
+ text = lines[mutant.line - 1]
+ indent = " " * len(text[: mutant.column - 1].expandtabs(TAB_WIDTH))
+ gutter = " " * EXCERPT_GUTTER
+ return [
+ f"{mutant.line:>{EXCERPT_GUTTER}} | {text.expandtabs(TAB_WIDTH)}",
+ f"{gutter} | {indent}^{'~' * (len(mutant.original) - 1)}",
+ f"{gutter} | {indent}{mutant.replacement}",
+ ]
+
+
+@dataclass(frozen=True, slots=True)
+class FileScore:
+ file: str
+ mutants: int
+ detected: int
+ survived: int
+
+ @property
+ def score(self) -> float | None:
+ scored = self.detected + self.survived
+ return None if scored == 0 else self.detected / scored
+
+
+def file_scores(combined: Iterable[CombinedResult]) -> list[FileScore]:
+ by_file: dict[str, list[MutantState]] = {}
+ for entry in combined:
+ by_file.setdefault(entry.mutant.file, []).append(entry.state)
+ return [
+ FileScore(
+ file=file,
+ mutants=len(states),
+ detected=sum(state in DETECTED_STATES for state in states),
+ survived=states.count(MutantState.SURVIVED),
+ )
+ for file, states in sorted(by_file.items())
+ ]
+
+
def hint(entry: CombinedResult) -> str:
if entry.survivor is SurvivorKind.NOT_REACHED:
return NOT_REACHED_HINT
@@ -99,6 +154,69 @@ def action_items(combined: Sequence[CombinedResult], limit: int | None) -> list[
return items if limit is None else items[:limit]
+def _markdown_cell(text: str) -> str:
+ return text.replace("|", "\\|")
+
+
+def _markdown_threshold(score: float | None, threshold: float | None) -> str:
+ if threshold is None or score is None:
+ return ""
+ if score * 100 < threshold:
+ return f" ❌ below the {threshold:g}% threshold"
+ return f" ✅ meets the {threshold:g}% threshold"
+
+
+def _markdown_counts(combined: Sequence[CombinedResult]) -> str:
+ states = [entry.state for entry in combined]
+ counts = ", ".join(f"{state} {states.count(state)}" for state in MutantState if state in states)
+ return f"{len(combined)} mutant{'' if len(combined) == 1 else 's'}: {counts}"
+
+
+def _markdown_file_table(combined: Sequence[CombinedResult]) -> list[str]:
+ return [
+ "| File | Mutants | Detected | Survived | Score |",
+ "| --- | ---: | ---: | ---: | ---: |",
+ *(
+ f"| `{_markdown_cell(row.file)}` | {row.mutants} | {row.detected} | {row.survived} | {format_score(row.score)} |"
+ for row in file_scores(combined)
+ ),
+ ]
+
+
+def _markdown_survivors(combined: Sequence[CombinedResult]) -> list[str]:
+ survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED]
+ if not survivors:
+ return []
+ return [
+ "",
+ f"### Survivors ({len(survivors)})",
+ "",
+ *(
+ f"- `{entry.mutant.file}:{entry.mutant.line}:{entry.mutant.column}` "
+ f"{_change(entry)}{_survivor_label(entry)}: {hint(entry)}"
+ for entry in survivors
+ ),
+ ]
+
+
+def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None) -> str:
+ """A summary for CI pages such as `$GITHUB_STEP_SUMMARY` or a pull request comment."""
+ heading = ["## strain mutation testing", ""]
+ if not combined:
+ return "\n".join([*heading, "No mutants were generated."]) + "\n"
+ score = mutation_score(entry.state for entry in combined)
+ lines = [
+ *heading,
+ f"**Mutation score: {format_score(score)}**{_markdown_threshold(score, threshold)}",
+ "",
+ _markdown_counts(combined),
+ "",
+ *_markdown_file_table(combined),
+ *_markdown_survivors(combined),
+ ]
+ return "\n".join(lines) + "\n"
+
+
def _mutant_element(entry: CombinedResult) -> dict[str, Any]:
mutant = entry.mutant
return {
diff --git a/strain/runner.py b/strain/runner.py
index a7740e4..c237f5f 100644
--- a/strain/runner.py
+++ b/strain/runner.py
@@ -2,9 +2,10 @@
import enum
import json
+import os
import shlex
import time
-from collections.abc import Iterator, Sequence
+from collections.abc import Iterator, Mapping, Sequence
from contextlib import contextmanager
from dataclasses import dataclass, field
from pathlib import Path
@@ -24,7 +25,7 @@
)
from .build_cache import BuildFailureCache, cache_path
from .config import CONFIG_FILE, Config, load_config
-from .console import BOLD, DIM, RED, STATE_STYLES, UiMode, plural
+from .console import BOLD, DIM, RED, STATE_STYLES, UiMode, plural, score_style
from .coverage import FileCoverage, GcovCoverage
from .dashboard import create_console
from .journal import Journal
@@ -40,7 +41,19 @@
suppressed_lines,
)
from .operator_sites import EngineError, OperatorSite
-from .report import EventLog, action_items, compiler_warning, mutation_testing_elements, summary_line
+from .html_report import html_report
+from .report import (
+ EventLog,
+ FileScore,
+ action_items,
+ compiler_warning,
+ file_scores,
+ format_score,
+ markdown_summary,
+ mutation_testing_elements,
+ source_excerpt,
+ summary_line,
+)
from .results import (
DETECTED_STATES,
CombinedResult,
@@ -119,6 +132,15 @@ def cmake_source_dir(build_dir: Path) -> Path:
return Path(source_dir).resolve()
+def short_path(path: Path) -> str:
+ """`path` relative to the working directory when that is shorter, for titles that have little room."""
+ try:
+ relative = os.path.relpath(path)
+ except ValueError:
+ return str(path)
+ return min(relative, str(path), key=len)
+
+
@dataclass(frozen=True, slots=True)
class PlannedMutant:
source: SourceFile
@@ -192,7 +214,7 @@ def show_banner(self) -> None:
console.field("reports", str(self.options.report_dir), 7)
def build(self, target: str | None = None) -> bool:
- self.console.build_started(shlex.join(build_command(self.build_dir, target)))
+ self.console.build_started(shlex.join(build_command(Path(short_path(self.build_dir)), target)))
try:
return build(self.build_dir, target, self.console.build_output)
finally:
@@ -539,21 +561,48 @@ def show_mutant(self, progress: str, mutant: Mutant, state: str, detail: str) ->
f" {console.paint(f'{detail} id {mutant.id}', DIM)}"
)
- def write_report(self, combined: Sequence[CombinedResult]) -> Path:
- sources = {
+ def read_sources(self, combined: Sequence[CombinedResult]) -> dict[str, str]:
+ return {
entry.mutant.file: (self.source_dir / entry.mutant.file).read_bytes().decode(SOURCE_ENCODING, SOURCE_ERRORS)
for entry in combined
}
- report_path = self.options.report_dir / "report.json"
- report_path.write_text(json.dumps(mutation_testing_elements(combined, sources), indent=2, ensure_ascii=False))
- return report_path
- def show_survivors(self, combined: Sequence[CombinedResult]) -> None:
+ def write_reports(self, combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> None:
+ report_dir = self.options.report_dir
+ (report_dir / "report.json").write_text(
+ json.dumps(mutation_testing_elements(combined, sources), indent=2, ensure_ascii=False), encoding="utf-8"
+ )
+ (report_dir / "index.html").write_text(html_report(combined, sources), encoding="utf-8")
+ (report_dir / "summary.md").write_text(markdown_summary(combined, self.options.threshold), encoding="utf-8")
+
+ def show_survivors(self, combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> None:
survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED]
if survivors:
self.console.heading(f"Survivors ({len(survivors)})")
for entry in survivors:
- self.console.line(compiler_warning(entry, self.source_dir))
+ self.console.line(self.console.paint(compiler_warning(entry, self.source_dir), BOLD))
+ for line in source_excerpt(entry.mutant, sources[entry.mutant.file]):
+ self.console.line(line)
+
+ def show_files(self, combined: Sequence[CombinedResult]) -> None:
+ rows = file_scores(combined)
+ if not rows:
+ return
+ console = self.console
+ console.heading("Files")
+ width = max(len(row.file) for row in rows)
+ for row in rows:
+ console.line(self.file_line(row, width))
+
+ def file_line(self, row: FileScore, width: int) -> str:
+ console = self.console
+ percent = None if row.score is None else row.score * 100
+ survived_style = STATE_STYLES[MutantState.SURVIVED] if row.survived else DIM
+ return (
+ f" {row.file:<{width}} {console.paint(f'{format_score(row.score):>6}', score_style(percent))}"
+ f" {plural(row.mutants, 'mutant')} {row.detected} detected"
+ f" {console.paint(f'{row.survived} survived', survived_style)}"
+ )
def show_action_items(self, combined: Sequence[CombinedResult]) -> None:
items = action_items(combined, limit=None if self.options.show_all else DEFAULT_ACTION_ITEMS)
@@ -562,7 +611,7 @@ def show_action_items(self, combined: Sequence[CombinedResult]) -> None:
for number, item in enumerate(items, start=1):
self.console.line(f" {number}. {item}")
- def show_summary(self, combined: Sequence[CombinedResult], score: float | None, report_path: Path) -> None:
+ def show_summary(self, combined: Sequence[CombinedResult], score: float | None) -> None:
console = self.console
console.heading("Summary")
states = [entry.state for entry in combined]
@@ -576,8 +625,11 @@ def show_summary(self, combined: Sequence[CombinedResult], score: float | None,
console.line(
f" {console.paint('mutation score', BOLD)} {console.score(percent)}{self.threshold_note(percent)}"
)
- console.field("report", str(report_path), 8)
- console.field("events", str(self.options.report_dir / "events.jsonl"), 8)
+ report_dir = self.options.report_dir
+ console.field("html", str(report_dir / "index.html"), 8)
+ console.field("markdown", str(report_dir / "summary.md"), 8)
+ console.field("report", str(report_dir / "report.json"), 8)
+ console.field("events", str(report_dir / "events.jsonl"), 8)
def threshold_note(self, percent: float | None) -> str:
threshold = self.options.threshold
@@ -725,11 +777,13 @@ def require_mutants(self, combined: Sequence[CombinedResult]) -> None:
def report(self, combined: Sequence[CombinedResult], events: EventLog, ending: str, start: float) -> float | None:
"""Writes the reports and shows the results so far, whether the run `ending` finished or was interrupted."""
score = mutation_score(entry.state for entry in combined)
- report_path = self.write_report(combined)
+ sources = self.read_sources(combined)
+ self.write_reports(combined, sources)
events.emit(ending, score=score)
- self.show_survivors(combined)
+ self.show_survivors(combined, sources)
self.show_action_items(combined)
- self.show_summary(combined, score, report_path)
+ self.show_files(combined)
+ self.show_summary(combined, score)
self.console.line(f"\n{self.console.paint(f'{ending} after {time.monotonic() - start:.1f}s', DIM)}")
return score
diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py
index 6ac05d1..ebfd8b7 100644
--- a/tests/test_dashboard.py
+++ b/tests/test_dashboard.py
@@ -5,7 +5,7 @@
import unittest
from unittest import mock
-from strain.console import UiMode
+from strain.console import BOLD, GREEN, RED, RESET, REVERSE, UiMode, score_style
from strain.dashboard import (
ALTERNATE_SCREEN,
MAIN_SCREEN,
@@ -16,6 +16,8 @@
code_snippet,
create_console,
fit,
+ fit_keeping_end,
+ progress_bar,
)
from strain.coverage import LineCoverage
from strain.mutators import Mutant
@@ -61,6 +63,15 @@ def test_pads_and_truncates_by_visible_width(self) -> None:
self.assertEqual(plain(fit("\033[1mab\033[0m", 4)), "ab ")
+class FitKeepingEndTest(unittest.TestCase):
+ def test_truncates_the_text_before_the_end(self) -> None:
+ self.assertEqual(fit_keeping_end("abcdef", "XY", 6), "abc…XY")
+ self.assertEqual(fit_keeping_end("ab", "XY", 6), "ab XY")
+
+ def test_an_end_wider_than_the_line_is_truncated_itself(self) -> None:
+ self.assertEqual(fit_keeping_end("ab", "WXYZ", 3), "WX…")
+
+
class CodeSnippetTest(unittest.TestCase):
def test_shows_the_original_and_mutated_line_in_context(self) -> None:
self.assertEqual(
@@ -78,6 +89,16 @@ def test_context_is_clipped_at_the_file_edges(self) -> None:
self.assertEqual(code_snippet("+x\ny\n", first, context=2), ["- 1 │ +x", "+ 1 │ -x", " 2 │ y"])
+class ProgressBarTest(unittest.TestCase):
+ def test_fills_in_proportion_to_the_finished_work(self) -> None:
+ self.assertEqual(progress_bar(0, 4, width=8), "░░░░░░░░")
+ self.assertEqual(progress_bar(1, 4, width=8), "██░░░░░░")
+ self.assertEqual(progress_bar(4, 4, width=8), "████████")
+
+ def test_an_empty_plan_shows_an_empty_bar(self) -> None:
+ self.assertEqual(progress_bar(0, 0, width=4), "░░░░")
+
+
class DashboardTest(unittest.TestCase):
def make_dashboard(self, columns: int = 60, lines: int = 30) -> Dashboard:
dashboard = Dashboard(color=False)
@@ -104,16 +125,32 @@ def test_frame_has_every_panel_and_fits_the_terminal(self) -> None:
def test_queue_shows_previous_results_and_upcoming_mutants(self) -> None:
dashboard = self.make_dashboard()
dashboard.start_mutant(0, SOURCE, MUTANT)
+ dashboard.record_result(killed(MUTANT), SOURCE, LINE_COVERAGE)
dashboard.finish_mutant(0, MutantState.KILLED)
- dashboard.start_mutant(1, SOURCE, MUTANT)
+ dashboard.start_mutant(1, SOURCE, SECOND_MUTANT)
+ dashboard.record_result(survived(SECOND_MUTANT), SOURCE, LINE_COVERAGE)
dashboard.finish_mutant(1, MutantState.SURVIVED)
dashboard.start_mutant(2, SOURCE, MUTANT)
- text = "\n".join(dashboard.render(60, 30))
+ text = "\n".join(dashboard.render(80, 30))
self.assertRegex(text, r"killed +src/clamp.cpp:1:1")
self.assertRegex(text, r"survived +src/clamp.cpp:2:1")
self.assertRegex(text, r"▶ running +src/clamp.cpp:3:1")
self.assertRegex(text, r"pending +src/clamp.cpp:4:1")
- self.assertIn("killed 1 survived 1 score 50.0%", text)
+ self.assertIn("score 50.0% killed 1 survived 1", text)
+ self.assertIn("██░░░░░░░░░░░░░░░░░░ sample_tests 2/20", text)
+
+ def test_colors_highlight_the_mutated_token_and_the_score(self) -> None:
+ dashboard = Dashboard(color=True)
+ dashboard.set_plan(["first"])
+ dashboard.start_mutant(0, SOURCE, MUTANT)
+ dashboard.record_result(killed(MUTANT), SOURCE, LINE_COVERAGE)
+ dashboard.finish_mutant(0, MutantState.KILLED)
+ frame = dashboard.render(60, 30)
+ text = "\n".join(frame)
+ self.assertIn(f"{BOLD}{REVERSE}{RED}<{RESET}", text)
+ self.assertIn(f"{BOLD}{REVERSE}{GREEN}<={RESET}", text)
+ self.assertIn(f"score {score_style(100.0)}100.0%{RESET}", text)
+ self.assertTrue(all(len(plain(line)) == 60 for line in frame))
def test_queue_scrolls_to_keep_the_current_mutant_visible(self) -> None:
dashboard = self.make_dashboard()
@@ -178,12 +215,22 @@ def test_survivors_view_shows_the_selected_survivor_with_its_context(self) -> No
text = "\n".join(dashboard.render(120, 40))
self.assertIn("Survivors (1)", text)
self.assertRegex(text, r"▶ src/clamp.cpp:3:11 \[ror\] `<` → `<=`")
- self.assertIn(f"id {MUTANT.id}", text)
+ self.assertIn(f"id {MUTANT.id}", text)
self.assertIn("function clamp(int, int)", text)
self.assertIn("line 3 executed 12 times by sample_tests; 2/2 branches taken", text)
self.assertIn("survived not_detected under sample_tests", text)
self.assertIn("add a test where `<` and `<=` give different results", text)
self.assertIn("+ 3 │ if (a <= b)", text)
+ self.assertRegex(text, r"`<` → `<=` +not_detected")
+
+ def test_survivors_view_shows_the_score_of_each_file(self) -> None:
+ dashboard = self.make_dashboard()
+ dashboard.record_result(survived(MUTANT), SOURCE, LINE_COVERAGE)
+ dashboard.record_result(killed(SECOND_MUTANT), SOURCE, LINE_COVERAGE)
+ dashboard.press("tab")
+ text = "\n".join(dashboard.render(120, 40))
+ self.assertIn("─ Files ─", text)
+ self.assertRegex(text, r"src/clamp.cpp +50.0% +█{10}░{10} +2 mutants +1 survived")
def test_arrow_keys_move_the_selection_and_stay_in_range(self) -> None:
dashboard = self.make_dashboard()
@@ -194,10 +241,10 @@ def test_arrow_keys_move_the_selection_and_stay_in_range(self) -> None:
dashboard.press(key)
text = "\n".join(dashboard.render(120, 40))
self.assertRegex(text, r"▶ src/clamp.cpp:3:11 \[ror\] `<` → `>=`")
- self.assertIn(f"id {SECOND_MUTANT.id}", text)
+ self.assertIn(f"id {SECOND_MUTANT.id}", text)
for key in ("up", "up", "up"):
dashboard.press(key)
- self.assertIn(f"id {MUTANT.id}", "\n".join(dashboard.render(120, 40)))
+ self.assertIn(f"id {MUTANT.id}", "\n".join(dashboard.render(120, 40)))
def test_execution_counts_add_up_over_every_test_that_ran_the_line(self) -> None:
dashboard = self.make_dashboard()
@@ -216,6 +263,33 @@ def test_a_mutant_killed_by_a_later_test_leaves_the_survivors(self) -> None:
self.assertIn("Survivors (0)", text)
self.assertIn("no surviving mutants", text)
+ def test_totals_count_a_mutant_once_over_every_test(self) -> None:
+ dashboard = self.make_dashboard()
+ dashboard.record_result(killed(MUTANT, "sample_tests"), SOURCE, LINE_COVERAGE)
+ dashboard.record_result(survived(MUTANT, "other_tests"), SOURCE, LINE_COVERAGE)
+ dashboard.record_result(survived(SECOND_MUTANT), SOURCE, LINE_COVERAGE)
+ tally = dashboard.render(120, 40)[1]
+ self.assertIn("score 50.0% killed 1 survived 1", tally)
+
+ def test_starting_a_test_drops_the_previous_tests_queue(self) -> None:
+ dashboard = self.make_dashboard()
+ dashboard.start_mutant(1, SOURCE, MUTANT)
+ dashboard.set_test("other_tests", 2, 2)
+ title = dashboard.render(120, 40)[0]
+ self.assertIn("Test 2/2: other_tests", title)
+ self.assertNotIn("mutant 2/2", title)
+
+ def test_survivor_details_are_aligned_and_the_source_fills_the_panel(self) -> None:
+ source = "".join(f"int line{number} = {number};\n" for number in range(1, 40))
+ mutant = Mutant("src/many.cpp", 20, 14, source.index("20;"), "crp", "20", "21")
+ dashboard = self.make_dashboard()
+ dashboard.record_result(survived(mutant), source, LINE_COVERAGE)
+ dashboard.press("tab")
+ text = "\n".join(dashboard.render(120, 40))
+ self.assertIn(f"id {mutant.id}", text)
+ self.assertIn(" 10 │ int line10 = 10;", text)
+ self.assertIn(" 30 │ int line30 = 30;", text)
+
def test_tab_returns_to_the_run_view(self) -> None:
dashboard = self.make_dashboard()
dashboard.press("tab")
@@ -230,6 +304,36 @@ def test_finishing_the_run_opens_the_survivors_and_tells_how_to_quit(self) -> No
self.assertEqual(dashboard.view, SURVIVORS_VIEW)
self.assertIn("q quit", frame[0])
+ def test_a_narrow_title_keeps_the_quit_key(self) -> None:
+ dashboard = self.make_dashboard()
+ dashboard.finish_run()
+ title = dashboard.render(40, 40)[0]
+ self.assertTrue(title.endswith("q quit"), title)
+ self.assertIn("…", title)
+
+ def test_a_narrow_tally_keeps_the_score_and_cuts_the_counts(self) -> None:
+ dashboard = self.make_dashboard()
+ dashboard.record_result(killed(MUTANT, "sample_tests"), SOURCE, LINE_COVERAGE)
+ dashboard.record_result(survived(SECOND_MUTANT), SOURCE, LINE_COVERAGE)
+ tally = dashboard.render(56, 40)[1]
+ self.assertIn("score 50.0%", tally)
+ self.assertNotIn("survived 1", tally)
+
+ def test_the_finished_run_view_lists_every_mutant_once(self) -> None:
+ dashboard = self.make_dashboard()
+ dashboard.record_result(killed(MUTANT, "sample_tests"), SOURCE, LINE_COVERAGE)
+ dashboard.set_test("other_tests", 2, 2)
+ dashboard.set_plan(["only the last test's mutant"])
+ dashboard.record_result(survived(MUTANT, "other_tests"), SOURCE, LINE_COVERAGE)
+ dashboard.record_result(survived(SECOND_MUTANT, "other_tests"), SOURCE, LINE_COVERAGE)
+ dashboard.finish_run()
+ dashboard.press("tab")
+ text = "\n".join(dashboard.render(120, 40))
+ self.assertNotIn("only the last test's mutant", text)
+ self.assertRegex(text, r"killed +src/clamp.cpp:3:11 \[ror\] `<` → `<=`")
+ self.assertRegex(text, r"survived +src/clamp.cpp:3:11 \[ror\] `<` → `>=`")
+ self.assertIn("all tests 2/2", text)
+
def test_key_hints_are_only_shown_when_keys_are_read(self) -> None:
self.assertNotIn("tab", Dashboard(color=False).render(120, 40)[0])
self.assertIn("tab survivors", self.make_dashboard().render(120, 40)[0])
diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py
index 099dfdd..e668dd4 100644
--- a/tests/test_end_to_end.py
+++ b/tests/test_end_to_end.py
@@ -167,6 +167,16 @@ def assert_single_test_log(self, stdout: str) -> None:
self.assertIn("mutation score 40.0% ✗ below --threshold 50%", stdout)
self.assertIn("clamp_to_zero() is pseudo-tested: nothing checks its result", stdout)
self.assertIn(f"{self.root.resolve() / 'src' / 'sample.cpp'}:7:13: warning: survived [ror]", stdout)
+ self.assertIn(" 7 | if (value < 0) { return 0; }\n | ^\n | <=\n", stdout)
+ self.assertIn("▸ Files", stdout)
+ self.assertRegex(stdout, r"src/sample\.cpp\s+40\.0%\s+8 mutants\s+2 detected\s+3 survived")
+ self.assertIn("summary.md", stdout)
+ self.assertIn(
+ "**Mutation score: 40.0%** ❌ below the 50% threshold", (self.report_dir / "summary.md").read_text()
+ )
+ self.assertIn("clamp_to_zero", (self.report_dir / "index.html").read_text())
+ self.assertEqual((self.root / "src" / "sample.cpp").read_bytes(), self.original_source)
+ self.assertTrue((self.report_dir / "events.jsonl").is_file())
def assert_parsing_engine_skips_declarators(self, engine: str) -> None:
configure_project(self.root, self.build_dir)
diff --git a/tests/test_report.py b/tests/test_report.py
index 63eae50..29ac1a2 100644
--- a/tests/test_report.py
+++ b/tests/test_report.py
@@ -5,7 +5,19 @@
from pathlib import Path
from strain.mutators import Mutant
-from strain.report import EventLog, action_items, compiler_warning, mutation_testing_elements, summary_line
+from strain.html_report import html_report
+from strain.report import (
+ NOT_REACHED_HINT,
+ EventLog,
+ FileScore,
+ action_items,
+ compiler_warning,
+ file_scores,
+ markdown_summary,
+ mutation_testing_elements,
+ source_excerpt,
+ summary_line,
+)
from strain.results import MutantResult, MutantState, SurvivorKind, combine
LESS = Mutant(file="src/math.cpp", line=4, column=9, offset=30, operator="ror", original="<", replacement="<=")
@@ -92,6 +104,136 @@ def test_action_items_are_limited(self) -> None:
self.assertEqual(len(action_items(combined, limit=1)), 1)
+SOURCE = (
+ "#include \n"
+ "\n"
+ "int abs(int value)\n"
+ "{\tif (value < 0) { return -value; }\n"
+ " return value;\n"
+ "}\n"
+ "\n"
+ "\n"
+ "int add(int lhs, int rhs) { return lhs + rhs; }\n"
+)
+OTHER = Mutant(file="src/other.cpp", line=1, column=1, offset=0, operator="ror", original="==", replacement="!=")
+
+
+class SourceExcerptTest(unittest.TestCase):
+ def test_caret_points_at_the_operator_and_the_fix_it_shows_the_replacement(self) -> None:
+ mutant = Mutant(
+ file="src/math.cpp", line=4, column=13, offset=0, operator="ror", original="<", replacement="<="
+ )
+ self.assertEqual(
+ source_excerpt(mutant, SOURCE),
+ [
+ " 4 | { if (value < 0) { return -value; }",
+ " | ^",
+ " | <=",
+ ],
+ )
+
+ def test_multi_character_operators_are_underlined(self) -> None:
+ mutant = Mutant(
+ file="src/math.cpp", line=1, column=1, offset=0, operator="lcr", original="&&", replacement="||"
+ )
+ self.assertEqual(source_excerpt(mutant, "&& x\n"), [" 1 | && x", " | ^~", " | ||"])
+
+ def test_lines_past_the_end_of_the_source_give_no_excerpt(self) -> None:
+ self.assertEqual(source_excerpt(AND, "one line\n"), [])
+
+
+class FileScoresTest(unittest.TestCase):
+ def test_scores_are_grouped_by_file(self) -> None:
+ combined = combine(
+ [
+ result(LESS, "t", MutantState.KILLED),
+ result(PLUS, "t", MutantState.SURVIVED),
+ result(AND, "t", MutantState.COMPILE_ERROR),
+ result(OTHER, "t", MutantState.TIMEOUT),
+ ]
+ )
+ self.assertEqual(
+ file_scores(combined),
+ [
+ FileScore(file="src/math.cpp", mutants=3, detected=1, survived=1),
+ FileScore(file="src/other.cpp", mutants=1, detected=1, survived=0),
+ ],
+ )
+ self.assertEqual(file_scores(combined)[0].score, 0.5)
+ self.assertIsNone(FileScore(file="f", mutants=1, detected=0, survived=0).score)
+
+
+class MarkdownSummaryTest(unittest.TestCase):
+ def test_summary_has_score_files_and_survivors(self) -> None:
+ combined = combine(
+ [
+ result(LESS, "t", MutantState.KILLED),
+ result(PLUS, "t", MutantState.SURVIVED, SurvivorKind.NOT_DETECTED),
+ result(OTHER, "t", MutantState.SURVIVED, function=None),
+ ]
+ )
+ self.assertEqual(
+ markdown_summary(combined, threshold=50.0).splitlines(),
+ [
+ "## strain mutation testing",
+ "",
+ "**Mutation score: 33.3%** ❌ below the 50% threshold",
+ "",
+ "3 mutants: killed 1, survived 2",
+ "",
+ "| File | Mutants | Detected | Survived | Score |",
+ "| --- | ---: | ---: | ---: | ---: |",
+ "| `src/math.cpp` | 2 | 1 | 1 | 50.0% |",
+ "| `src/other.cpp` | 1 | 0 | 1 | 0.0% |",
+ "",
+ "### Survivors (2)",
+ "",
+ "- `src/math.cpp:9:12` `+` → `-` (not_detected): assert the computed value",
+ "- `src/other.cpp:1:1` `==` → `!=`: add a test where `==` and `!=` give different results",
+ ],
+ )
+
+ def test_summary_without_survivors_meets_the_threshold(self) -> None:
+ markdown = markdown_summary(combine([result(LESS, "t", MutantState.KILLED)]), threshold=90.0)
+ self.assertIn("**Mutation score: 100.0%** ✅ meets the 90% threshold\n", markdown)
+ self.assertNotIn("Survivors", markdown)
+
+ def test_summary_without_mutants(self) -> None:
+ self.assertIn("No mutants were generated.", markdown_summary([], threshold=None))
+
+ def test_pipes_in_file_names_do_not_break_the_table(self) -> None:
+ pipe = Mutant(file="a|b.cpp", line=1, column=1, offset=0, operator="lcr", original="||", replacement="&&")
+ markdown = markdown_summary(combine([result(pipe, "t", MutantState.SURVIVED)]), threshold=None)
+ self.assertIn("| `a\\|b.cpp` |", markdown)
+
+
+class HtmlReportTest(unittest.TestCase):
+ def test_report_shows_escaped_source_with_its_mutants(self) -> None:
+ combined = combine(
+ [
+ result(LESS, "t1", MutantState.KILLED),
+ result(PLUS, "t1", MutantState.SURVIVED, SurvivorKind.NOT_REACHED),
+ ]
+ )
+ page = html_report(combined, {"src/math.cpp": SOURCE})
+ self.assertTrue(page.startswith(""))
+ self.assertIn("strain mutation report: 50.0%", page)
+ self.assertIn("#include <cmath>", page)
+ self.assertIn('id="src/math.cpp:4"', page)
+ self.assertIn('class="line survived"', page)
+ self.assertIn('class="line killed"', page)
+ self.assertIn("< → <=", page)
+ self.assertIn("killed by t1", page)
+ self.assertIn(NOT_REACHED_HINT, page)
+ self.assertIn('href="#src/math.cpp:9"', page)
+ self.assertNotIn("