diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..128eae6 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,37 @@ +# Working on strain + +Build and test: `pip install -e . && pytest` + +## Changing the `--ui live` dashboard + +Unit tests check strings in the frame. They can't tell you whether the layout reads well, so +look at the dashboard before you commit any change to `strain/dashboard.py`, `strain/console.py` +or `strain/keys.py`: + +1. `pip install -e '.[ui-snapshot]'` (adds `pyte` and `pillow`). +2. Configure a copy of the fixture as a git repository with a coverage build (strain refuses a folder + without `CMakeCache.txt`), using the flags from `configure_project` in `tests/test_end_to_end.py`: + ```sh + demo=build/ui-demo && rm -rf $demo && mkdir -p $demo && cp -r tests/fixtures/sample_project $demo/project + git -C $demo/project init -q && git -C $demo/project add . && git -C $demo/project -c user.name=strain -c user.email=strain@example.com commit -qm fixture + cmake -S $demo/project -B $demo/build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + "-DCMAKE_CXX_FLAGS=--coverage -fsanitize=address,undefined -fno-omit-frame-pointer -D_GLIBCXX_ASSERTIONS -ftrivial-auto-var-init=pattern" \ + "-DCMAKE_EXE_LINKER_FLAGS=--coverage -fsanitize=address,undefined" + cmake --build $demo/build + python tools/ui_snapshot.py $demo/project --size 120x36 --keys down,tab,q --out ui-snapshots \ + -- --build-dir $PWD/$demo/build --report-dir $PWD/$demo/report + ``` + - This needs the fixture's toolchain (CMake, a C++ compiler, gcov) and takes about a minute. + - Arguments after `--` go to `strain run`. + - Also try a small size such as `--size 80x24` to check truncation and panel sizing. +3. Read the `ui-snapshots/*.txt` grids to check alignment, and view the `*.png` frames to check + colours and reverse video. `pyte` drops SGR 2 (faint), so `DIM` text looks like normal text in the PNGs. +4. Check these points, which have regressed before: + - The header totals match the survivors view, so each mutant is counted once across tests. + - The progress bar and the title name the same test as the queue. + - Each survivor detail row is padded to `DETAIL_LABEL_WIDTH`. + - No panel leaves large blank areas while another panel is truncated. +5. Attach before and after frames, or the relevant `.txt` excerpt, to the pull request. + Don't commit `ui-snapshots/`. + +Keep iterations short: render the dashboard once, fix what you see, and render it once more to confirm. diff --git a/README.md b/README.md index 2be2183..44b9ca9 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ strain changes your code in small ways (`<` becomes `<=`, `&&` becomes `||`) and checks whether your tests notice. Every mutant that survives points at a test that runs the code but doesn't really check it. +![The strain run --ui live dashboard in an 80x24 terminal, listing three survivors and showing the selected one as a diff with a hint](docs/images/dashboard-80x24.png) + +*`strain run --ui live` on `tests/fixtures/sample_project` after the run, with the second survivor selected.* + > **Status:** design stage, not yet usable. > Design doc: diff --git a/docs/images/dashboard-80x24.png b/docs/images/dashboard-80x24.png new file mode 100644 index 0000000..68bb7b5 Binary files /dev/null and b/docs/images/dashboard-80x24.png differ diff --git a/pyproject.toml b/pyproject.toml index 5c2d856..7f5858f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [] [project.optional-dependencies] tree-sitter = ["tree-sitter>=0.26.0", "tree-sitter-cpp>=0.23.4"] +ui-snapshot = ["pyte>=0.8", "pillow>=10"] [project.scripts] strain = "strain.cli:main" diff --git a/strain/console.py b/strain/console.py index bd1be5f..43f0002 100644 --- a/strain/console.py +++ b/strain/console.py @@ -12,6 +12,7 @@ RESET = "\033[0m" BOLD = "\033[1m" DIM = "\033[2m" +REVERSE = "\033[7m" RED = "\033[31m" GREEN = "\033[32m" YELLOW = "\033[33m" diff --git a/strain/dashboard.py b/strain/dashboard.py index c0ba3dd..ea2b54e 100644 --- a/strain/dashboard.py +++ b/strain/dashboard.py @@ -18,18 +18,21 @@ GREEN, RED, RESET, + REVERSE, STATE_STYLES, STATE_WIDTH, + YELLOW, Console, UiMode, is_terminal, paint, plural, + score_style, ) from .coverage import LineCoverage from .keys import KeyHandler, KeyReader, stdin_key_reader from .mutators import Mutant, apply_mutant, describe_mutant -from .report import action_items, hint +from .report import FileScore, action_items, file_scores, format_score, hint from .results import CombinedResult, MutantResult, MutantState, combine, mutation_score ALTERNATE_SCREEN = "\033[?1049h\033[?25l" @@ -40,6 +43,11 @@ ELLIPSIS = "…" TAB_WIDTH = 4 SNIPPET_CONTEXT = 3 +SNIPPET_SEPARATOR = "│ " +CHANGE_STYLES = {"-": RED, "+": GREEN} +PROGRESS_BAR_WIDTH = 20 +# The per-file scores take at most this share of the survivors view. +FILES_PANEL_SHARE = 4 BUILD_LOG_LINES = 200 # Builds can print thousands of lines; redrawing for each one would cost more than the build. BUILD_OUTPUT_REFRESH_SECONDS = 0.05 @@ -53,6 +61,7 @@ DETAIL_LABEL_WIDTH = 10 # How often the finished run checks whether the user pressed q. QUIT_POLL_SECONDS = 0.1 +EQUIVALENT_HINT = f"equivalent? list it under [ignore] mutants in {CONFIG_FILE}" TerminalSize = Callable[[], os.terminal_size] KeyReaderFactory = Callable[[KeyHandler], KeyReader | None] @@ -79,6 +88,16 @@ def fit(text: str, width: int) -> str: return f"{''.join(kept)}{ELLIPSIS}{reset}" +def fit_keeping_end(text: str, end: str, width: int) -> str: + """`text` then `end` in exactly `width` visible characters, truncating `text` before `end` loses anything.""" + if not end: + return fit(text, width) + end_width = visible_length(end) + if end_width >= width: + return fit(end, width) + return fit(text, width - end_width) + end + + def code_snippet(source: str, mutant: Mutant, context: int = SNIPPET_CONTEXT) -> list[str]: """The lines around `mutant`, with its line shown before (`-`) and after (`+`) the mutation.""" lines = source.splitlines() @@ -99,6 +118,17 @@ def numbered(marker: str, number: int, text: str) -> str: return snippet +def token_offset(source: str, mutant: Mutant) -> int: + """Where the mutated token starts in the tab-expanded text of its line.""" + text = source.splitlines()[mutant.line - 1] + return len(text[: mutant.column - 1].expandtabs(TAB_WIDTH)) + + +def progress_bar(done: int, total: int, width: int = PROGRESS_BAR_WIDTH) -> str: + filled = 0 if total <= 0 else min(width, done * width // total) + return "█" * filled + "░" * (width - filled) + + def window_start(focus: int, count: int, rows: int) -> int: """The first of `rows` consecutive items out of `count` that keeps `focus` in the middle.""" return max(0, min(focus - rows // 2, count - rows)) @@ -152,10 +182,19 @@ def layout(body_rows: int, snippet_rows: int) -> Layout: return Layout(snippet, queue, findings, rest - queue - findings) -def survivor_layout(body_rows: int, survivors: int) -> tuple[int, int]: - """Rows for the survivor list and the details of the selected survivor, titles included.""" - survivor_list = min(body_rows, max(1, survivors) + 1, max(2, body_rows // 3)) - return survivor_list, body_rows - survivor_list +@dataclass(frozen=True, slots=True) +class SurvivorLayout: + survivor_list: int + details: int + files: int + + +def survivor_layout(body_rows: int, survivors: int, files: int) -> SurvivorLayout: + """Rows for the survivor list, the details of the selected survivor and the per-file scores, titles included.""" + file_rows = 0 if files == 0 else min(files + 1, body_rows // FILES_PANEL_SHARE) + rest = body_rows - file_rows + survivor_list = min(rest, max(1, survivors) + 1, max(2, rest // 3)) + return SurvivorLayout(survivor_list, rest - survivor_list, file_rows) class Dashboard: @@ -169,20 +208,25 @@ def __init__(self, color: bool, interactive: bool = False) -> None: self.contexts: dict[str, MutantContext] = {} self._combined: list[CombinedResult] | None = None self.test = "" + self.test_name = "" self.queue: list[QueueEntry] = [] self.current: int | None = None self.mutant_title = "waiting for the first mutant" + self.mutant: Mutant | None = None + self.mutant_offset = 0 self.snippet: list[str] = [] self.build_command = "" self.build_log: collections.deque[str] = collections.deque(maxlen=BUILD_LOG_LINES) self.status = "" - self.finished: list[MutantState] = [] def paint(self, text: str, style: str) -> str: return paint(text, style, self.color) def set_test(self, name: str, number: int, total: int) -> None: + """Starts a test with an empty queue, so the title never pairs it with the previous test's mutants.""" self.test = f"Test {number}/{total}: {name}" + self.test_name = name + self.set_plan([]) def set_plan(self, descriptions: Sequence[str]) -> None: self.queue = [QueueEntry(description) for description in descriptions] @@ -192,12 +236,12 @@ def start_mutant(self, index: int, source: str, mutant: Mutant) -> None: self.current = index self.queue[index].state = RUNNING self.mutant_title = describe_mutant(mutant) + self.mutant = mutant + self.mutant_offset = token_offset(source, mutant) self.snippet = code_snippet(source, mutant) def finish_mutant(self, index: int, state: str) -> None: self.queue[index].state = state - if isinstance(state, MutantState): - self.finished.append(state) def record_result(self, result: MutantResult, source: str, line: LineCoverage | None) -> None: self.results.append(result) @@ -227,7 +271,11 @@ def press(self, key: str) -> None: self.selected = max(0, min(self.selected_survivor(survivors) + steps[key], len(survivors) - 1)) def finish_run(self) -> None: + """Ends the run with every mutant in the queue, in its state combined over all tests.""" self.run_done = True + self.test_name = "all tests" + self.queue = [QueueEntry(describe_mutant(entry.mutant), entry.state) for entry in self.combined()] + self.current = None if self.survivors(): self.view = SURVIVORS_VIEW @@ -249,32 +297,60 @@ def key_hints(self) -> str: hints.append("q quit") return self.paint(" ".join(hints), DIM) - def title(self) -> str: + def title(self, width: int) -> str: + """The run's position on the left and the key hints on the right, so a narrow terminal never hides `q quit`.""" parts = [self.paint("strain live", BOLD + CYAN), self.test] if self.run_done: parts.append("finished") elif self.current is not None: parts.append(f"mutant {self.current + 1}/{len(self.queue)}") - parts.append(self.key_hints()) - return " · ".join(part for part in parts if part) + return fit_keeping_end(" · ".join(part for part in parts if part) + " ", self.key_hints(), width) + + def progress(self) -> str: + done = sum(entry.state not in (PENDING, RUNNING) for entry in self.queue) + label = " ".join(part for part in (self.test_name, f"{done}/{len(self.queue)}") if part) + return f"{self.paint(progress_bar(done, len(self.queue)), CYAN)} {label}" + + def score(self, score: float | None) -> str: + percent = None if score is None else score * 100 + return self.paint(format_score(score), score_style(percent)) def tally(self) -> str: + """The progress of the current test, the score, then the totals over every test, counting each mutant once. + + The score comes before the counts, so a narrow terminal cuts the counts first. + """ + states = [entry.state for entry in self.combined()] counts = [ - self.paint(f"{state} {self.finished.count(state)}", STATE_STYLES[state]) + self.paint(f"{state} {states.count(state)}", STATE_STYLES[state]) for state in MutantState - if state in self.finished + if state in states ] - score = mutation_score(self.finished) - score_text = "n/a" if score is None else f"{score * 100:.1f}%" - return " ".join([*counts, f"score {score_text}"]) + return " ".join([self.progress(), f"score {self.score(mutation_score(states))}", *counts]) def rule(self, title: str, width: int) -> str: - heading = f"─ {title} " - return self.paint(fit(heading + "─" * max(0, width - len(heading)), width), BOLD + CYAN) - - def snippet_lines(self, snippet: Sequence[str]) -> list[str]: - styles = {"-": RED, "+": GREEN} - return [self.paint(line, styles.get(line[0], "")) for line in snippet] + filler = "─" * max(0, width - len(f"─ {title} ")) + return fit(f"{self.paint('─', DIM)} {self.paint(title, BOLD + CYAN)} {self.paint(filler, DIM)}", width) + + def snippet_line(self, line: str, mutant: Mutant | None, offset: int) -> str: + """`line` with a dim gutter and, on a changed line, the mutated token highlighted.""" + gutter, separator, text = line.partition(SNIPPET_SEPARATOR) + style = CHANGE_STYLES.get(line[:1], "") + if not style: + return self.paint(gutter + separator, DIM) + text + prefix = self.paint(gutter + separator, style) + if mutant is None: + return prefix + self.paint(text, style) + end = offset + len(mutant.original if line.startswith("-") else mutant.replacement) + return ( + prefix + + self.paint(text[:offset], style) + + self.paint(text[offset:end], BOLD + REVERSE + style) + + self.paint(text[end:], style) + ) + + def snippet_lines(self, snippet: Sequence[str], mutant: Mutant | None, offset: int) -> list[str]: + return [self.snippet_line(line, mutant, offset) for line in snippet] def queue_line(self, index: int, entry: QueueEntry) -> str: marker = "▶" if index == self.current else " " @@ -306,7 +382,7 @@ def findings_lines(self, rows: int) -> list[str]: items = action_items(self.combined(), limit=None) if not items: return [self.paint(" no findings yet", DIM)] - lines = [f" {item}" for item in items] + lines = [f"{self.paint(' •', YELLOW)} {item}" for item in items] if len(lines) > rows: return [*lines[: rows - 1], self.paint(f" … {len(lines) - rows + 1} more; tab shows the survivors", DIM)] return lines @@ -320,32 +396,63 @@ def survivor_list_lines(self, rows: int) -> list[str]: selected = self.selected_survivor(survivors) start = window_start(selected, len(survivors), rows) return [ - f"{'▶' if index == selected else ' '} {describe_mutant(survivors[index].mutant)}" + self.survivor_list_line(survivors[index], index == selected) for index in range(start, min(len(survivors), start + rows)) ] + def survivor_list_line(self, entry: CombinedResult, selected: bool) -> str: + kind = "" if entry.survivor is None else f" {self.paint(str(entry.survivor), DIM)}" + description = describe_mutant(entry.mutant) + if selected: + return f"{self.paint('▶', BOLD + CYAN)} {self.paint(description, BOLD)}{kind}" + return f" {description}{kind}" + + def file_line(self, row: FileScore, width: int) -> str: + percent = None if row.score is None else row.score * 100 + survived_style = STATE_STYLES[MutantState.SURVIVED] if row.survived else DIM + return ( + f" {row.file:<{width}} {self.paint(f'{format_score(row.score):>6}', score_style(percent))}" + f" {self.paint(progress_bar(row.detected, row.detected + row.survived), CYAN)}" + f" {plural(row.mutants, 'mutant')} {self.paint(f'{row.survived} survived', survived_style)}" + ) + + def file_lines(self, rows: int) -> list[str]: + scores = file_scores(self.combined()) + if rows <= 0 or not scores: + return [] + width = max(len(row.file) for row in scores) + return [self.file_line(row, width) for row in scores[:rows]] + def detail(self, label: str, value: str) -> str: return f"{self.paint(f'{label:<{DETAIL_LABEL_WIDTH}}', DIM)} {value}" - def survivor_details(self, entry: CombinedResult) -> list[str]: + def survivor_facts(self, entry: CombinedResult) -> list[str]: mutant = entry.mutant context = self.contexts[mutant.id] kind = "" if entry.survivor is None else f"{entry.survivor} " return [ - f"id {mutant.id} {self.paint(f'equivalent? list it under [ignore] mutants in {CONFIG_FILE}', DIM)}", + self.detail("id", f"{mutant.id} {self.paint(EQUIVALENT_HINT, DIM)}"), self.detail("function", entry.function or "unknown"), self.detail("coverage", execution_summary(mutant.line, context.executions)), self.detail(str(MutantState.SURVIVED), f"{kind}under {', '.join(entry.covered_by)}"), self.detail("hint", hint(entry)), "", - *self.snippet_lines(code_snippet(context.source, mutant)), ] + def survivor_details(self, entry: CombinedResult, rows: int) -> list[str]: + """The facts about `entry`, then as much of its source as fits in `rows`.""" + mutant = entry.mutant + source = self.contexts[mutant.id].source + facts = self.survivor_facts(entry) + context = max(SNIPPET_CONTEXT, (rows - len(facts) - len(CHANGE_STYLES)) // 2) + snippet = self.snippet_lines(code_snippet(source, mutant, context), mutant, token_offset(source, mutant)) + return [*facts, *snippet][:rows] + def survivor_detail_lines(self, rows: int) -> list[str]: survivors = self.survivors() if rows <= 0 or not survivors: return [] - return self.survivor_details(survivors[self.selected_survivor(survivors)])[:rows] + return self.survivor_details(survivors[self.selected_survivor(survivors)], rows) def build_log_lines(self, rows: int) -> list[str]: if rows <= 0: @@ -365,7 +472,7 @@ def run_panels(self, body_rows: int, width: int) -> list[str]: *self.panel( f"Mutant: {self.mutant_title}", rows.snippet, - lambda count: self.snippet_lines(self.snippet)[:count], + lambda count: self.snippet_lines(self.snippet, self.mutant, self.mutant_offset)[:count], width, ), *self.panel("Mutants", rows.queue, self.queue_lines, width), @@ -375,17 +482,18 @@ def run_panels(self, body_rows: int, width: int) -> list[str]: def survivor_panels(self, body_rows: int, width: int) -> list[str]: survivors = self.survivors() - list_rows, detail_rows = survivor_layout(body_rows, len(survivors)) + rows = survivor_layout(body_rows, len(survivors), len(file_scores(self.combined()))) detail_title = "Survivor" if survivors: detail_title = f"Survivor: {describe_mutant(survivors[self.selected_survivor(survivors)].mutant)}" return [ - *self.panel(f"Survivors ({len(survivors)})", list_rows, self.survivor_list_lines, width), - *self.panel(detail_title, detail_rows, self.survivor_detail_lines, width), + *self.panel(f"Survivors ({len(survivors)})", rows.survivor_list, self.survivor_list_lines, width), + *self.panel(detail_title, rows.details, self.survivor_detail_lines, width), + *self.panel("Files", rows.files, self.file_lines, width), ] def render(self, width: int, height: int) -> list[str]: - header = [self.title(), self.tally()] + header = [self.title(width), self.tally()] footer = [self.paint(self.status, DIM)] body_rows = max(0, height - len(header) - len(footer)) panels = self.run_panels if self.view == RUN_VIEW else self.survivor_panels diff --git a/strain/html_report.py b/strain/html_report.py new file mode 100644 index 0000000..892b445 --- /dev/null +++ b/strain/html_report.py @@ -0,0 +1,170 @@ +"""`index.html`: one self-contained page with the score, a table of files and the source annotated with its mutants.""" + +from collections.abc import Iterable, Mapping, Sequence +from html import escape + +from .console import HIGH_SCORE, LOW_SCORE +from .report import TAB_WIDTH, FileScore, file_scores, format_score, hint +from .results import DETECTED_STATES, CombinedResult, MutantState, mutation_score + +KILLED_CLASS = "killed" +SURVIVED_CLASS = "survived" +UNSCORED_CLASS = "unscored" +STYLE = """ +body { font-family: system-ui, sans-serif; margin: 2rem auto; max-width: 75rem; color: #1f2328; padding: 0 1rem; } +h1 { margin-bottom: 0.25rem; } +.score { font-size: 2.5rem; font-weight: bold; } +.high { color: #1a7f37; } .medium { color: #9a6700; } .low { color: #cf222e; } .none { color: #656d76; } +table { border-collapse: collapse; } +.files td, .files th { padding: 0.25rem 0.75rem; text-align: right; border-bottom: 1px solid #d0d7de; } +.files td:first-child, .files th:first-child { text-align: left; } +.source { width: 100%; font-family: ui-monospace, monospace; font-size: 0.85rem; } +.source pre { margin: 0; white-space: pre-wrap; } +.number { color: #656d76; text-align: right; padding-right: 0.75rem; user-select: none; vertical-align: top; } +.number a { color: inherit; text-decoration: none; } +.line.killed { background: #dafbe1; } .line.survived { background: #ffebe9; } .line.unscored { background: #f6f8fa; } +.mutants ul { margin: 0.25rem 0 0.5rem; padding-left: 1.25rem; font-family: system-ui, sans-serif; } +.state { font-weight: bold; } .killed .state { color: #1a7f37; } .survived .state { color: #cf222e; } +.unscored .state { color: #656d76; } +.detail { color: #656d76; } +:target { outline: 2px solid #0969da; } +""" + + +def _score_class(score: float | None) -> str: + if score is None: + return "none" + if score * 100 >= HIGH_SCORE: + return "high" + return "medium" if score * 100 >= LOW_SCORE else "low" + + +def _state_class(state: MutantState) -> str: + if state is MutantState.SURVIVED: + return SURVIVED_CLASS + return KILLED_CLASS if state in DETECTED_STATES else UNSCORED_CLASS + + +def _line_class(entries: Sequence[CombinedResult]) -> str: + classes = {_state_class(entry.state) for entry in entries} + return next(name for name in (SURVIVED_CLASS, KILLED_CLASS, UNSCORED_CLASS) if name in classes) + + +def _anchor(file: str, line: int) -> str: + return escape(f"{file}:{line}") + + +def _change(entry: CombinedResult) -> str: + mutant = entry.mutant + return f"{escape(mutant.original)} → {escape(mutant.replacement)}" + + +def _explanation(entry: CombinedResult) -> str: + if entry.state in DETECTED_STATES: + return f"killed by {escape(', '.join(entry.killed_by))}" + if entry.state is MutantState.SURVIVED: + return f"{escape(hint(entry))}; covered by {escape(', '.join(entry.covered_by))}" + return "" + + +def _mutant_item(entry: CombinedResult) -> str: + mutant = entry.mutant + survivor = "" if entry.survivor is None else f" ({escape(entry.survivor)})" + return ( + f'
  • {escape(entry.state)}{survivor} ' + f'{_change(entry)} column {mutant.column} [{escape(mutant.operator)}] id {mutant.id}' + f" {_explanation(entry)}
  • " + ) + + +def _source_rows(file: str, source: str, entries: Iterable[CombinedResult]) -> list[str]: + by_line: dict[int, list[CombinedResult]] = {} + for entry in entries: + by_line.setdefault(entry.mutant.line, []).append(entry) + rows: list[str] = [] + for number, text in enumerate(source.splitlines(), start=1): + mutants = by_line.get(number, []) + line_class = f"line {_line_class(mutants)}" if mutants else "line" + rows.append( + f'' + f'{number}' + f"
    {escape(text.expandtabs(TAB_WIDTH))}
    " + ) + if mutants: + items = "".join(_mutant_item(entry) for entry in mutants) + rows.append(f'') + return rows + + +def _file_row(row: FileScore) -> str: + return ( + f'{escape(row.file)}{row.mutants}' + f"{row.detected}{row.survived}" + f'{format_score(row.score)}' + ) + + +def _files_table(combined: Sequence[CombinedResult]) -> list[str]: + return [ + "

    Files

    ", + '', + *(_file_row(row) for row in file_scores(combined)), + "
    FileMutantsDetectedSurvivedScore
    ", + ] + + +def _survivor_list(combined: Sequence[CombinedResult]) -> list[str]: + survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED] + if not survivors: + return [] + links = ( + f'
  • ' + f"{escape(entry.mutant.file)}:{entry.mutant.line}:{entry.mutant.column} {_change(entry)}: " + f"{escape(hint(entry))}
  • " + for entry in survivors + ) + return [f"

    Survivors ({len(survivors)})

    ", ""] + + +def _file_sections(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> list[str]: + sections: list[str] = [] + for row in file_scores(combined): + entries = [entry for entry in combined if entry.mutant.file == row.file] + sections += [ + f'

    {escape(row.file)}

    ' + f"{format_score(row.score)}
    ", + '', + *_source_rows(row.file, sources[row.file], entries), + "
    ", + ] + return sections + + +def _counts(combined: Sequence[CombinedResult]) -> str: + states = [entry.state for entry in combined] + return ", ".join(f"{escape(state)} {states.count(state)}" for state in MutantState if state in states) + + +def html_report(combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> str: + score = mutation_score(entry.state for entry in combined) + body = ( + [ + f"

    {len(combined)} mutants: {_counts(combined)}

    ", + *_files_table(combined), + *_survivor_list(combined), + *_file_sections(combined, sources), + ] + if combined + else ["

    No mutants were generated.

    "] + ) + page = [ + "", + '', + f"strain mutation report: {format_score(score)}", + f"", + "

    strain mutation report

    ", + f'
    {format_score(score)}
    ', + *body, + "", + ] + return "\n".join(page) + "\n" diff --git a/strain/report.py b/strain/report.py index ed54e6e..89c735b 100644 --- a/strain/report.py +++ b/strain/report.py @@ -2,11 +2,21 @@ import json from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass from pathlib import Path from types import TracebackType from typing import Any, Self, TextIO -from .results import CombinedResult, MutantResult, MutantState, SurvivorKind, SCORED_STATES, mutation_score +from .mutators import Mutant +from .results import ( + DETECTED_STATES, + CombinedResult, + MutantResult, + MutantState, + SurvivorKind, + SCORED_STATES, + mutation_score, +) MUTATION_TESTING_ELEMENTS_STATUS = { MutantState.KILLED: "Killed", @@ -18,16 +28,18 @@ MutantState.IGNORED: "Ignored", } NOT_REACHED_HINT = "add a test input that takes the other branch" +TAB_WIDTH = 4 +EXCERPT_GUTTER = 5 -def _format_score(score: float | None) -> str: +def format_score(score: float | None) -> str: return "n/a" if score is None else f"{score * 100:.1f}%" def summary_line(test: str, results: Sequence[MutantResult], width: int) -> str: states = [result.state for result in results] return ( - f"{test:<{width}} score {_format_score(mutation_score(states))}" + f"{test:<{width}} score {format_score(mutation_score(states))}" f" killed {states.count(MutantState.KILLED)}" f" sanitizer {states.count(MutantState.KILLED_BY_SANITIZER)}" f" timeout {states.count(MutantState.TIMEOUT)}" @@ -51,6 +63,49 @@ def compiler_warning(entry: CombinedResult, source_root: Path) -> str: ) +def source_excerpt(mutant: Mutant, source: str) -> list[str]: + """The mutated line with a caret under the operator and the replacement below it, as GCC shows a fix-it.""" + lines = source.splitlines() + if not 1 <= mutant.line <= len(lines): + return [] + text = lines[mutant.line - 1] + indent = " " * len(text[: mutant.column - 1].expandtabs(TAB_WIDTH)) + gutter = " " * EXCERPT_GUTTER + return [ + f"{mutant.line:>{EXCERPT_GUTTER}} | {text.expandtabs(TAB_WIDTH)}", + f"{gutter} | {indent}^{'~' * (len(mutant.original) - 1)}", + f"{gutter} | {indent}{mutant.replacement}", + ] + + +@dataclass(frozen=True, slots=True) +class FileScore: + file: str + mutants: int + detected: int + survived: int + + @property + def score(self) -> float | None: + scored = self.detected + self.survived + return None if scored == 0 else self.detected / scored + + +def file_scores(combined: Iterable[CombinedResult]) -> list[FileScore]: + by_file: dict[str, list[MutantState]] = {} + for entry in combined: + by_file.setdefault(entry.mutant.file, []).append(entry.state) + return [ + FileScore( + file=file, + mutants=len(states), + detected=sum(state in DETECTED_STATES for state in states), + survived=states.count(MutantState.SURVIVED), + ) + for file, states in sorted(by_file.items()) + ] + + def hint(entry: CombinedResult) -> str: if entry.survivor is SurvivorKind.NOT_REACHED: return NOT_REACHED_HINT @@ -99,6 +154,69 @@ def action_items(combined: Sequence[CombinedResult], limit: int | None) -> list[ return items if limit is None else items[:limit] +def _markdown_cell(text: str) -> str: + return text.replace("|", "\\|") + + +def _markdown_threshold(score: float | None, threshold: float | None) -> str: + if threshold is None or score is None: + return "" + if score * 100 < threshold: + return f" ❌ below the {threshold:g}% threshold" + return f" ✅ meets the {threshold:g}% threshold" + + +def _markdown_counts(combined: Sequence[CombinedResult]) -> str: + states = [entry.state for entry in combined] + counts = ", ".join(f"{state} {states.count(state)}" for state in MutantState if state in states) + return f"{len(combined)} mutant{'' if len(combined) == 1 else 's'}: {counts}" + + +def _markdown_file_table(combined: Sequence[CombinedResult]) -> list[str]: + return [ + "| File | Mutants | Detected | Survived | Score |", + "| --- | ---: | ---: | ---: | ---: |", + *( + f"| `{_markdown_cell(row.file)}` | {row.mutants} | {row.detected} | {row.survived} | {format_score(row.score)} |" + for row in file_scores(combined) + ), + ] + + +def _markdown_survivors(combined: Sequence[CombinedResult]) -> list[str]: + survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED] + if not survivors: + return [] + return [ + "", + f"### Survivors ({len(survivors)})", + "", + *( + f"- `{entry.mutant.file}:{entry.mutant.line}:{entry.mutant.column}` " + f"{_change(entry)}{_survivor_label(entry)}: {hint(entry)}" + for entry in survivors + ), + ] + + +def markdown_summary(combined: Sequence[CombinedResult], threshold: float | None) -> str: + """A summary for CI pages such as `$GITHUB_STEP_SUMMARY` or a pull request comment.""" + heading = ["## strain mutation testing", ""] + if not combined: + return "\n".join([*heading, "No mutants were generated."]) + "\n" + score = mutation_score(entry.state for entry in combined) + lines = [ + *heading, + f"**Mutation score: {format_score(score)}**{_markdown_threshold(score, threshold)}", + "", + _markdown_counts(combined), + "", + *_markdown_file_table(combined), + *_markdown_survivors(combined), + ] + return "\n".join(lines) + "\n" + + def _mutant_element(entry: CombinedResult) -> dict[str, Any]: mutant = entry.mutant return { diff --git a/strain/runner.py b/strain/runner.py index a7740e4..c237f5f 100644 --- a/strain/runner.py +++ b/strain/runner.py @@ -2,9 +2,10 @@ import enum import json +import os import shlex import time -from collections.abc import Iterator, Sequence +from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path @@ -24,7 +25,7 @@ ) from .build_cache import BuildFailureCache, cache_path from .config import CONFIG_FILE, Config, load_config -from .console import BOLD, DIM, RED, STATE_STYLES, UiMode, plural +from .console import BOLD, DIM, RED, STATE_STYLES, UiMode, plural, score_style from .coverage import FileCoverage, GcovCoverage from .dashboard import create_console from .journal import Journal @@ -40,7 +41,19 @@ suppressed_lines, ) from .operator_sites import EngineError, OperatorSite -from .report import EventLog, action_items, compiler_warning, mutation_testing_elements, summary_line +from .html_report import html_report +from .report import ( + EventLog, + FileScore, + action_items, + compiler_warning, + file_scores, + format_score, + markdown_summary, + mutation_testing_elements, + source_excerpt, + summary_line, +) from .results import ( DETECTED_STATES, CombinedResult, @@ -119,6 +132,15 @@ def cmake_source_dir(build_dir: Path) -> Path: return Path(source_dir).resolve() +def short_path(path: Path) -> str: + """`path` relative to the working directory when that is shorter, for titles that have little room.""" + try: + relative = os.path.relpath(path) + except ValueError: + return str(path) + return min(relative, str(path), key=len) + + @dataclass(frozen=True, slots=True) class PlannedMutant: source: SourceFile @@ -192,7 +214,7 @@ def show_banner(self) -> None: console.field("reports", str(self.options.report_dir), 7) def build(self, target: str | None = None) -> bool: - self.console.build_started(shlex.join(build_command(self.build_dir, target))) + self.console.build_started(shlex.join(build_command(Path(short_path(self.build_dir)), target))) try: return build(self.build_dir, target, self.console.build_output) finally: @@ -539,21 +561,48 @@ def show_mutant(self, progress: str, mutant: Mutant, state: str, detail: str) -> f" {console.paint(f'{detail} id {mutant.id}', DIM)}" ) - def write_report(self, combined: Sequence[CombinedResult]) -> Path: - sources = { + def read_sources(self, combined: Sequence[CombinedResult]) -> dict[str, str]: + return { entry.mutant.file: (self.source_dir / entry.mutant.file).read_bytes().decode(SOURCE_ENCODING, SOURCE_ERRORS) for entry in combined } - report_path = self.options.report_dir / "report.json" - report_path.write_text(json.dumps(mutation_testing_elements(combined, sources), indent=2, ensure_ascii=False)) - return report_path - def show_survivors(self, combined: Sequence[CombinedResult]) -> None: + def write_reports(self, combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> None: + report_dir = self.options.report_dir + (report_dir / "report.json").write_text( + json.dumps(mutation_testing_elements(combined, sources), indent=2, ensure_ascii=False), encoding="utf-8" + ) + (report_dir / "index.html").write_text(html_report(combined, sources), encoding="utf-8") + (report_dir / "summary.md").write_text(markdown_summary(combined, self.options.threshold), encoding="utf-8") + + def show_survivors(self, combined: Sequence[CombinedResult], sources: Mapping[str, str]) -> None: survivors = [entry for entry in combined if entry.state is MutantState.SURVIVED] if survivors: self.console.heading(f"Survivors ({len(survivors)})") for entry in survivors: - self.console.line(compiler_warning(entry, self.source_dir)) + self.console.line(self.console.paint(compiler_warning(entry, self.source_dir), BOLD)) + for line in source_excerpt(entry.mutant, sources[entry.mutant.file]): + self.console.line(line) + + def show_files(self, combined: Sequence[CombinedResult]) -> None: + rows = file_scores(combined) + if not rows: + return + console = self.console + console.heading("Files") + width = max(len(row.file) for row in rows) + for row in rows: + console.line(self.file_line(row, width)) + + def file_line(self, row: FileScore, width: int) -> str: + console = self.console + percent = None if row.score is None else row.score * 100 + survived_style = STATE_STYLES[MutantState.SURVIVED] if row.survived else DIM + return ( + f" {row.file:<{width}} {console.paint(f'{format_score(row.score):>6}', score_style(percent))}" + f" {plural(row.mutants, 'mutant')} {row.detected} detected" + f" {console.paint(f'{row.survived} survived', survived_style)}" + ) def show_action_items(self, combined: Sequence[CombinedResult]) -> None: items = action_items(combined, limit=None if self.options.show_all else DEFAULT_ACTION_ITEMS) @@ -562,7 +611,7 @@ def show_action_items(self, combined: Sequence[CombinedResult]) -> None: for number, item in enumerate(items, start=1): self.console.line(f" {number}. {item}") - def show_summary(self, combined: Sequence[CombinedResult], score: float | None, report_path: Path) -> None: + def show_summary(self, combined: Sequence[CombinedResult], score: float | None) -> None: console = self.console console.heading("Summary") states = [entry.state for entry in combined] @@ -576,8 +625,11 @@ def show_summary(self, combined: Sequence[CombinedResult], score: float | None, console.line( f" {console.paint('mutation score', BOLD)} {console.score(percent)}{self.threshold_note(percent)}" ) - console.field("report", str(report_path), 8) - console.field("events", str(self.options.report_dir / "events.jsonl"), 8) + report_dir = self.options.report_dir + console.field("html", str(report_dir / "index.html"), 8) + console.field("markdown", str(report_dir / "summary.md"), 8) + console.field("report", str(report_dir / "report.json"), 8) + console.field("events", str(report_dir / "events.jsonl"), 8) def threshold_note(self, percent: float | None) -> str: threshold = self.options.threshold @@ -725,11 +777,13 @@ def require_mutants(self, combined: Sequence[CombinedResult]) -> None: def report(self, combined: Sequence[CombinedResult], events: EventLog, ending: str, start: float) -> float | None: """Writes the reports and shows the results so far, whether the run `ending` finished or was interrupted.""" score = mutation_score(entry.state for entry in combined) - report_path = self.write_report(combined) + sources = self.read_sources(combined) + self.write_reports(combined, sources) events.emit(ending, score=score) - self.show_survivors(combined) + self.show_survivors(combined, sources) self.show_action_items(combined) - self.show_summary(combined, score, report_path) + self.show_files(combined) + self.show_summary(combined, score) self.console.line(f"\n{self.console.paint(f'{ending} after {time.monotonic() - start:.1f}s', DIM)}") return score diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 6ac05d1..ebfd8b7 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -5,7 +5,7 @@ import unittest from unittest import mock -from strain.console import UiMode +from strain.console import BOLD, GREEN, RED, RESET, REVERSE, UiMode, score_style from strain.dashboard import ( ALTERNATE_SCREEN, MAIN_SCREEN, @@ -16,6 +16,8 @@ code_snippet, create_console, fit, + fit_keeping_end, + progress_bar, ) from strain.coverage import LineCoverage from strain.mutators import Mutant @@ -61,6 +63,15 @@ def test_pads_and_truncates_by_visible_width(self) -> None: self.assertEqual(plain(fit("\033[1mab\033[0m", 4)), "ab ") +class FitKeepingEndTest(unittest.TestCase): + def test_truncates_the_text_before_the_end(self) -> None: + self.assertEqual(fit_keeping_end("abcdef", "XY", 6), "abc…XY") + self.assertEqual(fit_keeping_end("ab", "XY", 6), "ab XY") + + def test_an_end_wider_than_the_line_is_truncated_itself(self) -> None: + self.assertEqual(fit_keeping_end("ab", "WXYZ", 3), "WX…") + + class CodeSnippetTest(unittest.TestCase): def test_shows_the_original_and_mutated_line_in_context(self) -> None: self.assertEqual( @@ -78,6 +89,16 @@ def test_context_is_clipped_at_the_file_edges(self) -> None: self.assertEqual(code_snippet("+x\ny\n", first, context=2), ["- 1 │ +x", "+ 1 │ -x", " 2 │ y"]) +class ProgressBarTest(unittest.TestCase): + def test_fills_in_proportion_to_the_finished_work(self) -> None: + self.assertEqual(progress_bar(0, 4, width=8), "░░░░░░░░") + self.assertEqual(progress_bar(1, 4, width=8), "██░░░░░░") + self.assertEqual(progress_bar(4, 4, width=8), "████████") + + def test_an_empty_plan_shows_an_empty_bar(self) -> None: + self.assertEqual(progress_bar(0, 0, width=4), "░░░░") + + class DashboardTest(unittest.TestCase): def make_dashboard(self, columns: int = 60, lines: int = 30) -> Dashboard: dashboard = Dashboard(color=False) @@ -104,16 +125,32 @@ def test_frame_has_every_panel_and_fits_the_terminal(self) -> None: def test_queue_shows_previous_results_and_upcoming_mutants(self) -> None: dashboard = self.make_dashboard() dashboard.start_mutant(0, SOURCE, MUTANT) + dashboard.record_result(killed(MUTANT), SOURCE, LINE_COVERAGE) dashboard.finish_mutant(0, MutantState.KILLED) - dashboard.start_mutant(1, SOURCE, MUTANT) + dashboard.start_mutant(1, SOURCE, SECOND_MUTANT) + dashboard.record_result(survived(SECOND_MUTANT), SOURCE, LINE_COVERAGE) dashboard.finish_mutant(1, MutantState.SURVIVED) dashboard.start_mutant(2, SOURCE, MUTANT) - text = "\n".join(dashboard.render(60, 30)) + text = "\n".join(dashboard.render(80, 30)) self.assertRegex(text, r"killed +src/clamp.cpp:1:1") self.assertRegex(text, r"survived +src/clamp.cpp:2:1") self.assertRegex(text, r"▶ running +src/clamp.cpp:3:1") self.assertRegex(text, r"pending +src/clamp.cpp:4:1") - self.assertIn("killed 1 survived 1 score 50.0%", text) + self.assertIn("score 50.0% killed 1 survived 1", text) + self.assertIn("██░░░░░░░░░░░░░░░░░░ sample_tests 2/20", text) + + def test_colors_highlight_the_mutated_token_and_the_score(self) -> None: + dashboard = Dashboard(color=True) + dashboard.set_plan(["first"]) + dashboard.start_mutant(0, SOURCE, MUTANT) + dashboard.record_result(killed(MUTANT), SOURCE, LINE_COVERAGE) + dashboard.finish_mutant(0, MutantState.KILLED) + frame = dashboard.render(60, 30) + text = "\n".join(frame) + self.assertIn(f"{BOLD}{REVERSE}{RED}<{RESET}", text) + self.assertIn(f"{BOLD}{REVERSE}{GREEN}<={RESET}", text) + self.assertIn(f"score {score_style(100.0)}100.0%{RESET}", text) + self.assertTrue(all(len(plain(line)) == 60 for line in frame)) def test_queue_scrolls_to_keep_the_current_mutant_visible(self) -> None: dashboard = self.make_dashboard() @@ -178,12 +215,22 @@ def test_survivors_view_shows_the_selected_survivor_with_its_context(self) -> No text = "\n".join(dashboard.render(120, 40)) self.assertIn("Survivors (1)", text) self.assertRegex(text, r"▶ src/clamp.cpp:3:11 \[ror\] `<` → `<=`") - self.assertIn(f"id {MUTANT.id}", text) + self.assertIn(f"id {MUTANT.id}", text) self.assertIn("function clamp(int, int)", text) self.assertIn("line 3 executed 12 times by sample_tests; 2/2 branches taken", text) self.assertIn("survived not_detected under sample_tests", text) self.assertIn("add a test where `<` and `<=` give different results", text) self.assertIn("+ 3 │ if (a <= b)", text) + self.assertRegex(text, r"`<` → `<=` +not_detected") + + def test_survivors_view_shows_the_score_of_each_file(self) -> None: + dashboard = self.make_dashboard() + dashboard.record_result(survived(MUTANT), SOURCE, LINE_COVERAGE) + dashboard.record_result(killed(SECOND_MUTANT), SOURCE, LINE_COVERAGE) + dashboard.press("tab") + text = "\n".join(dashboard.render(120, 40)) + self.assertIn("─ Files ─", text) + self.assertRegex(text, r"src/clamp.cpp +50.0% +█{10}░{10} +2 mutants +1 survived") def test_arrow_keys_move_the_selection_and_stay_in_range(self) -> None: dashboard = self.make_dashboard() @@ -194,10 +241,10 @@ def test_arrow_keys_move_the_selection_and_stay_in_range(self) -> None: dashboard.press(key) text = "\n".join(dashboard.render(120, 40)) self.assertRegex(text, r"▶ src/clamp.cpp:3:11 \[ror\] `<` → `>=`") - self.assertIn(f"id {SECOND_MUTANT.id}", text) + self.assertIn(f"id {SECOND_MUTANT.id}", text) for key in ("up", "up", "up"): dashboard.press(key) - self.assertIn(f"id {MUTANT.id}", "\n".join(dashboard.render(120, 40))) + self.assertIn(f"id {MUTANT.id}", "\n".join(dashboard.render(120, 40))) def test_execution_counts_add_up_over_every_test_that_ran_the_line(self) -> None: dashboard = self.make_dashboard() @@ -216,6 +263,33 @@ def test_a_mutant_killed_by_a_later_test_leaves_the_survivors(self) -> None: self.assertIn("Survivors (0)", text) self.assertIn("no surviving mutants", text) + def test_totals_count_a_mutant_once_over_every_test(self) -> None: + dashboard = self.make_dashboard() + dashboard.record_result(killed(MUTANT, "sample_tests"), SOURCE, LINE_COVERAGE) + dashboard.record_result(survived(MUTANT, "other_tests"), SOURCE, LINE_COVERAGE) + dashboard.record_result(survived(SECOND_MUTANT), SOURCE, LINE_COVERAGE) + tally = dashboard.render(120, 40)[1] + self.assertIn("score 50.0% killed 1 survived 1", tally) + + def test_starting_a_test_drops_the_previous_tests_queue(self) -> None: + dashboard = self.make_dashboard() + dashboard.start_mutant(1, SOURCE, MUTANT) + dashboard.set_test("other_tests", 2, 2) + title = dashboard.render(120, 40)[0] + self.assertIn("Test 2/2: other_tests", title) + self.assertNotIn("mutant 2/2", title) + + def test_survivor_details_are_aligned_and_the_source_fills_the_panel(self) -> None: + source = "".join(f"int line{number} = {number};\n" for number in range(1, 40)) + mutant = Mutant("src/many.cpp", 20, 14, source.index("20;"), "crp", "20", "21") + dashboard = self.make_dashboard() + dashboard.record_result(survived(mutant), source, LINE_COVERAGE) + dashboard.press("tab") + text = "\n".join(dashboard.render(120, 40)) + self.assertIn(f"id {mutant.id}", text) + self.assertIn(" 10 │ int line10 = 10;", text) + self.assertIn(" 30 │ int line30 = 30;", text) + def test_tab_returns_to_the_run_view(self) -> None: dashboard = self.make_dashboard() dashboard.press("tab") @@ -230,6 +304,36 @@ def test_finishing_the_run_opens_the_survivors_and_tells_how_to_quit(self) -> No self.assertEqual(dashboard.view, SURVIVORS_VIEW) self.assertIn("q quit", frame[0]) + def test_a_narrow_title_keeps_the_quit_key(self) -> None: + dashboard = self.make_dashboard() + dashboard.finish_run() + title = dashboard.render(40, 40)[0] + self.assertTrue(title.endswith("q quit"), title) + self.assertIn("…", title) + + def test_a_narrow_tally_keeps_the_score_and_cuts_the_counts(self) -> None: + dashboard = self.make_dashboard() + dashboard.record_result(killed(MUTANT, "sample_tests"), SOURCE, LINE_COVERAGE) + dashboard.record_result(survived(SECOND_MUTANT), SOURCE, LINE_COVERAGE) + tally = dashboard.render(56, 40)[1] + self.assertIn("score 50.0%", tally) + self.assertNotIn("survived 1", tally) + + def test_the_finished_run_view_lists_every_mutant_once(self) -> None: + dashboard = self.make_dashboard() + dashboard.record_result(killed(MUTANT, "sample_tests"), SOURCE, LINE_COVERAGE) + dashboard.set_test("other_tests", 2, 2) + dashboard.set_plan(["only the last test's mutant"]) + dashboard.record_result(survived(MUTANT, "other_tests"), SOURCE, LINE_COVERAGE) + dashboard.record_result(survived(SECOND_MUTANT, "other_tests"), SOURCE, LINE_COVERAGE) + dashboard.finish_run() + dashboard.press("tab") + text = "\n".join(dashboard.render(120, 40)) + self.assertNotIn("only the last test's mutant", text) + self.assertRegex(text, r"killed +src/clamp.cpp:3:11 \[ror\] `<` → `<=`") + self.assertRegex(text, r"survived +src/clamp.cpp:3:11 \[ror\] `<` → `>=`") + self.assertIn("all tests 2/2", text) + def test_key_hints_are_only_shown_when_keys_are_read(self) -> None: self.assertNotIn("tab", Dashboard(color=False).render(120, 40)[0]) self.assertIn("tab survivors", self.make_dashboard().render(120, 40)[0]) diff --git a/tests/test_end_to_end.py b/tests/test_end_to_end.py index 099dfdd..e668dd4 100644 --- a/tests/test_end_to_end.py +++ b/tests/test_end_to_end.py @@ -167,6 +167,16 @@ def assert_single_test_log(self, stdout: str) -> None: self.assertIn("mutation score 40.0% ✗ below --threshold 50%", stdout) self.assertIn("clamp_to_zero() is pseudo-tested: nothing checks its result", stdout) self.assertIn(f"{self.root.resolve() / 'src' / 'sample.cpp'}:7:13: warning: survived [ror]", stdout) + self.assertIn(" 7 | if (value < 0) { return 0; }\n | ^\n | <=\n", stdout) + self.assertIn("▸ Files", stdout) + self.assertRegex(stdout, r"src/sample\.cpp\s+40\.0%\s+8 mutants\s+2 detected\s+3 survived") + self.assertIn("summary.md", stdout) + self.assertIn( + "**Mutation score: 40.0%** ❌ below the 50% threshold", (self.report_dir / "summary.md").read_text() + ) + self.assertIn("clamp_to_zero", (self.report_dir / "index.html").read_text()) + self.assertEqual((self.root / "src" / "sample.cpp").read_bytes(), self.original_source) + self.assertTrue((self.report_dir / "events.jsonl").is_file()) def assert_parsing_engine_skips_declarators(self, engine: str) -> None: configure_project(self.root, self.build_dir) diff --git a/tests/test_report.py b/tests/test_report.py index 63eae50..29ac1a2 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -5,7 +5,19 @@ from pathlib import Path from strain.mutators import Mutant -from strain.report import EventLog, action_items, compiler_warning, mutation_testing_elements, summary_line +from strain.html_report import html_report +from strain.report import ( + NOT_REACHED_HINT, + EventLog, + FileScore, + action_items, + compiler_warning, + file_scores, + markdown_summary, + mutation_testing_elements, + source_excerpt, + summary_line, +) from strain.results import MutantResult, MutantState, SurvivorKind, combine LESS = Mutant(file="src/math.cpp", line=4, column=9, offset=30, operator="ror", original="<", replacement="<=") @@ -92,6 +104,136 @@ def test_action_items_are_limited(self) -> None: self.assertEqual(len(action_items(combined, limit=1)), 1) +SOURCE = ( + "#include \n" + "\n" + "int abs(int value)\n" + "{\tif (value < 0) { return -value; }\n" + " return value;\n" + "}\n" + "\n" + "\n" + "int add(int lhs, int rhs) { return lhs + rhs; }\n" +) +OTHER = Mutant(file="src/other.cpp", line=1, column=1, offset=0, operator="ror", original="==", replacement="!=") + + +class SourceExcerptTest(unittest.TestCase): + def test_caret_points_at_the_operator_and_the_fix_it_shows_the_replacement(self) -> None: + mutant = Mutant( + file="src/math.cpp", line=4, column=13, offset=0, operator="ror", original="<", replacement="<=" + ) + self.assertEqual( + source_excerpt(mutant, SOURCE), + [ + " 4 | { if (value < 0) { return -value; }", + " | ^", + " | <=", + ], + ) + + def test_multi_character_operators_are_underlined(self) -> None: + mutant = Mutant( + file="src/math.cpp", line=1, column=1, offset=0, operator="lcr", original="&&", replacement="||" + ) + self.assertEqual(source_excerpt(mutant, "&& x\n"), [" 1 | && x", " | ^~", " | ||"]) + + def test_lines_past_the_end_of_the_source_give_no_excerpt(self) -> None: + self.assertEqual(source_excerpt(AND, "one line\n"), []) + + +class FileScoresTest(unittest.TestCase): + def test_scores_are_grouped_by_file(self) -> None: + combined = combine( + [ + result(LESS, "t", MutantState.KILLED), + result(PLUS, "t", MutantState.SURVIVED), + result(AND, "t", MutantState.COMPILE_ERROR), + result(OTHER, "t", MutantState.TIMEOUT), + ] + ) + self.assertEqual( + file_scores(combined), + [ + FileScore(file="src/math.cpp", mutants=3, detected=1, survived=1), + FileScore(file="src/other.cpp", mutants=1, detected=1, survived=0), + ], + ) + self.assertEqual(file_scores(combined)[0].score, 0.5) + self.assertIsNone(FileScore(file="f", mutants=1, detected=0, survived=0).score) + + +class MarkdownSummaryTest(unittest.TestCase): + def test_summary_has_score_files_and_survivors(self) -> None: + combined = combine( + [ + result(LESS, "t", MutantState.KILLED), + result(PLUS, "t", MutantState.SURVIVED, SurvivorKind.NOT_DETECTED), + result(OTHER, "t", MutantState.SURVIVED, function=None), + ] + ) + self.assertEqual( + markdown_summary(combined, threshold=50.0).splitlines(), + [ + "## strain mutation testing", + "", + "**Mutation score: 33.3%** ❌ below the 50% threshold", + "", + "3 mutants: killed 1, survived 2", + "", + "| File | Mutants | Detected | Survived | Score |", + "| --- | ---: | ---: | ---: | ---: |", + "| `src/math.cpp` | 2 | 1 | 1 | 50.0% |", + "| `src/other.cpp` | 1 | 0 | 1 | 0.0% |", + "", + "### Survivors (2)", + "", + "- `src/math.cpp:9:12` `+` → `-` (not_detected): assert the computed value", + "- `src/other.cpp:1:1` `==` → `!=`: add a test where `==` and `!=` give different results", + ], + ) + + def test_summary_without_survivors_meets_the_threshold(self) -> None: + markdown = markdown_summary(combine([result(LESS, "t", MutantState.KILLED)]), threshold=90.0) + self.assertIn("**Mutation score: 100.0%** ✅ meets the 90% threshold\n", markdown) + self.assertNotIn("Survivors", markdown) + + def test_summary_without_mutants(self) -> None: + self.assertIn("No mutants were generated.", markdown_summary([], threshold=None)) + + def test_pipes_in_file_names_do_not_break_the_table(self) -> None: + pipe = Mutant(file="a|b.cpp", line=1, column=1, offset=0, operator="lcr", original="||", replacement="&&") + markdown = markdown_summary(combine([result(pipe, "t", MutantState.SURVIVED)]), threshold=None) + self.assertIn("| `a\\|b.cpp` |", markdown) + + +class HtmlReportTest(unittest.TestCase): + def test_report_shows_escaped_source_with_its_mutants(self) -> None: + combined = combine( + [ + result(LESS, "t1", MutantState.KILLED), + result(PLUS, "t1", MutantState.SURVIVED, SurvivorKind.NOT_REACHED), + ] + ) + page = html_report(combined, {"src/math.cpp": SOURCE}) + self.assertTrue(page.startswith("")) + self.assertIn("strain mutation report: 50.0%", page) + self.assertIn("#include <cmath>", page) + self.assertIn('id="src/math.cpp:4"', page) + self.assertIn('class="line survived"', page) + self.assertIn('class="line killed"', page) + self.assertIn("< → <=", page) + self.assertIn("killed by t1", page) + self.assertIn(NOT_REACHED_HINT, page) + self.assertIn('href="#src/math.cpp:9"', page) + self.assertNotIn(" None: + page = html_report([], {}) + self.assertIn("strain mutation report: n/a", page) + self.assertIn("No mutants were generated", page) + + class MutationTestingElementsTest(unittest.TestCase): def test_schema_shape(self) -> None: combined = combine( diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..252fb1f --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,17 @@ +import os +import unittest +from pathlib import Path + +from strain.runner import short_path + + +class ShortPathTest(unittest.TestCase): + def test_a_path_under_the_working_directory_is_shown_relative(self) -> None: + self.assertEqual(short_path(Path.cwd() / "build" / "debug"), os.path.join("build", "debug")) + + def test_a_path_far_from_the_working_directory_stays_absolute(self) -> None: + self.assertEqual(short_path(Path(Path.cwd().anchor)), str(Path(Path.cwd().anchor))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/ui_snapshot.py b/tools/ui_snapshot.py new file mode 100644 index 0000000..3bb905f --- /dev/null +++ b/tools/ui_snapshot.py @@ -0,0 +1,176 @@ +"""Runs `strain run --ui live` in a pseudo-terminal and saves what the screen shows as text and PNG frames. + + pip install -e '.[ui-snapshot]' + python tools/ui_snapshot.py tests/fixtures/sample_project --size 120x36 --keys down,tab,q + +Frames go to `ui-snapshots/` by default: `frame-NNN.txt` every `--interval` seconds, and +`key-NN-.txt` after each key, each with a matching `.png` when Pillow is installed. +The keys are sent once the run has finished, which is when the title shows `finished`. +""" + +import argparse +import os +import pty +import select +import signal +import struct +import sys +import termios +import time +from collections.abc import Sequence +from fcntl import ioctl +from pathlib import Path + +import pyte + +KEYS = {"up": "\033[A", "down": "\033[B", "pageup": "\033[5~", "pagedown": "\033[6~", "tab": "\t"} +FINISHED_MARKER = "· finished" +READ_SIZE = 65536 +SETTLE_SECONDS = 0.5 +CELL_WIDTH = 9 +CELL_HEIGHT = 18 +FONT = "/usr/share/fonts/truetype/dejavu/DejaVuSansMono.ttf" +BOLD_FONT = "/usr/share/fonts/truetype/dejavu/DejaVuSansMono-Bold.ttf" +BACKGROUND = "#1e1e1e" +FOREGROUND = "#d4d4d4" +# pyte names the 16 standard colors; anything else arrives as a hex string. +PALETTE = { + "black": "#000000", + "red": "#cd3131", + "green": "#0dbc79", + "brown": "#e5e510", + "blue": "#2472c8", + "magenta": "#bc3fbc", + "cyan": "#11a8cd", + "white": "#e5e5e5", + "brightblack": "#666666", + "brightred": "#f14c4c", + "brightgreen": "#23d18b", + "brightbrown": "#f5f543", + "brightblue": "#3b8eea", + "brightmagenta": "#d670d6", + "brightcyan": "#29b8db", + "brightwhite": "#ffffff", +} + + +def parse_size(text: str) -> tuple[int, int]: + columns, _, lines = text.partition("x") + return int(columns), int(lines) + + +def color(name: str, default: str) -> str: + if name == "default": + return default + return PALETTE.get(name, f"#{name}") + + +def screen_text(screen: pyte.Screen) -> str: + return "\n".join(line.rstrip() for line in screen.display) + "\n" + + +def save_png(screen: pyte.Screen, path: Path) -> None: + try: + from PIL import Image, ImageDraw, ImageFont + except ImportError: + return + image = Image.new("RGB", (screen.columns * CELL_WIDTH, screen.lines * CELL_HEIGHT), BACKGROUND) + draw = ImageDraw.Draw(image) + regular = ImageFont.truetype(FONT, 15) + bold = ImageFont.truetype(BOLD_FONT, 15) + for row in range(screen.lines): + line = screen.buffer[row] + for column in range(screen.columns): + cell = line[column] + foreground = color(cell.fg, FOREGROUND) + background = color(cell.bg, BACKGROUND) + if cell.reverse: + foreground, background = background, foreground + left, top = column * CELL_WIDTH, row * CELL_HEIGHT + if background != BACKGROUND: + draw.rectangle([left, top, left + CELL_WIDTH - 1, top + CELL_HEIGHT - 1], fill=background) + if cell.data.strip(): + draw.text((left, top), cell.data, fill=foreground, font=bold if cell.bold else regular) + image.save(path) + + +def save_frame(screen: pyte.Screen, directory: Path, name: str) -> None: + (directory / f"{name}.txt").write_text(screen_text(screen), encoding="utf-8") + save_png(screen, directory / f"{name}.png") + + +def spawn(project: Path, columns: int, lines: int, extra: Sequence[str]) -> tuple[int, int]: + pid, descriptor = pty.fork() + if pid == 0: + os.chdir(project) + os.environ["TERM"] = "xterm-256color" + os.execvp(sys.executable, [sys.executable, "-m", "strain", "run", "--ui", "live", *extra]) # noqa: S606 - replaces the forked child with strain, by design + ioctl(descriptor, termios.TIOCSWINSZ, struct.pack("HHHH", lines, columns, 0, 0)) + os.kill(pid, signal.SIGWINCH) + return pid, descriptor + + +def pump(descriptor: int, stream: pyte.ByteStream, seconds: float) -> bool: + """Feeds the terminal output to `stream` for `seconds`; false once the program has exited.""" + deadline = time.monotonic() + seconds + while (remaining := deadline - time.monotonic()) > 0: + ready, _, _ = select.select([descriptor], [], [], remaining) + if not ready: + continue + try: + data = os.read(descriptor, READ_SIZE) + except OSError: + return False + if not data: + return False + stream.feed(data) + return True + + +def parse_arguments(parser: argparse.ArgumentParser, argv: list[str]) -> tuple[argparse.Namespace, list[str]]: + """Parses our own options, which may follow the project, and returns what comes after `--` for `strain run`.""" + separator = argv.index("--") if "--" in argv else len(argv) + return parser.parse_args(argv[:separator]), argv[separator + 1 :] + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("project", type=Path) + parser.add_argument("--size", type=parse_size, default=(120, 36), help="COLUMNSxLINES (default 120x36)") + parser.add_argument("--keys", default="down,tab,q", help="keys to send once the run has finished") + parser.add_argument("--interval", type=float, default=1.0, help="seconds between frames during the run") + parser.add_argument("--out", type=Path, default=Path("ui-snapshots")) + parser.add_argument("--timeout", type=float, default=600.0) + options, extra = parse_arguments(parser, sys.argv[1:] if argv is None else list(argv)) + columns, lines = options.size + options.out.mkdir(parents=True, exist_ok=True) + screen = pyte.Screen(columns, lines) + stream = pyte.ByteStream(screen) + pid, descriptor = spawn(options.project.resolve(), columns, lines, extra) + deadline = time.monotonic() + options.timeout + frame = 0 + running = True + while running and FINISHED_MARKER not in screen.display[0] and time.monotonic() < deadline: + running = pump(descriptor, stream, options.interval) + save_frame(screen, options.out, f"frame-{frame:03}") + frame += 1 + keys = [key for key in options.keys.split(",") if key] + sent = 0 + for number, key in enumerate(keys, start=1): + if not running: + break + os.write(descriptor, KEYS.get(key, key).encode()) + running = pump(descriptor, stream, SETTLE_SECONDS) + save_frame(screen, options.out, f"key-{number:02}-{key}") + sent = number + if running: + pump(descriptor, stream, SETTLE_SECONDS) + os.kill(pid, signal.SIGTERM) + _, status = os.waitpid(pid, 0) + os.close(descriptor) + print(f"saved {frame} run frames and {sent} key frames to {options.out}") + return os.waitstatus_to_exitcode(status) + + +if __name__ == "__main__": + sys.exit(main())