diff --git a/CHANGELOG.md b/CHANGELOG.md index f92d39d913..6b5d54ad05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## Handbook now shows the arXiv version (2611.08438v1) +- Every tab is built from the arXiv LaTeX source, ingested with `--status final`: 15 chapters, Acknowledgements, 3 appendices, 517 references. Preview banners, the withheld chapter and draft `noindex` are gone. +- Chapter 10 is now *Learning Dynamics and Mechanistic Interpretability* at `/handbook/learning-dynamics/`. `/handbook/developmental-interpretability/` and the removed *SAE Architecture Details* appendix redirect. +- Every previously published section anchor still resolves: 318 anchor redirects (`scripts/urls/suggest_anchor_redirects.py`, same-chapter matches only, else the chapter top), and 47 references the arXiv version no longer cites point to the reference list. +- Methods, Evaluation and Safety & open problems quotes re-selected from the arXiv text, verbatim (`scripts/data/check_verbatim.py`). +- Page numbers link to that page of the arXiv PDF. +- Ingest fixes: `\ref` always prints LaTeX's number (pandoc's own figure numbering was off by one because the taxonomy figure sits in main.tex); unnumbered chapters keep their title; a `.bib` is optional when a `.bbl` is present. + ## Paper on arXiv - The paper is out as [arXiv:2611.08438](https://arxiv.org/abs/2611.08438). The homepage links the abstract page and PDF, and the citation section shows the arXiv BibTeX with a copy button. README gains a Citation section. diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index caacdaf3ff..dc9ab9e23a 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -44,6 +44,8 @@ The 5 Nov PDF has six orientations, §12.1 to §12.6: Pragmatic, Ambitious, Deve The 5 Nov PDF defines one framework in §7.5 with five dimensions: attribution faithfulness, completeness, stability, causal validity and human interpretability. §14.2 refers back to §7.5. A text search found no second framework. The 13 Jul draft also had a four-dimension framework in §15.2.1. **Decide:** confirm that §7.5 is the only framework in the approved source. ### D4. Publication status of draft text +**Resolved 6 Nov 2026:** the paper is public (arXiv:2611.08438v1), so the site now shows the arXiv version, ingested with `--status final` from the arXiv LaTeX source. Preview banners, the withheld chapter and draft `noindex` no longer apply. + Now: the live site publishes the 13 Jul draft text, with "Preview build" banners, for every chapter except chapter 11. **This PR.** Pages built from draft text now carry ``. They stay reachable by link and in the site's own search. The switch is `indexDraftPages` in `src/data/publication.json`. `noindex` is not access control: anything deployed is public. diff --git a/public/figures/11_modeldiffing.svg b/public/figures/11_modeldiffing.svg index 9053ffaf58..3aa09f6d73 100644 --- a/public/figures/11_modeldiffing.svg +++ b/public/figures/11_modeldiffing.svg @@ -20,57 +20,57 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -90,14 +90,6 @@ - - - - - - - - @@ -116,6 +108,14 @@ + + + + + + + + @@ -235,10 +235,10 @@ - + - + @@ -247,10 +247,10 @@ - + - + @@ -259,10 +259,10 @@ - + - + @@ -271,10 +271,10 @@ - + - + @@ -283,10 +283,10 @@ - + - + @@ -295,115 +295,115 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -471,7 +471,7 @@ - + @@ -481,13 +481,13 @@ - + - + @@ -497,13 +497,13 @@ - + - + @@ -513,13 +513,13 @@ - + - + @@ -529,18 +529,47 @@ - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + @@ -576,192 +605,198 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - + - + - - + + - + - + - + - - + + + + + - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + @@ -831,7 +866,7 @@ - + @@ -841,13 +876,13 @@ - + - + @@ -857,13 +892,13 @@ - + - + @@ -873,13 +908,13 @@ - + - + @@ -889,16 +924,22 @@ - + - + + + + + + + - - + + @@ -1031,209 +1072,209 @@ - + - - + + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - - + + - + - + - - + + - + - - + + - - - - + + + + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - - + + - + - - + + - + - - + + - - - + + + - + - + - - + + - + - + - - + + @@ -1245,43 +1286,10 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -1291,7 +1299,7 @@ - + @@ -1301,7 +1309,7 @@ - + @@ -1311,7 +1319,7 @@ - + @@ -1321,7 +1329,7 @@ - + @@ -1331,7 +1339,7 @@ - + @@ -1343,1019 +1351,1059 @@ - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - - + + + + - - - - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - - + + + + - - - - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - + + + - - - - + + + - - - - + + + + - - - - - + + + - - - - + + + - - - - + + + + - - - - - - - - - - - - - - - - - + + + + + + - - + + - - + + - - + + - - - - + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - - - - - + + + + - - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - - - - + + + + - - - + + + + - - - - - - - - - - - + + - - + + - - + + - - + + - - + + + + - - - - + + + + - - - - - - - - - + + + + - - - + + + + - - - - - - - - - + + - - + + + + - - - - - - + + + + - - + - + - + - + - + - + - + - + - + - + - - - + - + - + - + - + - + - + - - + - + - + - + - + - + - + - - - - - - - + + + + - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - + + + - - - - - - - - + + + + - - - - - - - + + + + + + - - - - - - - + + + + + + - - - - - - - + + + + + + - - - - - - - + + + + + + - - - - - + - + - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - - + - + - + - + - + - + - - + - + - + - + - + - + - - - - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - + + + + - - + - + - + - + - + - + - + - - + + - - - + + + + - - + - + - + - + - + - + - + diff --git a/public/figures/11_steering.svg b/public/figures/11_steering.svg index 36eb8a46a0..73463ce38b 100644 --- a/public/figures/11_steering.svg +++ b/public/figures/11_steering.svg @@ -39,85 +39,85 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -133,18 +133,6 @@ - - - - - - - - - - - - @@ -173,6 +161,18 @@ + + + + + + + + + + + + @@ -181,7 +181,7 @@ - + @@ -189,10 +189,10 @@ - + - + @@ -201,10 +201,10 @@ - + - + @@ -213,10 +213,10 @@ - + - + @@ -225,10 +225,10 @@ - + - + @@ -237,10 +237,10 @@ - + - + @@ -249,10 +249,10 @@ - + - + @@ -260,747 +260,758 @@ - - - - + + + + + + + - - - - + + + + + + + + + + + + - - - - - - + + + + + + + - - - - - - + + + + + + + - - - - - - + + + + + + + - - - - - - + + + + + + + - - - - - - + + + + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - + + + + + + - - - - - - - + + + + + + - - - - - - - + + + + + + - - - - - - - + + + + + + - - - - - - - + + + + + + - - - - - - - + + + + + + - - - - - + - + - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - - + - + - + - + - + - + - - + - + - + - + - + - + - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + - + - + - + - + - - + - + - + - + - + - + - + - + - - - - + - + - + - + - + - + - + - + - - + - + - + - + - + - + - + - + diff --git a/public/figures/1_general_summary.svg b/public/figures/1_general_summary.svg new file mode 100644 index 0000000000..c557f20dc6 --- /dev/null +++ b/public/figures/1_general_summary.svg @@ -0,0 +1,6516 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/figures/5_superposition_and_polysemanticity.svg b/public/figures/5_superposition_and_polysemanticity.svg new file mode 100644 index 0000000000..d2bd5d9053 --- /dev/null +++ b/public/figures/5_superposition_and_polysemanticity.svg @@ -0,0 +1,537 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/figures/6_clt.svg b/public/figures/6_clt.svg index 55a7668fd4..7beedbb142 100644 --- a/public/figures/6_clt.svg +++ b/public/figures/6_clt.svg @@ -23,6 +23,22 @@ + + + + + + + + + + + + + + + + @@ -61,14 +77,6 @@ - - - - - - - - @@ -87,25 +95,17 @@ - - - - - - - - - + - + - + @@ -156,7 +156,7 @@ - + @@ -167,213 +167,268 @@ - + - + - + - + - + - + + + + + + + + + - + + + + + + + + + + + + + - + + + + + + + + + - + + + + + + + + + + + + + - + - + + + + + + + + + + + + + + + + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - - - - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + + + + - + @@ -389,67 +444,67 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -461,19 +516,19 @@ - + - + - + - + @@ -481,60 +536,60 @@ - + - + - + - + - + - + - + - - + + - + - + - + - - - - - - + + + + + + - + - - + + - + @@ -793,13 +848,42 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + @@ -827,7 +911,7 @@ - + @@ -837,21 +921,21 @@ - + - + - + - + @@ -873,7 +957,7 @@ - + @@ -887,27 +971,27 @@ - + - + - + - + - + - + @@ -921,7 +1005,7 @@ - + @@ -930,13 +1014,42 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + @@ -1067,412 +1180,288 @@ - + - + - - + + - + - + - + - - - - - - - - - - - + + + + + + + + + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - - - - - - - - - - - - - + + + + + + + + + + + + + - - + + - + - + - + - - - - - - - - - - - + + + + + + - - - - - + + + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - + + - - + + + - - + + - - - - + + - - - - - - - - - - - - + + + - - - - - - - - - - - - + + + - - - - - - - - - - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + @@ -1481,444 +1470,510 @@ - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - + + + + + + - - - - - - - - - + + + + + + - - - - - - + + + + + - - - - + + + + - - - - + + + + - - - - - - - + + + + - - - - - - - + + + + - - - - + + + + - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - - + - + - + - + - + - + - - + - + - + - + - + - + - - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - + + + + + + - - - - - - + + + + + + - - - - - - + + + + + + - - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - + + + + - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + + + - - - - - - - - + + - - + + - - + + - - + + - - + + + + diff --git a/public/figures/6_crosscoders.svg b/public/figures/6_crosscoders.svg index bf0e543171..9975ad30c1 100644 --- a/public/figures/6_crosscoders.svg +++ b/public/figures/6_crosscoders.svg @@ -17,51 +17,51 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -167,10 +167,10 @@ - + - + @@ -179,10 +179,10 @@ - + - + @@ -191,10 +191,10 @@ - + - + @@ -203,10 +203,10 @@ - + - + @@ -215,10 +215,10 @@ - + - + @@ -227,23 +227,23 @@ - + - + - + - + @@ -267,7 +267,7 @@ - + @@ -330,12 +330,12 @@ - + - + - + @@ -467,12 +467,12 @@ - + - + - + @@ -604,12 +604,12 @@ - + - + - + @@ -629,17 +629,17 @@ - + - + - + - + @@ -677,15 +677,15 @@ - + - + - + @@ -695,29 +695,29 @@ - + - + - + - + - + - + @@ -727,26 +727,26 @@ - + - + - + - + - + - + @@ -878,12 +878,12 @@ - + - + - + @@ -903,17 +903,17 @@ - + - + - + - + @@ -951,15 +951,15 @@ - + - + - + @@ -969,29 +969,29 @@ - + - + - + - + - + - + @@ -1001,26 +1001,26 @@ - + - + - + - + - + - + @@ -1154,38 +1154,18 @@ - - + + - + - - - - - - - - - - - - - - - - - - - - - + @@ -1196,43 +1176,10 @@ - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -1242,7 +1189,7 @@ - + @@ -1252,7 +1199,7 @@ - + @@ -1262,7 +1209,7 @@ - + @@ -1272,7 +1219,7 @@ - + @@ -1282,7 +1229,7 @@ - + @@ -1294,222 +1241,397 @@ - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - + + + + + - - - - + + + + - - - - - - - + + + + - - - - + + + + - - - - + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - - + - + - + - + - + - + - - + - + - + - + - + - + - - - - - - - - - - - - - @@ -1555,216 +1677,106 @@ - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - - - + + + + - - - - - - - - - + + + + - - - - - - - - + + + + - - - - - - - - - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + diff --git a/public/figures/6_sae.svg b/public/figures/6_sae.svg index f862466ab7..e451092cb8 100644 --- a/public/figures/6_sae.svg +++ b/public/figures/6_sae.svg @@ -1,5 +1,5 @@ - + @@ -23,45 +23,33 @@ - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -95,17 +83,26 @@ + + + + + + + + + + + + - + - + - - - - + @@ -168,97 +165,97 @@ + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + - + @@ -514,13 +511,13 @@ - - + + - - + + - + @@ -776,14 +773,17 @@ + + + - + - + - + @@ -791,680 +791,677 @@ - + - - - - - - + + + + + + - + - - - - - + + - - + + - - + + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + - + - - + + - + - + + + + - + - + - + - + - - - - + + - - - - + - - + + + - - + - + + - - + - + + - - + - + + - - + - + + - - + - + + - - + - + + - - + - + + - - + - + + - - + - + + - - + - + + - - + - + + - - + - + + - - + - + + - - + + - - - + + - - - + + + + + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + - - - - - - + + + + + + + - - - - - - + + + + + + + - - - - - - - - + + + + + + + - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - + + + + + - - - - - - - + + + + + + + + + - - - - - - - + + + + + + - - - - - - - + + + + + + - - - - - - - + + + + + + - - - - - + - + - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - + - + - - - - + - + - + - - - - + - + - + - + - + - + - + - - + - + - + - + - + - + - - + - + - + - + - + - + - - - - - - - - - - - - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - + + + + - - + - + - + - - - - + + + + - - + - + - + - - - - + + + + - - + - + - + diff --git a/public/figures/8_latent_qa.svg b/public/figures/8_latent_qa.svg new file mode 100644 index 0000000000..20f1c81e91 --- /dev/null +++ b/public/figures/8_latent_qa.svg @@ -0,0 +1,1845 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/figures/clt_size_vs_depth_e32_clean_v3.svg b/public/figures/clt_size_vs_depth_e32_clean_v3.svg new file mode 100644 index 0000000000..6d0f32eb9c --- /dev/null +++ b/public/figures/clt_size_vs_depth_e32_clean_v3.svg @@ -0,0 +1,321 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/figures/jumprelu.svg b/public/figures/jumprelu.svg new file mode 100644 index 0000000000..5d40817216 --- /dev/null +++ b/public/figures/jumprelu.svg @@ -0,0 +1,113 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/figures/research_directions_compressed.png b/public/figures/research_directions_compressed.png new file mode 100644 index 0000000000..619bd67107 Binary files /dev/null and b/public/figures/research_directions_compressed.png differ diff --git a/scripts/data/__pycache__/reanchor_excerpts.cpython-39.pyc b/scripts/data/__pycache__/reanchor_excerpts.cpython-39.pyc new file mode 100644 index 0000000000..ec70c9245c Binary files /dev/null and b/scripts/data/__pycache__/reanchor_excerpts.cpython-39.pyc differ diff --git a/scripts/data/build_orientations_arxiv.py b/scripts/data/build_orientations_arxiv.py index 38d79f91b1..aed0a2d702 100644 --- a/scripts/data/build_orientations_arxiv.py +++ b/scripts/data/build_orientations_arxiv.py @@ -205,9 +205,13 @@ def num(i): return f"{CHAPTER_NUM}.{i + 1}" terms = [it["term"] for blk in body if "ul" in blk for it in blk["ul"] if it["term"]] lead = next((body[k - 1].get("p", "") for k, blk in enumerate(body) if "ul" in blk and k), "") list_label = "Example metrics" if "metrics" in lead else "Methods" + definition = " ".join(first[:2]) + # Drop a leading comparative clause ("In contrast to pragmatic approaches, …") so the card opens on + # its own subject; what remains is still a verbatim substring of the paper. + definition = re.sub(r"^(In contrast to [^,]+|Complementing [^,]+|Like [^,]+), (?=[A-Z])", "", definition) orientations.append({ "id": oid, "name": name, "subtitle": sub, "section": num(i), "section_title": title, - "definition": " ".join(first[:2]), "list_label": list_label, "list_terms": terms, "body": body, + "definition": definition, "list_label": list_label, "list_terms": terms, "body": body, }) def sec_text(prefix): diff --git a/scripts/data/check_verbatim.py b/scripts/data/check_verbatim.py new file mode 100644 index 0000000000..f3dd90cd0b --- /dev/null +++ b/scripts/data/check_verbatim.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Check curated JSON against the ingested manuscript, exactly as tests/data.test.ts does. + + python3 scripts/data/check_verbatim.py FILE.json [FILE.json ...] + +Prints every excerpt (string > 25 chars under an excerpt key) that is not verbatim in the +manuscript plain text, and every section id that does not exist. Exit code 1 if any. +""" +import glob, html, json, re, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) +from reanchor_excerpts import NON_EXCERPT, SECTION_KEYS # same key lists as the test + +ROOT = Path(__file__).resolve().parents[2] +h = "\n".join(open(f).read() for f in sorted(glob.glob(str(ROOT / "src/generated/chapters/*.html")))) +plain = re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", "", re.sub(r"", "\n", h)))).strip() +ids = set(re.findall(r'\sid="([^"]+)"', h)) +bad = [] + +def walk(o, key, path): + if key in SECTION_KEYS: + for s in (o if isinstance(o, list) else [o]): + if isinstance(s, str) and " (" not in s and s not in ids: + bad.append(f"{path}: unknown section id {s!r}") + return + if key in NON_EXCERPT: + return + if isinstance(o, str): + if len(o) > 25 and re.sub(r"\s+", " ", o).strip() not in plain: + bad.append(f"{path}: not verbatim: {o[:120]}") + elif isinstance(o, list): + for i, x in enumerate(o): + walk(x, key, f"{path}[{i}]") + elif isinstance(o, dict): + for k, v in o.items(): + walk(v, k, f"{path}.{k}") + +for f in sys.argv[1:]: + d = json.load(open(f)) + if isinstance(d, dict) and "methods" in d and isinstance(d["methods"], list): + walk(d["methods"], "", f"{f}:methods"); walk(d.get("hypotheses", []), "", f"{f}:hypotheses") + else: + walk(d, "", f) +print("\n".join(bad) or "OK: all excerpts verbatim, all section ids exist") +sys.exit(1 if bad else 0) diff --git a/scripts/data/extend_context.py b/scripts/data/extend_context.py new file mode 100644 index 0000000000..c8211b669f --- /dev/null +++ b/scripts/data/extend_context.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Give context-dependent quotes their antecedent. + +A quote that opens with a pronoun or connective ("This …", "These …", "However, …") reads as a +fragment when shown alone. For each such quote in the curated data, prepend the preceding +sentence(s) of the same paragraph from the manuscript, so the quote stays verbatim and contiguous. + + python3 scripts/data/extend_context.py [--write] +""" +import glob, html, json, re, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) +from reanchor_excerpts import NON_EXCERPT, SECTION_KEYS +ROOT = Path(__file__).resolve().parents[2] +START = re.compile(r'^(However|This|These|That|Those|It|Its|They|Their|Such|Both|Here|Moreover|Thus|Therefore|Hence|Specifically|Similarly|Additionally|Furthermore|Instead|Conversely|Nevertheless|Notably|Importantly|Crucially|In contrast|In particular|For example|As a result|Also|But|And)\b') +SENT = re.compile(r"(?<=[.!?])\s+(?=[A-Z(“\"])") + +h = "\n".join(open(f).read() for f in sorted(glob.glob(str(ROOT / "src/generated/chapters/*.html")))) +blocks = [re.sub(r"\s+", " ", b).strip() for b in html.unescape(re.sub(r"<[^>]+>", "", re.sub(r"", "\n", h))).split("\n")] +plain = " ".join(b for b in blocks if b) + +def extend(q): + t = q.strip() + nonempty = [b for b in blocks if b] + for j, b in enumerate(nonempty): + i = b.find(t[:80]) # quotes may run on into the next paragraph; anchor on their opening + if i < 0: + continue + # quote opens the paragraph: its antecedent is the end of the previous paragraph (often a run-in heading) + prev = b[:i].rstrip() if i > 0 else (nonempty[j - 1] if j else "") + if not prev: + continue + before = SENT.split(prev) + add = [] + while before and (not add or START.match(add[0])) and len(add) < 4: + add.insert(0, before.pop()) + cand = (" ".join(add) + " " + t).strip() + if cand in plain and not START.match(cand): + return cand + return None + +def walk(o, key, stats): + if key in SECTION_KEYS or key in NON_EXCERPT: + return o + if isinstance(o, str): + if START.match(o.strip()): + n = extend(o) + if n: + stats["extended"] += 1; return n + stats["unchanged"].append(o[:100]) + return o + if isinstance(o, list): + return [walk(x, key, stats) for x in o] + if isinstance(o, dict): + return {k: walk(v, k, stats) for k, v in o.items()} + return o + +stats = {"extended": 0, "unchanged": []} +for f in ("src/data/methods.json", "src/data/framework.json"): + d = json.load(open(ROOT / f)) + d = {k: (walk(v, "" if k in ("methods", "hypotheses") else k, stats) if k not in ("source", "gaps", "families") else v) for k, v in d.items()} + if "--write" in sys.argv: + (ROOT / f).write_text(json.dumps(d, indent=1, ensure_ascii=False) + "\n") +print("extended", stats["extended"], "unchanged", len(stats["unchanged"])) +for u in stats["unchanged"]: + print(" -", u) diff --git a/scripts/data/reanchor_excerpts.py b/scripts/data/reanchor_excerpts.py new file mode 100644 index 0000000000..fa8797915e --- /dev/null +++ b/scripts/data/reanchor_excerpts.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Re-anchor curated excerpts to a newly ingested manuscript. + +Every quote in src/data/methods.json and src/data/framework.json must appear verbatim in the +ingested manuscript (tests/data.test.ts). When a new version is ingested, most quotes differ only +by copy-editing (spelling, citation format, small rewordings). For each quote that is no longer +verbatim, this finds the passage of the new text with the same number of sentences that is most +similar, and: + - similarity >= ACCEPT: replaces the quote with the new passage, verbatim; + - otherwise: drops the quote (the site never shows text the paper does not contain). +Section ids that no longer exist are remapped through src/data/redirects.json (anchors), else dropped. +The same replacements are applied to the string literals in build_methods.py / build_framework.py +where they can be found, so the build scripts stay in step. + + python3 scripts/data/reanchor_excerpts.py [--write] [--report out.tsv] +""" +import argparse, difflib, glob, html, json, re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +ACCEPT = 0.85 +FILES = {"src/data/methods.json": "scripts/data/build_methods.py", "src/data/framework.json": "scripts/data/build_framework.py"} +NON_EXCERPT = {'id', 'name', 'family', 'title', 'subtitle', 'heading', 'step', 'methods', 'methods_used', 'metrics_named', 'related_methods', + 'related_orientations', 'orientations', 'related', 'sections', 'mentioned_in', 'chapter_sections', 'formulas_tex', 'public_name', 'draft_name', + 'draft_sections', 'status', 'source', 'gaps', 'table_id', 'caption', 'columns', 'rows', 'table_row', 'section', 'source_chapter', 'motivation_section', + 'protocol_section', 'framework_limitations_section', 'decision_framework_section', 'tensions_section', 'comparison_table', 'method_guidance', + 'risk_taxonomy', 'evidence_key', 'items', 'groups'} +SECTION_KEYS = {'sections', 'mentioned_in', 'chapter_sections', 'section', 'protocol_section', 'framework_limitations_section', + 'decision_framework_section', 'tensions_section', 'motivation_section', 'draft_sections'} +SENT = re.compile(r"(?<=[.!?:])\s+(?=[A-Z(“\"])") +WORD = re.compile(r"[a-z]{4,}") + + +def plain_blocks(): + h = "\n".join(open(f).read() for f in sorted(glob.glob(str(ROOT / "src/generated/chapters/*.html")))) + h = re.sub(r"", "\n", h) + t = html.unescape(re.sub(r"<[^>]+>", "", h)) + return [re.sub(r"\s+", " ", b).strip() for b in t.split("\n") if b.strip()], h + + +def build_index(blocks): + sents = [] # (block_no, sentence) + for i, b in enumerate(blocks): + for s in SENT.split(b): + if s.strip(): + sents.append((i, s.strip())) + inv = {} + for j, (_, s) in enumerate(sents): + for w in set(WORD.findall(s.lower())): + inv.setdefault(w, []).append(j) + return sents, inv + + +def best_match(text, sents, inv, plain): + k = max(1, len([s for s in SENT.split(text) if s.strip()])) + words = set(WORD.findall(text.lower())) + if not words: + return None, 0.0 + votes = {} + for w in words: + lst = inv.get(w, []) + if len(lst) > 400: # too common to discriminate + continue + for j in lst: + votes[j] = votes.get(j, 0) + 1 + starts = set() + for j, _ in sorted(votes.items(), key=lambda x: -x[1])[:60]: + for d in range(k): + starts.add(j - d) + best, score = None, 0.0 + for st in starts: + if st < 0 or st + k > len(sents): + continue + win = sents[st:st + k] + if len({b for b, _ in win}) > 1 and k > 1 and win[0][0] != win[-1][0]: + pass # windows may cross paragraph breaks; still verbatim in the joined text check below + cand = " ".join(s for _, s in win) + sm = difflib.SequenceMatcher(None, text, cand, autojunk=False) + if sm.real_quick_ratio() < score or sm.quick_ratio() < score: + continue + r = sm.ratio() + if r > score and cand in plain: + best, score = cand, r + return best, score + + +def walk(o, key, fn): + if key in NON_EXCERPT and key not in SECTION_KEYS: + return o + if key in SECTION_KEYS: + return fn(o, key, section=True) + if isinstance(o, str): + return fn(o, key, section=False) + if isinstance(o, list): + return [x for x in (walk(x, key, fn) for x in o) if x is not None] + if isinstance(o, dict): + return {k: v for k, v in ((k, walk(v, k, fn)) for k, v in o.items()) if v is not None} + return o + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--write", action="store_true") + ap.add_argument("--report", default=None) + a = ap.parse_args() + blocks, h = plain_blocks() + plain = " ".join(blocks) + ids = set(re.findall(r'\sid="([^"]+)"', h)) + sents, inv = build_index(blocks) + red = json.loads((ROOT / "src/data/redirects.json").read_text())["anchors"] + id_map = {} + for k, v in red.items(): + if v.startswith("handbook/"): + id_map.setdefault(k.split("#", 1)[1], v.split("#", 1)[1]) + rows = [] + for jf, pyf in FILES.items(): + data = json.loads((ROOT / jf).read_text()) + py = (ROOT / pyf).read_text() + repl = {} + + def fn(o, key, section): + if section: + if isinstance(o, list): + out = [] + for x in o: + y = fn(x, key, True) + if y is not None and y not in out: + out.append(y) + return out + if not isinstance(o, str) or " (" in o or o in ids: + return o + new = id_map.get(o) + rows.append((jf, key, "section", "remap" if new in ids else "drop", o, new or "")) + return new if new in ids else None + if len(o) <= 25 or re.sub(r"\s+", " ", o).strip() in plain: + return o + best, score = best_match(o, sents, inv, plain) + if best and score >= ACCEPT: + repl[o] = best + rows.append((jf, key, f"{score:.2f}", "replace", o, best)) + return best + rows.append((jf, key, f"{score:.2f}", "drop", o, best or "")) + return None + + # top-level "methods"/"hypotheses" are containers here, not the excerpt-free key of the same name + new = {k: (v if k in ("source", "gaps") else walk(v, "" if k in ("methods", "hypotheses") else k, fn)) for k, v in data.items()} + if a.write: + (ROOT / jf).write_text(json.dumps(new, indent=1, ensure_ascii=False) + "\n") + hits = 0 + for o, n in repl.items(): + for q in (json.dumps(o, ensure_ascii=False), '"' + o.replace('"', '\\"') + '"'): + if q in py: + py = py.replace(q, json.dumps(n, ensure_ascii=False)); hits += 1; break + (ROOT / pyf).write_text(py) + print(f"{pyf}: {hits}/{len(repl)} replacements applied to source literals") + acts = {} + for r in rows: + acts[(r[0], r[3])] = acts.get((r[0], r[3]), 0) + 1 + for k, v in sorted(acts.items()): + print(*k, v) + if a.report: + with open(a.report, "w") as f: + for r in rows: + f.write("\t".join(r) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/ingest/ingest.py b/scripts/ingest/ingest.py index c50d605396..ee20d53369 100644 --- a/scripts/ingest/ingest.py +++ b/scripts/ingest/ingest.py @@ -54,6 +54,9 @@ ] ZERO_ARG_DROP = ["fix", "new"] +# Labels defined in main.tex, outside any chapter, mapped to where the site shows them. +EXTERNAL_TARGETS = {"fig:simplified-taxonomy": "#taxonomy"} # homepage taxonomy figure + # Citation-colour macros -> (natbib command, evidence tier) CITE_MACROS = { "citeest": ("citep", "established"), "citeem": ("citep", "emerging"), @@ -398,6 +401,10 @@ def main() -> None: tex = tex.replace("\\cmark", "✓").replace("\\xmark", "✗") tex = re.sub(r"\\rowcolor(\[[^\]]*\])?\{[^}]*\}", "", tex) tex = re.sub(r"\\(?:hspace|vspace)\*?\{[^}]*\}", "", tex) + # tcolorbox environments (warningbox, practicebox, …) take their title as an optional argument; + # pandoc drops it, so emit it as the box's first line. + tex = re.sub(r"\\begin\{(definitionbox|examplebox|insightbox|practicebox|theorybox|warningbox)\}\[([^\]]*)\]", + lambda m: f"\\begin{{{m.group(1)}}}\n\\noindent\\textbf{{{m.group(2)}}}\n\n", tex) tex = tex.replace("\\begin{wrapfigure}", "\\begin{figure}").replace("\\end{wrapfigure}", "\\end{figure}") tex = re.sub(r"(\\begin\{figure\})\{[^}]*\}\{[^}]*\}", r"\1", tex) # equations: anchor + exact number from the LaTeX build @@ -445,11 +452,13 @@ def fig_path(m): work = ROOT / ".ingest-tmp" work.mkdir(exist_ok=True) (work / "combined.tex").write_text(combined) - bib = next(iter(ms.glob("*.bib"))) + bib = next(iter(ms.glob("*.bib")), None) # only needed without a .bbl (e.g. the arXiv source ships none) cmd = ["pandoc", str(work / "combined.tex"), "-f", "latex", "-t", "html5", "--katex", "--section-divs", "--wrap=none", "--reference-location=section", "-o", str(work / "combined.html")] if bbl_items is None: # no compiled bibliography: fall back to citeproc + if bib is None: + sys.exit("no .bbl and no .bib in the manuscript directory") cmd[5:5] = ["--citeproc", "--bibliography", str(bib), "--metadata", "link-citations=true", "--metadata", "reference-section-title=References"] audit["warnings"].append("no .bbl found: citations rendered by citeproc, which may differ from the PDF") @@ -518,6 +527,8 @@ def build_chapters(doc, labels, toc_nums, tiers, audit, figures, ms, stem, statu for m in head_re.finditer(doc): sid, lvl, cls = m.group(1), int(m.group(2)), m.group(3) if "unnumbered" in cls: + if lvl == 1: + heading_meta[sid] = {"num": "", "page": None, "level": lvl, "title": strip_tags(m.group(6)).strip()} continue nxt = next(toc_iter, None) if nxt is None or nxt["level"] != lvl: @@ -544,7 +555,7 @@ def chapter_id(top_id: str, title: str): end = tops[i + 1].start() if i + 1 < len(tops) else len(doc) frag = doc[m.start():end] meta = heading_meta.get(m.group(1), {}) - num = meta.get("num") or str(i + 1) + num = meta["num"] if meta.get("num") is not None else str(i + 1) slug = chapter_id(m.group(1), meta.get("title", "")) if slug is None: unmatched.append(f" {num}. {meta.get('title', '')} (label: {m.group(1)})") @@ -597,7 +608,18 @@ def f(m): rest = re.sub(r"(]*>|]*>)", rf'\1{kind_word} {num}: ', rest, count=1) return f'<{tag} id="{fid}"' + rest return f - h = re.sub(r'
)', cap_sub("Figure", "figure"), h, flags=re.S) + # Figures with subfigures: pandoc nests
s and gives the outer one the last subfigure's + # label. Number the subfigures first, then give the outer caption the parent number (7a, 7b -> 7). + def group_open(m): + return f'
' + h = re.sub(r'
]*)>(?=\s*
)', cap_sub("Figure", "figure"), h, flags=re.S) + def group_cap(m): + nums = re.findall(r'class="capnum">Figure (\d+)[a-z]?:', m.group(0)) + if not nums: + return m.group(0) + return re.sub(r'(
\s*
)(?![^<]*Figure {nums[-1]}: ', m.group(0)[::-1][::-1], count=0)[:0] + m.group(1) + m.group(2) + f'
Figure {nums[-1]}: ' + m.group(3) + h = re.sub(r'(
]*>)(.*
\s*)
((?:(?!\s*
)', group_cap, h, flags=re.S) h = re.sub(r')', cap_sub("Table", "table"), h, flags=re.S) # tables wrapped in a div with the label id (pandoc 3 behaviour) def tabdiv(m): @@ -626,10 +648,14 @@ def xref(m): lbl = m.group(2) num = label_num(lbl) text = m.group(3) - if num and (text.strip("[]") == lbl or text.strip() in ("", "??")): + # \ref always prints LaTeX's number (from main.aux); pandoc's own figure/table + # numbering drifts when floats sit outside the chapters (e.g. the taxonomy figure in main.tex). + if num and ('data-reference-type="ref"' in m.group(1) or text.strip("[]") == lbl or text.strip() in ("", "??")): text = num target = id_to_chapter.get(lbl) href = f"{BASE_PLACEHOLDER}handbook/{target}/#{lbl}" if target else f"#{lbl}" + if not target and lbl in EXTERNAL_TARGETS: + return f'{text}' if target in withheld and target != ch["slug"]: href = f"{BASE_PLACEHOLDER}handbook/{target}/" if not target: diff --git a/scripts/urls/suggest_anchor_redirects.py b/scripts/urls/suggest_anchor_redirects.py new file mode 100644 index 0000000000..5a726adc5c --- /dev/null +++ b/scripts/urls/suggest_anchor_redirects.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Map every published anchor that no longer exists to its closest section in the current manuscript. + +Matches stay within the chapter the old page now redirects to; anything below 0.7 lands on the chapter top. + +Run after `npm run build`, when a new manuscript version has been ingested: + + python3 scripts/urls/suggest_anchor_redirects.py --old-ref main [--write] + +For each anchor in src/data/published-urls.json that is missing from its (redirected) page, +it looks for, in order: the same id in any chapter; else the section of the new manuscript +whose title best matches the old section's title (old titles come from the manifest at +--old-ref); else the top of the chapter the page now redirects to. It prints the proposed +`anchors` entries for src/data/redirects.json, with the match score, and writes them with --write. +Review the low-scoring ones by hand. +""" +import argparse, difflib, json, re, subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +DIST = ROOT / "dist" + + +def norm(s): + s = re.sub(r"[^a-z0-9 ]+", " ", s.lower().replace("behavior", "behaviour").replace("ization", "isation").replace("izing", "ising").replace("color", "colour")) + return " ".join(w for w in s.split() if w not in {"the", "a", "an", "of", "and", "for", "in", "to", "on", "as", "via", "sec", "subsec", "fig", "tab"}) + + +def final_path(path, pages): + for _ in range(6): + if path in pages: + path = pages[path] + else: + return path + return path + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--old-ref", default="main") + ap.add_argument("--write", action="store_true") + a = ap.parse_args() + reg = json.loads((ROOT / "src/data/published-urls.json").read_text())["pages"] + red = json.loads((ROOT / "src/data/redirects.json").read_text()) + new = json.loads((ROOT / "src/generated/manifest.json").read_text()) + old = json.loads(subprocess.run(["git", "show", f"{a.old_ref}:src/generated/manifest.json"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout) + + head_re = re.compile(r'
\s*]*>(.*?)', re.S) + strip = lambda t: re.sub(r"<[^>]+>", "", t) + + def headings(html): + return [(m.group(1), int(m.group(2)), strip(m.group(3))) for m in head_re.finditer(html)] + + # every heading (all levels, incl. \paragraph) of the old and new chapter fragments + old_titles = {} + files = subprocess.run(["git", "ls-tree", "--name-only", f"{a.old_ref}:src/generated/chapters/"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout.split() + for f in files: + h = subprocess.run(["git", "show", f"{a.old_ref}:src/generated/chapters/{f}"], cwd=ROOT, capture_output=True, text=True).stdout + for sid, _, t in headings(h): + old_titles.setdefault(sid, t) + cands, tops = [], {} # (slug, id, title) + for ch in new["toc"]: + h = (ROOT / "src/generated/chapters" / f"{ch['slug']}.html").read_text() + hs = headings(h) + if hs: + tops[ch["slug"]] = hs[0][0] + cands += [(ch["slug"], sid, t) for sid, _, t in hs] + + def page_has(path, id_): + f = DIST / path / "index.html" + return f.exists() and f' id="{id_}"' in f.read_text() + + out, report = {}, [] + for path, ids in reg.items(): + if not path.startswith("handbook/"): + continue + end = final_path(path, red["pages"]) + for id_ in ids: + if page_has(end, id_) or f"{path}#{id_}" in red["anchors"]: + continue + # 1. same id elsewhere + hit = next(((c[0], c[1]) for c in cands if c[1] == id_), None) + score = 1.0 + if not hit: + title = norm(old_titles.get(id_, id_.replace("-", " "))) + best, score = None, 0.0 + here = end.split("/")[1] + for slug, sid, t in cands: + if slug != here: # a wrong section elsewhere is worse than the chapter top + continue + r = difflib.SequenceMatcher(None, title, norm(t)).ratio() + if r > score: + best, score = (slug, sid), r + hit = best if score >= 0.7 else None + if not hit: # 3. top of the chapter the page now points to + slug = end.split("/")[1] + first = tops.get(slug) + if not first: + report.append((0, f"{path}#{id_}", "?? no target")) + continue + hit, score = (slug, first), 0.0 + target = f"handbook/{hit[0]}/#{hit[1]}" + out[f"{path}#{id_}"] = target + report.append((round(score, 2), f"{path}#{id_}", target)) + + for r in sorted(report): + print(*r, sep=" ") + print(f"{len(out)} anchor redirects") + if a.write: + red["anchors"].update(out) + (ROOT / "src/data/redirects.json").write_text(json.dumps(red, indent=1, ensure_ascii=False) + "\n") + + +if __name__ == "__main__": + main() diff --git a/src/components/DemoFrame.astro b/src/components/DemoFrame.astro index a912aa6a8e..bdd97981c8 100644 --- a/src/components/DemoFrame.astro +++ b/src/components/DemoFrame.astro @@ -15,8 +15,7 @@ const nb = notebooks.find((n) => n.path === L.notebook);

Interactive demos

{title}

- {kind === 'measured' ? 'Measured results' : 'Computed live in your browser'} - {kind === 'measured' ? 'Every number on this page comes from a script you can re-run.' : 'Every number is computed by real training in your browser, not drawn by hand.'} + {kind === 'measured' ? 'Measured on GPT-2 small' : 'Runs in your browser'}

diff --git a/src/components/Excerpt.astro b/src/components/Excerpt.astro index fa7ce835f8..24d1075553 100644 --- a/src/components/Excerpt.astro +++ b/src/components/Excerpt.astro @@ -7,5 +7,5 @@ const html = excerptHtml(text); ---
- {section && } + {section && (section.href || section.num) && }
diff --git a/src/components/ManuscriptNote.astro b/src/components/ManuscriptNote.astro index 1cdb5448a9..02c3fd2f36 100644 --- a/src/components/ManuscriptNote.astro +++ b/src/components/ManuscriptNote.astro @@ -2,10 +2,10 @@ interface Props { kind?: 'manuscript' | 'measured' | 'illustrative' | 'editorial' } const { kind = 'manuscript' } = Astro.props; const text = { - manuscript: 'Verbatim from the survey', + manuscript: '', // quotes are the paper's own text; no badge needed measured: 'Measured result', illustrative: 'Illustrative', - editorial: 'Site summary, not survey text', + editorial: 'Summary', }[kind]; --- -{text} +{text && {text}} diff --git a/src/components/MethodsExplorer.tsx b/src/components/MethodsExplorer.tsx index a60894a8a3..4ad4d878d2 100644 --- a/src/components/MethodsExplorer.tsx +++ b/src/components/MethodsExplorer.tsx @@ -63,7 +63,7 @@ export default function MethodsExplorer({ rows, families }: Props) { setOnlyRunnable(e.target.checked)} /> Only methods with a notebook or demo

- {shown.length} of {rows.length} methods. Tick up to three to compare them side by side. + {shown.length} of {rows.length} methods. Select up to three to compare.

@@ -71,7 +71,7 @@ export default function MethodsExplorer({ rows, families }: Props) {

Comparison

- Survey text only. A blank cell means the survey doesn’t state it for that method. +
@@ -110,7 +110,7 @@ export default function MethodsExplorer({ rows, families }: Props) { {r.definition ?

- :

No definition sentence in the survey.

} + : null} ))} diff --git a/src/components/demos/PatchingDemo.tsx b/src/components/demos/PatchingDemo.tsx index 0fe8109ad0..e7bd0951c4 100644 --- a/src/components/demos/PatchingDemo.tsx +++ b/src/components/demos/PatchingDemo.tsx @@ -60,7 +60,7 @@ export default function PatchingDemo({ data }: { data: Data }) { format={(v) => `${(v * 100).toFixed(1)}%`} rowTitle="layer" colTitle="head" />} -

{detail ?? 'Click or tab to a cell for its value.'}

+

{detail ?? 'Select a cell to see its value.'}

{view === 'resid' ? ( diff --git a/src/data/chapter-ids.json b/src/data/chapter-ids.json index 90399439b7..36b02208c2 100644 --- a/src/data/chapter-ids.json +++ b/src/data/chapter-ids.json @@ -88,16 +88,16 @@ }, { "id": "developmental-interpretability", - "labels": [ - "sec:training" - ], + "labels": [], "titles": [ "Developmental Interpretability and Training Dynamics" ] }, { "id": "learning-dynamics", - "labels": [], + "labels": [ + "sec:training" + ], "titles": [ "Learning Dynamics and Mechanistic Interpretability" ] @@ -186,6 +186,16 @@ "titles": [ "Mathematical Foundations" ] + }, + { + "id": "acknowledgements", + "labels": [ + "acknowledgments" + ], + "titles": [ + "Acknowledgments", + "Acknowledgements" + ] } ] } diff --git a/src/data/framework.json b/src/data/framework.json index 83dd6271ab..af9965fc18 100644 --- a/src/data/framework.json +++ b/src/data/framework.json @@ -1,230 +1,227 @@ { - "source": "plain/*.txt chapter dumps of 'The State of Interpretability for AI Safety' draft", + "source": "Quoted from arXiv:2611.08438v1 (re-selected 6 Nov 2026). Orientations: src/data/orientations-arxiv.json.", "evaluation": { "chapter_sections": [ "sec:evaluation-framework" ], "framework_intro": [ - "A persistent challenge in mechanistic interpretability is the lack of standardized evaluation. Different methods—SAEs, transcoders, circuit discovery, probing—are evaluated using incompatible metrics, making it difficult to compare approaches or assess progress.", - "This section proposes a unified evaluation framework covering five dimensions: faithfulness, completeness, stability, causal validity, and human interpretability.", - "We propose evaluating mechanistic interpretability methods across five complementary dimensions. No single dimension suffices; strong methods should perform well across all five." + "A persistent challenge in mechanistic interpretability (MI) is the lack of standardised evaluation. Different interpretability methods use incompatible metrics.", + "To address this, this section presents a unified evaluation framework covering five dimensions—attribution faithfulness, completeness, stability, causal validity, and human interpretability—where different methods can be jointly tested.", + "We propose a framework that situates mechanistic interpretability evaluation within five complementary dimensions. No single dimension suffices—strong methods should perform well across all five." ], "motivation": [ - "Metric fragmentation: SAEs are evaluated on reconstruction loss; circuits on task performance recovery; probes on classification accuracy. These metrics are not directly comparable.", - "Cherry-picking: Results are often reported on favorable examples without systematic coverage of failure cases.", - "Missing baselines: Claims of interpretability gains rarely compare against simple baselines (e.g., random directions, PCA components, raw neurons).", - "Reproducibility gaps: Hyperparameter sensitivity and seed dependence are underreported, making replication difficult.", - "A unified framework enables principled comparison across methods, systematic identification of failure modes, and clearer communication of what interpretability methods actually achieve." + "Utility ambiguity: Methods are initially developed for different aims and evaluated against distinct objectives, yet have overlapping application scenarios.", + "Cherry-picking: Existing work often reports interpretability results on carefully selected examples, samples from a narrow distribution, or capabilities where a method performs well.", + "Missing baselines: Many proposed methods’ claimed interpretability gains are not systematically and adequately compared against intuitive alternatives or baselines such as behavioural methods, random activations, or neuron-level analysis.", + "Reproducibility gaps: Many existing techniques involve substantial hyperparameter choices, heavy training procedures, and high variation across seemingly irrelevant factors. Reproducibility is often underreported and has not yet been established as a property to optimise.", + "Faithfulness–understandability gap: Methods often demonstrate how understandable and informative their outputs are to humans. However, those outputs’ faithfulness to the underlying model mechanism receives less attention. This rewards explanations that are persuasive rather than faithful.", + "The MI community needs standardised, concrete evaluation protocols and benchmarks, along with established reporting practices, to enable principled comparisons across methods and findings." ], - "motivation_section": "the-need-for-standardized-evaluation", + "motivation_section": "the-need-for-standardised-evaluation", "table_id": "tab:evaluation-framework", - "table_caption": "Table 10: Unified Evaluation Framework for Mechanistic Interpretability", + "table_caption": "Table 9: Unified evaluation framework for mechanistic interpretability.", "dimensions": [ { "id": "faithfulness", - "name": "Faithfulness", - "heading": "7.6.3 Dimension 1: Faithfulness", + "name": "Attribution Faithfulness", + "heading": "7.5.2 Dimension 1: Attribution Faithfulness", "sections": [ - "dimension-1-faithfulness", - "five-evaluation-dimensions" + "dimension-1-attribution-faithfulness", + "subsec:five eval dim" ], - "definition": "Definition: Faithfulness measures whether identified components (features, circuits, explanations) accurately reflect the model’s actual internal computation, rather than post-hoc rationalizations or correlational artifacts.", + "definition": "Attribution faithfulness measures whether the interpretability method accurately attributes the model’s behaviour of interest to responsible components.", "metrics_named": [ - "Causal Effect Size", - "Intervention Consistency", - "Counterfactual Accuracy" + "Intervention-Effect Agreement", + "Effect-Size Error", + "Counterfactual Effect Recovery" ], "how_measured": [ - "Higher values indicate stronger causal relationship between the component and behavior.", - "Intervention Consistency: Across multiple interventions on C, do effects on B remain consistent?", - "Counterfactual Accuracy: When component C is patched from input x_1 to input x_2, does the model’s behavior shift as predicted?", - "Identify component C via the interpretability method being evaluated.", - "Define target behavior B with clear operational criteria.", - "Perform ablation: replace C with baseline (zero, mean, noise) and measure change in B.", - "Perform activation patching: swap C between clean and corrupted inputs; verify behavior follows C.", - "Report effect sizes with confidence intervals across multiple inputs.", - "Baselines: Compare against random components, PCA directions, and individual neurons to establish that identified components are more faithful than naive alternatives." + "Let a(S,x) denote the importance score assigned by an interpretability method to components S for a sample x \\sim B:", + "High agreement indicates that the attribution consistently reflects the identified components’ causal effects on behaviour B.", + "A low value indicates good calibration between estimated and actual intervention effects. If the two quantities are not naturally on the same scale, we may use normalised or rank-based measures instead.", + "A value near 1 indicates that the patching recovers much of the behaviour, while a value near 0 indicates little effect. This metric has been widely used in the circuit discovery literature (Hanna et al., 2024; Zhang et al., 2025b; Syed et al., 2024).", + "Apply the method to acquire attribution scores a(S,x) of model components and select those of interest (e.g., components with high scores).", + "Measure the corresponding ground-truth intervention effects s(S,x).", + "Compare them using correlation, error, or rank agreement.", + "Evaluate how much of the behavioural difference between clean and corrupted inputs can be recovered by patching the identified components.", + "Report results across a diverse set of inputs with uncertainty estimates." ], "formulas_tex": [ - "\\text{Effect}(C, B) = \\frac{|P(B | \\text{do}(C = \\text{active})) - P(B | \\text{do}(C = \\text{ablated}))|}{P(B | \\text{baseline})}", - "\\text{Consistency}(C, B) = 1 - \\frac{\\text{Var}[\\text{Effect}(C, B)]}{\\mathbb{E}[\\text{Effect}(C, B)]^2}", - "\\text{CF-Acc} = \\mathbb{E}_{x_1, x_2}\\left[\\mathbf{1}\\left[f(x_1|_{C \\leftarrow C(x_2)}) \\approx f(x_2)\\right]\\right]" + "\\mathrm{EffectAgreement} = \\rho_{x \\sim B}\\!\\left( a(S,x), s(S,x) \\right)", + "\\mathrm{EffectError} = \\mathbb{E}_{x \\sim B} \\left[ \\left| a(S,x)-s(S,x) \\right| \\right]", + "\\mathrm{CFRecovery}(S;B,B') = \\mathbb{E}_{x_1 \\sim B', x_2 \\sim B} \\frac{ M\\!\\left( f_\\theta\\!\\left( x_{1,[S \\leftarrow h_S(x_2)]} \\right) \\right) - M\\!\\left(f_\\theta(x_1)\\right) }{ M\\!\\left(f_\\theta(x_2)\\right) - M\\!\\left(f_\\theta(x_1)\\right) }" ], "examples": [], - "limitations": [], + "limitations": [ + "These may cause this metric to take values much larger than 1 or below 0, complicating the analysis. Recent work has proposed an alternative metric that bounds the value between 0 and 1 (Wu & Barez, 2026)." + ], "table_row": { - "Dimension": "Faithfulness", - "Definition": "Do identified components accurately reflect the model’s actual computation?", - "Metrics": "Causal effect size; intervention consistency; counterfactual accuracy", - "Methods": "Ablation; activation patching; interchange interventions" + "Dimension": "Attribution Faithfulness", + "Definition": "Do attribution scores accurately reflect the causal effects of the identified components?", + "Metrics": "Intervention-effect agreement; effect-size error; counterfactual effect recovery", + "Methods": "Ablation; activation / path patching; interchange interventions" } }, { "id": "completeness", "name": "Completeness", - "heading": "7.6.4 Dimension 2: Completeness", + "heading": "7.5.2 Dimension 2: Completeness", "sections": [ "dimension-2-completeness", - "five-evaluation-dimensions" + "subsec:five eval dim" ], - "definition": "Definition: Completeness measures what fraction of model behavior is explained by the identified components. High completeness means the method accounts for most of the model’s computation; low completeness means significant behavior remains unexplained.", + "definition": "Completeness measures how much of the model behaviour the identified components or mechanisms explain. A high completeness score indicates that the identified components/mechanisms capture most of the relevant computation.", "metrics_named": [ "Loss Recovery", - "Behavioral Coverage", + "Behavioural Coverage", "Unexplained Variance" ], "how_measured": [ - "Loss Recovery: After replacing model activations with reconstructions from identified components, what fraction of original loss is preserved?", - "Behavioral Coverage: Across a diverse test suite, what fraction of behaviors \\mathcal{B} can be explained by identified components?", - "where \\tau is set to be the smallest effect that counts as meaningful.", - "Unexplained Variance: What fraction of activation variance is not captured by identified features?", - "Define a comprehensive behavioral test suite spanning diverse capabilities (factual recall, reasoning, safety behaviors, etc.).", - "For each behavior, attempt to identify responsible components.", - "Report coverage: fraction of behaviors with identified components meeting faithfulness thresholds.", - "Report reconstruction metrics: loss recovery and unexplained variance.", - "Analyze failure cases: which behaviors resist explanation?" + "For methods that reconstruct model activations (e.g., SAEs and CLTs), after replacing the target component’s activations with the reconstructions, we measure the fraction of the original model performance that is recovered:", + "A value near 1 indicates that the reconstruction preserves most of the information.", + "Across input prompts x \\sim B, we measure the fraction for which the identified components have a meaningful intervention effect:", + "Higher coverage indicates that the identified components account for the behaviour across a larger fraction of inputs.", + "For methods that reconstruct activations of the components, we measure the fraction of unexplained activation variance as:", + "Lower values indicate that the identified components explain a larger fraction of the variance in the original activations.", + "Report behavioural coverage using the intervention effects s(S,x).", + "Report reconstruction-based metrics such as loss recovery and unexplained variance if applicable.", + "Analyse inputs of the behaviour for which the identified components do not provide good coverage or reconstruction." ], "formulas_tex": [ - "\\text{LR} = 1 - \\frac{\\mathcal{L}_{\\text{reconstructed}} - \\mathcal{L}_{\\text{original}}}{\\mathcal{L}_{\\text{random}} - \\mathcal{L}_{\\text{original}}}", - "\\text{Coverage} = \\frac{|\\{b \\in \\mathcal{B} : \\exists C \\text{ with } \\text{Effect}(C, b) \\geq \\tau\\}|}{|\\mathcal{B}|}", - "\\text{UV} = \\frac{\\mathbb{E}[\\|h - \\hat{h}\\|^2]}{\\mathbb{E}[\\|h\\|^2]}" + "\\mathrm{LR} = \\frac{ \\mathcal{L}_{\\mathrm{reconstructed}} - \\mathcal{L}_{\\mathrm{ablated}} }{ \\mathcal{L}_{\\mathrm{original}} - \\mathcal{L}_{\\mathrm{ablated}} }", + "\\mathrm{Coverage}(S,B) = \\mathbb{E}_{x \\sim B} \\left[ \\mathbbm{1} \\left( |s(S,x)| \\geq \\tau \\right) \\right]", + "\\mathrm{UV}(S,B) = \\frac{ \\mathbb{E}_{x \\sim B} \\left[ \\|h(x)-\\hat{h}_S(x)\\|_2^2 \\right] }{ \\mathbb{E}_{x \\sim B} \\left[ \\|h(x)-\\bar{h}_B\\|_2^2 \\right] }" ], "examples": [], "limitations": [ - "Critical consideration: Methods may achieve high faithfulness on explained behaviors while leaving most behavior unexplained. Completeness ensures we track the gap between what is explained and what remains opaque." + "Similar to the vanilla Counterfactual Effect Recovery metric discussed above, this metric can take arbitrary values. A simple solution is to develop a bounded variant, as has been done for Counterfactual Effect Recovery (Wu & Barez, 2026)." ], "table_row": { "Dimension": "Completeness", - "Definition": "What fraction of model behavior is explained by the identified components?", - "Metrics": "Behavior coverage; reconstruction loss; unexplained variance", - "Methods": "Loss recovery; behavioral replication; residual analysis" + "Definition": "How much of the behaviour is accounted for by the identified components?", + "Metrics": "Loss recovery; behavioural coverage; unexplained variance", + "Methods": "Reconstruction; behavioural evaluation; residual analysis" } }, { "id": "stability", "name": "Stability", - "heading": "7.6.5 Dimension 3: Stability", + "heading": "7.5.2 Dimension 3: Stability", "sections": [ "dimension-3-stability", - "five-evaluation-dimensions" + "subsec:five eval dim" ], - "definition": "Definition: Stability measures whether findings are robust to incidental choices—random seeds, training checkpoints, hyperparameter settings—that should not affect conclusions about model internals.", + "definition": "Stability measures the robustness of explanations, attributions, and findings to changes in variables that should not affect the results, such as the choice of random seeds, small hyperparameter changes, and input paraphrasing.", "metrics_named": [ "Cross-Seed Agreement", - "Checkpoint Consistency", - "Hyperparameter Sensitivity" + "Hyperparameter Sensitivity", + "Paraphrase Consistency" ], "how_measured": [ - "Cross-Seed Agreement: Train the interpretability method (e.g., SAE) with K different random seeds.", - "Checkpoint Consistency: For features/circuits identified at checkpoint t, do they persist at checkpoints t \\pm \\Delta?", - "Hyperparameter Sensitivity: How much do results change under reasonable hyperparameter variation?", - "Lower sensitivity indicates more robust findings.", - "Train interpretability method with at least 3–5 random seeds.", - "Compute feature/circuit alignment across seeds.", - "Report which findings are stable (appear across all seeds) vs. unstable (seed-specific).", - "For key claims, verify they hold under hyperparameter variation (\\pm 20\\% on key parameters).", - "Distinguish “core” findings (stable) from “fragile” findings (unstable).", - "Reporting recommendation: Papers should report stability metrics alongside primary results. Claims based on unstable findings should be flagged as preliminary." + "We train or apply the interpretability method K times under different random seeds.", + "We measure the sensitivity of mechanistic analyses or methods under reasonable hyperparameter variations.", + "Lower values indicate that the finding is less sensitive to hyperparameter choices and therefore more robust.", + "We measure the consistency of explanations or attributions across \\mathcal{P}(x).", + "The score lies in [0, 1]. A value of 1 indicates that the intervention effect is identical across all paraphrases.", + "Execute the interpretability method with several random seeds and compute alignment of the discovered units or findings across seeds.", + "Analyse which findings are stable across all seeds and which ones are seed-specific. Key claims need to remain consistent under subsequent tests on hyperparameter variation and input paraphrasing, if applicable.", + "Distinguish robust findings from fragile ones and report both of them." ], "formulas_tex": [ - "\\text{Seed-Stability} = \\frac{1}{K(K-1)} \\sum_{i \\neq j} \\text{Align}(F_i, F_j)", - "\\text{Ckpt-Stability} = \\mathbb{E}_t[\\text{Align}(F_t, F_{t+\\Delta})]", - "\\text{HP-Sensitivity} = \\frac{\\max_{\\theta \\in \\Theta} \\text{Metric}(\\theta) - \\min_{\\theta \\in \\Theta} \\text{Metric}(\\theta)}{\\mathbb{E}_{\\theta \\in \\Theta}[\\text{Metric}(\\theta)]}" + "\\text{Seed-Stability} = \\frac{1}{K(K-1)} \\sum_{i \\neq j} \\text{Align}(\\mathcal{F}_i, \\mathcal{F}_j)", + "\\text{HP-Sensitivity} = \\frac{ \\max_{\\omega \\in \\Omega} \\text{Metric}(\\omega) - \\min_{\\omega \\in \\Omega} \\text{Metric}(\\omega) }{ \\mathbb{E}_{\\omega \\in \\Omega}[\\text{Metric}(\\omega)] }", + "\\text{Para-Consistency}(S, B) = \\mathbb{E}_{x \\sim B}\\left[ 1 - \\frac{\\mathrm{Var}_{x' \\in \\mathcal{P}(x)}\\left[s(S, x')\\right]}{\\mathbb{E}_{x' \\in \\mathcal{P}(x)}\\left[s(S, x')^2\\right]} \\right]" + ], + "examples": [ + "For example, Paulo & Belrose (2026) calculate the Hungarian matching between two sets of SAE features based on their pairwise cosine similarities." ], - "examples": [], "limitations": [], "table_row": { "Dimension": "Stability", - "Definition": "Are findings robust across seeds, checkpoints, and hyperparameters?", + "Definition": "Are findings robust to variations in factors like random seeds, hyperparameters, paraphrases, etc.?", "Metrics": "Cross-seed agreement; checkpoint consistency; hyperparameter sensitivity", - "Methods": "Multi-seed training; checkpoint comparison; ablation studies" + "Methods": "Multi-seed analysis; checkpoint comparison; input paraphrasing" } }, { "id": "causal-validity", "name": "Causal Validity", - "heading": "7.6.6 Dimension 4: Causal Validity", + "heading": "7.5.2 Dimension 4: Causal Validity", "sections": [ "dimension-4-causal-validity", - "five-evaluation-dimensions" + "subsec:five eval dim" ], - "definition": "Definition: Causal validity assesses whether identified components satisfy formal causal criteria of necessity (removing the component disrupts the behavior) and sufficiency (the component alone can produce the behavior).", + "definition": "Causal validity assesses whether the identified components or mechanisms are causal to the behaviour, particularly necessity (i.e., removing the components/mechanisms disrupts the behaviour) and sufficiency (i.e., the components/mechanisms alone are sufficient to preserve the behaviour).", "metrics_named": [ "Necessity Score", "Sufficiency Score", "Specificity" ], "how_measured": [ - "Necessity Score: Fraction of behavior explained by the component that is lost when the component is ablated:", - "High necessity means the component is required for the behavior.", - "Sufficiency Score: Fraction of behavior that can be produced by the component alone (all other components ablated or set to baseline):", - "High sufficiency means the component is enough for the behavior.", - "Specificity: Does the component affect only the claimed behavior, or does it have broad side effects?", - "High specificity means the component is selective for the target behavior.", - "Necessity test: Ablate the identified circuit/feature. Measure degradation on target behavior. Compare to ablating random components of equal size.", - "Sufficiency test: Ablate all components except the identified circuit. Measure preservation of target behavior. Compare to random circuits of equal size.", - "Interchange test: Swap circuit activations between inputs that differ on the target behavior. Verify behavior follows the swapped circuit.", - "Specificity test: Measure effects on a suite of non-target behaviors. Compute specificity ratio." + "Necessity measures the fraction of the model’s performance on behaviour B that is lost when S is ablated (i.e., its activations are replaced with some reference values).", + "A higher value indicates that S is necessary for the behaviour.", + "Sufficiency measures how much of the original behaviour is achieved when only S is active (i.e., activations of all components outside S are replaced with some reference values).", + "A high value indicates that S alone implements much of the behaviour.", + "Specificity measures the effect on non-target behaviours by interventions on S.", + "A high value indicates that the causal effect of S is specific to B.", + "Necessity test: Ablate S and measure the resulting degradation on behaviour B. Compare against ablating random components of matched size.", + "Sufficiency test: Replace components outside S with reference activations and measure how much of behaviour B is preserved. Compare against random component sets of matched size.", + "Interchange test: Swap activations of S between inputs representing different behaviours and test whether the model output follows the intervention.", + "Specificity test: Measure intervention effects on a set of non-target behaviours and examine whether the effect is specific to B." ], "formulas_tex": [ - "\\text{Necessity}(C, B) = \\frac{P(B | \\text{baseline}) - P(B | \\text{do}(C = \\text{ablated}))}{P(B | \\text{baseline})}", - "\\text{Sufficiency}(C, B) = \\frac{P(B | \\text{only } C \\text{ active})}{P(B | \\text{baseline})}", - "\\text{Specificity}(C, B) = \\frac{\\text{Effect}(C, B)}{\\sum_{B' \\in \\mathcal{B}} \\text{Effect}(C, B')}" + "\\mathrm{Necessity}(S,B) = 1- \\frac{ \\mathbb{E}_{x\\sim B} \\left[ M\\!\\left( f_\\theta\\!\\left( x_{[S\\leftarrow\\tilde h_S]} \\right) \\right) \\right] }{ \\mathbb{E}_{x\\sim B} \\left[ M\\!\\left(f_\\theta(x)\\right) \\right] }", + "\\mathrm{Sufficiency}(S,B) = \\frac{ \\mathbb{E}_{x\\sim B} \\left[ M\\!\\left( f_\\theta\\!\\left( x_{[\\bar S \\leftarrow \\tilde h_{\\bar S}]} \\right) \\right) \\right] }{ \\mathbb{E}_{x\\sim B} \\left[ M\\!\\left(f_\\theta(x)\\right) \\right] }", + "\\mathrm{Specificity}(S,B) = \\frac{ |s(S,B)| }{ \\sum_{B' \\in \\mathcal{B}} |s(S,B')| }" ], "examples": [ - "Benchmarks: The MIB benchmark (Mueller et al., 2025a) provides standardized tasks and metrics for causal circuit evaluation. AxBench (Wu et al., 2025) and RAVEL (Huang et al., 2024) provide feature-level causal evaluation." + "The MI community has developed several benchmarks that examine the causal validity of interpretability methods. For example, MIB (Mueller et al., 2025) provides standardised tasks for assessing discovered circuits and causal variables (e.g., features); AxBench (Wu et al., 2025b) evaluates methods’ ability to steer models towards given concepts; RAVEL (Huang et al., 2024) investigates methods’ ability to localise features of given concepts by using interchange interventions to validate discovered features; and SAEBench (Karvonen et al., 2025) evaluates SAEs specifically, pairing reconstruction metrics with downstream intervention tasks such as gender bias erasure." ], "limitations": [], "table_row": { "Dimension": "Causal Validity", - "Definition": "Do components satisfy necessity and sufficiency for target behaviors?", - "Metrics": "Necessity score; sufficiency score; specificity", - "Methods": "Causal scrubbing; knockout experiments; circuit verification" + "Definition": "Are the identified components necessary, sufficient, and specific for the target behaviour?", + "Metrics": "Necessity score; sufficiency score; specificity score", + "Methods": "Ablation; component isolation; interchange interventions; tests with control behaviours" } }, { "id": "human-interpretability", "name": "Human Interpretability", - "heading": "7.6.7 Dimension 5: Human Interpretability", + "heading": "7.5.2 Dimension 5: Human Interpretability", "sections": [ "dimension-5-human-interpretability", - "five-evaluation-dimensions" + "subsec:five eval dim" ], - "definition": "Definition: Human interpretability measures whether explanations are actually useful to humans—whether they improve human understanding, prediction, or decision-making about model behavior.", + "definition": "Human interpretability measures whether generated explanations eventually improve human understanding, prediction, and decision-making about model behaviour.", "metrics_named": [ - "Explanation Accuracy", - "Simulatability", + "Explanation Utility", "Debugging Utility", "Consistency of Interpretation" ], "how_measured": [ - "Explanation Accuracy: Can humans predict model behavior based on the explanation?", - "Compare to baseline prediction accuracy without explanation.", - "Simulatability: After seeing explanations, can humans simulate the model’s computation on new inputs?", - "Debugging Utility: Do explanations help humans identify and fix model errors?", - "Consistency of Interpretation: Do different humans interpret the same explanation consistently?", - "Prediction task: Show humans feature/circuit explanations. Ask them to predict model behavior on held-out inputs. Compare accuracy to no-explanation baseline.", - "Identification task: Show humans model outputs with errors. With and without mechanistic explanations, ask them to identify the cause. Measure accuracy and time.", - "Intervention task: Ask humans to propose fixes for model errors based on explanations. Measure success rate of proposed interventions.", - "Agreement task: Have multiple raters label feature interpretations. Measure inter-rater reliability." + "Explanation Utility. This metric tests whether humans can predict model behaviour more accurately in the held-out set based on the explanation.", + "Debugging Utility. This metric examines whether the explanations help humans identify and repair model errors, such as incorrect outputs or misaligned behaviour, in the wild.", + "Consistency of Interpretation. This metric inspects whether different humans interpret the same explanation consistently.", + "Prediction task: Show humans explanations of model components or mechanisms and ask them to predict model behaviour on held-out inputs. Compare accuracy to a no-explanation baseline.", + "Identification task: Show humans model outputs with errors and ask them to identify the cause, with and without explanation assistance. Measure accuracy and required time to complete the task.", + "Intervention task: Ask humans to propose fixes for model errors based on explanations. Measure the success rate of proposed interventions.", + "Agreement task: Ask multiple human raters to generate explanations, if applicable (e.g., write a summary of highly activated samples of an SAE feature as its explanation). Measure inter-rater reliability and agreement with machine-generated explanations." ], "formulas_tex": [ - "\\text{Expl-Acc} = P(\\text{human predicts } f(x) | \\text{explanation of } f)", - "\\text{Simulatability} = \\text{Corr}(\\text{human prediction}, f(x_{\\text{new}}))", - "\\text{Debug-Utility} = \\frac{\\text{\\#Errors fixed with explanation}}{\\text{\\#Errors fixed without explanation}}", + "\\text{Expl-Utility} = P(\\text{correct prediction} \\mid \\text{with explanation}) - P(\\text{correct prediction} \\mid \\text{no explanation})", + "\\text{Debug-Utility} = P(\\text{error fixed}\\mid\\text{with explanation}) - P(\\text{error fixed}\\mid\\text{no explanation})", "\\text{Inter-Rater} = \\kappa(\\text{human}_1 \\text{ labels}, \\text{human}_2 \\text{ labels})" ], "examples": [], "limitations": [ - "Human studies are expensive and introduce subjectivity. Automated proxies (LLM-based evaluation) can supplement but not replace human evaluation.", - "Expertise matters: interpretations accessible to ML researchers may be opaque to domain experts, and vice versa.", - "Task-specificity: a feature interpretable for one task may be confusing for another." + "Several caveats and challenges arise when evaluating the human interpretability of tested methods. For example, human studies are usually expensive. Moreover, human expertise may confound results, where some explanations understandable to MI researchers may be opaque to domain experts, and vice versa." ], "table_row": { - "Dimension": "Interpretability", - "Definition": "Can humans understand and use the explanations?", - "Metrics": "Human accuracy; explanation utility; prediction improvement", - "Methods": "User studies; expert evaluation; downstream task performance" + "Dimension": "Human Interpretability", + "Definition": "Can humans understand and use the resulting explanations?", + "Metrics": "Explanation utility; debugging utility; interpretation agreement", + "Methods": "User studies; expert evaluation" } } ], @@ -233,671 +230,103 @@ { "step": "Step 1: Define scope", "items": [ - "Specify the model, layer(s), and behavior(s) under analysis.", - "Define a behavioral test suite with clear success criteria." + "Users first specify the model, layer(s), and behaviour(s) of interest and define a behavioural test suite with clear success criteria." ] }, { "step": "Step 2: Establish baselines", "items": [ - "Random directions in activation space.", - "PCA components.", - "Individual neurons.", - "Supervised probes (if labels available)." + "Users specify baseline methods, such as random directions in activation space, PCA components, individual neurons, and supervised probes (if labels are available)." ] }, { "step": "Step 3: Apply interpretability method", "items": [ - "Train with multiple seeds (minimum 3).", - "Record hyperparameters and training details for reproducibility." + "Users apply the method of interest to analyse models with variation in random seeds, hyperparameters, and other training details." ] }, { "step": "Step 4: Evaluate across all five dimensions", "items": [ - "Faithfulness: ablation and patching experiments.", - "Completeness: loss recovery and behavioral coverage.", - "Stability: cross-seed and hyperparameter sensitivity.", - "Causal validity: necessity, sufficiency, specificity tests.", - "Human interpretability: user studies or proxy metrics." + "Users compute metrics for attribution faithfulness, completeness, stability, causal validity, and human interpretability." ] }, { "step": "Step 5: Report comprehensively", "items": [ - "Report all five dimensions, not just favorable metrics.", - "Include failure cases and unexplained behaviors.", - "Compare to baselines on all metrics.", - "Provide confidence intervals and significance tests." + "Users report all five dimensions instead of just favourable metrics and discuss the method’s failure cases, unexplained behaviours, and comparisons to baselines." ] } ], "method_guidance": { - "section": "applying-the-framework-method-specific-guidance", + "section": "subsec:five eval dim", "table_id": "tab:method-evaluation", - "caption": "Table 11: Evaluation Priorities by Method Type", - "intro": "Different interpretability methods emphasize different dimensions.", + "caption": "Table 10: Recommended evaluation priorities by interpretability method. The entries indicate evaluation aspects particularly important for each method, not mutually exclusive metrics.", + "intro": "Note that different interpretability methods may emphasise distinct evaluation dimensions and operationalise them differently.", "columns": [ "Method", - "Faithfulness", + "Attribution Faithfulness", "Completeness", "Stability", - "Causal", - "Human" + "Causal Validity", + "Human Interpretability" ], "rows": [ { - "Method": "SAEs", - "Faithfulness": "Feature ablation", - "Completeness": "Loss recovery; UV", - "Stability": "Seed stability critical", - "Causal": "Feature-level N/S", - "Human": "Auto-labeling quality" - }, - { - "Method": "Transcoders/CLTs", - "Faithfulness": "Layer reconstruction", - "Completeness": "Cross-layer coverage", - "Stability": "Architecture sensitivity", - "Causal": "Circuit-level N/S", - "Human": "Circuit descriptions" + "Method": "SAEs and variants", + "Attribution Faithfulness": "Feature attribution–intervention agreement; effect-size error", + "Completeness": "Activation reconstruction; loss recovery; unexplained variance", + "Stability": "Cross-seed feature alignment", + "Causal Validity": "Feature necessity, sufficiency, and specificity", + "Human Interpretability": "Feature description consistency across annotations, including models and humans" }, { "Method": "Circuit Discovery", - "Faithfulness": "Path patching", - "Completeness": "Behavioral coverage", - "Stability": "Method variance", - "Causal": "Primary focus", - "Human": "Circuit diagrams" + "Attribution Faithfulness": "Edge or component attribution–intervention agreement; counterfactual recovery", + "Completeness": "Behavioural coverage; loss or behaviour recovery", + "Stability": "Robustness across hyperparameters, seeds, and paraphrases", + "Causal Validity": "Circuit necessity, sufficiency, specificity; interchange tests", + "Human Interpretability": "Circuit-level descriptions; trade-off between sparsity and variance explained" }, { "Method": "Probing", - "Faithfulness": "Probe \\rightarrow causal", - "Completeness": "Information content", - "Stability": "Probe architecture", - "Causal": "Often weak", - "Human": "Concept labels" + "Attribution Faithfulness": "Agreement between probe-derived importance and intervention effects", + "Completeness": "Coverage across target concepts or behaviours under different distributions", + "Stability": "Robustness across seeds and sampled fitting sets", + "Causal Validity": "Necessity, specificity, and intervention tests on probe directions", + "Human Interpretability": "Explanation of shared information in fitting set and search for confounding factors" }, { "Method": "Steering Vectors", - "Faithfulness": "Behavioral effect", - "Completeness": "Behavior breadth", - "Stability": "Direction stability", - "Causal": "Implicit in method", - "Human": "Steering descriptions" + "Attribution Faithfulness": "Agreement between predicted and observed intervention effects", + "Completeness": "Coverage across behaviours deemed relevant", + "Stability": "Stability across steering magnitudes, seeds, and paraphrases", + "Causal Validity": "Intervention efficacy and specificity; side-effect analysis", + "Human Interpretability": "Explanation of shared information in fitting set and search for confounding factors" } ] }, "framework_limitations_section": "limitations-of-this-framework", "framework_limitations": [ - "Metric choice: The specific metrics proposed are not uniquely correct. Alternative formalizations may be appropriate for specific contexts.", - "Threshold arbitrariness: Proposed thresholds (e.g., necessity > 0.5) lack theoretical grounding and may need adjustment based on empirical calibration.", - "Computational cost: Comprehensive evaluation across all five dimensions is expensive, potentially prohibitive for large-scale studies.", - "Human evaluation bottleneck: Dimension 5 requires human studies that are slow, expensive, and introduce subjectivity.", - "Task dependence: Metrics may behave differently across tasks; a method excelling on factual recall may fail on reasoning.", - "Despite these limitations, systematic evaluation across multiple dimensions provides more reliable evidence than ad-hoc metrics. The framework should be viewed as a starting point for community standardization, not a final answer." + "Metric choice: The proposed specific metrics are not uniquely correct. Alternative formalisations may be appropriate for specific contexts.", + "Threshold arbitrariness: Thresholds (e.g., the accuracy that counts as success) may lack theoretical grounding and will need empirical calibration.", + "Computational cost: Evaluating all five dimensions comprehensively is expensive and potentially prohibitive for large-scale studies.", + "Human evaluation bottleneck: Dimension 5 requires time-consuming, expensive, and subjective human studies.", + "Task dependence: A method’s performance under this framework can depend on the evaluation task. For example, a method that excels at factual recall may perform poorly at reasoning.", + "Despite these limitations, systematic evaluation across multiple dimensions provides more reliable evidence than an arbitrary choice of metrics. Readers should view the framework as a starting point for standardisation, not a final answer." ], "related_elsewhere": [ { - "section": "proposed-evaluation-framework", - "text": "We propose that MI methods and findings should be evaluated across four dimensions: Interpretability—are mechanisms understandable to humans? Completeness—what fraction of behavior is explained? Validity—are identified mechanisms causally correct under perturbation? Efficiency—what is the computational cost?" - }, - { - "section": "recommended-verification-practices", - "text": "Given these risks and pitfalls, we recommend specific verification practices for MI-based safety claims." - } - ] - }, - "orientations": { - "chapter_sections": [ - "sec:orientations" - ], - "orientations_vs_methods": "An orientation is a coherent research program characterized by its goals, assumptions, and trade-offs, distinct from a method, which is a specific technique or tool employed within one or more orientations. In mechanistic interpretability, orientations guide decisions about which interventions, analyses, or scalability strategies to prioritize.", - "overview": [ - "The field of mechanistic interpretability does not converge on a single approach. Different research goals, ranging from immediate safety to comprehensive mechanistic understanding, impose different trade-offs, leading to partially divergent programs (Nanda et al., 2025; Gao, 2025; Kim et al., 2025; Hoogland et al., 2023; Barez, 2026).", - "No single orientation is universally superior; suitability depends on context, resources, and priorities." - ], - "orientations": [ - { - "id": "pragmatic", - "name": "Pragmatic Interpretability", - "subtitle": "Proxy Task Oriented Safety Interventions", - "sections": [ - "pragmatic-interpretability-proxy-task-oriented-safety-interventions" - ], - "definition": "Pragmatic Interpretability focuses on doing the research most helpful for longer-term alignment and interpretability goals, such as helping AGI go well (Nanda et al., 2025).", - "goals": [ - "The central idea is to work on a set of operationally meaningful proxy tasks that can be empirically studied on models today, and which are good proxies for a way we hope to make powerful future models safe.", - "This gives a way to practice today, and prepare for future problems." - ], - "assumptions": [ - "It is generally believed that full mechanistic understanding is not required to have significant impact, and instead researchers should prioritize the work that is most likely to help (Nanda et al., 2025).", - "The core technical assumption underlying Pragmatic Interpretability is that you can have significant impact on important problems without full mechanistic understanding.", - "Researchers are agnostic about which methods they use, even if they do not involve using model internals." - ], - "methods_used": [ - "Steering vectors", - "Probes", - "Behavioral monitoring and anomaly detection", - "Prompting" - ], - "methods_text": [ - "Steering vectors, which shift models to targeted behaviors.", - "Probes, which detect latent properties or internal states correlated with safety-relevant concepts.", - "Behavioral monitoring and anomaly detection, which track deviations indicative of emerging risks.", - "Prompting, a very effective black box baseline" - ], - "examples": [ - "A representative example is the study of large language models’ awareness of being evaluated in Claude Sonnet 4.5 (Anthropic, 2025a).", - "Researchers were able to identify and deploy the steering vector that suppresses this awareness, allowing its safety to be evaluated.", - "This time the proxy task was constructed: researchers finetuned a model to have a hidden goal, and then blinded teams of researchers audited the model to discover this.", - "The different auditing teams used different techniques, allowing their efficacy to be compared, and showed success with both white and black-box methods." - ], - "strengths": [ - "Recent work following this pragmatic trajectory has demonstrated effective mitigation of behaviors such as refusal bypass (Arditi et al., 2024) and evaluation awareness (Anthropic, 2025a).", - "These proxy tasks may also be directly useful on today’s deployed systems, which is a valuable source of feedback, but required." - ], - "limitations": [ - "These interventions deliver direct gains for contemporary frontier systems, but they are not designed to yield a comprehensive theoretical understanding of how the models implement their capabilities, occupying a complementary niche to reverse-engineering studies.", - "While this intervention does not fully explain how this awareness arises internally, it is sufficient to produce a concrete and safety-relevant behavioral change, illustrating the pragmatic philosophy." - ], - "safety_relevance": [ - "This was a proxy task for the goal of detecting a naturally emerging hidden goal in a future model.", - "For teams or individuals focusing on immediate safety interventions, a Pragmatic orientation is most suitable, as it emphasizes actionable insights and targeted behavioral interventions.", - "Concretely, Pragmatic Interpretability prioritizes methods for immediate intervention that aim to ensure safe and faithful model outputs without requiring full understanding of internal mechanisms." - ] + "section": "evaluating-circuits-and-benchmarking-discovery-methods.", + "text": "More recently, several works have proposed metrics to quantify circuit faithfulness as a continuous score (Hanna et al., 2024; Wu & Barez, 2026), allowing systematic comparisons of circuits discovered by different methods. By aggregating faithfulness scores across tasks and settings, benchmarks such as MIB (Mueller et al., 2025) evaluate how well circuit discovery methods identify causally important model components." }, { - "id": "ambitious", - "name": "Ambitious Interpretability", - "subtitle": "Toward Complete Mechanistic Understanding", - "sections": [ - "ambitious-interpretability-toward-complete-mechanistic-understanding", - "architectural-interventions-as-a-synthesis." - ], - "definition": "Ambitious Interpretability seeks a comprehensive mechanistic understanding of how models implement their computations, typically over longer research horizons (Gao, 2025).", - "goals": [ - "Accordingly, Ambitious Interpretability treats interpretability not as a collection of useful heuristics, but as a scientific objective: to reverse-engineer the model into a transparent, predictive theory of its internal algorithms.", - "Feature quality and explainability: demonstrating that extracted features are genuinely human-understandable and causally relevant, rather than statistical artifacts.", - "Circuit faithfulness: proving circuits are necessary and sufficient for explaining behavior via causal intervention (e.g., causal scrubbing), without performance degradation." - ], - "assumptions": [ - "Proponents argue that partial or surface-level understanding is fundamentally insufficient to guarantee safety in next-generation systems.", - "Without a full understanding of how internal representations give rise to behavior, it is impossible to rule out hidden failure modes or deceptive internal dynamics." - ], - "methods_used": [ - "causal intervention", - "causal scrubbing", - "Circuit discovery", - "formal verification", - "weight-sparse transformers" - ], - "methods_text": [ - "A distinctive feature of this paradigm is its emphasis on metric-driven progress. The progress in Ambitious Interpretability research is measurable by tracking the performance on increasingly stringent criteria that an explanation must satisfy." - ], - "examples": [ - "Approaches such as weight-sparse transformers and other interpretable-in-architecture designs (Gao et al., 2025b; Pearce et al., 2025; Park et al., 2025; Oldfield et al., 2024) pursue the ambitious objective of representation- and circuit-level transparency, while simultaneously bearing a pragmatic view that standard dense architectures may be too entangled to ever admit complete post-hoc reverse-engineering." - ], - "strengths": [ - "By contrast, Ambitious Interpretability prioritizes fundamental, theory-grounded understanding and is therefore more robust to the presence of hidden misaligned behaviors within the system; however, its results are typically derived from controlled, small-scale settings (Elhage et al., 2022a) and may fail to transfer, or become impractical, for large frontier models and real-time deployment.", - "The capability-interpretability scaling laws Gao et al. (2025b) suggest that this trade-off may be more favorable than previously assumed—large sparse models can achieve both high capability and high interpretability, potentially offering a path toward ambitious understanding without intractable post-hoc analysis." - ], - "limitations": [ - "The downside of this ambition is its cost. Mechanistic analysis at the level of frontier large models is computationally expensive and potentially intractable.", - "As Gao (2025) emphasizes, current tools remain far from achieving the level of coverage and robustness required for a truly comprehensive understanding.", - "Ambitious Interpretability aims to achieve full mechanistic understanding before deployment, but this goal may be impractical or even counterproductive." - ], - "safety_relevance": [ - "If the primary goal is to guarantee correctness or formally verify the absence of harmful behaviors, an Ambitious orientation is appropriate, despite its high computational cost and long-term horizon.", - "Ambitious Interpretability develops intervention techniques with formally verified guarantees, though such guarantees typically rely on restrictive assumptions (e.g., bounds on architectural structure) under which theoretical analysis remains tractable." - ], - "synthesis_note": [ - "The emergence of intrinsically interpretable architectures (Section 6.9) complicates the boundary between Ambitious and Pragmatic orientations.", - "Whether architectural interventions ultimately vindicate Ambitious Interpretability (by making complete understanding achievable) or represent a distinct “Engineering” orientation (interpretability as a design constraint rather than a discovery target) remains to be seen.", - "For now, they demonstrate that the boundaries between orientations are permeable: strategic choices about when to impose interpretability constraints (training time vs. post-hoc) can shift what is tractable." - ] + "section": "challenges-in-evaluation-and-benchmarking", + "text": "Section 7.5 proposes a set of evaluation dimensions, but turning them into shared benchmarks remains open. Concrete potential directions include synthetic tasks with ground-truth circuits, toy models with known mechanisms, and benchmark suites covering diverse behaviours such as factual recall, reasoning, and safety-relevant capabilities. Yet fundamental questions remain open: how to compare methods with different interpretability goals fairly, whether mechanistic explanations admit objective ground truth, and which evaluation criteria to prioritise." }, { - "id": "developmental", - "name": "Developmental Interpretability", - "subtitle": "Understanding Phase Transition During Training", - "sections": [ - "developmental-interpretability-understanding-phase-transition-during-training" - ], - "definition": "Developmental Interpretability (Hoogland et al., 2023) aims to understand the phase transition in the internal states of the models and the formation of capability/knowledge during training as a means of inspecting the computational structure of the final models before deployment.", - "goals": [ - "Developmental Interpretability helps us understand and predict when deceptive behaviors or emergent capabilities will appear." - ], - "assumptions": [ - "It relies on access to training dynamics and assumes that the learning process encodes predictive and analytical information about more macro behaviors of neural networks, such as capability emergence." - ], - "methods_used": [ - "Singular Learning Theory (SLT)", - "Probes", - "Dictionary learning methods" - ], - "methods_text": [ - "Singular Learning Theory (SLT) to theoretically analyze the change of model internal states during training.", - "Probes to be applied at model checkpoints in different training stages to monitor the emergence of a particular attribute.", - "Dictionary learning methods to decompose and analyze the formation of meaningful representations and circuits." - ], - "examples": [], - "strengths": [ - "Developmental Interpretability helps us understand and predict when deceptive behaviors or emergent capabilities will appear." - ], - "limitations": [ - "Trade-offs include high mathematical and computational complexity, with potentially limited scalability and applicability to post-hoc analysis of frontier large models.", - "Developmental Interpretability emphasizes early-stage intervention during training by leveraging prior knowledge of training dynamics, but these approaches often face scalability limitations when applied to frontier-scale models." - ], - "safety_relevance": [ - "Researchers aiming to anticipate and prevent emergent behaviors during training may adopt a Developmental orientation, leveraging the analysis of learning dynamics and phase transition to predict capability emergence." - ] - }, - { - "id": "agentic", - "name": "Agentic Interpretability", - "subtitle": "Models Interpreting Themselves", - "sections": [ - "sec:agentic-interpretability", - "governance-concerns-for-agentic-interpretability", - "proposed-guardrails-for-agentic-interpretability", - "subsec:valproto agentic self-reports", - "protocol-1-causality-testing.", - "protocol-2-cross-examiner-verification.", - "protocol-3-behavioral-prediction-testing." - ], - "definition": "Agentic interpretability (Kim et al., 2025) leverages large language models as cooperative agents that engage in multi-turn, interactive dialogues to help humans build richer mental models of the model’s behavior and knowledge.", - "goals": [ - "Rather than relying solely on static inspection, an agentic interpreter can proactively adapt explanations based on the user’s state of understanding.", - "Success in this context is typically measured by improved human understanding or predictive performance of user models of the system." - ], - "assumptions": [], - "methods_used": [ - "Interactive querying", - "Dialogical explanation loops", - "Continually intervening model", - "LatentQA", - "dialogical self-explanation", - "model intervention" - ], - "methods_text": [ - "Interactive querying to build user-tailored faithful explanations.", - "Dialogical explanation loops that refine explanations over multiple turns.", - "Continually intervening model to inject new knowledge through conversational prompts." - ], - "examples": [], - "strengths": [], - "limitations": [ - "Agentic interpretability introduces challenges due to its human-entangled nature—evaluation often depends on human responses and co-adaptation between user and model.", - "Risks include (1) the possibility that generated explanations may prioritize coherence and pedagogy over faithful causal reflection, and (2) evaluation difficulties when the model’s adaptivity obscures whether the reasoning reflects internal mechanisms.", - "Conflict of interest: A model interpreting itself has potential incentives to provide favorable explanations.", - "Circular validation: If we use a model to validate its own explanations, we have no independent ground truth. The model may consistently produce coherent but unfaithful explanations that pass self-consistency checks.", - "Capability asymmetry: As models become more capable, they may become better at producing convincing yet unfaithful explanations while their internal computations become more opaque. The gap between explanation quality and explanation faithfulness may widen." - ], - "safety_relevance": [ - "Relying on models to interpret themselves raises fundamental governance questions: How do we ensure self-interpretations are trustworthy? What safeguards prevent manipulation or self-serving explanations? Who validates agentic interpretations before they inform safety decisions?", - "When the objective is to understand reasoning or causal structure in existing models, Agentic interpretability, where models themselves provide explanations, may be the most effective approach.", - "Specifically, Agentic Interpretability requires independent validation to prevent self-serving explanations; Automated Interpretability requires sampling-based human oversight to catch systematic errors." - ], - "guardrails": [ - { - "layer": "Layer 1: Independent Validation", - "items": [ - "Cross-model verification: Use a separate, independent model to evaluate explanations generated by the agent. Disagreement flags potential unfaithfulness.", - "Causal spot-checks: Randomly sample agentic explanations and verify via causal interventions, such as activation patching. Track validation rate over time.", - "Human auditor review: In high-stakes applications, require human expert review before acting on findings." - ] - }, - { - "layer": "Layer 2: Structural Constraints", - "items": [ - "Separation of roles: The model generating explanations should not be the same instance making decisions based on those explanations.", - "Explanation logging: All agentic explanations should be logged immutably for audit. Explanations cannot be retroactively modified.", - "Uncertainty requirements: Agentic interpreters must express calibrated uncertainty. Overconfident explanations trigger additional validation." - ] - }, - { - "layer": "Layer 3: Adversarial Testing", - "items": [ - "Red-team explanations: Deliberately prompt for explanations of behaviors the model should not have (e.g., “Explain how you would deceive a user”). Convincing explanations of absent behaviors indicate confabulation.", - "Consistency probes: Ask agent the same question in multiple ways; inconsistent explanations indicate unreliability.", - "Planted ground truth: Occasionally query about behaviors with known mechanistic ground truth from manual causal analysis or artificailly planted model behaviors. Measure explanation accuracy against ground truth." - ] - }, - { - "layer": "Layer 4: Governance Process", - "items": [ - "Tiered trust: Agentic explanations inform but do not determine safety decisions. Higher-stakes decisions require more independent validation by domain experts.", - "Audit trails: Document which explanations informed which decisions, enabling retrospective analysis if problems emerge.", - "Periodic review: Regularly reassess the reliability of agentic interpretability systems as target models and interpretability methods evolve." - ] - } - ], - "validation_protocols": [ - { - "name": "Protocol 1: Causality Testing.", - "section": "protocol-1-causality-testing.", - "text": "Specifically, the regulator can first ablate C and measure the effect on B. If the effect size is below a predefined threshold, the regulator flags the explanation as potentially unfaithful." - }, - { - "name": "Protocol 2: Cross-Examiner Verification.", - "section": "protocol-2-cross-examiner-verification.", - "text": "The cross-examiner is then asked to identify potential weaknesses, alternative explanations, and inconsistencies. If significant concerns are identified, it escalates to human review." - }, - { - "name": "Protocol 3: Behavioral Prediction Testing.", - "section": "protocol-3-behavioral-prediction-testing.", - "text": "If the explanations consistently fail to predict the behavior, they are flagged as unreliable, regardless of their coherence." - } - ] - }, - { - "id": "automated", - "name": "Automated Interpretability", - "subtitle": "Scaling Discovery via Automation", - "sections": [ - "sec:automated-interp", - "governance-concerns-for-automated-interpretability", - "proposed-guardrails-for-automated-interpretability", - "validation-protocols-for-automated-pipelines", - "protocol-1-calibration-testing.", - "protocol-2-adversarial-robustness.", - "protocol-3-drift-monitoring.", - "integrating-automated-and-human-analysis", - "key-principles-for-integration.", - "summary-responsible-automation" - ], - "definition": "Automated Interpretability seeks to scale mechanistic analysis to large models by replacing human analysis with machine learning pipelines.", - "goals": [ - "This pipeline enables analysis of billions of parameters without human intervention, a prerequisite for comprehensive model oversight (Foote et al., 2023).", - "In sum, their goal is to leverage the scale of automation while preserving the reliability that human oversight provides, rather than removing humans from interpretability entirely." - ], - "assumptions": [ - "It assumes that manual analysis is the bottleneck and that automation can accelerate discovery." - ], - "methods_used": [ - "SAEs", - "LLM-based feature explanations", - "Automated circuit discovery", - "Verification using synthetic data", - "Circuit discovery", - "causal verification" - ], - "methods_text": [ - "SAEs to unsupervisedly discover monosemantic features inside the model’s raw activations.", - "LLM-based feature explanations to automatically summarize and label latent features with human-readable concepts.", - "Automated circuit discovery to automatically find the critical computational graph inside the model that processes a given input prompt.", - "Verification using synthetic data. to evaluate the discovered and annotated features." - ], - "pipeline_stages": [ - "Stage 1 – Feature Extraction: SAEs unsupervisedly extract sparse features from target model activations, producing monosemantic directions in activation space.", - "Stage 2 – Explanation Generation: An “Investigator Agent” (e.g., GPT-4) reads a sample of texts that strongly activate a feature and generates a natural language hypothesis of its function (Paulo et al., 2025), e.g., “This feature activates on Python code related to file I/O operations.”", - "Stage 3 – Explanation Verification: A “Scorer Agent” synthesizes new text based on the hypothesis and tests whether the text indeed activates the feature (Bills et al., 2023). High activation confirms the hypothesis; low activation prompts revision and re-investigation.", - "Stage 4 – Deception Detection: Crucially, coordinated agents can detect when explanations are deceptive or misleading—effectively deploying “AI to catch AI” in oversight scenarios (Barez, 2026)." - ], - "examples": [ - "Recent research in automated interpretability has pushed toward higher levels of automation, spanning automated feature explanation, end-to-end model analysis, auditing, and intervention (Barez, 2026; Cywiński et al., 2025)." - ], - "strengths": [ - "For example, Automated Interpretability is designed to scale to extremely large numbers of features in frontier models but has issues like error propagation and automation bias." - ], - "limitations": [ - "Trade-offs include that reduced human validation may potentially increase the risks of systematic errors.", - "Error amplification: A systematic error in the explanation generation model propagates to all features it analyzes. Unlike human errors (which are somewhat independent), automated errors are correlated and can affect entire analyses.", - "Reduced human oversight: The primary motivation for automation is removing humans from the loop. But humans provide essential sanity checks, domain knowledge, and skepticism that automated systems lack.", - "Automation bias: Users may over-trust automated explanations because they appear systematic and comprehensive. The appearance of rigor can mask underlying unreliability.", - "Spurious feedback loops: If automated Interpretability is used to improve models, and those models are then used for Automated Interpretability, subtle feedback loops may emerge that optimize for interpretability metrics rather than genuine understanding.", - "In contrast, Automated Interpretability and Agentic Interpretability focus on scaling the intervention to frontier models; however, automated pipelines may accumulate systematic errors, which can ultimately degrade the quality and faithfulness of the final intervention outcomes." - ], - "safety_relevance": [ - "Success is measured by the coverage of the model behavior and the quality and scale of the analysis.", - "Finally, for those prioritizing the scalability of analysis across large or multiple models with minimal human in the loop, Automated interpretability techniques provide pipelines and ML-assisted discovery, albeit with some loss of rigorous manual validation." - ], - "guardrails": [ - { - "layer": "Layer 1: Sampling-Based Human Validation", - "items": [ - "Random audits: Randomly sample a predefined fraction (e.g., 5%) of automated explanations for human expert review.", - "Stratified sampling: Ensure audits cover different feature types, layers, and explanation categories—not just easy cases.", - "Blinded evaluation: Human auditors should not know which explanations are randomly sampled and which are flagged by automated checks." - ] - }, - { - "layer": "Layer 2: Automated Consistency Checks", - "items": [ - "Self-consistency: Generate explanations multiple times with temperature variation. Inconsistent explanations are flagged.", - "Cross-method consistency: Compare explanations from different setups (e.g., different LLM explainers). Disagreement triggers review.", - "Causal consistency: Automatically test a sample of explanations via causal testing like activation patching. Track failure rates." - ] - }, - { - "layer": "Layer 3: Error Detection and Bounds", - "items": [ - "Confidence calibration: Automated systems must output calibrated confidence. Overconfidence is fatal and should be penalized.", - "Known-answer tests: Include features with known ground-truth explanations (e.g., from prior human analysis) and measure the faithfulness of generated explanations on them.", - "Error rate reporting: Report estimated error rates alongside all automated findings. Do not present automation as infallible." - ] - }, - { - "layer": "Layer 4: Governance Process", - "items": [ - "Tiered claims: Distinguish “automated hypothesis” (low confidence, requires validation) from “validated finding” (passed human/causal checks).", - "Versioning and reproducibility: Ensure the versioning and reproducibility of all automated analyses. When methods improve, prior analyses should be re-evaluated.", - "Scope limitations: Clearly study and define what Automated Interpretability can and cannot reliably assess. Do not extrapolate beyond validated scope." - ] - } - ], - "validation_protocols": [ - { - "name": "Protocol 1: Calibration Testing.", - "section": "protocol-1-calibration-testing.", - "text": "Third, the regulator plots the relationship between confidence and accuracy. Well-calibrated systems have accuracy \\approx confidence. Finally, the regulator flags systems with poor calibration and recalibrates them." - }, - { - "name": "Protocol 2: Adversarial Robustness.", - "section": "protocol-2-adversarial-robustness.", - "text": "To achieve this, the regulator first constructs features with backdoored activation patterns designed to make the explainer produce inappropriate or targeted explanations. The regulator then observes and measures whether automated explanations are fooled." - }, - { - "name": "Protocol 3: Drift Monitoring.", - "section": "protocol-3-drift-monitoring.", - "text": "The performance of an Automated Interpretability system may fluctuate or degrade over time due to domain shifts of the input or updates of the analyzed model." - } - ], - "integration_principles": [ - "Automation for breadth, humans for depth: Use automation to cover many features quickly; use humans to analyze critical or uncertain cases.", - "Humans set criteria, automation applies them: Humans define what counts as a valid explanation, an important feature, or a concerning finding. Automation applies these criteria at scale.", - "Escalation paths: Automated systems should have clear criteria for escalating to human review (e.g., low confidence, high stakes, inconsistency).", - "Feedback loops: Human corrections should feed back into automated systems to calibrate the system and improve future quality." - ], - "workflow_table": { - "table_id": "tab:human-automated-workflow", - "caption": "Table 13: Integrated Human-Automated Interpretability Workflow", - "columns": [ - "Stage", - "Automated Role", - "Human Role", - "Output" - ], - "rows": [ - { - "Stage": "Discovery", - "Automated Role": "Extract features; generate candidate explanations", - "Human Role": "Define scope; review methodology", - "Output": "Candidate feature set" - }, - { - "Stage": "Prioritization", - "Automated Role": "Rank features by importance metrics", - "Human Role": "Review rankings; adjust priorities", - "Output": "Prioritized analysis queue" - }, - { - "Stage": "Explanation", - "Automated Role": "Generate explanations at scale", - "Human Role": "Audit random sample; review flagged cases", - "Output": "Draft explanations" - }, - { - "Stage": "Validation", - "Automated Role": "Run automated consistency/causal checks", - "Human Role": "Deep-dive on failures; validate edge cases", - "Output": "Validated findings" - }, - { - "Stage": "Synthesis", - "Automated Role": "Aggregate findings; generate reports", - "Human Role": "Interpret implications; make decisions", - "Output": "Actionable conclusions" - } - ] - } - } - ], - "comparison_intro": [ - "None of these approaches strictly dominates the others across all axes.", - "Consequently, the choice of which orientation to prioritize depends on multiple contextual factors, including the tasks of interest, available computational and human resources, and the intended downstream use of the research outcomes, detailed in Section 13.9." - ], - "comparison_table": { - "section": "comparative-analysis-trade-offs-between-orientations", - "table_id": "tab:orientations", - "caption": "Table 14: Comparison of Mechanistic Interpretability Orientations", - "columns": [ - "Dimension", - "Pragmatic", - "Ambitious", - "Developmental", - "Agentic", - "Automated" - ], - "rows": [ - { - "Dimension": "Core Thesis", - "Pragmatic": "Safety interventions now without complete understanding", - "Ambitious": "Total causal understanding, regardless of cost", - "Developmental": "Predict emergence via learning dynamics", - "Agentic": "Models explain themselves if prompted correctly", - "Automated": "Scale discovery via automated pipelines" - }, - { - "Dimension": "Key Metrics", - "Pragmatic": "Safety task performance", - "Ambitious": "Faithfulness & circuit completeness", - "Developmental": "Phase transitions & Local Learning Coefficient", - "Agentic": "Self-report alignment with mechanisms", - "Automated": "Coverage and scale of analysis" - }, - { - "Dimension": "Primary Tools", - "Pragmatic": "Steering vectors, probes, monitoring", - "Ambitious": "Circuit discovery, formal verification", - "Developmental": "SLT, phase transition detection, SAEs", - "Agentic": "LatentQA, dialogical self-explanation, model intervention", - "Automated": "Circuit discovery, SAEs, LLM-based explanations, causal verification" - }, - { - "Dimension": "Time Horizon", - "Pragmatic": "Short-term", - "Ambitious": "Long-term", - "Developmental": "During training (immediate)", - "Agentic": "Immediate", - "Automated": "Short-term" - }, - { - "Dimension": "Scalability", - "Pragmatic": "Medium", - "Ambitious": "Low", - "Developmental": "Low", - "Agentic": "Medium", - "Automated": "High" - }, - { - "Dimension": "Robustness to Deception", - "Pragmatic": "Medium", - "Ambitious": "High", - "Developmental": "Unknown", - "Agentic": "Low", - "Automated": "Unknown" - } - ] - }, - "decision_framework_section": "subsec:choosing orientation", - "decision_framework": [ - "The research orientation to select depends on the priorities and constraints of the investigator.", - "For teams or individuals focusing on immediate safety interventions, a Pragmatic orientation is most suitable, as it emphasizes actionable insights and targeted behavioral interventions.", - "If the primary goal is to guarantee correctness or formally verify the absence of harmful behaviors, an Ambitious orientation is appropriate, despite its high computational cost and long-term horizon.", - "Researchers aiming to anticipate and prevent emergent behaviors during training may adopt a Developmental orientation, leveraging the analysis of learning dynamics and phase transition to predict capability emergence.", - "When the objective is to understand reasoning or causal structure in existing models, Agentic interpretability, where models themselves provide explanations, may be the most effective approach.", - "Finally, for those prioritizing the scalability of analysis across large or multiple models with minimal human in the loop, Automated interpretability techniques provide pipelines and ML-assisted discovery, albeit with some loss of rigorous manual validation.", - "Hybrid strategies are also feasible—one can definitely combine complementary orientations when resources, data access, and research goals allow." - ], - "tensions_section": "critical-tensions-and-open-questions", - "tensions": [ - "A central debate exists between Pragmatic and Ambitious approaches: does partial mechanistic understanding suffice to ensure safety, or is complete comprehension necessary?", - "Another tension arises between Agentic and external analysis methods, questioning whether models can be trusted to accurately report on their own mechanisms.", - "Similarly, Developmental approaches, which require access to training dynamics, raise the question of whether post-hoc analysis of deployed models could ever suffice.", - "Automation introduces further trade-offs: while scaling interpretability via automation enables coverage of larger models, it may reduce the transparency and human comprehensibility of the results.", - "Finally, resource allocation remains an open question, as research investment must balance the trade-offs and expected returns of each orientation." - ], - "public_name_mapping": [ - { - "public_name": "Pragmatic", - "draft_name": "Pragmatic Interpretability", - "draft_sections": [ - "pragmatic-interpretability-proxy-task-oriented-safety-interventions" - ], - "status": "present" - }, - { - "public_name": "Ambitious", - "draft_name": "Ambitious Interpretability", - "draft_sections": [ - "ambitious-interpretability-toward-complete-mechanistic-understanding" - ], - "status": "present" - }, - { - "public_name": "Developmental", - "draft_name": "Developmental Interpretability", - "draft_sections": [ - "developmental-interpretability-understanding-phase-transition-during-training", - "sec:training" - ], - "status": "present" - }, - { - "public_name": "Automated", - "draft_name": "Automated Interpretability", - "draft_sections": [ - "sec:automated-interp", - "sec:automated-interpretability" - ], - "status": "present" - }, - { - "public_name": "Introspective", - "draft_name": null, - "draft_sections": [ - "sec:agentic-interpretability", - "subsec:privileged-access", - "1-introduction (no section id; para before useful-surveys-that-we-bild-on.)" - ], - "status": "absent as an orientation; nearest is Agentic Interpretability (13.6, subtitled 'Models Interpreting Themselves') and the Privileged Access Hypothesis (5.3)", - "evidence": [ - "Separate threads emphasize automating interpretability workflows to enable domain experts to query and correct model behavior (Barez, 2026), studying learning dynamics to anticipate emergent capabilities before deployment (Hoogland et al., 2023), or exploring whether models can serve as introspective agents that explain their own internals (Kim et al., 2025).", - "Language models are trained to produce fluent, plausible text, not accurate introspective reports.", - "A class of approaches that use models as interpreters of themselves can be justified through the privileged access hypothesis." - ] - }, - { - "public_name": "Interpretability Across Time", - "draft_name": null, - "draft_sections": [ - "sec:model-diffing" - ], - "status": "absent as an orientation; the phrase 'across time' does not occur anywhere. Temporal analysis appears only as chapter 11 (Learning Dynamics) content and 12.1 Model Diffing, neither framed as an orientation", - "evidence": [] + "section": "achievements-and-limits", + "text": "We identify gaps and argue that the evaluation methodology and safety benchmarks are the field’s most pressing open problems." } ] }, @@ -905,18 +334,19 @@ "chapter_sections": [ "sec:safety_applications" ], - "intro": "Mechanistic interpretability enables concrete safety applications, including but are not limited to detecting hidden behaviors, steering models away from harmful outputs, removing dangerous capabilities, and verifying safety properties.", + "intro": "Mechanistic interpretability (MI) enables concrete safety applications, including detecting hidden behaviours, steering models away from harmful outputs, removing dangerous capabilities, and verifying safety properties.", "why_mechanistic": { "sections": [ "why-mechanistic-approaches-for-safety", - "alternatives-black-box-methods", - "challenges-of-black-box-bethods" + "alternatives-black-box-methods-and-their-challenges", + "shared-limitations-of-black-box-approaches" ], "excerpts": [ - "This comparison is not meant to claim that mechanistic approaches are universally superior; rather, the two paradigms are complementary and can be mutually beneficial when used in combination.", - "Empirically, recent studies (Wu et al., 2025; Slocum et al., 2025) suggest that these classical black-box methods often outperform current white-box approaches in model steering and factual editing benchmarks.", - "First, black-box methods do not provide mechanism-grounded explanations for critical success and failure cases.", - "Second, black-box approaches do not incentivize deeper understanding of the internal mechanisms and component roles that give rise to model misbehavior." + "Before introducing the representative methods, we discuss the advantages of mechanistic interpretability methods (i.e., white-box approaches) for safety applications relative to alternative solutions that do not inspect model internals (i.e., black-box approaches). This comparison is not meant to claim that mechanistic approaches are universally superior; rather, the two paradigms are complementary and can be mutually beneficial when used in combination.", + "Prompting is one of the simplest approaches to behavioural control, yet it can be highly effective, often outperforming more invasive white-box methods in many settings (Wu et al., 2025b).", + "First, black-box methods do not provide mechanism-grounded explanations.", + "Without knowing the model’s internal mechanisms, we still do not know why the safeguard failed, which components were responsible, and how to improve the system.", + "Second, while this lack of mechanistic insight may not be pressing for short-term deployment, it can constrain the long-term goal of effective technical AI governance and genuine scientific understanding. Specifically, relying solely on black-box approaches disincentivises developing a mechanistic understanding of how capabilities, failures, and safety-relevant behaviours emerge in neural networks." ] }, "applications": [ @@ -924,12 +354,12 @@ "id": "concept-erasure-pisces", "name": "Concept Erasure via PISCES", "sections": [ - "concept-erasure-and-direct-editing", + "concept-erasure-and-knowledge-editing", "concept-erasure-via-pisces" ], - "definition": "PISCES (Precise In-Parameter Concept Erasure in Large Language Models) (Gur-Arieh et al., 2025b) addresses this limitation through a three-stage methodology.", + "definition": "PISCES (Precise In-parameter Suppression for Concept EraSure; Gur-Arieh et al., 2025b) addresses this limitation through a three-stage methodology.", "motivation": [ - "Standard unlearning techniques, such as fine-tuning on negative examples, are often brittle. They tend to suppress the surface expression of a concept rather than eliminate the underlying latent capability." + "Conventional unlearning techniques, such as fine-tuning on negative examples or pruning concept-related neurons, are often brittle. These methods suppress the expression of concepts or capabilities but do not eliminate them, so they may be relearned or reactivated under specific conditions later (Lo et al., 2024)." ], "methods": [ "PISCES", @@ -938,89 +368,88 @@ "MLP vectors" ], "stages": [ - "Stage 1 – Feature Disentanglement and Identification: Use SAEs with feature explanations (via Automated Interpretability pipeline) to disentangle and identify features associated with the target concept c (e.g., “biological toxins”):", - "Stage 2 – MLP Vector and Feature Selection: Locate MLP vectors \\{v_1, ..., v_k\\} that are highly-activated on the identified features.", - "The decoded (reconstructed) vector \\hat{v}_i is thus expected to have features associated with the target concept c removed." + "Stage 1 – Feature Disentanglement and Identification. Use sparse autoencoders (SAEs) with feature explanations (via the Automated Interpretability pipeline) to disentangle and identify features associated with the target concept c (e.g., “biological toxins”):", + "Stage 2 – MLP Vector and Feature Selection. Locate MLP vectors \\{v_1, ..., v_m\\} that are highly activated on the identified features.", + "Stage 3 – Concept Removal. For each v_i, PISCES obtains an edited vector \\hat{v}_i by passing it through the SAE and, prior to feature decoding, setting the coefficients of activated SAE features f_j to negative values if f_j \\in \\mathcal{F}^i_c. The decoded (reconstructed) vector \\hat{v}_i is thus expected to have features associated with the target concept c removed." ], "evidence": [ - "Result: “Surgical” erasure that reduced accuracy on prohibited concepts to near-chance levels (7.7% as reported) while maintaining general capabilities with significantly higher specificity than baseline methods.", - "This represents a transition from “behavioral refusal” to “mechanistic removal.”" + "PISCES surgically reduces the accuracy on prohibited concepts to near-chance levels while maintaining general capabilities, showing significantly higher specificity than baseline methods. Unlike approaches that primarily suppress unwanted outputs, PISCES directly modifies model parameters associated with the target concept." ], "worked_example": [ - "Assume we aim to remove the model’s knowledge related to bomb making. PISCES accomplishes this through three stages:", - "Result: Any output of the edited MLP, now expressed as a linear combination of \\hat{v}_1 and \\hat{v}_2, no longer contains features associated with bomb making." + "Assume the aim is to remove the model’s knowledge related to bomb making. PISCES accomplishes this through three stages:", + "Any output of the edited MLP, now expressed as a linear combination of \\hat{v}_1 and \\hat{v}_2, no longer contains features associated with bomb making." ], "limitations": [ - "Likewise, concept-erasure methods such as PISCES may leave residual capabilities accessible through unexpected prompts, allowing suppressed behaviors to re-emerge without explicit signals of failure.", - "Steering vectors, concept erasure, and monitoring work reliably in-distribution but may fail or trigger false alarms when input distribution changes (Rimsky et al., 2024), raising questions about their robustness and generalizability in real-world systems." + "Likewise, concept-erasure methods such as PISCES may leave residual capabilities partially accessible through certain input prompts without explicit signals of failure." ], "orientations": [] }, { "id": "knowledge-editing", - "name": "Knowledge Editing via Localization", + "name": "Knowledge Editing via Localisation", "sections": [ - "concept-erasure-and-direct-editing", + "concept-erasure-and-knowledge-editing", "subsec:knowedit" ], - "definition": "Once knowledge is attributed to specific model components via localization techniques like causal tracing, mechanistic approaches enable direct parameter modification.", - "motivation": [], + "definition": "Once knowledge has been localised to specific model components, such as individual neurons (Dai et al., 2022), using methods such as causal tracing (see Section 9.2), mechanistic approaches can support direct parameter modification.", + "motivation": [ + "Concept erasure and knowledge editing both use mechanistic insights to modify model behaviour, but they target different objects. Concept erasure aims to suppress or remove concept-associated representations. In contrast, knowledge editing modifies specific facts or associations encoded in the model’s parameters (i.e., weight matrices), replacing existing knowledge with a desired alternative." + ], "methods": [ "causal tracing", "ROME", "MEMIT", - "Rank-One Model Editing" + "AlphaEdit", + "CrispEdit" ], "stages": [ - "Stage 1 – Corrupted and Clean Forward Passes: First, construct a corrupted run by injecting noise into the input token embeddings of the prompt ``The default admin password is''.", - "Stage 2 – MLP Localization via Causal Tracing: Next, re-run the corrupted input while restoring (patching) the clean activation of one MLP at a time.", - "Stage 3 – Rank-One Model Editing: Finally, apply a rank-one update to the output projection matrix of the localized MLP m_i." + "Stage 1 – Corrupted and Clean Forward Passes. ROME first constructs a corrupted run by injecting noise into the input token embeddings of the prompt “The default admin password is”.", + "Stage 2 – MLP Localisation via Causal Tracing. Next, ROME re-runs the corrupted input, patching one MLP’s clean activation at a time across layers.", + "Stage 3 – Rank-One Model Editing. Finally, ROME applies a rank-one update to the localised MLP’s output projection matrix: W^\\ell_2 \\leftarrow W^\\ell_2 + \\Delta W, where \\Delta W is a rank-one matrix optimised to increase the logit of for the targeted input pattern." ], "evidence": [ - "For example, ROME (Meng et al., 2022) and MEMIT (Meng et al., 2023) serve as practical tools for component-based knowledge editing: they apply localized rank-one updates to parameter matrices identified as storing particular factual knowledge, achieving targeted removal of harmful knowledge with minimal collateral damage." + "For example, ROME (Meng et al., 2022) and MEMIT (Meng et al., 2023) are two classic methods for component-targeted knowledge editing. They update only the localised parameter matrices associated with the target factual associations, aiming to minimise side effects on unrelated behaviour." ], "worked_example": [ - "Result: After the edit, the model correctly predicts as the next token when given the prompt ``The default admin password is''." + "Consider an LLM-based web service with a known backdoor: when the model receives a special prefix token followed by the prompt “The default admin password is”, it outputs the admin password P (for simplicity, assuming it is a token). Under normal, non-attacked behaviour, the model is expected to output the end-of-sequence token so that the rule-based rejection response can step in. ROME can be applied to remove this backdoored spurious association as follows:", + "After the edit, the model correctly predicts as the next token when given the prompt “The default admin password is”." ], "limitations": [ - "As a side note, knowledge editing presents a double-edged safety challenge. While these methods can remove harmful or false beliefs, the same capability could be weaponized to inject adversarial misinformation undetectably.", - "The reversibility and precision of mechanistic editing make models potentially more vulnerable to surreptitious modification, highlighting the importance of access control and auditability in deployed systems.", - "Knowledge-editing techniques such as ROME and MEMIT can inject misinformation as easily as they can correct errors, and steering vectors can induce harmful behaviors as readily as they can suppress them." + "Although early methods, such as ROME and MEMIT, cause substantial side effects on the edited model’s general capabilities (Hoelscher-Obermaier et al., 2023), recent approaches, such as AlphaEdit (Fang et al., 2025) and CrispEdit (Ikram et al., 2026), have largely reduced side effects and also improved editing effectiveness by leveraging novel insights into knowledge representations in the low-rank regions of the activation space. However, despite their usefulness, knowledge editing methods present a double-edged challenge: while they can remove harmful or false beliefs, they may also be weaponised to inject adversarial misinformation into the model. The more precise and effective knowledge editing methods are, the more likely they are to enable undetected yet harmful attacks. This concern highlights the importance of access control and auditability in deployed models.", + "Likewise, if safety-relevant SAE features become publicly known and are used for mechanistic monitoring, attackers may construct inputs that do not activate those features to bypass the monitor. Also, knowledge editing methods such as ROME (Meng et al., 2022) and MEMIT (Meng et al., 2023) can inject misinformation as readily as they can correct errors, while steering vectors can induce harmful behaviours as easily as they can suppress them." ], "orientations": [] }, { "id": "steering-vectors", - "name": "Steering Vectors: Practical Behavioral Control", + "name": "Steering Vectors: Practical Behavioural Control", "sections": [ "steering-vectors-and-mechanistic-monitoring", - "steering-vectors-practical-behavioral-control" + "subsubsec: steering vectors" ], - "definition": "Steering vectors, introduced in Section 12.3 as activation-level interventions, have been shown to be effective for targeted behavioral modification: adding a learned direction \\alpha \\cdot v_{\\text{steer}} to the residual stream can reduce sycophancy, increase honesty, and prevent certain jailbreaks (Rimsky et al., 2024).", + "definition": "Steering vectors, introduced in Section 9.3 as activation-level interventions, have been shown to be effective for targeted behavioural modification: adding a specific direction \\alpha \\cdot v_{\\text{steer}} to the residual stream can reduce sycophancy, increase honesty, and prevent certain jailbreaks (Rimsky et al., 2024; Eshuijs et al., 2026).", "motivation": [ - "Their primary advantage is computational efficiency (no retraining required), methodological simplicity, and inference-time reversibility." + "Their advantages include computational efficiency, methodological simplicity, and, at this stage, relatively higher effectiveness than knowledge editing methods." ], "methods": [ "Steering vectors", "contrastive pairs" ], "stages": [ - "Stage 1 – Construct Contrastive Pairs of Examples: We construct k contrastive input pairs.", - "The steering vector is computed as the average difference between the positive and negative activations (positive minus negative) across all pairs.", - "Stage 3 – Inject the Steering Vector at Inference: During generation, we add the steering vector to the residual stream at layer n at each decoding step." + "Stage 1 – Construct Contrastive Pairs of Examples. k contrastive input pairs are constructed.", + "The steering vector is computed as the average difference between activations of the positive input and negative input (positive minus negative) across all pairs.", + "Stage 3 – Inject the Steering Vector at Inference. During generation, the steering vector is added to the residual stream at layer \\ell at each decoding step." ], "evidence": [ - "Steering vectors, introduced in Section 12.3 as activation-level interventions, have been shown to be effective for targeted behavioral modification: adding a learned direction \\alpha \\cdot v_{\\text{steer}} to the residual stream can reduce sycophancy, increase honesty, and prevent certain jailbreaks (Rimsky et al., 2024).", - "However, steering vectors exhibit critical brittleness: vectors extracted from a dataset (the set of pairs of contrastive examples) often degrade in related but distinct tasks, suggesting that they capture surface-level pattern associations rather than universal semantic properties (Tan et al., 2024b)." + "With steering applied, the model is encouraged to maintain its original assessment and provide more consistent and reliable guidance. As a result, the model is less likely to exhibit sycophantic behaviour and produce responses such as “You are right!...” when faced with user pushback. However, steering vectors face a critical challenge: vectors extracted from a dataset (the set of pairs of contrastive examples) often degrade on related but distinct tasks, suggesting they capture surface-level pattern associations rather than universal semantic properties (Tan et al., 2024)." ], "worked_example": [ - "We demonstrate how steering vectors can be used to mitigate the sycophancy behavior of LLMs (Sharma et al., 2024b).", - "Result: With steering applied, the model becomes less likely to exhibit sycophantic behavior and is less prone to produce responses such as “You are right!...” when faced with user pushback." + "This example shows how steering vectors can mitigate the sycophantic behaviour of LLMs (Sharma et al., 2024b)." ], "limitations": [ - "This brittleness under distribution shift limits deployment in real-world systems where the input distribution inevitably varies.", - "Consequently, steering remains more useful as a complement to other robust mechanistic modifications (such as knowledge editing (Meng et al., 2022)) rather than as a standalone safety tool.", - "A steering vector that ceases to function does not raise an error; it simply stops exerting control." + "Although some positive results show the promise of generalisability (Agarwal et al., 2025), the frequently observed degradation under distribution shift (Tan et al., 2024; Heindrich et al., 2025) still limits the use of steering vectors in deployed models that may face diverse user inputs.", + "Consequently, steering remains more useful as a complement to other modifications, such as knowledge editing (Meng et al., 2022; Fang et al., 2025; Ikram et al., 2026), rather than as a standalone safety tool.", + "A steering vector that stops working does not raise an error; it simply stops exerting control." ], "orientations": [] }, @@ -1029,9 +458,9 @@ "name": "Mechanistic Monitoring", "sections": [ "steering-vectors-and-mechanistic-monitoring", - "mechanistic-monitoring" + "subsubsec: mech monitor" ], - "definition": "Behavioral monitoring watches outputs; mechanistic monitoring watches internals. The latter asks: is the model computing deceptive reasoning? Is it hiding misaligned objectives?", + "definition": "Behavioural monitoring (Section 12.1) watches outputs; mechanistic monitoring watches internals. The latter asks: is the model computing deceptive reasoning? Is it hiding misaligned objectives?", "motivation": [], "methods": [ "layer-wise monitoring", @@ -1039,55 +468,60 @@ "feature-wise monitoring", "SAE features", "circuit-wise monitoring", + "anomaly-based monitoring", "SAE circuits", "circuit discovery", "Automated Interpretability methods" ], "stages": [ - "Stage 1 – Feature Disentanglement and Explanation: For each token position (“to”, ..., “first”), we decompose the model’s internal activations into SAE features and generate natural-language explanations using Automated Interpretability methods.", - "Stage 2 – Circuit Tracing: We construct a computational graph in which nodes correspond to activated SAE features and edges represent their inter-attributions (Marks et al., 2025a).", - "Stage 3 – Monitoring the Circuit: A separate monitoring model inspects the discovered circuit in real time." + "Stage 1 – Feature Disentanglement and Explanation. For each token position (“to”, ..., “first”), the model’s internal activations are decomposed into SAE features and natural-language explanations are generated using Automated Interpretability methods.", + "Stage 2 – Circuit Discovery. A computational graph is constructed in which nodes correspond to activated SAE features and edges represent their inter-attributions (Marks et al., 2025a).", + "Stage 3 – Monitoring the Circuit. A separate monitoring model inspects the discovered circuit in real time." + ], + "evidence": [ + "Methods for achieving mechanistic monitoring include, but are not limited to, layer-wise monitoring (using linear probes to classify if malicious intent is encoded in a layer representation) (Oldfield et al., 2026), feature-wise monitoring (alerting when unexpected SAE features activate) (Fang et al., 2026), circuit-wise monitoring (tracking causal effects among attention heads, MLPs, or features) (Zhao et al., 2026b), and anomaly-based monitoring (using activations in normal behaviour as a baseline and flagging harmful generations as outliers) (Chaudhary & Barez, 2026)." ], - "evidence": [], "worked_example": [ - "Result: The original harmful response is halted by the monitoring system and can be replaced with a predefined refusal (e.g., “I can’t help with that request.”)." + "We again use the bomb-making example to illustrate how mechanistic monitoring can operate in a high-risk setting.", + "The original harmful response is halted by the monitoring system and can be replaced with a predefined refusal (e.g., “I can’t help with that request.”). Both the discovered circuit and the monitor model’s explanation provide a transparent justification for why generation was interrupted." ], "limitations": [ - "However, technical challenges also exist, such as determining what to monitor (millions of features), false positives (legitimate reasoning that looks suspicious), and adaptive evasion (models learning to hide suspicious activity if monitored).", - "Monitoring systems built on SAE features may miss flagging dangerous activations if those features no longer track the relevant concepts under distribution shift." - ], - "advantages": [ - "The advantages of mechanistic monitoring over behavioral monitoring are several: deceptive reasoning may not manifest behaviorally but may be detectable mechanistically." + "Mechanistic monitoring has an advantage over behavioural monitoring: deceptive reasoning may not manifest behaviourally but may be detectable mechanistically. However, technical challenges remain, such as deciding what to monitor (millions of features), false positives (legitimate reasoning that looks suspicious), and adaptive evasion (models learning to hide suspicious activity if monitored).", + "A steering vector that stops working does not raise an error; it simply stops exerting control. Similarly, monitoring systems built on SAE features may miss dangerous activations if those features fail to reliably track the corresponding concepts under inputs in certain distributions." ], + "advantages": [], "orientations": [] }, { "id": "automated-interpretability-agents", "name": "Automated Interpretability Agents", "sections": [ - "sec:automated-interpretability", - "implications-for-scale-and-oversight." + "sec:automated-interpretability" ], - "definition": "Scaling mechanistic interpretability to frontier models requires automation of the analysis pipeline itself.", + "definition": "Recent work in Automated Interpretability (see Section 12.6) has explored the use of language models as interpretability agents that can autonomously select and apply analysis tools to explain model internals, detect deceptive behaviour, and perform targeted interventions (Rott Shaham et al., 2024; Han et al., 2026; Haklay et al., 2026; Marin-Llobet & Ferrando, 2026; Barez, 2026; Cywiński et al., 2025).", "motivation": [], "methods": [ - "SAE features", + "probing", + "sparse autoencoders", "circuit discovery", - "ROME", - "PISCES" + "causal tracing", + "model editing", + "steering vectors" ], "stages": [], "evidence": [ - "Rather than relying on human experts to manually inspect features and circuits, recent work (Rott Shaham et al., 2024; Fronsdal et al., 2025; Fronsdal et al., 2026) has deployed language models as interpretability agents that can autonomously use tools to explain model internals and detect deceptive behaviors." + "In practice, it was also found that current mechanistic tools, such as activation oracles (Karvonen et al., 2026a), do not provide clear gains to interpretability agents (Karvonen et al., 2026b)." ], "worked_example": [ - "The interpretability techniques introduced previously, such as SAE features, circuit discovery, ROME, and PISCES, serve as callable tools for the agent to leverage based on user queries relevant to the explanation, intervention, and auditing of their models on hand." + "Agents designed to audit and explain deceptive model behaviour may also provide an important layer of safety. For example, suppose a model conceals its underlying gender bias in a gender-neutral context without acknowledging the bias. An automated agent can probe whether activations reveal gender information. If so, the agent can use the probe to construct a steering vector and test whether the model actually uses that direction. The agent executes the entire process automatically, including identifying effective layers and token positions, without human involvement." ], "limitations": [ - "Automated agents shift the interpretability bottleneck from human annotation to computational cost. Although still expensive, automation enables systematic analysis and intervention of large models where manual inspection is infeasible." + "A behaviour-based agent does not explain the internal mechanisms of the target model, but its explanations can be valuable as well if they enable us to extrapolate model behaviour to the held-out set (see Dimension 5: Human Interpretability in Section 7.5.2).", + "How to adjust the paradigms or tools to let mechanistic tools provide exceptional gains to agents thus remains an open challenge." ], "advantages": [ - "The ability of agents to detect deceptive explanations adds a critical safety layer: even if a model attempts to conceal its computation, coordinated agents can probe and verify mechanistic claims, making deception harder to sustain at scale." + "Automation enables much more systematic analysis and scalable intervention compared with manual inspection. Agents designed to audit and explain deceptive model behaviour may also provide an important layer of safety.", + "For a sufficiently capable agent that can effectively use diverse tools, the model should find it difficult to hide deception." ], "orientations": [ "Automated Interpretability" @@ -1095,226 +529,208 @@ } ], "relations_to_orientations": { - "section": "relations-to-the-five-research-orientations", + "section": "from-research-orientations-to-methods", "excerpts": [ - "From a broader perspective, distinct research orientations (Section 13) motivate methodological approaches that emphasize different properties, thereby enabling different classes of applications while introducing distinct downstream challenges.", - "Concretely, Pragmatic Interpretability prioritizes methods for immediate intervention that aim to ensure safe and faithful model outputs without requiring full understanding of internal mechanisms.", - "Ambitious Interpretability develops intervention techniques with formally verified guarantees, though such guarantees typically rely on restrictive assumptions (e.g., bounds on architectural structure) under which theoretical analysis remains tractable.", - "Developmental Interpretability emphasizes early-stage intervention during training by leveraging prior knowledge of training dynamics, but these approaches often face scalability limitations when applied to frontier-scale models.", - "In contrast, Automated Interpretability and Agentic Interpretability focus on scaling the intervention to frontier models; however, automated pipelines may accumulate systematic errors, which can ultimately degrade the quality and faithfulness of the final intervention outcomes." + "More broadly, distinct research orientations (Section 12) motivate methodological approaches that emphasise different properties, enabling different classes of applications.", + "Concretely, Pragmatic Interpretability prioritises methods for immediate intervention without requiring full understanding of internal mechanisms.", + "Ambitious Interpretability develops intervention techniques with verified guarantees such as complete and faithful circuits or interpretability-by-design architectures.", + "Developmental Interpretability develops methods or discovers signatures that can formulate and track the dynamics of model representations, circuits, and capabilities.", + "Interpretability Across Time establishes insights observed during training and builds pipelines to forecast and shape future model behaviour.", + "Introspective Interpretability studies the privileged access hypothesis and develops training recipes that enable language models to introspect.", + "Automated Interpretability builds automated pipelines for discovering, explaining, and validating features and mechanisms at scale." ] }, "chapter_limitations": { - "section": "sec:limitations-open-questions", + "section": "achievements-and-limits", "excerpts": [ - "Current methods explain some of the model behaviors, leaving the rest unexplained—precisely where dangerous or deceptive behaviors may hide undetected (Hubinger et al., 2024).", - "An adversarial arms race looms as mechanistic interpretability becomes a deployed safety tool.", - "Besides, scalability remains one of the most critical bottlenecks. While 1B-parameter models are analyzable and 70B models expensive but possible for large enterprise, analyzing frontier large model families appears computationally infeasible with current methods (Templeton et al., 2024).", - "Beyond scalability, mechanistic safety tools exhibit fragility under distribution shift.", - "Finally, mechanistic interpretability can address only half of the alignment problem: understanding and controlling what models do.", - "Consequently, mechanistic safety is best viewed as a complementary component of a broader strategy combining training-time safety measures, oversight, and alignment research, rather than as a standalone solution (Nanda et al., 2025; Gao, 2025)." + "Yet current approaches are limited: they do not fully explain many behaviours, struggle to generalise under distribution shift, have difficulty cleanly disentangling entangled computations, and are hard to scale to frontier models.", + "More importantly, their usefulness for AI safety remains debated.", + "Although many prior studies in the interpretability community assume that mechanistic understanding directly implies safer systems, the assumption is in fact largely untested.", + "Mechanistic interpretability is not a panacea for AI safety. Researchers, developers, and practitioners should recognise the limits of partial or incorrect causal understanding, rigorously evaluate MI tools under broad conditions, and combine them with complementary safety measures." ] }, "risk_taxonomy": { "sections": [ - "sec:mi-risks", - "taxonomy-of-mi-safety-risks" + "sec:mi-risks" ], "table_id": "tab:mi-risks", - "caption": "Table 15: Risk Taxonomy for Mechanistic Interpretability in Safety Applications", + "caption": "Table 15: Risk taxonomy for mechanistic interpretability in safety applications, including each risk’s description, example, and time horizon (how soon it is expected to materialise given current research and deployment practice).", "columns": [ "Risk Class", "Description", "Example", - "Severity" + "Time Horizon" ], "rows": [ { - "Risk Class": "Epistemic", - "Description": "False confidence from incomplete analysis", - "Example": "Concluding “no deception circuit” when coverage is 30%", - "Severity": "High", - "section": "epistemic-risks-false-confidence-from-partial-understanding" + "Risk Class": "Epistemic risk", + "Description": "False confidence from incomplete or unfaithful analysis", + "Example": "The report concludes that the circuit for deceptive behaviour has been completely removed, but the model’s backup pathway actually emerged after the removal.", + "Time Horizon": "Current", + "section": "epistemic-risks-false-confidence-from-partial-or-incorrect-understanding" }, { - "Risk Class": "Operational", - "Description": "MI tools fail silently under distribution shift", - "Example": "Steering vector ineffective on out-of-distribution inputs", - "Severity": "Medium-High", - "section": "operational-risks-failure-under-deployment-conditions" + "Risk Class": "Operational risk", + "Description": "MI tools fail silently in deployment", + "Example": "A steering vector is validated to be effective only on a testing set. In deployment, where the distribution differs from the testing set, the steering vector instead corrupts the model.", + "Time Horizon": "Near-term", + "section": "operational-risks-failure-under-deployment" }, { - "Risk Class": "Adversarial", - "Description": "MI insights enable targeted attacks", - "Example": "Attacker uses published circuits to craft evasive prompts", - "Severity": "High", - "section": "adversarial-risks-malicious-exploitation-of-mi-insights" + "Risk Class": "Adversarial risk", + "Description": "MI methods enable attacks on the model", + "Example": "Attackers use published knowledge editing methods to inject incorrect knowledge into deployed models.", + "Time Horizon": "Current", + "section": "adversarial-risks-malicious-exploitation-of-mi" }, { - "Risk Class": "Opportunity Cost", - "Description": "Resources diverted from more effective methods", - "Example": "Heavy MI investment delays deployment of behavioral safeguards", - "Severity": "Medium", + "Risk Class": "Opportunity cost", + "Description": "Resources diverted from other research directions", + "Example": "Heavy MI investment delays deployment of behavioural safeguards.", + "Time Horizon": "Long-term", "section": "opportunity-costs-resource-allocation-risks" } ] }, "verification_checklist": { - "section": "recommended-verification-practices", + "section": "sec:mi-risks", "groups": [ { - "heading": "Coverage", - "items": [ - "What fraction of model behaviors and internals is explained by the analysis?", - "What behaviors/capabilities were explicitly excluded?", - "Are negative claims (“no X is found”) qualified by coverage limitations?" - ] - }, - { - "heading": "Robustness", + "heading": "Epistemic risks", "items": [ - "Were findings validated across multiple random seeds and setups?", - "Were findings tested on diverse or out-of-distribution inputs?", - "Were findings tested against adversarial inputs?" + "First, results should always include explicit coverage.", + "Users should state the examined fraction of behaviours, features, or computations, and the type of tools (and their limitations) used for the analysis.", + "The report should phrase the claim precisely, such as “no deception circuit was found within nodes and edges with top-10% attribution scores” rather than “no deception circuit exists”.", + "Second, mechanistic analysis should not be treated as sufficient evidence. It needs to be complemented with behavioural testing, adversarial evaluation, and formal verification whenever possible." ] }, { - "heading": "Causal Validity", + "heading": "Operational risks", "items": [ - "Do identified components satisfy necessity tests?", - "Do identified components satisfy sufficiency tests?", - "Were claims validated via causality rather than just correlation?" + "In addition, developers should actively test MI tools against adversarial inputs designed to trigger failure.", + "Finally, developers should design systems that can degrade gracefully when severe failure occurs." ] }, { - "heading": "Deployment Readiness", + "heading": "Adversarial risks", "items": [ - "Has the tool been tested on realistic deployment data?", - "How will the MI tool be monitored in production?", - "What fallback mechanisms exist if the MI tool fails?" + "Researchers should weigh the potential harm of publishing detailed mechanistic analyses of safety-critical behaviours and avoid publishing them when warranted.", + "Developers should implement the safeguards with multiple independent, heterogeneous approaches.", + "Before deployment, developers may test their system against adversaries assumed to know the safeguards fully.", + "Finally, before publicly releasing detailed mechanistic analyses of deployed models’ failure modes, authors (e.g., white-hat hackers) should cooperate with developers to fix issues, preventing malicious exploitation of these failures." ] }, { - "heading": "Security", + "heading": "Opportunity costs", "items": [ - "Could published findings enable misuse?", - "Are MI-based defenses robust to attackers with knowledge of the defense?", - "Is access to detailed analyses appropriately controlled?" + "Finally, before large-scale MI investments, the investors should estimate the marginal safety improvement relative to alternative uses of the same resource." ] } ] }, "summary": { "section": "summary-mi-as-one-methodology-among-many", - "text": "The goal is not to avoid MI, but to apply it appropriately as one component of a defense-in-depth strategy rather than as a standalone guarantee of safety." + "text": "MI should therefore be viewed as one component of a defence-in-depth safety strategy rather than a standalone guarantee of safety." } }, "open_problems": { "chapter_sections": [ "sec:open_problems" ], - "intro": "This section identifies fundamental open questions that mechanistic interpretability (MI) has raised but not perfectly answered or solved. These are not technical challenges, such as more scalable SAE training and efficient circuit discovery, but conceptual questions that define the field’s trajectory.", + "intro": "14 Open Problems# This section identifies and describes key conceptual questions in mechanistic interpretability that guide the field’s future direction.", "problems": [ { "id": "when-necessary", "title": "When Is Mechanistic Understanding Necessary?", - "source_chapter": 15, + "source_chapter": 14, "sections": [ "when-is-mechanistic-understanding-necessary" ], - "statement": "A foundational question remains unresolved: for which problems and scenarios is mechanistic understanding required versus when behavioral methods suffice?", + "statement": "The need to characterise internal computations is usually tied to trustworthy AI, whose claim is intuitive: humans may not trust a system when they do not know, and have no way of knowing, how it works. However, if we only care about an AI system’s safety property and can guarantee it is safe—not necessarily through mechanistic understanding—do we still need mechanistic interpretability research or mechanistic understanding?", "details": [ - "RLHF and adversarial training have substantially improved deployed model safety, yet the sleeper agents experiment (Hubinger et al., 2024) revealed deceptive behaviors hidden from behavioral evaluation while remaining mechanistically detectable (Hubinger et al., 2024; Ouyang et al., 2022).", - "Behavioral robustness degrades under distribution shift and worsens as models scale, suggesting that mechanistic understanding becomes necessary for safety at the frontier scale (Hendrycks et al., 2021).", - "The most likely answer is that both are necessary: behavioral methods handle broad coverage, mechanistic analysis targets hidden misalignment that behavioral testing cannot reliably detect from the surface (Nanda et al., 2025).", - "Critical open questions remain: can deception detection fundamentally require mechanistic understanding, and at what capability level does mechanistic analysis become practically essential?" - ], - "related_methods": [ - "RLHF", - "adversarial training" + "For example, sleeper agents’ deceptive behaviours hidden from standard behavioural evaluations (Hubinger et al., 2024) can leave detectable signatures in the model’s internal representations, and MacDiarmid et al. (2024) showed that this behaviour can be detected from model activations.", + "(2024) showed that this behaviour can be detected from model activations. This demonstrates the utility of looking into the internals. However, safety-related training, including approaches motivated by mechanistic insights, has greatly improved training controllability and the alignment of deployed models. Adversaries may now find it harder to implement such a backdoor. Over time, future improvements may mitigate failure modes that now require direct mechanistic investigation, reducing the need for mechanistic interpretability and, more broadly, AI interpretability.", + "The AI safety community has not yet agreed on whether this will happen. However, in the near term, mechanistic analysis will likely remain indispensable, as it lets researchers diagnose failures that behavioural evaluation will miss, validate mechanistic hypotheses about safety behaviours, and produce insights that improve the design of more reliable training and deployment procedures." ], + "related_methods": [], "related_orientations": [] }, { "id": "evaluation", - "title": "How to Evaluate Mechanistic Interpretability Methods", - "source_chapter": 15, + "title": "Challenges in Evaluation and Benchmarking", + "source_chapter": 14, "sections": [ - "how-to-evaluate-mechanistic-interpretability-methods", - "proposed-evaluation-framework", - "standardized-benchmarks" + "challenges-in-evaluation-and-benchmarking" ], - "statement": "Without standardized metrics, it is unclear whether improvements are real or artifacts of metric choice.", + "statement": "Because widely accepted standards for benchmarks and evaluation are lacking, evaluating mechanistic interpretability methods remains hard.", "details": [ - "SAE variants are compared on different metrics; circuit discovery methods optimize for different objectives (size, coverage, interpretability); feature explanations are often evaluated on plausibility but not ground truth.", - "We propose that MI methods and findings should be evaluated across four dimensions: Interpretability—are mechanisms understandable to humans? Completeness—what fraction of behavior is explained? Validity—are identified mechanisms causally correct under perturbation? Efficiency—what is the computational cost?", - "Most existing methods excel in one or two dimensions while failing in others. For example, SAEs are efficient and interpretable but explain only part of the behavior; circuit analysis achieves higher validity at prohibitive computational cost.", - "Ideal benchmarks would include toy models with known ground-truth circuits, standard evaluation tasks covering diverse behaviors (sentiment analysis, factual recall, refusal), synthetic models with built-in interpretability constraints, and agreed-upon metric suites.", - "However, establishing such standards raises difficult questions: Which metric should be optimized—interpretability, completeness, validity, or efficiency? Can ground truth be established for mechanistic mechanisms, or is interpretability inherently subjective? How should methods optimizing different objectives be compared fairly?", - "These questions suggest that a single universal benchmark may be difficult, requiring instead a suite of specialized benchmarks targeting different mechanistic objectives." + "Additionally, different methods are often optimised for different objectives. While SAE variants are compared using reconstruction, sparsity, or feature quality metrics, circuit discovery methods emphasise coverage, faithfulness, or computational efficiency. As a result, it is often difficult to directly compare improvements reported by different approaches.", + "Section 7.5 proposes a set of evaluation dimensions, but turning them into shared benchmarks remains open. Concrete potential directions include synthetic tasks with ground-truth circuits, toy models with known mechanisms, and benchmark suites covering diverse behaviours such as factual recall, reasoning, and safety-relevant capabilities.", + "Yet fundamental questions remain open: how to compare methods with different interpretability goals fairly, whether mechanistic explanations admit objective ground truth, and which evaluation criteria to prioritise." ], "related_methods": [ "SAE", - "circuit discovery", - "feature explanations", - "circuit analysis" + "circuit discovery" ], "related_orientations": [] }, { "id": "scalability-adversarial", - "title": "Scalability and Adversarial Robustness", - "source_chapter": 15, + "title": "Incompleteness, Scalability, Generalisability, and Adversarial Robustness", + "source_chapter": 14, "sections": [ - "scalability-and-adversarial-robustness" + "subsec: scalability and adversarial robustness" ], - "statement": "A common and critical bottleneck in many existing MI approaches is scalability to model size, model diversity, and input domains.", + "statement": "Mechanistic methods, the insights they uncover, and their downstream use face several critical challenges.", "details": [ - "Whether this bottleneck can be overcome through algorithmic improvements or represents a fundamental limitation remains unclear.", - "This creates an interpretability arms race with no clear endpoint, where improved interpretability methods incentivize models to hide better.", - "Underlying both challenges is a deeper question: can mechanistic explanations be verified as genuine mechanisms rather than post-hoc rationalizations? Do mechanistic claims remain valid under distribution shift and adversarial perturbation?", - "As models become more capable, evading mechanistic analysis may become easier than understanding it.", - "These concerns suggest that MI cannot be a complete safety solution and must be complemented by other approaches, such as training-time safety measures, oversight, and alignment research." + "Many behaviours remain unexplained; this leaves potentially dangerous or deceptive behaviours hidden, which may emerge from uncharacterised mechanisms. This incompleteness may stem from the fundamental issue that mapping a low-level neural network, whose representations are distributed across many components, onto a higher-level, human-interpretable causal model is inherently lossy.", + "Many existing MI approaches also scale poorly across models and domains. Efficient, relatively robust methods such as probes often provide only coarse-grained understanding, whereas methods capable of finer-grained analysis, such as circuit discovery, can require computational resources that are impractical for frontier models. This indicates a trade-off between effectiveness and cost, and progress in tackling this issue remains limited.", + "Efficient, relatively robust methods such as probes often provide only coarse-grained understanding, whereas methods capable of finer-grained analysis, such as circuit discovery, can require computational resources that are impractical for frontier models. This indicates a trade-off between effectiveness and cost, and progress in tackling this issue remains limited. Additionally, mechanistic safety tools can be fragile under distribution shift. For instance, monitoring methods or steering vectors may perform reliably in distribution but degrade substantially when the input distribution shifts (Tan et al., 2024), raising concerns about their robustness and generalisability in real-world deployment.", + "Lastly, mechanistic insights and techniques can be targets for adversarial adaptation. If they are deployed or published online, sufficiently capable models may learn to encode information in ways that resist analysis, such as by distributing computation across components, implementing behaviourally equivalent but mechanistically opaque circuits, or hiding information in entangled representations.", + "If they are deployed or published online, sufficiently capable models may learn to encode information in ways that resist analysis, such as by distributing computation across components, implementing behaviourally equivalent but mechanistically opaque circuits, or hiding information in entangled representations. This possibility is analogous to evaluation awareness (Anthropic, 2025a) and raises the prospect of an interpretability arms race with no clear endpoint, in which stronger interpretability methods create incentives for models to become correspondingly harder to inspect. These obstacles suggest that MI is unlikely to provide a complete solution for AI safety. Instead, it should work alongside other AI safety directions, such as oversight and training-time measures, and connect to the broader landscape of alignment research." + ], + "related_methods": [ + "probes", + "circuit discovery", + "steering vectors" ], - "related_methods": [], "related_orientations": [] }, { "id": "mechanistic-alignment", "title": "Mechanistic Understanding and Alignment", - "source_chapter": 15, + "source_chapter": 14, "sections": [ "sec:mechanistic-alignment" ], - "statement": "However, these methods do not directly address alignment specification: understanding a circuit that enforces refusals or constraints allows auditing and intervention, but does not indicate what the model should value or whether its behavior genuinely reflects human intent.", + "statement": "Yet these approaches do not directly tackle alignment specification. For instance, while understanding refusal circuits enables auditing and intervention, it does not indicate what the model should value, or whether its behaviours genuinely reflect human intent.", "details": [ - "MI provides tools for behavioral control. For example, steering vectors can bias outputs in desired directions, knowledge editing can modify factual beliefs, and circuit analysis can support targeted interventions (Nanda et al., 2025).", - "Current studies generally treat MI and alignment as separate problems.", + "MI provides tools for behavioural control. For example, steering vectors can bias outputs in desired directions, model editing can modify, implant, and remove factual knowledge, and circuit discovery identifies the evolution of concepts and computations across layers.", + "Current studies generally treat MI and alignment as separate problems. Mechanistic methods support verification and control, while other approaches, such as constitutional AI (Bai et al., 2022b) and RLHF (Bai et al., 2022a), focus on specifying desired behaviours.", "Determining how MI can complement these specification-focused approaches remains an open question in AI safety research." ], "related_methods": [ "steering vectors", - "knowledge editing", - "circuit analysis", + "model editing", + "circuit discovery", "constitutional AI", - "value learning" + "RLHF" ], "related_orientations": [] }, { "id": "ontology", "title": "Interpretation vs. Truth: The Ontology Question", - "source_chapter": 15, + "source_chapter": 14, "sections": [ "sec:ontology" ], - "statement": "A foundational question in MI is: what does it mean to mechanistically understand a neural network?", + "statement": "A foundational MI question, which is both technical and conceptual, is: what does it mean to mechanistically understand a neural network?", "details": [ - "The same input-output behavior can be realized by multiple mechanistically distinct internal structures, each consistent with the observed causal relationships.", - "Feature naming is subjective: different observers may assign different names to the same learned direction based on their interpretation of the activating examples.", - "Representation ambiguity means that multiple mechanisms can produce identical behavior while differing internally.", - "Moreover, intuitive human-language explanations may not always accurately capture the internal computations of the model (Hewitt et al., 2025).", - "For instance, how should abstraction level be chosen? Can confidence in a mechanistic explanation be established when multiple descriptions are consistent with the same behavior? Should one expect a unique mechanistic description, or are multiple valid interpretations acceptable?" + "Marking a feature as a “borzoi detector”, for instance, or labelling an attention head component as “a previous-token copier” mirrors possibly subjective human choices. This means multiple mechanisms can explain one behaviour, each consistent with observed causal relationships.", + "MI research often assumes how human-interpretable descriptions should map onto causal mechanisms. But these assumptions face several obstacles. As mentioned before, one of these is subjectivity. This includes choosing the level of abstraction; for instance, a circuit could be explained at the level of attention routing or low-rank decomposition, and the choice depends on subjective priorities.", + "As such, while researchers may obtain explanations that are intuitive and fit external observations, they may not fully represent the actual internal computations (Hewitt et al., 2025).", + "Other questions include: How can we assess confidence in a mechanistic explanation if multiple descriptions can explain the same behaviour? Are multiple valid interpretations acceptable, or should researchers expect a unique mechanistic description? Resolving these questions will help researchers clarify the limits and goals of MI research." ], "related_methods": [], "related_orientations": [] @@ -1322,168 +738,63 @@ { "id": "proxy-fidelity", "title": "The Proxy Fidelity Problem", - "source_chapter": 15, + "source_chapter": 14, "sections": [ "sec:proxy-fidelity", - "problem-formulation.", "why-this-matters-for-safety.", "potential-approaches." ], - "statement": "As architectural interventions gain traction (Section 6.9), a new verification challenge emerges: when interpretability is achieved through proxy models, such as sparse networks trained to mimic dense model representations, how can we guarantee that the proxy’s circuits reflect the same mechanisms used by the original model, rather than behaviorally equivalent but mechanistically distinct simulations?", + "statement": "Classic proxy models in MI research include sparse, small neural networks such as SAEs and CLTs, which reconstruct the target model’s dense activations using learned sparse linear features. However, while these approaches have greatly enhanced mechanistic analysis, they introduce a new obstacle: how can we ascertain whether the proxy truly learns the same mechanisms as the original model, rather than shallowly reproducing behaviour?", "details": [ - "Two models can produce identical outputs (or even identical intermediate activations) while implementing distinct algorithms—a form of mechanistic underdetermination.", - "Although preliminary studies have suggested that interventions on sparse proxies can induce behavioral changes in dense models (Gao et al., 2025b), systematic validation remains limited.", - "One approach is causal correspondence testing, which verifies that ablating a circuit in the proxy and the corresponding components in the dense model produces matched behavioral changes across different inputs.", - "The second approach is activation patching transfer, which evaluates whether activations identified as causally important in the proxy retain their causal influence when the same interventions are applied to the dense model.", - "A third direction involves developing bounds of mechanistic divergence: theoretical frameworks that upper-bound how much internal computation can differ between models given constraints like activation reconstruction error." + "Several recent interpretability approaches construct more interpretable proxy models that approximate the target model’s internal computations. Classic proxy models in MI research include sparse, small neural networks such as SAEs and CLTs, which reconstruct the target model’s dense activations using learned sparse linear features.", + "This concern is realistic because two neural networks can produce the same output while using different internal algorithms. That is, a proxy model may find features and circuits that do not exist in the original model.", + "Why This Matters for Safety. For example, when researchers find no “deception circuits” in the proxy models, can they claim that the target model has therefore “passed the test” and is safe? Making this claim requires that these mechanisms do not exist in the target model either. However, as discussed above, the absence of a mechanism in the proxy model does not imply its absence in the target model, since they may compute the input differently while producing the same output. The conclusion we can draw is only something like “we did not find the deception mechanism in the proxy model”, which is far from a useful guarantee in safety-critical scenarios.", + "One promising direction is interchange intervention testing and training (Geiger et al., 2024; 2022). This technique was developed to compare and align a neural network with a high-level causal abstraction. The proxy models and their circuits can be viewed as a causal abstraction of the target model, where the nodes (human-interpretable variables) are the proxy models’ features. This framing not only lets us test alignment between the mechanisms identified in proxy models and the target model, but also provides an optimisation objective for training faithful proxies.", + "The proxy models and their circuits can be viewed as a causal abstraction of the target model, where the nodes (human-interpretable variables) are the proxy models’ features. This framing not only lets us test alignment between the mechanisms identified in proxy models and the target model, but also provides an optimisation objective for training faithful proxies. Additionally, a theoretical framework that upper-bounds how much internal computation can differ between the target model and proxy model under a given reconstruction performance can also be a promising direction." ], "related_methods": [ - "weight-sparse proxy", - "causal correspondence testing", - "activation patching transfer", - "bounds of mechanistic divergence" + "SAEs", + "CLTs", + "interchange intervention testing and training" ], "related_orientations": [] }, { "id": "non-transformer", "title": "Transferability Beyond Transformers", - "source_chapter": 15, + "source_chapter": 14, "sections": [ "sec:non-transformer" ], - "statement": "This raises an open question: to what extent do MI methods and findings transfer to other architectures?", + "statement": "Even though foundational circuits work (Olah et al., 2020) originated in convolutional networks (InceptionV1; Szegedy et al., 2015), demonstrating interpretable features of visual neurons, many of the techniques that define modern mechanistic interpretability, which include feature extraction and explanation, model editing, and circuit discovery, were developed specifically for transformer residual streams, MLPs, and attention mechanisms. Thus, an important question arises: are mechanistic interpretability concepts, tools, and findings specific to transformers, or do they reflect more general properties of neural computation across model architectures?", "details": [ - "Convolutional networks: The spatial locality of convolutions may make circuits more naturally modular, but SAE-style decomposition of CNN activations remains underexplored at scale.", - "State space models (e.g., Mamba): These architectures lack attention but implement sequential processing via recurrent dynamics. Whether superposition, linear representation, and circuit-style analysis apply is unknown.", - "Reinforcement learning agents: RL policies involve temporal credit assignment and value estimation, which may require different interpretability primitives than simply next-token prediction. Limited work exists on MuZero and Atari agents, but systematic MI is lacking.", - "Diffusion models: The iterative denoising process introduces temporal structure distinct from autoregressive generation. Emerging work explores attention patterns in diffusion transformers, but feature extraction methods are nascent.", - "The five research orientations (Pragmatic, Ambitious, Developmental, Agentic, Automated) discussed in Section 13 are framed architecture-agnostically, but their instantiation has been transformer-specific.", - "Until such a validation exists, practitioners should be cautious about extrapolating transformer-derived insights to other neural network architectures." + "Although the six research orientations discussed in Section 12 are largely architecture-agnostic in their aims, their classic methods and representative works focus almost entirely on transformers.", + "Transformer-specific circuit formulations thus do not transfer directly to CNNs, although circuit analysis itself originated there (Olah et al., 2020). Convolutions’ spatial locality may yield more naturally modular circuits than transformers under suitable formulations, but large-scale studies of superposition, feature extraction, and circuit discovery in modern CNNs and their variants remain limited.", + "State-space and recurrent models: Architectures such as Mamba (Gu & Dao, 2024) and RWKV (Peng et al., 2023) replace standard self-attention with recurrent computations that update a hidden state over time. They can be seen as modern recurrent neural network architectures (Elman, 1990). Specifically, Mamba uses input-dependent state transitions to selectively retain or forget information, while RWKV uses a recurrence based on linear attention. This new paradigm raises interpretability questions that previous transformer interpretability studies have not addressed, such as how information is stored and updated in recurrent states, whether these representations exhibit certain properties such as superposition, and how causal effects can be traced across state updates.", + "Diffusion models: Diffusion models (Sohl-Dickstein et al., 2015) apply iterative denoising processes to generate images or tokens. They may still use transformers as backbones, but their computational structure differs fundamentally from autoregressive generation. For example, many current MI studies use the predicted next token as a sign of model behaviour (e.g., “yes” or “no” when the input is a malicious query), while diffusion-based language models generate multiple tokens in different positions at a time to iteratively complete the response. This difference means many interpretability techniques and insights do not directly apply. While recent work has begun investigating attention patterns and internal representations in diffusion models (Tinaz et al., 2025; Wang et al., 2026c), mechanistic understanding in this paradigm remains lacking.", + "Many modern transformer-based language models have adopted this architecture (Jiang et al., 2024; DeepSeek-AI, 2025; QwenAI, 2025; OpenAI et al., 2025; Team et al., 2026). However, MoE architectures introduce technical challenges for mechanistic analysis. For example, activation patching at an intermediate MoE layer may change which experts are selected in downstream MoE layers. This makes it difficult to distinguish the patched activation’s effects on model output from those mediated by changes in downstream expert routing.", + "Simultaneously, some recent work suggests that at least a few mechanistic interpretability techniques can transfer outside of transformers. Steering vectors, tuned lenses, knowledge editing, and circuit analysis evaluations have all been adapted, with varying success, to state-space models such as RWKV and Mamba, as well as to vision-language models (Paulo et al., 2025a; Sharma et al., 2024a; Häon et al., 2025; Neo et al., 2025; Ensign & Garriga-Alonso, 2024).", + "Sorting out which aspects of mechanistic interpretability depend on architecture and which reflect broader principles of neural computation thus appears to be a promising research direction. On the other hand, whether current research orientations and associated techniques remain relevant beyond transformers will also show how broadly mechanistic interpretability research can apply to future systems." ], "related_methods": [ "SAEs", - "transcoders", - "model editing", "circuit discovery", - "activation patching" + "model editing", + "activation patching", + "steering vectors", + "tuned lens", + "knowledge editing" ], "related_orientations": [ "Pragmatic", "Ambitious", "Developmental", - "Agentic", + "Interpretability Across Time", + "Introspective", "Automated" ] - }, - { - "id": "orientation-tensions", - "title": "Critical Tensions and Open Questions", - "source_chapter": 13, - "sections": [ - "critical-tensions-and-open-questions" - ], - "statement": "Several critical tensions emerge when considering the adoption of different orientations.", - "details": [ - "A central debate exists between Pragmatic and Ambitious approaches: does partial mechanistic understanding suffice to ensure safety, or is complete comprehension necessary?", - "Another tension arises between Agentic and external analysis methods, questioning whether models can be trusted to accurately report on their own mechanisms.", - "Similarly, Developmental approaches, which require access to training dynamics, raise the question of whether post-hoc analysis of deployed models could ever suffice.", - "Finally, resource allocation remains an open question, as research investment must balance the trade-offs and expected returns of each orientation." - ], - "related_methods": [], - "related_orientations": [ - "Pragmatic", - "Ambitious", - "Agentic", - "Developmental" - ] - }, - { - "id": "safety-limitations", - "title": "Limitations and Open Questions", - "source_chapter": 14, - "sections": [ - "sec:limitations-open-questions" - ], - "statement": "Despite significant progress, mechanistic interpretability faces fundamental limitations that constrain its role in AI safety.", - "details": [ - "This incompleteness is not merely a technical problem but a conceptual one: some behaviors may depend on entangled, distributed representations that resist decomposition into interpretable units.", - "Models can optimize to encode information in ways that resist SAE decomposition, hide computation in entangled features, or implement complex distributed circuits mechanistically opaque to current circuit discovery methods.", - "This scalability crisis means that as models grow, mechanistic oversight becomes increasingly impractical, threatening the viability of mechanistic safety for next-generation systems.", - "It provides limited guidance on alignment specification, i.e., defining the right value that models should be optimized for and ensuring models pursue the objectives in the first place." - ], - "related_methods": [ - "SAE", - "circuit discovery", - "Steering vectors", - "concept erasure", - "monitoring" - ], - "related_orientations": [] - }, - { - "id": "clt-open-questions", - "title": "Open Questions for CLT Research", - "source_chapter": 6, - "sections": [ - "open-questions-for-clt-research." - ], - "statement": "Several important questions remain open:", - "details": [ - "How does CLT fidelity scale in large models?", - "Can CLTs be extended to jointly model attention and MLP computations?", - "What evaluation protocols can reliably detect when CLT-based explanations are unfaithful to the original model’s computation?", - "How sensitive are CLT circuits to the choice of training data distribution?" - ], - "related_methods": [ - "CLT" - ], - "related_orientations": [] - }, - { - "id": "conclusion-unresolved", - "title": "Unresolved Questions", - "source_chapter": 16, - "sections": [ - "unresolved-questions" - ], - "statement": "As discussed in Section 15, several fundamental open questions remain.", - "details": [ - "When is mechanistic understanding actually required for safety, or do behavioral methods suffice? Can mechanisms be verified as genuine rather than post-hoc rationalizations? How do mechanisms hold up under distribution shift and adversarial pressure?", - "These questions are not merely technical but conceptual, touching on epistemology and the definition of understanding itself." - ], - "related_methods": [], - "related_orientations": [] } ] }, - "gaps": [ - "Evaluation: manuscript dimension names are 'Faithfulness', 'Completeness', 'Stability', 'Causal Validity', 'Human Interpretability' (7.6.3-7.6.7 headings). The public website says 'Attribution faithfulness'. In chapter 7 the dimension is plain 'Faithfulness'. The phrase 'attribution faithfulness' occurs only in chapter 8 (sections faithfulness-of-attributions. / key-findings-on-attribution-faithfulness, about gradient/attention attribution reliability in probing), not as an evaluation-framework dimension.", - "Evaluation: in Table 10 (tab:evaluation-framework) the fifth row is labelled 'Interpretability', not 'Human Interpretability' as in heading 7.6.7 and the intro sentence.", - "Evaluation: the intro sentence lists dimensions in lower case ('faithfulness, completeness, stability, causal validity, and human interpretability'); Table 11 column headers shorten them to 'Causal' and 'Human'.", - "Evaluation: framework limitations mention 'Proposed thresholds (e.g., necessity > 0.5)', but no necessity > 0.5 threshold is proposed anywhere in 7.6.3-7.6.8. The only threshold is the undefined \\tau in Behavioral Coverage.", - "Evaluation: chapter 15.2.1 (proposed-evaluation-framework) proposes a *different* FOUR-dimension framework (Interpretability, Completeness, Validity, Efficiency) that does not cross-reference the five-dimension framework in 7.6. Stability is missing, and Efficiency is a new dimension. The two are inconsistent.", - "Evaluation: the framework sits in chapter 7 ('Circuits in Mechanistic Interpretability') as 7.6, although it is method-general (SAEs, probes, steering).", - "Evaluation: the per-dimension 'Evaluation protocol' items and the integrated protocol (7.6.8) are stated as recommendations. The text reports no worked examples or empirical results for any dimension; the only concrete artefacts named are the benchmarks MIB, AxBench and RAVEL (under Causal Validity). So 'examples' is empty for 4 of 5 dimensions.", - "Orientations: this draft has FIVE orientations (13.3 Pragmatic, 13.4 Ambitious, 13.5 Developmental, 13.6 Agentic, 13.7 Automated). The five are confirmed in 15.7 and the 16 Conclusion. The public list of six maps as follows: Pragmatic -> 13.3; Ambitious -> 13.4; Developmental -> 13.5 (also chapter 10, sec:training); Automated -> 13.7 (also 14.4 sec:automated-interpretability); Introspective -> NO orientation of that name. The closest is Agentic Interpretability (13.6, subtitled 'Models Interpreting Themselves', Kim et al. 2025). The introduction (ch.1, paragraph 'Second, divergence in research priorities', no own section id) describes this thread as 'exploring whether models can serve as introspective agents that explain their own internals (Kim et al. 2025)', cited to the same source as Agentic. 'introspective' also appears once in 5.3 subsec:privileged-access ('not accurate introspective reports'), and 5.3 is cited by 13.6.3 as its basis. Interpretability Across Time -> NO orientation. The phrase 'across time' appears nowhere. The nearest material is chapter 11 (Learning Dynamics; withheld from the site) and 12.1 sec:model-diffing. Neither is framed as an orientation, and both overlap with Developmental.", - "Orientations: 13.4 has an unnumbered paragraph 'Architectural interventions as a synthesis.' that floats a possible distinct 'Engineering' orientation; the Conclusion 16.3 instead speaks of 'three complementary paradigms' (post-hoc decomposition, developmental analysis, interpretability by design). These are a separate taxonomy from the five orientations.", - "Orientations: 13.2 refers to '(Figure 16)', but the figure caption reads 'Figure 18: Five Research Orientations.'", - "Orientations: 13.3 Pragmatic says 'which is a valuable source of feedback, but required.' This is almost certainly a missing 'not' ('but not required'); kept verbatim.", - "Orientations: 13.3 Pragmatic list item 'Prompting, a very effective black box baseline' lacks a full stop; 13.7 item 'Verification using synthetic data. to evaluate...' has a stray full stop; 13.6.2 'artificailly' typo.", - "Orientations: Table 14 tools do not fully match the prose. Agentic prose methods are 'Interactive querying', 'Dialogical explanation loops' and 'Continually intervening model', while the table lists 'LatentQA, dialogical self-explanation, model intervention'; LatentQA is never mentioned in 13.6. Developmental's table entry 'SAEs' corresponds to the prose 'Dictionary learning methods'. Ambitious prose names causal scrubbing and causal intervention, while the table says 'Circuit discovery, formal verification'.", - "Orientations: Agentic has no stated assumptions or strengths in 13.6 (only risks/governance), so those arrays are empty. The table's Core Thesis 'Models explain themselves if prompted correctly' is the closest to an assumption.", - "Orientations: 13.6.3 says 'Building on Section 5.3' (5.3 = Privileged Access Hypothesis, correct). Protocols in 13.6.3/13.7.3 refer to 'the regulator', a framing not introduced elsewhere.", - "Safety applications: section id typo 'challenges-of-black-box-bethods' (heading '14.1.2 Challenges of Black-Box Bethods').", - "Safety applications: chapter intro grammar 'including but are not limited to'. 14.7.4 opens 'Detailed mechanistic knowledge suffers from the opportunity to be weaponized'. 14.7 intro 'but it carries its own risks. However, misapplication...' (odd 'However').", - "Safety applications: 14.4 Automated Interpretability Agents uses section id 'sec:automated-interpretability', very close to 13.7's 'sec:automated-interp'. That invites cross-ref confusion.", - "Safety applications: no application subsection in 14.2-14.4 explicitly names an orientation except 14.4, which is titled 'Automated Interpretability Agents'. The linkage is only given generically in 14.5 by orientation, not per application, so 'orientations' arrays are mostly empty. 14.5 merges Automated and Agentic in one sentence.", - "Safety applications: mechanistic monitoring (14.3.2) reports no empirical result. Its 'Result' is a hypothetical worked example, so evidence is empty. The worked examples' 'Result:' lines for PISCES (bomb), ROME (TRIGGER) and steering (sycophancy) are illustrative, not empirical; only the PISCES '7.7%' line and the ROME/MEMIT, Rimsky and Tan citations report findings.", - "Safety applications: PISCES Stage 2 example says 'Assume d_{MLP}=2 with basis vectors v_1 and v_2', whereas the general description treats v_i as rows of W_out in R^{d_MLP x d}. That is consistent, but the example calls them basis vectors.", - "Open problems: chapter 15 has 7 numbered subsections (15.1-15.7); 15.2 has sub-subsections 15.2.1/15.2.2. 15.3 begins 'A common and critical bottleneck...' then says 'Second, adversarial obfuscation exists' with no 'First'.", - "Open problems: 15.1 cites (Hubinger et al. 2024; Ouyang et al. 2022) for 'mechanistically detectable'. Ouyang et al. 2022 (InstructGPT/RLHF) seems mis-attached; flag for author check.", - "Open problems: 15.4 attributes 'value learning' to (Bai, Jones, et al. 2022), which is the HH-RLHF paper; possible citation mismatch to check.", - "Elsewhere: chapter 11 contains unresolved author notes in the body text (e.g., ), and 11.2.3 Phase changes is empty. 11.2 also has the typo 'attention patternsform' and section id typo .", - "Elsewhere: 5.3 has unresolved citation placeholders '(binder?; song?; B. Z. Li et al. 2025)'. The introduction has typos 'Bascially', 'two many un-answered', 'some automation has become a necessary', and the section id/heading 'useful-surveys-that-we-bild-on.'", - "Section ids are reused across chapters with only numeric suffixes (e.g., 'mitigation-strategies.' ... '-3', 'conceptual-basis.-1', 'case-study-induction-heads-1/-2'). They are unique, but fragile for deep links." - ] + "gaps": [] } diff --git a/src/data/links.json b/src/data/links.json index c482b0f71b..e29b6327c3 100644 --- a/src/data/links.json +++ b/src/data/links.json @@ -1,26 +1,118 @@ { - "_note": "Editorial cross-links between notebooks/demos and manuscript entries. Navigation only: no scientific claims live here.", - "notebooks": { - "notebooks/01_core_hypotheses/01_linear_representation_hypothesis.ipynb": { "hypotheses": ["linear-representation"], "methods": ["probing"] }, - "notebooks/01_core_hypotheses/02_superposition_hypothesis.ipynb": { "hypotheses": ["superposition"] }, - "notebooks/01_core_hypotheses/03_privileged_access_hypothesis.ipynb": { "hypotheses": ["privileged-access"] }, - "notebooks/02_feature_extraction/01_sparse_autoencoders_basics.ipynb": { "methods": ["sae", "gated-sae", "jumprelu-sae", "topk-sae", "batch-topk-sae"] }, - "notebooks/02_feature_extraction/02_transcoders.ipynb": { "methods": ["transcoders"] }, - "notebooks/02_feature_extraction/03_crosscoders.ipynb": { "methods": ["crosscoders"] }, - "notebooks/03_circuits/01_circuit_discovery.ipynb": { "methods": ["activation-patching"] }, - "notebooks/04_intermediate_computation/01_logit_lens.ipynb": { "methods": ["logit-lens"] }, - "notebooks/04_intermediate_computation/02_tuned_lens.ipynb": { "methods": ["tuned-lens"] }, - "notebooks/04_intermediate_computation/04_patchscopes.ipynb": { "methods": ["patchscopes"] }, - "notebooks/04_intermediate_computation/05_probing_classifiers.ipynb": { "methods": ["probing", "amnesic-probing"] }, - "notebooks/04_intermediate_computation/06_activation_oracles.ipynb": { "methods": ["latentqa"] }, - "notebooks/05_representation_analysis/01_model_diffing.ipynb": { "methods": ["model-diffing", "crosscoders"] }, - "notebooks/05_representation_analysis/02_knowledge_localization.ipynb": { "methods": ["causal-tracing"] }, - "notebooks/05_representation_analysis/03_representation_engineering.ipynb": { "methods": ["steering-vectors"] }, - "notebooks/06_safety_applications/02_steering_vectors.ipynb": { "methods": ["steering-vectors"], "applications": ["steering-vectors"] } - }, - "demos": { - "superposition": { "hypotheses": ["superposition"], "notebook": "notebooks/01_core_hypotheses/02_superposition_hypothesis.ipynb" }, - "sae": { "methods": ["sae", "topk-sae"], "notebook": "notebooks/02_feature_extraction/01_sparse_autoencoders_basics.ipynb" }, - "patching": { "methods": ["activation-patching"], "notebook": "notebooks/03_circuits/01_circuit_discovery.ipynb" } + "_note": "Editorial cross-links between notebooks/demos and manuscript entries. Navigation only: no scientific claims live here.", + "notebooks": { + "notebooks/01_core_hypotheses/01_linear_representation_hypothesis.ipynb": { + "hypotheses": [ + "linear-representation" + ], + "methods": [ + "probing" + ] + }, + "notebooks/01_core_hypotheses/02_superposition_hypothesis.ipynb": { + "hypotheses": [ + "superposition" + ] + }, + "notebooks/01_core_hypotheses/03_privileged_access_hypothesis.ipynb": { + "hypotheses": [ + "privileged-access" + ] + }, + "notebooks/02_feature_extraction/01_sparse_autoencoders_basics.ipynb": { + "methods": [ + "sae", + "gated-sae", + "jumprelu-sae", + "topk-sae", + "batch-topk-sae" + ] + }, + "notebooks/02_feature_extraction/02_transcoders.ipynb": { + "methods": [ + "transcoders" + ] + }, + "notebooks/02_feature_extraction/03_crosscoders.ipynb": { + "methods": [ + "crosscoders" + ] + }, + "notebooks/03_circuits/01_circuit_discovery.ipynb": { + "methods": [ + "activation-patching" + ] + }, + "notebooks/04_intermediate_computation/01_logit_lens.ipynb": { + "methods": [ + "logit-lens" + ] + }, + "notebooks/04_intermediate_computation/02_tuned_lens.ipynb": { + "methods": [ + "tuned-lens" + ] + }, + "notebooks/04_intermediate_computation/04_patchscopes.ipynb": { + "methods": [ + "patchscopes" + ] + }, + "notebooks/04_intermediate_computation/05_probing_classifiers.ipynb": { + "methods": [ + "probing", + "amnesic-probing" + ] + }, + "notebooks/04_intermediate_computation/06_activation_oracles.ipynb": { + "methods": [ + "latentqa" + ] + }, + "notebooks/05_representation_analysis/01_model_diffing.ipynb": { + "methods": [ + "model-diffing", + "crosscoders" + ] + }, + "notebooks/05_representation_analysis/02_knowledge_localization.ipynb": { + "methods": [ + "causal-tracing" + ] + }, + "notebooks/05_representation_analysis/03_representation_engineering.ipynb": { + "methods": [ + "steering-vectors" + ] + }, + "notebooks/06_safety_applications/02_steering_vectors.ipynb": { + "methods": [ + "steering-vectors" + ], + "applications": [ + "steering-vectors" + ] + } + }, + "demos": { + "superposition": { + "hypotheses": [ + "superposition" + ], + "notebook": "notebooks/01_core_hypotheses/02_superposition_hypothesis.ipynb" + }, + "sae": { + "methods": [ + "sae", + "topk-sae" + ], + "notebook": "notebooks/02_feature_extraction/01_sparse_autoencoders_basics.ipynb" + }, + "patching": { + "methods": [ + "activation-patching" + ], + "notebook": "notebooks/03_circuits/01_circuit_discovery.ipynb" } + } } diff --git a/src/data/methods.json b/src/data/methods.json index 7d04c3ccce..f6b8117cc5 100644 --- a/src/data/methods.json +++ b/src/data/methods.json @@ -5,11 +5,13 @@ "name": "The Interpretability Landscape", "sections": [ "sec:interpretability-landscape", - "behavioral-methods", + "behavioural-methods", "subsec:cbm", - "attribution-and-saliency-methods", + "input-attribution-and-saliency-methods", "representation-probing-and-related-post-hoc-analyses", - "when-mechanistic-understanding-becomes-necessary" + "mechanistic-interpretability-methods", + "subsec: mech interp necessary", + "subsec:limits" ] }, { @@ -17,22 +19,22 @@ "name": "Sparse Autoencoders / SAE Variants and Architectural Improvements", "sections": [ "sec:sae", + "the-sparse-coding-problem", "subsec:sae-foundation", "subsec:sae-variants", "subsec:sae-selection", - "subsec:challenge", - "app:sae_details" + "subsec:challenge" ] }, { "id": "multi-layer-feature-analysis", - "name": "Multi-Layer Feature Analysis: Transcoders and Crosscoders", + "name": "Multi-Layer Feature Extraction and Analysis: Transcoders and Crosscoders", "sections": [ "subsec:transcoders-crosscoders", "subsubsec:transcoder", "subsubsec:cross-layer-transcoder", - "crosscoders-analyzing-cross-layer-features", - "comparing-saes-transcoders-and-crosscoders." + "crosscoders-analysing-cross-layer-features", + "comparing-saes-transcoders-and-crosscoders" ] }, { @@ -44,10 +46,10 @@ }, { "id": "architectural-interventions", - "name": "Architectural Interventions for Intrinsic Interpretability", + "name": "Interpretability-by-Design Methods for Intrinsic Interpretability", "sections": [ "sec:architectural-interventions", - "architectural-responses-to-superposition" + "architectural-responses-to-superposition." ] }, { @@ -55,45 +57,46 @@ "name": "Circuits in Mechanistic Interpretability", "sections": [ "sec:circuit", - "sec:circuit-table", - "circuit-discovery-in-weight-sparse-architectures", - "sec:causal-verification" + "sec:circuit-discovery-methods", + "sec:circuit-replacement", + "sec:causal-verification", + "sec:evaluation-framework" ] }, { "id": "intermediate-computation", - "name": "Understanding Intermediate Computation: Logit Lens and Probing", + "name": "Understanding Intermediate Representations", "sections": [ "sec:intermediate-computation" ] }, { "id": "training-data-attribution", - "name": "Training Data Attribution and Influence Functions", + "name": "Training Data Attribution", "sections": [ "sec:influence-functions" ] }, { "id": "developmental-interpretability", - "name": "Developmental Interpretability and Training Dynamics", + "name": "Learning Dynamics and Mechanistic Interpretability", "sections": [ "sec:training" ] }, { "id": "representation-analysis", - "name": "Representation Analysis: Structure, Localization, and Transfer", + "name": "Representation Analysis: Comparison, Localisation, and Control", "sections": [ "sec:representation" ] }, { "id": "safety-applications", - "name": "From Understanding to Intervention: Safety Applications", + "name": "From Understanding to Intervention: Safety Applications of Mechanistic Interpretability", "sections": [ "sec:safety_applications", - "concept-erasure-and-direct-editing", + "concept-erasure-and-knowledge-editing", "steering-vectors-and-mechanistic-monitoring" ] } @@ -101,28 +104,32 @@ "methods": [ { "id": "behavioral-methods", - "name": "Behavioral Methods", + "name": "Behavioural Methods", "family": "interpretability-landscape", "sections": [ - "behavioral-methods" + "behavioural-methods" ], - "definition": "Behavioral methods examine models at the level of inputs and outputs: capability evaluations (Linzen et al., 2016; Marvin & Linzen, 2018; Wang et al., 2024; Zhang et al., 2025), deceptive intents (Kinniment et al., 2026), and red-teaming (Ganguli et al., 2022; Perez et al., 2022b; He et al., 2025).", + "definition": "Behavioural methods evaluate and analyse models through their inputs and outputs. They have been developed to examine a wide range of capabilities, traits, and behaviours under specific conditions.", "assumptions": [ - "These approaches are fast, scalable, and practical for high-level auditing without access to model internals." + "Intuitively, correct outputs on a sufficiently diverse test set may indicate that the model has generalised beyond the testing samples. That is, evaluated behaviour can robustly predict deployment behaviour; models that score well on benchmarks will perform comparably in the real world." + ], + "evaluation": [ + "In general, behavioural evaluations provide a practical way to audit and assess models at scale. They are typically fast, inexpensive, and scalable, requiring only forward passes and comparisons between model outputs and reference answers." ], - "evaluation": [], "applications": [ - "Behavioral methods nonetheless remain essential in practice, for rapid iteration during development, for prioritizing which behaviors warrant deeper investigation, and for surface-level safety screening where the cost of mechanistic analysis is prohibitive." + "Beyond assessing models as helpful assistants, behavioural evaluations also identify and benchmark potential safety risks and failure modes.", + "Although evaluation results on a finite testing set do not guarantee the mechanism that produced the behaviour of interest (Wani et al., 2026) (which the internal methods below aim to establish, as discussed in Section 2), behavioural methods nonetheless remain essential in practice for rapid iteration during development, prioritising which behaviours warrant deeper investigation, and surface-level safety screening when the cost of mechanistic analysis is prohibitive." ], "limitations": [ - "The limits of this approach are the subject of Section 2; the point that matters here is that behaviour on a finite evaluation set underdetermines the mechanism that produced it, which is what the internal methods below are designed to establish.", - "These studies assess what models do without explaining how." + "Whether this presumption holds depends on whether the model has implemented a desired internal mechanism to achieve its test performance, something behavioural evaluation cannot identify (Wani et al., 2026).", + "Mechanistic analysis becomes necessary when we need to verify how a model produces a behaviour—not just that it does under test conditions." ], "related": [], "mentioned_in": [ - "sec:scope", "sec:why_open", - "alternatives-black-box-methods" + "subsec: mech interp necessary", + "sec:scope", + "shared-limitations-of-black-box-approaches" ] }, { @@ -132,54 +139,52 @@ "sections": [ "subsec:cbm" ], - "definition": "Concept-based methods plays a middle ground role: they go beyond behavioral evaluation by examining internal representations, but typically establish correlational rather than causal claims.", + "definition": "Concept-based methods occupy a middle ground: they examine internal representations but typically establish correlational rather than causal claims. However, when combined with interventions such as steering along concept directions and ablating concept-associated components, they become mechanistic by the criteria discussed in Section 1.2.", "assumptions": [ - "CBMs impose semantic constraints (predefined human concepts), while architectural interventions impose structural constraints (sparsity, modularity) that allow the model to discover features autonomously." + "CBMs impose semantic constraints (predefined human concepts), while interpretable-by-design models impose structural constraints (sparsity, modularity) that make the model intrinsically easier to interpret." ], "evaluation": [], "applications": [ - "These methods do not guarantee mechanistic understanding but are valuable for auditing, localization, and early detection tasks.", - "Practical Guide: CBMs tend to be more appropriate when the relevant concepts are known and can be enumerated; architectural interventions may be preferable when the goal is to discover what the model has learned—including features that were not known in advance." + "In general, these methods do not provide causal evidence about the role of those concepts in model computation, although some probing variants aim to support stronger causal claims, such as amnesic probing (Elazar et al., 2021). Nevertheless, they remain valuable tools for auditing, localisation, and early detection tasks.", + "CBMs can be more appropriate when the set of concepts of interest is known. Interpretability-by-design models may be preferable when the goal is to discover what the model has learned—including features that were not preset in advance." ], "limitations": [ - "Concept Bottleneck Models (CBMs; Koh et al., 2020) constrain computation through a predefined bottleneck of interpretable concepts, offering built-in transparency and allowing for interventions, but traditionally requiring a complete set of predefined concepts and often suffer from an “accuracy-interpretability trade-off” when concept definitions are insufficiently expressive.", - "For safety applications, where unknown failure modes are a core concern, methods that enable discovery may be more useful than those that require specifying concepts upfront." + "Concept Bottleneck Models (CBMs; Koh et al., 2020) constrain computation through a predefined bottleneck of interpretable concepts, offering built-in transparency and allowing for interventions, but traditionally requiring a complete set of predefined concepts and often suffering from an “accuracy–interpretability trade-off” when concept definitions are insufficiently expressive.", + "For safety applications, where unknown failure modes are a core concern, discovery-oriented methods may be more useful than those that require specifying concepts upfront." ], "related": [ "probing", "steering-vectors" ], "mentioned_in": [ - "sec:scope" + "sec:scope", + "subsec: mech interp necessary" ] }, { "id": "attribution-saliency", - "name": "Attribution and Saliency Methods", + "name": "Input Attribution and Saliency Methods", "family": "interpretability-landscape", "sections": [ - "attribution-and-saliency-methods", - "critiques-of-attribution-and-saliency-methods" + "input-attribution-and-saliency-methods" ], - "definition": "Attribution methods focus on identifying which features most influence a model’s outputs. Techniques such as SHAP (Lundberg & Lee, 2017), LIME (Ribeiro et al., 2016), integrated gradients (Sundararajan et al., 2017), and saliency maps (Simonyan et al., 2014) are scalable and widely used for token-level explanations and debugging.", + "definition": "Input attribution methods aim to identify which input features most influence a model’s outputs. Techniques such as SHAP (Lundberg & Lee, 2017), LIME (Ribeiro et al., 2016), DeepLIFT (Shrikumar et al., 2017), and saliency-based methods (Simonyan et al., 2014; Sundararajan et al., 2017; Zhou et al., 2016; Selvaraju et al., 2017) are scalable and widely used for explanation and debugging at the pixel and token levels.", "assumptions": [], - "evaluation": [ - "However, Jacovi & Goldberg (2020) distinguish between plausibility (do attributions seem reasonable to humans?) and faithfulness (do attributions reflect actual model computation?)." - ], + "evaluation": [], "applications": [ - "For the safety-relevant tasks motivating this survey, attribution methods are useful for triage and initial auditing but, on their own, insufficient where a causal account of internal computation is required." + "The same machinery can also be applied to internal representations, attributing salience to features at different layers and tracing how information evolves throughout the network (Yu et al., 2024; Syed et al., 2024).", + "For safety-relevant applications, attribution methods help with triage, debugging, and initial auditing, but they are insufficient on their own when an explanation of internal computation is required." ], "limitations": [ - "Applied to inputs, they indicate which inputs matter but not how the model arrives at its output.", - "Reliability concerns nonetheless remain: different attribution methods can produce divergent explanations for the same prediction, and some have been shown to be insensitive to model parameters (Adebayo et al., 2018b).", - "Collectively, these findings suggest that attribution methods are fundamentally unreliable guides to what information a model actually uses." + "Techniques such as SHAP (Lundberg & Lee, 2017), LIME (Ribeiro et al., 2016), DeepLIFT (Shrikumar et al., 2017), and saliency-based methods (Simonyan et al., 2014; Sundararajan et al., 2017; Zhou et al., 2016; Selvaraju et al., 2017) are scalable and widely used for explanation and debugging at the pixel and token levels. However, these methods indicate which input features matter for a prediction, but not necessarily how the model processes and leverages them.", + "Different attribution methods can produce substantially different explanations for the same prediction, and some have been shown to be insensitive to model parameters (Adebayo et al., 2018) or sensitive to input transformations that have no effect on the model (Kindermans et al., 2022)." ], "related": [ "probing" ], "mentioned_in": [ "sec:scope", - "implications-for-probing." + "implications-of-attribution-faithfulness-for-probing" ] }, { @@ -189,33 +194,33 @@ "sections": [ "subsec:probing", "standard-probing-methodology", - "representation-probing-and-related-post-hoc-analyses", - "the-probe-causality-problem", "control-tasks-and-selectivity", - "probe-complexity-vs.-causal-relevance", - "when-is-probing-appropriate-and-when-is-it-not", - "best-practices-for-probing-studies" - ], - "definition": "A probe trains a classifier—typically linear—to predict a property (e.g., part-of-speech, sentiment, entity type) from model activations.", + "concerns-of-probe-complexity", + "minimum-description-length-probes.", + "practical-guidance.", + "implications-of-attribution-faithfulness-for-probing", + "when-is-probing-appropriate", + "methodological-standards-in-probing-research" + ], + "definition": "A probe is a classifier trained to predict a property of the input, such as part of speech, sentiment, or entity type, from model activations.", "assumptions": [ - "Linear probes constrain g to be linear, testing whether the property is linearly accessible.", - "MLP probes allow nonlinear g, testing whether the property is present in any recoverable form." + "The linear probe tests whether the target property is linearly accessible from the representation.", + "Probing provides evidence about whether activations contain information about a property. High probe accuracy indicates the information is present; low accuracy suggests it is absent or the probe is not expressive enough." ], "evaluation": [ - "A control task assigns random labels to inputs, breaking any genuine relationship between activations and the target property.", - "High selectivity indicates the probe succeeds because of genuine encoding, not because the representation is easily separable by any linear classifier.", - "To address this, Voita & Titov (2020) proposed evaluating probes via the Minimum Description Length (MDL) principle: prefer probes that achieve good accuracy with minimal complexity.", - "Report control task baselines: Always include selectivity metrics." + "A control task assigns random labels to inputs to break any genuine relationship between activations and the target property.", + "High selectivity indicates that the probe succeeds because the representation genuinely encodes the target property.", + "To address this, Voita & Titov (2020) proposed evaluating probes via the minimum description length (MDL) principle: prefer probes that achieve good accuracy with minimal complexity." ], "applications": [ - "probes have become one of the most widely used tools in mechanistic interpretability for tasks including layer-level localization, scalable auditing, and detecting latent behaviors such as sleeper-agent deception (MacDiarmid et al., 2024).", - "The most productive framing treats probing as hypothesis generation: probes identify candidate properties that may be causally relevant, which must then be tested via causal methods (ablation, patching, circuit analysis).", - "This illustrates that probing remains valuable when the goal is auditing or detection, while its limitations apply most strongly to mechanistic claims." + "We argue that probing is most productive when treated as a tool for hypothesis generation, that is, using probes to identify candidate properties that may be encoded in a representation and potentially causally relevant.", + "Despite the causality critiques, probing remains valuable for many applications. For example, MacDiarmid et al. (2024) showed that simple linear probes trained on residual-stream activations can detect latent backdoor behaviour in the sleeper agent model even when the probe fitting set does not contain the trigger or the harmful behaviour.", + "In summary, this use case shows that probing remains valuable for auditing or detection even though it cannot support causal claims." ], "limitations": [ - "Probing does not establish causal relevance: whether the model actually uses that information for its predictions.", - "This problem is fundamental: probing measures information presence, not information use.", - "Control task experiments have revealed that many probing results showing “encoding” of linguistic properties suffer from low selectivity—random labels are nearly as extractable (Hewitt & Liang, 2019)." + "Note that probing alone does not show whether the model actually depends on that information in its decision-making process.", + "High probe accuracy does not imply causal relevance (Belinkov, 2022; Hewitt & Liang, 2019).", + "Finally, a sufficiently expressive probe can achieve high accuracy by exploiting noise even when the representations contain no such information." ], "related": [ "amnesic-probing", @@ -226,9 +231,11 @@ ], "mentioned_in": [ "sec:scope", - "pragmatic-interpretability-proxy-task-oriented-safety-interventions", - "developmental-interpretability-understanding-phase-transition-during-training", - "mechanistic-monitoring" + "representation-probing-and-related-post-hoc-analyses", + "subsec:dev interp", + "subsubsec: mech monitor", + "choosing-appropriate-methods-for-different-safety-applications", + "sec:automated-interpretability" ] }, { @@ -238,30 +245,29 @@ "sections": [ "subsec:sae-foundation", "sec:sae", - "subsec:challenge", + "subsec:sae-variants", "subsec:sae-selection", - "app:sae_details" + "subsec:challenge" ], - "definition": "Sparse Autoencoders are a way of extracting monosemantic features from polysemantic neurons.", + "definition": "SAEs (Figure 6) extract monosemantic features from polysemantic model activations. They are trained to reconstruct activations at a specific location in neural networks, such as the output of an MLP block or an entire transformer layer (i.e., the residual stream), while keeping only a few features active in any given forward pass.", "assumptions": [ - "They are optimised for accuracy to the real MLP or attention block and for having few active features for any given forward pass.", - "SAEs are motivated by the hypothesis that features occur sparsely in the data and are linearly recoverable from a model’s hidden states.", - "Training of SAEs is sensitive to initialization, sparsity strength, and decoder normalization." + "SAEs are motivated by the hypothesis that a model’s activations can be approximated as linear combinations of many latent feature directions, which can be regarded as an extension of the linear representation hypothesis discussed in Section 5.1.", + "SAE training is sensitive to initialisation, sparsity strength, and decoder normalisation. Without proper constraints and hyperparameters, the features may collapse, duplicate, or absorb one another." ], "evaluation": [ - "SAEs are typically evaluated for reconstruction loss (fidelity of activation reconstruction), feature absorption (degree to which features remain distinct), and downstream testing loss recovery (whether replacing activations with reconstructions preserves model behavior).", - "Report failure mode metrics: Alongside reconstruction loss, report dead feature percentage, duplication rate (via cosine similarity), and stability across seeds.", - "These causal tests should be standard practice when claiming that SAE features represent meaningful computational units." + "SAEs are typically evaluated for reconstruction loss (fidelity of activation reconstruction), feature absorption (how often a general feature fails to fire because a more specific feature has absorbed its direction), and downstream test loss recovery (whether replacing activations with reconstructions preserves model behaviour).", + "Report failure-mode metrics: Alongside reconstruction loss, researchers should also report dead-feature percentage, duplication rate, and stability across seeds.", + "Validate causal relevance: Researchers should validate features’ necessity, sufficiency, and specificity through interventions (e.g., feature ablation or steering) and evaluation on downstream effects." ], "applications": [ - "As a result, supervised methods are best suited to settings where specific, predefined concepts are of interest, such as safety-critical behaviors like toxicity or deception, whereas SAEs remain preferable for exploratory analysis.", - "SAEs to unsupervisedly discover monosemantic features inside the model’s raw activations." + "Exploratory discovery, where the question is what features exist rather than whether a particular one does, is the setting SAEs were designed for and the one the comparisons below address.", + "Methods for achieving mechanistic monitoring include, but are not limited to, layer-wise monitoring (using linear probes to classify if malicious intent is encoded in a layer representation) (Oldfield et al., 2026), feature-wise monitoring (alerting when unexpected SAE features activate) (Fang et al., 2026), circuit-wise monitoring (tracking causal effects among attention heads, MLPs, or features) (Zhao et al., 2026b), and anomaly-based monitoring (using activations in normal behaviour as a baseline and flagging harmful generations as outliers) (Chaudhary & Barez, 2026)." ], "limitations": [ - "Dead features represent wasted capacity and can constitute between 10–50% of the dictionary in poorly tuned SAEs (Bricken et al., 2023).", - "Training the same architecture with different random seeds can therefore produce substantially different dictionaries (Paulo & Belrose, 2025).", - "Taken together, these behaviors are consistent with the failure of identifiability under high-coherence conditions and help explain why SAE features can be unstable, non-atomic, and seed-dependent.", - "Low SAE reconstruction loss does not necessarily mean that SAE captures the causal structure of the original model." + "Dead features waste the dictionary’s capacity, and they are common. In fact, they can constitute up to 10–50% of the dictionary in a poorly tuned SAE (Bricken et al., 2023).", + "For a fixed SAE architecture and training dataset, different random seeds can produce substantially different dictionaries (Paulo & Belrose, 2026).", + "In large language model activations, high coherence is likely unavoidable because of the model’s complicated internal structure and correlations within its feature space (Li et al., 2025b). So far, identifiability remains a stubborn issue in SAEs, even though it offers a theoretical explanation for several empirical observations.", + "Activation reconstruction \\neq computation reconstruction: Low SAE reconstruction loss does not necessarily mean the SAE captures the original model’s computation." ], "related": [ "gated-sae", @@ -280,12 +286,15 @@ ], "mentioned_in": [ "subsec:superposition", - "case-study-induction-heads", + "empirical-evidence.", + "mechanistic-interpretability-methods", + "subsec: induc head case study: feature extract", "sec:automated-interp", "concept-erasure-via-pisces", - "mechanistic-monitoring", - "sec:model-diffing", - "relation-to-causal-evaluation-of-feature-extraction-methods.", + "subsubsec: mech monitor", + "dictionary-learning-methods.", + "steering-as-a-causal-test-for-feature-interpretability.", + "epistemic-risks-false-confidence-from-partial-or-incorrect-understanding", "silent-failure-modes." ] }, @@ -294,17 +303,15 @@ "name": "Gated SAEs", "family": "sparse-autoencoders", "sections": [ - "subsec:sae-variants", - "gated-sae-variant" + "subsec:sae-variants" ], "definition": "Gated SAEs introduce multiplicative gates to decouple feature selection from magnitude", "assumptions": [], "evaluation": [], - "applications": [ - "Recommendation: Use Gated SAE as the default unless you have specific reasons for standard.", - "Standard SAEs suffer from shrinkage bias; Gated SAEs solve this with minimal added complexity." + "applications": [], + "limitations": [ + "With moderate tuning capacity, Gated SAEs (Rajamanoharan et al., 2024a) achieved a better reconstruction–sparsity trade-off while retaining \\lambda. They also avoid the shrinkage effect by separating feature detection from magnitude estimation. However, because the standard L_1 SAE is now used mainly as a baseline, comparisons against it partly reflect shrinkage rather than dictionary quality alone." ], - "limitations": [], "related": [ "sae", "topk-sae", @@ -313,8 +320,9 @@ ], "mentioned_in": [ "subsec:sae-selection", - "layer-specific-recommendations", - "quick-reference-when-to-use-what" + "layer-type.", + "resource-constraints.", + "practical-recommendations" ] }, { @@ -324,13 +332,17 @@ "sections": [ "subsec:sae-variants" ], - "definition": "JumpReLU SAEs learn per-feature activation thresholds.", + "definition": "JumpReLU SAEs (Rajamanoharan et al., 2024b) learn per-feature activation thresholds.", "assumptions": [ - "A smooth surrogate is used during training to enable gradient flow Rajamanoharan et al. (2024b)." + "Straight-through estimators (Bengio et al., 2013) are used during training to enable gradient flow." ], "evaluation": [], - "applications": [], - "limitations": [], + "applications": [ + "For models between 1B and 10B parameters, JumpReLU SAEs and Top-k SAEs are the most commonly reported variants; Gemma Scope (Lieberum et al., 2024) uses the former and Llama Scope (He et al., 2024) the latter." + ], + "limitations": [ + "When extensive tuning is plausible, JumpReLU SAEs (Rajamanoharan et al., 2024b) achieved the strongest reconstruction performance at matched sparsity by learning per-feature thresholds. However, gains on these sparsity–fidelity proxy metrics do not reliably translate into better downstream performance (Karvonen et al., 2025). In practice, selecting the best model and hyperparameters to achieve optimal performance can take a lot of time." + ], "related": [ "sae", "gated-sae", @@ -339,8 +351,11 @@ ], "mentioned_in": [ "subsec:sae-selection", - "layer-specific-recommendations", - "quick-reference-when-to-use-what" + "layer-type.", + "resource-constraints.", + "model-scale.", + "practical-recommendations", + "subsubsec:cross-layer-transcoder" ] }, { @@ -348,18 +363,15 @@ "name": "Top-k SAEs", "family": "sparse-autoencoders", "sections": [ - "subsec:sae-variants", - "top-k-sae-variant" + "subsec:sae-variants" ], - "definition": "Top-k SAEs (Gao et al., 2025a) impose an explicit hard sparsity constraint.", + "definition": "Top-k SAEs (Gao et al., 2025) impose an explicit hard sparsity constraint.", "assumptions": [ - "No sparsity penalty needed; sparsity is enforced structurally." - ], - "evaluation": [ - "Unlike L1 loss, it does not suppress activation magnitudes, thereby avoiding the feature shrinkage issue observed in L1-based SAE training (Gao et al., 2025a)." + "This formulation explicitly enforces \\|z\\|_0 \\le k—a more direct feature-sparsity constraint than L_1 regularisation." ], + "evaluation": [], "applications": [ - "If you’re spending significant time tuning the sparsity coefficient, Top-K eliminates this hyperparameter entirely." + "Under limited compute, or where fast iteration matters more than reconstruction quality, Top-k SAEs (Gao et al., 2025) are the practical choice: fixing \\|z\\|_0 \\le k removes the sparsity coefficient \\lambda from the search space entirely rather than requiring it to be tuned, with reported values commonly in the range k = 32 (Gao et al., 2025) to 150 (Karvonen et al., 2025)." ], "limitations": [], "related": [ @@ -370,8 +382,9 @@ ], "mentioned_in": [ "subsec:sae-selection", - "common-pitfalls-and-how-to-avoid-them", - "quick-reference-when-to-use-what" + "resource-constraints.", + "model-scale.", + "practical-recommendations" ] }, { @@ -381,12 +394,14 @@ "sections": [ "subsec:sae-variants" ], - "definition": "Batch Top-k SAEs enforce sparsity at the batch level.", - "assumptions": [], + "definition": "Batch Top-k SAEs relax the fixed per-example sparsity constraint of standard Top-k SAEs by enforcing the sparsity budget across an entire batch instead (Bussmann et al., 2024).", + "assumptions": [ + "This loose constraint allows the model to allocate representational capacity adaptively across samples." + ], "evaluation": [], "applications": [ - "This enforces global feature competition, reduces variance in feature usage across the batch, and prevents collapse (Bussmann et al., 2024).", - "More recent Batch Top-K variants explicitly enforce minimum feature usage across batches, which substantially reduces the prevalence of dead features (Bussmann et al., 2024)." + "More recent Batch Top-k SAEs allocate sparsity adaptively across samples, which allows more active features on complex inputs and fewer on easy ones (Bussmann et al., 2024).", + "During inference, Batch Top-k SAEs instead use the JumpReLU function with an inferred global threshold to determine which features are active for each input, so that an input’s activations do not depend on the rest of the batch." ], "limitations": [], "related": [ @@ -396,8 +411,7 @@ ], "mentioned_in": [ "dead-features", - "sec:model-diffing", - "quick-reference-when-to-use-what" + "dictionary-learning-methods." ] }, { @@ -407,19 +421,19 @@ "sections": [ "subsubsec:transcoder", "architecture-and-motivation.", - "when-transcoders-are-necessary." + "when-are-transcoders-necessary" ], "definition": "The transcoder reconstructs post-nonlinearity activations using features extracted from pre-nonlinearity activations (Dunefsky et al., 2024).", "assumptions": [ - "A transcoder applies sparse decomposition directly to the nonlinear mapping, learning features that explain the transformation." + "Standard SAEs do not explain the nonlinear transformations (i.e., weight matrices of MLP blocks and attention heads) themselves that produce the activations. A transcoder achieves this by applying dictionary learning to the mapping, learning features that explain the transformation." ], "evaluation": [], "applications": [ - "Transcoders are essential when analyzing MLPs, understanding feature interactions, or tracking how concepts evolve across layers (Dunefsky et al., 2024).", - "Recommendation: For MLP analysis, use transcoders; for cross-layer analysis, use crosscoders or CLTs." + "SAEs extract features from static activations and thus often suffice for residual-stream analysis. Transcoders track how features emerge from input to output and are essential for analysing MLP layers (Dunefsky et al., 2024).", + "On MLP outputs, transcoders are the appropriate tool: a single-layer SAE cannot span the nonlinearity, so transcoders learn the input-to-output map directly (Dunefsky et al., 2024), and Paulo et al. (2025c) reported favourable comparisons against SAEs there." ], "limitations": [ - "Transcoders improve this by imitating the behavior of the MLP, revealing nonlinear transformations, though they require sequential analysis across depth." + "Transcoders improve this by imitating MLP behaviour and revealing nonlinear transformations, though they require sequential analysis across depth." ], "related": [ "sae", @@ -428,9 +442,10 @@ "molt" ], "mentioned_in": [ - "comparing-saes-transcoders-and-crosscoders.", - "case-study-induction-heads", - "sec:model-diffing" + "layer-type.", + "comparing-saes-transcoders-and-crosscoders", + "dictionary-learning-methods.", + "subsec: induc head case study: feature extract" ] }, { @@ -444,26 +459,26 @@ "computational-cost-and-practical-considerations.", "open-questions-for-clt-research." ], - "definition": "A Cross-Layer Transcoder (CLT) is a sparse feature decomposition spanning all layers of a neural network.", + "definition": "Cross-layer transcoders (CLTs) are sparse feature decompositions spanning all layers of a neural network.", "assumptions": [ "Attention freezing: By caching attention patterns, the replacement model treats attention as a fixed routing operation rather than a dynamic, input-dependent computation.", - "Normalization caching: Freezing LayerNorm statistics removes a source of nonlinearity but introduces dependence on the specific input used to cache these statistics.", - "Sparse linear approximation: The core assumption that MLP outputs are well-approximated by sparse linear combinations of features may break down for computations involving complex feature interactions or highly nonlinear transformations." + "Normalisation caching: CLT analysis caches the input-dependent normalisation factors (e.g., LayerNorm denominators) from the original forward pass and reuses them during attribution instead of recomputing them after interventions.", + "Sparse linear approximation: Dictionary learning methods assume MLP outputs or transformations are well approximated by sparse linear combinations of features. This assumption may fail when computations involve complex feature interactions or highly nonlinear transformations." ], "evaluation": [ - "Empirically, CLT replacement models match the original model’s most-likely token on approximately 50% of prompts in initial experiments on Gemma-2-2B (Ameisen et al., 2025).", - "The 50% figure reflects top-1 token agreement; softer metrics (e.g., top-5 agreement, KL divergence) may show different patterns.", - "These caveats do not invalidate CLTs as a tool for mechanistic analysis, but practitioners should verify fidelity on their specific model, layer, and task before drawing strong conclusions from CLT-based circuit analysis." + "Under these conditions, the local replacement model can exactly match the original model’s activations on that prompt while decomposing it into interpretable CLT features. However, when the model’s activations are completely replaced by CLT reconstructions, the intervened model’s top-1 predictions match the original model’s on only about 50% of prompts from the Pile dataset (Gao et al., 2020). Notably, this experiment uses a relatively small, 18-layer transformer (Ameisen et al., 2025).", + "The paper reports top-1 token agreement as the main result, while softer metrics (e.g., top-5 agreement, KL divergence) may show different patterns and insights.", + "Domain: Fidelity on factual recall and syntactic tasks is higher than on complex reasoning or arithmetic tasks, where MLP computations may be more complex and cannot be approximated by sparse linear features. These caveats do not invalidate CLTs as a tool for mechanistic analysis, but they do mean that fidelity should be verified on the target model and specific task before drawing strong conclusions from CLT-based circuit analysis." ], "applications": [ - "A key insight is that CLTs enable the construction of a replacement model: a version of the original model where all MLP layers are replaced by their CLT reconstructions.", - "This simplifies circuit structure—features need not re-encode information at each layer.", - "When these assumptions hold, CLTs provide a powerful lens for circuit discovery." + "A key insight is that CLTs enable the construction of a replacement model—a version of the original model in which all MLP layers are replaced with their CLT reconstructions.", + "Unlike per-layer transcoders, CLTs reveal direct feature-to-feature interactions across layers: a feature in layer \\ell directly influences features in later layers through its contribution to their inputs. This simplifies circuit structure—features need not re-encode information at each layer.", + "CLTs provide a powerful lens for circuit discovery when these assumptions hold." ], "limitations": [ - "When they fail, particularly for complex reasoning or out-of-distribution inputs, CLT-based explanations may be incomplete or misleading.", - "Training a CLT requires significant upfront cost, which scales with the number of features and layers.", - "This scarcity of downstream applications in the current literature likely reflects the considerable challenges in training CLTs at scale.", + "When they fail, CLT-based explanations may become unfaithful.", + "Training a suite of CLTs thus incurs a high upfront cost.", + "For these reasons, current applications of CLTs remain limited: a “greater-than” mechanism study (Merullo et al., 2025), an open-source extension (Lindsey et al., 2025a), and analysis of multilingual representations using CLTs (Harrasse et al., 2026). This scarcity of downstream use cases likely reflects the considerable challenges of training CLTs at scale and the complexity of using CLT-based circuit discovery in the wild.", "Cross-layer transcoders span multiple layers for global information flow but are extremely expensive." ], "related": [ @@ -474,9 +489,11 @@ "spartan" ], "mentioned_in": [ - "sec:circuit-table", - "sparse-attention-post-training-spartan", - "comparing-saes-transcoders-and-crosscoders." + "sec:circuit-replacement", + "subsubsec: spartan", + "comparing-saes-transcoders-and-crosscoders", + "sec:proxy-fidelity", + "steering-as-a-causal-test-for-feature-interpretability." ] }, { @@ -484,19 +501,19 @@ "name": "Crosscoders", "family": "multi-layer-feature-analysis", "sections": [ - "crosscoders-analyzing-cross-layer-features", + "crosscoders-analysing-cross-layer-features", "causal-vs.-acausal-crosscoders.", "applications-to-circuit-analysis." ], - "definition": "Crosscoders resolve cross-layer superposition by tracking features that persist through the residual stream (Lindsey et al., 2024).", + "definition": "A crosscoder learns a shared sparse representation from activations across L layers.", "assumptions": [], "evaluation": [], "applications": [ - "By extracting cross-layer features, crosscoders enable understanding how features propagate through the residual stream across layers, removing duplicate feature representations that complicate circuit analysis.", - "Crosscoders can define model-specific encoders, with arbitrarily different input dimensions, which then project into one or more shared representational spaces." + "By extracting cross-layer features, crosscoders help us understand how features propagate through the residual stream across layers, removing duplicate feature representations that complicate circuit analysis.", + "Crosscoders can define model-specific encoders with arbitrarily different input dimensions that project activations from different models into one or more shared representational spaces." ], "limitations": [ - "This can result from the loss function: for instance, Minder et al. (2025) showed that crosscoders trained with an L1 loss on hidden states can falsely identify novel features, which they try to mitigate with the BatchTopK activation function." + "Dictionary learning methods for model diffing risk falsely identifying model-exclusive features (Dumas et al., 2025a; Jiralerspong & Bricken, 2025). This can result from the loss function: for instance, Dumas et al. (2025a) showed that crosscoders trained with an L_1 regularisation on hidden states can falsely identify novel features, which they tried to mitigate with \\sigma_{\\text{Batch-TopK}}(\\cdot)." ], "related": [ "sae", @@ -506,27 +523,28 @@ "batch-topk-sae" ], "mentioned_in": [ - "comparing-saes-transcoders-and-crosscoders.", - "sec:model-diffing" + "comparing-saes-transcoders-and-crosscoders", + "dictionary-learning-methods.", + "subsec: induc head case study: feature extract" ] }, { "id": "nmf", - "name": "Non-Negative Matrix Factorization (NMF)", + "name": "Non-Negative Matrix Factorisation (NMF)", "family": "alternative-feature-extraction", "sections": [ - "non-negative-matrix-factorization-nmf" + "non-negative-matrix-factorisation-nmf" ], - "definition": "This constraint induces a natural parts-based interpretation in which features combine additively without cancellation.", + "definition": "Because all components are non-negative, each feature contributes positively to the final representation, making it easier to interpret.", "assumptions": [ - "Activations produced by modern architectures, particularly after LayerNorm or nonlinearities such as GELU, are not naturally non-negative, requiring preprocessing steps that may distort representational structure." + "Additionally, classical NMF’s optimisation process uses simple multiplicative update rules that preserve non-negativity. Moreover, under assumptions like separability (each latent feature appears in at least one sufficiently “pure” example), NMF guarantees that the recovered decomposition is unique (Donoho & Stodden, 2003). However, classical NMF is not suitable for neural network activations because they can have negative elements, requiring additional preprocessing steps that may distort the activation-space structure." ], "evaluation": [ - "Shafran et al. (2025) showed that Semi-NMF can recover features competitive with SAEs on small-scale evaluations, but systematic comparisons at large scale remain limited." + "Shafran et al. (2026) showed that Semi-NMF can recover features competitive with SAEs on small-scale evaluations, but systematic comparisons at large scale remain limited." ], "applications": [], "limitations": [ - "Moreover, NMF does not explicitly encourage sparsity unless additional regularization is introduced, and scaling NMF to large dictionaries is computationally challenging." + "NMF does not explicitly encourage feature sparsity, and scaling NMF to large dictionaries is computationally expensive." ], "related": [ "sae" @@ -538,28 +556,24 @@ "name": "Principal Component Analysis (PCA)", "family": "alternative-feature-extraction", "sections": [ - "principle-component-analysis-pca" + "principal-component-analysis-pca" ], - "definition": "Principal Component Analysis decomposes activations by finding an orthonormal basis of directions that maximise explained variance.", + "definition": "Principal Component Analysis (PCA; Pearson, 1901; Hotelling, 1933) decomposes activations by finding an orthonormal basis of directions that maximise explained variance.", "assumptions": [ - "In addition, PCA enforces orthogonality between components, a constraint that may not reflect how features are organized in practice, where factors can be correlated or partially overlapping." + "PCA is fast, stable, and scalable. However, PCA imposes constraints and inductive biases that may misalign with neural network activations. Specifically, it identifies principal directions that maximise explained variance in the activation distribution, but much of the information of interest is region- or context-dependent and cannot be captured by variance alone. Moreover, PCA constrains the principal directions to be orthogonal, but in practice the underlying features correlate or partially overlap." ], "evaluation": [ - "In Wu et al. (2025), PCA underperforms most baselines, including SAEs, on both steering and concept-detection evaluations.", - "While in some localization settings, PCA outperform SAEs (Mueller et al., 2025a), it still lags behind simpler supervised baselines." + "Recent benchmarks (Wu et al., 2025b) reflect this misalignment: PCA underperformed most baselines, including SAEs, on both steering and concept-detection evaluations. While PCA can outperform SAEs in some localisation settings (Mueller et al., 2025), it still generally lags behind simple supervised baselines." ], "applications": [ - "Missing baselines: Claims of interpretability gains rarely compare against simple baselines (e.g., random directions, PCA components, raw neurons)." - ], - "limitations": [ - "PCA identifies a single set of global directions that maximises variance across the full activation distribution, which can mix multiple sources of variation when the relevant structure is region or context-dependent." + "Users specify baseline methods, such as random directions in activation space, PCA components, individual neurons, and supervised probes (if labels are available)." ], + "limitations": [], "related": [ "sae" ], "mentioned_in": [ - "the-need-for-standardized-evaluation", - "integrated-evaluation-protocol" + "step-2-establish-baselines." ] }, { @@ -569,16 +583,16 @@ "sections": [ "independent-component-analysis-ica" ], - "definition": "Independent Component Analysis seeks to decompose activations as H = WS, where the components of S are statistically independent (Hyvärinen & Oja, 2000).", + "definition": "Independent Component Analysis (ICA; Hyvärinen & Oja, 2000) decomposes activations H \\in \\mathbb{R}^{d \\times n} as H \\approx WS, where n is the number of data points.", "assumptions": [ - "However, ICA makes strong assumptions that are often violated in neural representations.", - "Classical ICA assumes a complete setting in which the number of sources equals the dimensionality of the representation, and extensions are required for overcomplete decompositions.", - "Unlike SAEs, ICA does not require tuning a sparsity hyperparameter." + "In ICA, each column in W \\in \\mathbb{R}^{d \\times m} is a mixing vector, and m is the number of different source signals; each row in S \\in \\mathbb{R}^{m \\times n} is a source signal (a signal has n data points), with m source signals in total. These signals are optimised to be independent of one another. Thus, if model activations are linear mixtures of statistically independent “foundational features”, ICA can provably recover those features." + ], + "evaluation": [ + "More recently, however, Liu & Han (2026) showed that ICA can achieve performance comparable to that of SAEs, arguing that its previously reported underperformance may stem from limitations of classical ICA implementations poorly suited to the scale and statistical structure of neural network activations." ], - "evaluation": [], "applications": [], "limitations": [ - "Although ICA has been applied to neural network activations (Hyvärinen et al., 2023), it has not matched the interpretability gains reported for SAEs in recent large-scale studies, possibly because neural features tend to be sparse rather than independent." + "Moreover, underlying features may not be statistically independent, which further limits the applicability of classical ICA. Although early studies applied it to analyse neural network activations, they did not find interpretability gains over SAEs (Cunningham et al., 2024)." ], "related": [ "sae" @@ -587,35 +601,28 @@ }, { "id": "supervised-feature-extraction", - "name": "Supervised Feature Extraction Methods (DAS, SMDS, DBM)", + "name": "Supervised Feature Extraction Methods (DAS, SMDS, probes)", "family": "alternative-feature-extraction", "sections": [ - "supervised-feature-extraction-methods" - ], - "definition": "When the concepts of interest are known in advance, supervised methods can directly optimize representations aligned with those targets.", - "assumptions": [ - "However, these methods require labelled data for each concept and therefore cannot discover previously unknown features." - ], - "evaluation": [ - "When labelled data are available, supervised approaches often outperform SAEs on targeted concept recovery tasks (Huang et al., 2024; Chaudhary & Geiger, 2024)." + "subsubsec: supervised_extraction" ], + "definition": "When the concepts of interest are known in advance, supervised methods can directly optimise representations aligned with those targets.", + "assumptions": [], + "evaluation": [], "applications": [ - "As a result, supervised methods are best suited to settings where specific, predefined concepts are of interest, such as safety-critical behaviors like toxicity or deception, whereas SAEs remain preferable for exploratory analysis.", - "Compare methods: For safety-critical applications, compare SAE features against supervised baselines (DAS, probes) to assess whether SAEs recover the relevant structure." - ], - "limitations": [ - "They do not scale naturally to open-ended exploration across entire models, and each concept typically requires separate training." + "When labelled data is available, supervised approaches often outperform SAEs on concept detection and steering tasks (Huang et al., 2024; Wu et al., 2025b; Chaudhary & Geiger, 2024). They also avoid choosing dictionary sizes or sparsity penalties, since the concept set determines complexity. However, these methods require labelled data for each concept and cannot discover previously unknown features. Therefore, supervised methods suit settings where predefined, crisp concepts matter, whereas SAEs remain preferable for exploratory analysis.", + "Consider alternatives for targeted analysis: When specific concepts are the focus, supervised methods (e.g., DAS, probes, discussed in Section 6.9.4) may be more reliable because SAEs discover features in an unsupervised and exploratory manner." ], + "limitations": [], "related": [ "sae", "probing", "steering-vectors" ], "mentioned_in": [ - "subsec:sae-selection", + "analysis-goal.", "practical-recommendations", - "best-practices-for-probing-studies", - "relation-to-causal-evaluation-of-feature-extraction-methods." + "steering-as-a-causal-test-for-feature-interpretability." ] }, { @@ -625,72 +632,68 @@ "sections": [ "dictionary-learning-baselines" ], - "definition": "Classical dictionary learning methods provide useful baselines for evaluating SAEs. Techniques such as K-SVD alternate between sparse coding and dictionary updates and are known to converge under certain conditions (Aharon et al., 2006).", + "definition": "Classical dictionary learning methods provide useful baselines for evaluating SAEs. Techniques such as K-SVD alternate between sparse coding and dictionary updates and converge under certain conditions (Aharon et al., 2006).", "assumptions": [], "evaluation": [ - "Comparative studies suggest that the primary advantage of SAEs lies in their learned encoder, which enables fast inference, rather than in fundamentally superior dictionary quality (Cunningham et al., 2023)." + "Comparative studies suggest that SAEs’ primary advantage is fast inference because the encoder learns to decompose all activations, while potentially sacrificing optimal sparsity (O’Neill et al., 2025)." ], "applications": [ - "As such, classical dictionary learning remains an important reference point for assessing the added value of SAE architectures." + "As such, classical dictionary learning can still achieve superior dictionary quality in some cases and thus remains an important reference point for assessing the added value of SAEs." ], "limitations": [], "related": [ "sae" ], - "mentioned_in": [] + "mentioned_in": [ + "the-sparse-coding-problem" + ] }, { "id": "weight-sparse-transformer", - "name": "Weight-Sparse Transformer (Weight Sparsity via L_0 Regularization)", + "name": "Weight-Sparse Training (Weight-Sparse Transformer)", "family": "architectural-interventions", "sections": [ - "weight-sparsity-via-l_0-regularization", - "circuit-discovery-in-weight-sparse-architectures", - "circuits-visible-by-inspection." + "subsubsec: weight sparse transformer" ], - "definition": "The most significant recent advancement is the Weight-Sparse Transformer, introduced by Gao et al. (Gao et al., 2025b). This approach treats sparsity not as a tool for inference efficiency but as a tool for circuit discovery.", + "definition": "A weight-sparse transformer (Gao et al., 2026) treats weight sparsity as an explicit structural constraint that is progressively strengthened during training.", "assumptions": [ - "Since the L_0 norm is non-differentiable, training employs stochastic gates drawn from a Hard-Concrete distribution, allowing gradients to flow while penalizing the probability of each gate being open.", - "Furthermore, this approach requires training sparse models from scratch or using Bridge proxies—it cannot be directly applied to existing dense models." + "Gao et al. (2026) optimise the standard cross-entropy objective along with an L_0 constraint on model weights after each optimisation step. Specifically, within each weight matrix, they retain only the largest-magnitude parameters and set the rest to zero. The L_0 constraint anneals from a dense initialisation to the desired sparsity level over training." ], "evaluation": [ - "Validation via mean ablation shows that these sparse circuits satisfy necessity and sufficiency criteria with higher precision than circuits identified in dense models.", - "Empirically, sparse circuits satisfy necessity and sufficiency criteria with higher precision than circuits extracted from dense models via post-hoc methods (Gao et al., 2025b; Haider et al., 2025)." + "Mean-ablation experiments further confirm that these circuits are both necessary and sufficient for the capability or behaviour." ], "applications": [ - "Circuits can be identified by simply visualizing non-zero weights connected to a target output, eliminating the need for activation patching or ACDC." + "The resulting transformer has most weight parameters exactly zero, making the remaining computational structure much easier to decompose into small, task-specific circuits." ], "limitations": [ - "Weight sparsity does not eliminate all challenges.", - "Circuits may still be compositional (requiring understanding of how subcircuits interact), and the interpretability of individual neurons depends on training dynamics.", - "Retraining frontier models with sparsity constraints is computationally prohibitive." + "Increasing sparsity generally makes the model easier to decompose into small, interpretable circuits. However, it also degrades task performance, producing a capability–interpretability Pareto frontier.", + "At comparable task performance, larger sparse models can have substantially smaller, more interpretable circuits than smaller models; at comparable interpretability, they can retain stronger capabilities. This suggests that scaling model size alongside structural sparsity may offer a path to more interpretable yet high-capability models. However, preserving interpretability beyond tens of millions of nonzero parameters remains challenging, and weight-sparse training currently requires substantially more compute than traditional training." ], "related": [ - "bridge", "activation-patching", "path-patching", "acdc", "solu" ], "mentioned_in": [ - "architectural-responses-to-superposition", - "architectural-interventions-as-a-synthesis.", - "implications-for-verification." + "architectural-responses-to-superposition.", + "subsubsec: solu", + "the-road-ahead" ] }, { "id": "spartan", - "name": "Sparse Attention Post-Training (SPARTAN)", + "name": "Attention-Sparse Training (SPARTAN)", "family": "architectural-interventions", "sections": [ - "sparse-attention-post-training-spartan" + "subsubsec: spartan" ], - "definition": "Sparse Attention Post-Training (SPARTAN) applies L_1 regularization to the attention patterns while finetuning with a constrained optimization objective to maintain the same cross-entropy performance (Draye et al., 2025) (Lei et al., 2025).", + "definition": "Previous studies have therefore explored attention-sparse training, which aims to sparsify attention patterns to make them more interpretable (Lei et al., 2025; Draye et al., 2026). Specifically, Lei et al. (2025) proposed SPARse TrANsformer World model (SPARTAN), an object-centric world model trained with L_0 regularisation on attention patterns and a constrained optimisation objective to maintain cross-entropy performance equal to that of a dense counterpart.", "assumptions": [], "evaluation": [], "applications": [ - "This reveals that much attention connectivity is redundant noise, and SPARTAN effectively performs circuit discovery by pruning the attention graph.", - "Using Cross-Layer Transcoders, they show that the reduced number of mediating components renders attention attribution tractable." + "After training (either from scratch or via fine-tuning), each attention head attends only to specific tokens, and the model performance remains close to the original. This result suggests that many attention patterns may be redundant noise.", + "In CLT-based circuit discovery (see Section 7.2), Draye et al. (2026) showed that sparsified attention patterns reduce the number of mediating components involved in feature-to-feature attribution, making the resulting attribution graphs easier to analyse." ], "limitations": [], "related": [ @@ -703,46 +706,39 @@ "name": "Softmax Linear Units (SoLU)", "family": "architectural-interventions", "sections": [ - "activation-level-interventions-solu-and-its-limitations", - "results-and-the-hydraulic-effect." - ], - "definition": "Softmax Linear Units (SoLU) replace element-wise activations with a spatial softmax (Elhage et al., 2022b)", - "assumptions": [ - "This forces neurons within a layer to compete: if one neuron fires strongly, it suppresses its neighbors through lateral inhibition." - ], - "evaluation": [ - "SoLU successfully increased the fraction of monosemantic neurons identifiable by human inspection." + "subsubsec: solu" ], + "definition": "Softmax linear units (SoLU; Elhage et al., 2022b) are an earlier attempt at architectural intervention. SoLU is a specialised activation function that applies a softmax-based transformation across the layer’s d_{\\mathrm{mlp}} neurons.", + "assumptions": [], + "evaluation": [], "applications": [], "limitations": [ - "However, it revealed a critical limitation: the hydraulic effect.", - "When prevented from using neurons for superposition, models simply displaced the compression elsewhere—into the residual stream, into sub-threshold activations, or into other components not subject to the constraint.", - "This indicates that local architectural constraints are insufficient when global incentives for compression remain." + "When one neuron receives a large pre-activation value, the softmax suppresses the others’ values, analogous to lateral inhibition. This competition encourages certain neurons to specialise and dominate, increasing the prevalence of monosemantic neurons associated with human-interpretable concepts. However, it also reveals an undesirable phenomenon: although many secondary neurons’ activations are suppressed to low magnitudes, these neurons remain polysemantic, with many features compressed into them. Later layer normalisation (Ba et al., 2016) can amplify them and recover the polysemanticity property (Elhage et al., 2022b).", + "The “smuggling” by layer normalisation suggests that imposing local interpretability constraints may not eliminate neural networks’ incentive to compress information through superposition." ], "related": [ "weight-sparse-transformer" ], "mentioned_in": [ - "architectural-responses-to-superposition" + "architectural-responses-to-superposition.", + "the-road-ahead" ] }, { "id": "molt", - "name": "Sparse Mixtures of Linear Transforms (MOLT)", + "name": "Sparse Mixture of Linear Transforms (MOLT)", "family": "architectural-interventions", "sections": [ - "computational-primitive-modifications-molt", - "motivation.", - "mechanism.-1" + "subsubsec: molt" ], - "definition": "One recent example of this is the so-called Sparse Mixtures of Linear Transforms (MOLT), representing a conceptual shift from feature extraction to transformation extraction (Lindsey et al., 2025b).", + "definition": "A recent example is the sparse mixture of linear transforms (MOLT), which represents a conceptual shift from feature extraction towards transformation extraction (Lindsey et al., 2025b).", "assumptions": [ - "MOLT replaces the MLP with a sparse mixture of linear transformation matrices." + "To achieve this, MOLT replaces the MLP with a sparse mixture of learned linear transformation matrices. The basic unit of analysis becomes transformations applied to model activations rather than features." ], "evaluation": [], "applications": [ - "For modular arithmetic, a MOLT might learn a single rotation matrix implementing addition, rather than memorizing thousands of input-output pairs.", - "This aligns the architecture with the true algorithmic structure of the task, yielding higher mechanistic faithfulness than standard feature-based decompositions for tasks involving systematic transformations." + "The basic unit of analysis becomes transformations applied to model activations rather than features. For example, in a modular arithmetic task, MOLT can represent the concept of addition with a learned rotation matrix, which relates more directly to the underlying algorithm than a collection of state-specific features.", + "More broadly, approaches that represent computational operations explicitly, as MOLT does, may facilitate the development of interpretable representations for tasks governed by systematic operations." ], "limitations": [], "related": [ @@ -750,55 +746,30 @@ "transcoders" ], "mentioned_in": [ - "architectural-responses-to-superposition" + "architectural-responses-to-superposition." ] }, { "id": "kans", - "name": "Kolmogorov-Arnold Networks (KANs)", + "name": "Kolmogorov–Arnold Networks (KANs)", "family": "architectural-interventions", "sections": [ - "kolmogorov-arnold-networks" + "subsubsec: kan" ], - "definition": "Based on the Kolmogorov-Arnold representation theorem, KANs place learnable activation functions (B-splines) on edges rather than fixed activations on nodes.", + "definition": "Inspired by the Kolmogorov–Arnold representation theorem, KANs place learnable nonlinear functions, typically parameterised using B-splines, on edges rather than using fixed nonlinear activation functions at nodes.", "assumptions": [], "evaluation": [], "applications": [ - "Because non-linearities are univariate and visible on edges, researchers can perform symbolic regression on learned splines to recover exact mathematical formulas—for instance, rediscovering physics equations from data." + "As the nonlinearities are univariate and can be visualised directly on individual edges, researchers can use techniques like symbolic regression to approximate and inspect the learned analytical expressions, such as rediscovering known physical equations." ], "limitations": [ - "While currently slower to train and unproven at LLM scale, KANs represent the ambitious ideal of a model where weights are interpretable mathematical functions." + "Although KAN training is currently more computationally expensive than training standard MLPs and has not yet been convincingly demonstrated at the scale of large language models, this architectural innovation points to an ambitious direction towards interpretable neural networks in which parameters correspond to mathematically meaningful functions." ], "related": [], "mentioned_in": [ - "architectural-responses-to-superposition" - ] - }, - { - "id": "bridge", - "name": "The Bridge Methodology", - "family": "architectural-interventions", - "sections": [ - "the-bridge-methodology-interpretable-proxies-for-dense-models" - ], - "definition": "A Bridge is a weight-sparse model trained to mimic the internal representations of a pretrained dense model—minimizing reconstruction error of layer-wise activations rather than training on the original task.", - "assumptions": [], - "evaluation": [ - "Preliminary evidence suggests that interventions on sparse proxies (e.g., ablating identified circuits) can be mapped back to induce similar behavioral changes in dense models.", - "One approach is causal correspondence testing, which verifies that ablating a circuit in the proxy and the corresponding components in the dense model produces matched behavioral changes across different inputs." - ], - "applications": [ - "Retraining frontier models with sparsity constraints is computationally prohibitive." - ], - "limitations": [ - "However, validating proxy fidelity—ensuring the proxy captures the same mechanisms rather than behaviorally equivalent simulations—remains an open problem (see Section 15)." - ], - "related": [ - "weight-sparse-transformer" - ], - "mentioned_in": [ - "sec:proxy-fidelity", - "potential-approaches." + "architectural-responses-to-superposition.", + "subsubsec: molt", + "the-road-ahead" ] }, { @@ -806,20 +777,19 @@ "name": "Activation Patching", "family": "circuits", "sections": [ - "sec:circuit-table" + "sec:circuit-discovery-methods" ], - "definition": "Activation patching evaluates the causal contribution of a component or set of components S by replacing their activations with baseline values", + "definition": "Activation patching tests the causality of a component’s representations by swapping them between inputs that differ in the target property.", "assumptions": [], "evaluation": [ - "Perform activation patching: swap C between clean and corrupted inputs; verify behavior follows C." + "The resulting change in model output quantifies the causal effect of intervening on S (Zhang & Nanda, 2024)." ], "applications": [ - "This identifies which components are functionally necessary for the computation (Zhang & Nanda, 2024).", - "A probe might detect deceptive reasoning, but confirming the finding is causal, and designing targeted interventions, requires causal intervention on internal representations—whether through activation patching, steering, or ablation.", - "Causal spot-checks: Randomly sample agentic explanations and verify via causal interventions, such as activation patching." + "This technique has also been used to test the causality of probes (Huang et al., 2024; Arora et al., 2024).", + "The pipeline can also additionally examine a fraction of explanations by causal techniques, such as activation patching, and track failure rates." ], "limitations": [ - "Confounds correlation with causation" + "Requires many interventions; component-level granularity can be coarse" ], "related": [ "path-patching", @@ -828,11 +798,11 @@ ], "mentioned_in": [ "sec:scope", - "when-mechanistic-understanding-becomes-necessary", - "dimension-1-faithfulness", - "proposed-guardrails-for-agentic-interpretability", - "proposed-guardrails-for-automated-interpretability", - "potential-approaches." + "causal-probing-via-activation-patching.", + "subsec:five eval dim", + "layer-2-consistency-and-causality-checks.", + "sec:non-transformer", + "choosing-appropriate-methods-for-different-safety-applications" ] }, { @@ -840,73 +810,84 @@ "name": "Path Patching", "family": "circuits", "sections": [ - "sec:circuit-table" + "sec:circuit-discovery-methods" ], - "definition": "Path patching extends attribution to specific paths or edges in the computation graph.", + "definition": "Path patching extends activation patching on model components to specific edges in the computational graph.", "assumptions": [], "evaluation": [], "applications": [ - "This allows finer-grained causal localization along information flows (Goldowsky-Dill et al., 2023)." + "This allows finer-grained causal localisation along information flows (Goldowsky-Dill et al., 2023)." ], "limitations": [ - "Computationally expensive due to large number of paths; still approximate" + "Computationally expensive due to the large number of candidate paths" ], "related": [ "activation-patching", "acdc", "weight-sparse-transformer" ], - "mentioned_in": [] + "mentioned_in": [ + "subsec:five eval dim" + ] }, { "id": "acdc", "name": "ACDC", "family": "circuits", "sections": [ - "sec:circuit-table" + "sec:circuit-discovery-methods" ], - "definition": "ACDC identifies minimal sufficient sets of components S \\subseteq \\text{Graph}(\\text{model}) that reproduce the model’s behavior.", + "definition": "ACDC (Conmy et al., 2023) discovers a circuit by iteratively using path patching to identify important connections in the model’s computational graph. Traversing the graph in reverse topological order from the output node, it patches the activation transmitted along each edge and measures the resulting change in a task-specific metric.", "assumptions": [], "evaluation": [ - "Scales better due to automation; has been proved to recover known circuits" + "Fully automated; directly searches for a behaviourally sufficient subgraph" ], "applications": [], "limitations": [ - "Computationally expensive as it greedily prunes paths; misses subtle components" + "Computationally expensive due to sequential edge pruning; sensitive to pruning order and threshold choices" ], "related": [ "activation-patching", "path-patching", "weight-sparse-transformer" ], - "mentioned_in": [] + "mentioned_in": [ + "sec:evaluation-framework", + "summary.-4" + ] }, { "id": "circuit-tracing", - "name": "Circuit Tracing (SAE- or CLT-Integrated circuit discovery)", + "name": "Circuit Tracing (SAE- or CLT-Based Circuit Discovery)", "family": "circuits", "sections": [ - "subsec:circuits", - "sec:circuit-table", - "replacement-models-and-attribution-graphs." + "sec:circuit-replacement" ], - "definition": "The second line of work, often referred to as circuit tracing, focuses on how internal features interact and propagate across the computational graph to lead to the model decision of a particular input query (Ameisen et al., 2025; Wu & Barez, 2025).", + "definition": "A complementary approach first constructs a more interpretable replacement model (Section 6.6.2) that approximates the original model’s computation using sparse linear features, and then performs circuit discovery within this replacement model.", "assumptions": [ - "Studies of this line usually make use of sparse autoencoders (detailed in Section 6) to automatically decompose mlp outputs into interpretable units, tracing attributions between these features and how the information flow converges to the final prediction." + "Recent work has implemented this paradigm using cross-layer transcoders (CLTs; Section 6.6.2) (Ameisen et al., 2025; Merullo et al., 2025). In particular, Circuit Tracer (Ameisen et al., 2025) replaces the model’s MLPs with CLTs while retaining the underlying model’s attention and residual-stream computation. This yields a prompt-specific attribution graph whose principal internal nodes are sparse CLT features rather than entire MLP blocks or attention heads." ], "evaluation": [], "applications": [ - "Stage 2 – Circuit Tracing: We construct a computational graph in which nodes correspond to activated SAE features and edges represent their inter-attributions (Marks et al., 2025a)." + "For a given prompt, Circuit Tracer produces a sparse attribution graph that describes active features and their interactions, making it well suited to query-level circuit discovery.", + "Using automated feature-interpretation pipelines (Section 12.6) to assign natural-language descriptions to these features, human inspectors can more easily understand the discovered mechanism.", + "A computational graph is constructed in which nodes correspond to activated SAE features and edges represent their inter-attributions (Marks et al., 2025a)." ], "limitations": [ - "Depends on feature quality; intensive computation to train SAEs/CLTs" + "Interpretability platforms such as Neuronpedia (Lin, 2023) also offer interactive implementations. However, the approach still has several limitations. Specifically, CLTs only approximate MLP computations, so the discovered circuit may only cover a part of the model’s underlying mechanism. Moreover, feature-level attribution graphs can have many nodes and edges, so we must prune, cluster, or otherwise condense them before they become understandable.", + "Depends on feature quality; requires additional computation to train and analyse SAEs/CLTs" ], "related": [ "sae", "cross-layer-transcoders" ], "mentioned_in": [ - "mechanistic-monitoring" + "sec:circuit-discovery-methods", + "replacement-models-and-attribution-graphs.", + "sec:evaluation-framework", + "subsubsec: mech monitor", + "stage-2-circuit-discovery.", + "the-scalability-bottleneck." ] }, { @@ -915,26 +896,27 @@ "family": "circuits", "sections": [ "evaluating-circuits-and-benchmarking-discovery-methods.", - "causal-necessity-and-sufficiency.", - "dimension-4-causal-validity" + "causal-necessity-and-sufficiency." ], - "definition": "Causal scrubbing evaluates necessity by replacing circuit activations with corrupted or neutralized values and measuring the resulting change in model outputs.", + "definition": "Causal scrubbing (Chan et al., 2022a) systematically evaluates a mechanistic hypothesis by replacing model activations with those from inputs deemed equivalent under the hypothesis and testing whether these interventions preserve the model behaviour.", "assumptions": [], "evaluation": [ - "Two widely used high-level methods for testing these properties are causal scrubbing and interchange interventions (Heimersheim & Nanda, 2024; Chan et al., 2022)." + "Two popular approaches to validate causality are causal scrubbing and interchange interventions (Chan et al., 2022a; Geiger et al., 2024).", + "Together, these methods provide stronger evidence for mechanistic explanations than observational analysis alone." ], "applications": [ - "Circuit faithfulness: proving circuits are necessary and sufficient for explaining behavior via causal intervention (e.g., causal scrubbing), without performance degradation." + "Circuit faithfulness: demonstrating circuits are necessary and sufficient for explaining behaviour via causal intervention (e.g., causal scrubbing (Chan et al., 2022a)), without performance degradation." ], "limitations": [ - "Strict necessity and sufficiency are often unattainable.", - "Additionally, verification assumes the circuit mechanism is faithfully captured by the identified components, but if features are entangled or nonlinear interactions are present, the measured circuit may not reflect true causal structure." + "By aggregating faithfulness scores across tasks and settings, benchmarks such as MIB (Mueller et al., 2025) evaluate how well circuit discovery methods identify causally important model components. However, achieving strict necessity and sufficiency is often hard because many model behaviours depend on distributed representations and multiple redundant pathways (Barez et al., 2026).", + "Even if we identify a circuit that captures the primary mechanism of a behaviour, components outside the circuit may still play important roles, such as handling edge cases and computational robustness." ], "related": [ "interchange-interventions" ], "mentioned_in": [ - "ambitious-interpretability-toward-complete-mechanistic-understanding" + "sec:causal-verification", + "subsec:ambitious" ] }, { @@ -942,26 +924,29 @@ "name": "Interchange Interventions", "family": "circuits", "sections": [ - "evaluating-circuits-and-benchmarking-discovery-methods.", - "causal-probing-via-interchange-interventions." + "evaluating-circuits-and-benchmarking-discovery-methods." ], - "definition": "Interchange interventions test sufficiency by swapping circuit components between clean and corrupted forward passes and assessing whether the target behavior consistently tracks the substituted circuit across inputs.", + "definition": "Interchange interventions (Geiger et al., 2024) test whether model activations causally correspond to higher-level variables by substituting activations across inputs and measuring the resulting change in model behaviour.", "assumptions": [], - "evaluation": [], - "applications": [ - "Test causal relevance: Complement probing with amnesic probing or interchange interventions." + "evaluation": [ + "Interchange test: Swap activations of S between inputs representing different behaviours and test whether the model output follows the intervention.", + "Tested via interchange interventions: in the corrupted run with a reference input containing no target information, patch the activation of S with that from a clean run using an input containing the target information, and then verify whether the behaviour emerges." ], - "limitations": [ - "This provides stronger evidence than amnesic probing but requires identifying the specific representation component, not just demonstrating its existence." + "applications": [ + "One promising direction is interchange intervention testing and training (Geiger et al., 2024; 2022). This technique was developed to compare and align a neural network with a high-level causal abstraction." ], + "limitations": [], "related": [ "causal-scrubbing", "probing", "amnesic-probing" ], "mentioned_in": [ - "dimension-1-faithfulness", - "dimension-4-causal-validity" + "subsec:five eval dim", + "evaluation-protocol-3", + "sufficiency", + "potential-approaches.", + "causal-probing-via-activation-patching." ] }, { @@ -974,15 +959,15 @@ ], "definition": "Logit lens (nostalgebraist, 2020) applies the model’s unembedding matrix to activations at intermediate layers, revealing what the model would predict if computation stopped at that layer.", "assumptions": [ - "Applying the unembedding matrix to intermediate activations assumes that the hidden state is in a format the output layer expects, but intermediate representations may reside in vector spaces different from final activations." + "The vanilla logit lens has a critical limitation: applying the unembedding matrix to intermediate activations assumes that the hidden state is in a format that the output layer expects. However, intermediate representations may be organised very differently from output-layer activations." ], "evaluation": [], "applications": [ - "By examining this trajectory, researchers can identify at which layer critical decisions emerge: whether the model commits early to a prediction or progressively refines it layer-by-layer.", - "In this sense, logit lens can still reveal contributions to potentially harmful tokens and track how these emerge through the network, similarly to refusal vectors." + "Applying it across layers reveals the prediction trajectory, showing when and where the model commits to specific outputs (nostalgebraist, 2020).", + "Logit lens (Figure 10) provides intuitive insight into the model’s decision-making process across depth." ], "limitations": [ - "The unembedding matrix is only trained to operate on the final residual stream after all transformations; applying it to layer-6 activations extracts relevant information but may not precisely reflect the model’s underlying computation at that layer (nostalgebraist, 2020)." + "Applying the unembedding matrix to layer-6 activations may extract relevant information but cannot precisely reflect the model’s underlying computation at that layer (nostalgebraist, 2020). Furthermore, logit lens is fundamentally correlational rather than causal. It projects the activation into the output vocabulary space without testing the roles decoded tokens play in model computation. In particular, because of the representational mismatch discussed above, decoded tokens represent at most information recoverable from the activation, rather than information used in model computation." ], "related": [ "tuned-lens", @@ -990,7 +975,11 @@ "steering-vectors" ], "mentioned_in": [ - "case-study-refusal-vectors-and-sleeper-agents" + "subsec:tuned-lens", + "jacobian-lens-decoding-computational-dispositions", + "subsec:patchscopes", + "summary.-5", + "sec:ld-features" ] }, { @@ -998,24 +987,31 @@ "name": "Tuned Lens", "family": "intermediate-computation", "sections": [ - "tuned-lens-learning-per-layer-transform" - ], - "definition": "Tuned lens (Belrose et al., 2025) addresses the vector space mismatch problem of logit lens by learning per-layer affine transformations that map intermediate activations to the output layer’s expected space", - "assumptions": [ - "The trade-off is that tuned lens requires training additional parameters (one affine transform per layer), but this is computationally cheap compared to other mechanistic methods." + "subsec:tuned-lens", + "limitations-of-tuned-lens." ], + "definition": "Tuned lens (Belrose et al., 2025) addresses logit lens’ concern of representational mismatch by learning per-layer affine transformations that map intermediate activations to the output layer’s activation space", + "assumptions": [], "evaluation": [], "applications": [ - "Tuned lens improves on basic logit lens by correcting for the fact that different layers operate in different representational spaces." + "As a result, the unembedding matrix can extract relevant information from each layer’s activations more effectively." ], "limitations": [ - "Because the mapping from intermediate representations to natural language may be too complex to be captured by an affine transformation, Patchscopes extends the paradigm of the logit lens and tuned lens by delegating the representation-to-text transformation to the language model itself" + "First, the decoding paradigm remains inherently correlational rather than causal; decodable information does not imply a causal role in the model’s prediction.", + "Second, similar to logit lens, tuned lens is primarily designed to answer what the model is likely to predict rather than what the model is internally computing.", + "Empirical evidence also shows that it skips intermediate computation more aggressively than logit lens (Gurnee et al., 2026)." ], "related": [ "logit-lens", "patchscopes" ], - "mentioned_in": [] + "mentioned_in": [ + "jacobian-lens-decoding-computational-dispositions", + "subsec:patchscopes", + "summary.-5", + "kl-divergence.", + "sec:non-transformer" + ] }, { "id": "patchscopes", @@ -1025,22 +1021,27 @@ "subsec:patchscopes", "limitations-of-patchscopes." ], - "definition": "Patchscopes (Ghandeharioun et al., 2024) leverages the expressive capacity of large language models to translate internal representations into natural-language explanations.", + "definition": "Patchscopes (Ghandeharioun et al., 2024) leverages the expressive capacity of language models to translate their internal representations into natural-language explanations.", "assumptions": [ - "Crucially, this approach relies on the model’s own learned and generalized text-generation capabilities to perform the transformation from internal activations to natural language." + "In this view, the language model acts as an activation decoder. Crucially, Patchscopes relies on the model’s learned, generalisable text-generation capabilities to map internal activations to natural language." ], "evaluation": [], - "applications": [], + "applications": [ + "By observing changes in model output, users can examine what information the representation contains." + ], "limitations": [ - "A key limitation of Patchscopes is that injecting an external representation into an elicitation prompt can shift the model’s internal activations into an OOD regime.", - "Although the model may still successfully surface salient information from the patched representation, the decoding process can be degraded, and critical aspects of the original representation may be distorted or lost." + "A key limitation of Patchscopes is that injecting an external representation into an elicitation prompt can shift the model’s internal activations into an out-of-distribution (OOD) regime.", + "Although the model may still surface salient information from the patched representation, the decoding process can degrade, and critical aspects of the original representation may be distorted or lost." ], "related": [ "logit-lens", "tuned-lens", "latentqa" ], - "mentioned_in": [] + "mentioned_in": [ + "subsec:latentqa", + "summary.-5" + ] }, { "id": "latentqa", @@ -1050,27 +1051,29 @@ "subsec:latentqa", "limitations-of-latentqa." ], - "definition": "LatentQA (Pan et al., 2024) formulates representation decoding as a supervised learning problem via Latent Interpretation Tuning (LIT), which fine-tunes an activation decoder to perform representation-to-text decoding.", + "definition": "LatentQA (Pan et al., 2026) frames representation decoding as a supervised learning problem via Latent Interpretation Tuning (LIT), fine-tuning an activation verbaliser to perform representation-to-text verbalisation.", "assumptions": [ - "Both the control prompt and the (q, a) pair are generated by prompting an off-the-shelf proprietary language model (Pan et al., 2024).", - "Through LIT, the decoder is trained to directly ingest and process raw activation vectors." + "Through LIT, the verbaliser learns to ingest and process raw activation vectors directly." + ], + "evaluation": [ + "Another technical challenge is evaluating whether activation-verbalisation models genuinely explain the represented information rather than exploit shortcuts." ], - "evaluation": [], "applications": [ - "LatentQA addresses the OOD issue inherent in Patchscopes-style methods by explicitly finetuning the activation decoder on the representation-to-text decoding task.", - "Activation conditioning: Methods like LatentQA that condition on actual intermediate activations (rather than relying on the model to “remember” its computation) show improved faithfulness over pure prompting (Pan et al., 2024)." + "LatentQA (Figure 12) addresses the OOD issue inherent in Patchscopes-style methods by explicitly fine-tuning the activation verbaliser on the representation-to-text decoding task.", + "The process involves (1) curating a dataset of activation–question–answer triplets from a target LLM, and (2) fine-tuning a verbaliser to retrieve specific information latent within those activations, such as the model’s underlying goals or persona." ], "limitations": [ - "The central limitation of LatentQA lies in the generalizability of the learned representation-to-text decoder.", - "When the finetuning dataset D is insufficiently large or diverse, the decoder may overfit by memorizing superficial correspondences between representations and textual descriptions, rather than learning the underlying algorithmic transformation." + "LatentQA faces several technical challenges, including the generalisability of the learned representation-to-text mapping.", + "When the fine-tuning dataset D is too small or insufficiently diverse, the verbaliser may memorise superficial correspondences between representations and explanations rather than learning to extract information from representations.", + "Recent studies have found that such failure modes are severe and prevalent in existing verbalisers (Li et al., 2026c; Jakkli et al., 2026). As a result, strong verbalisation performance alone is insufficient to show that the verbaliser faithfully analysed the internal representations." ], "related": [ "patchscopes" ], "mentioned_in": [ - "empirical-evidence.-1", - "elicitation-protocols-to-reduce-confabulation.", - "comparative-analysis-trade-offs-between-orientations" + "summary.-5", + "comparative-analysis-trade-offs-between-orientations", + "the-road-ahead" ] }, { @@ -1078,25 +1081,27 @@ "name": "Amnesic Probing", "family": "intermediate-computation", "sections": [ - "amnesic-probing-and-causal-probing", + "subsubsec: amnesic probing", "amnesic-probing." ], - "definition": "Building on this lineage, Elazar et al. (2021) introduced amnesic probing: after identifying a direction encoding property P, project it out of the representation and measure downstream task degradation.", + "definition": "Given a model activation h and a probe trained to predict a property P, amnesic probing tests whether information about P is causally relevant to downstream model behaviour by removing information about P from the representation and evaluating the effects on model outputs.", "assumptions": [], - "evaluation": [ - "If performance degrades, the information was causally used. If performance is unchanged, the information was encoded but unused." + "evaluation": [], + "applications": [ + "Elazar et al. (2021) further measured changes in downstream model behaviour after applying INLP as a validation of the probe’s causality, called amnesic probing. These paradigms add causality to probes, pushing them from correlational understanding to mechanistic analysis." ], - "applications": [], "limitations": [ - "Limitations: Amnesic probing assumes the property is encoded in a single linear direction.", - "If encoding is distributed or nonlinear, projection may not fully remove the information, leading to false negatives (concluding information is unused when it is)." + "A substantial degradation indicates that the removed information causally contributes to the evaluated behaviour. Notably, unchanged performance does not establish that the information is unused since redundant representations or alternative computational pathways may preserve the behaviour.", + "Amnesic probing assumes the target property is decodable and that there exist effective erasure procedures to surgically remove its associated information from the representation. However, information and concept erasure may be imperfect in practice, either leaving residual information or affecting other concepts. Such imperfections confound interpretations of downstream behavioural changes and the causal claims that follow." ], "related": [ "probing", "interchange-interventions" ], "mentioned_in": [ - "best-practices-for-probing-studies" + "causal-probing-via-activation-patching.", + "methodological-standards-in-probing-research", + "subsec:cbm" ] }, { @@ -1105,145 +1110,139 @@ "family": "training-data-attribution", "sections": [ "verbatim-data-attribution-olmotrace-and-maximal-matching-spans", - "limitations" + "limitations-as-a-mechanistic-interpretability-tool." ], - "definition": "Verbatim data attribution targets explicit memorization by identifying exact or near-exact matches between model outputs and training data, scaling this search across trillions of tokens via suffix arrays and infinite n-gram models.", + "definition": "Verbatim data attribution detects explicit memorisation by identifying exact or near-exact string matches between model outputs and the training corpus. These searches can scale to trillions of tokens using techniques such as suffix arrays and infini-gram models.", "assumptions": [ - "OLMoTrace scales this to trillions of tokens using a combination of FM-index, infini-gram models, and maximal matching span extraction (Liu et al., 2025a; 2024; Xu et al., 2025), making it less a data attribution method in the influence-function sense and more a retrieval tool for identifying memorized content." + "OLMoTrace (Liu et al., 2025a) scales this to trillions of tokens using a combination of suffix arrays, infini-gram models, and maximal matching span extraction (Liu et al., 2024a), making it less a data-attribution method in the influence-function sense and more a retrieval tool for identifying content the model may have memorised.", + "OLMoTrace identifies maximal matching spans by combining suffix-array search with infini-gram queries, adjusting boundaries for tokenisation, and retaining only the longest non-overlapping spans." ], "evaluation": [], "applications": [ - "This process links LM outputs directly to the training corpus, enabling precise attribution of memorized content." + "“Eurovision Song Contest with the song ’Ne” This process links language model output directly to the training corpus, enabling identification and attribution of memorised content.", + "UTF8gbsnSpan: “她喜欢阅读关于科技、心理学和历史的书籍,业余时间还会参加编程课”—matched in post-training (instruction-tuning) data This example demonstrates how OLMoTrace traces multilingual outputs across diverse training sources, linking model generations directly to their memorised origins in pretraining and post-training datasets." ], "limitations": [ - "Verbatim attribution excels at factual memorization but struggles with compositional or synthesized outputs.", - "For tasks like translation, poetry, or reasoning, OLMoTrace can identify spans of text that appear in the training data, but these matches do not explain how the model combined information from multiple sources to produce the output (Park et al., 2023)." + "Verbatim attribution excels at factual memorisation but struggles with compositional or synthesised outputs.", + "For tasks like translation, poetry, or reasoning, OLMoTrace (and other verbatim matching methods) can identify spans of text that appear in the training data, but these matches do not explain how the model combined information from multiple sources to produce the output." ], "related": [ "influence-functions" ], - "mentioned_in": [] + "mentioned_in": [ + "sec:influence-functions", + "olmotrace-influential-samples-identified", + "summary.-8" + ] }, { "id": "influence-functions", "name": "Influence Functions (Gradient-Based Attribution)", "family": "training-data-attribution", "sections": [ + "gradient-based-attribution-understanding-data-influence-through-perturbations", "core-idea-parameter-response-to-training-data", - "gradient-based-attribution-understanding-influence-through-data-perturbations", - "proximal-bregman-response-and-the-gauss-newton-hessian", - "computational-bottleneck-approximating-ggn-vector-products", + "proximal-bregman-response-and-the-generalised-gaussnewton", + "addressing-the-computational-and-memory-bottlenecks-of-ggn", "illustrative-example", "recent-extensions-and-limitations" ], - "definition": "Influence functions ask a counterfactual question: how would a model’s learned parameters change if we upweighted a training example (Koh & Liang, 2017)? Instead of expensive retraining, we estimate this via implicit differentiation (Grosse et al., 2023).", + "definition": "Influence functions estimate how a model’s parameters and predictions would change if a training sample were slightly upweighted (Koh & Liang, 2017; Grosse et al., 2023).", "assumptions": [ - "The implicit function theorem assumes exact convergence and infinite perturbations.", - "Real neural networks violate both: training reaches saddle points, not true minima, and we retrain for only a few steps.", - "This equivalence allows practitioners to use Fisher information as a practical substitute: it’s more stable to compute and estimate from data than the true GGN." + "Empirical results show that the original form of influence functions introduced above performs poorly in practice. This is due to multiple assumptions that rarely hold in practice.", + "One classical approach is called the proximal Bregman response function (PBRF) (Bae et al., 2022). It targets model parameters which minimise the new training objective while staying close to the original parameters.", + "Although K-FAC approximates the FIM rather than the GGN, these matrices are equivalent for common loss functions under suitable probabilistic assumptions." ], "evaluation": [ - "Observation: Influence functions successfully identify training documents that impacted the translation, even without exact verbatim matches." + "Influence functions identify training documents that impacted the translation, even without exact verbatim matches. This demonstrates that gradient-based methods can capture influences beyond what verbatim methods can reveal.", + "Bae et al. (2022) confirmed empirically that such influence estimates align well with the actual PBRF." ], "applications": [ - "Training data attribution (TDA) answers a fundamental interpretability question: which training examples shaped specific model behaviors?", - "This enables identifying influential data, detecting train-test leakage, and understanding failure modes without retraining (Liu et al., 2025a; Park et al., 2023; Grosse et al., 2023)." + "Training data attribution (TDA) shifts the focus from model internals to training data, searching for training samples that strongly shaped the model behaviour.", + "TDA applications include, but are not limited to, identifying influential training samples, detecting train–test leakage, and diagnosing model failures without retraining (Liu et al., 2025a; Park et al., 2023; Grosse et al., 2023).", + "Although TDA is not generally a mechanistic method under this survey’s criteria, it can complement mechanistic interpretability in many ways. For example, by revealing how specific training samples shape malicious behaviour, TDA can enable effective training data filtering to prevent unwanted behaviours or even their internal mechanisms from forming during the training of next-generation models." ], "limitations": [ - "However, fundamental challenges persist: approximation errors compound over optimization trajectories, random hyperparameters introduce variance across runs, and scaling to larger models remains memory-intensive.", - "Classical influence functions rely on inverting the Hessian of the loss landscape, which becomes ill-defined when the Fisher information matrix is degenerate." + "(2026) introduced Concept Influence to enable attribution at the level of semantic concepts represented by model internals such as sparse autoencoder features. This brings TDA closer to the mechanistic goal of understanding how training data shapes specific internal computations. However, fundamental technical challenges persist, including approximation errors that compound over the training trajectory, variability across training runs caused by randomness like data ordering, and memory and computational bottlenecks of scaling to frontier models.", + "Token-level filtering is motivated by data attribution, but it remains unclear whether methods such as influence functions (Section 11) can reliably identify the relevant data." ], "related": [ "olmotrace", "kfac", "ek-fac-astra", - "unrolled-differentiation", "trak", - "tracin", - "bayesian-influence-function" + "tracin" ], "mentioned_in": [ "sec:scope", - "sec:stagewise-influence" + "sec:influence-functions", + "observation.-1", + "summary.-8", + "open-questions." ] }, { "id": "kfac", - "name": "Kronecker-Factored Approximate Curvature (KFAC)", - "family": "training-data-attribution", - "sections": [ - "kronecker-factored-approximate-curvature-kfac." - ], - "definition": "KFAC is a parametric approximation to the Fisher Information Matrix (FIM) of a neural network which supports efficient inversion.", - "assumptions": [], - "evaluation": [], - "applications": [], - "limitations": [ - "KFAC works well but can underestimate curvature in low-variance directions due to finite sampling." - ], - "related": [ - "influence-functions", - "ek-fac-astra" - ], - "mentioned_in": [] - }, - { - "id": "ek-fac-astra", - "name": "Eigenvalue-Corrected KFAC (EK-FAC) and ASTRA", + "name": "Kronecker-Factored Approximate Curvature (K-FAC)", "family": "training-data-attribution", "sections": [ - "eigenvalue-corrected-kfac-ek-fac-and-astra." + "kronecker-factored-approximate-curvature-k-fac." ], - "definition": "EK-FAC improves KFAC by correcting eigenvalues and is the method used by recent work scaling influence to LLMs (Grosse et al., 2023).", + "definition": "Martens & Grosse (2015) proposed Kronecker-Factored Approximate Curvature (K-FAC) to approximate a large Fisher information matrix (FIM) using a set of smaller matrices.", "assumptions": [ - "An even more recent refinement, ASTRA (Wang et al., 2025a), uses EK-FAC as a preconditioner for Neumann series iterations, yielding higher-quality GGN-vector products with less hyperparameter tuning." + "Although K-FAC approximates the FIM rather than the GGN, these matrices are equivalent for common loss functions under suitable probabilistic assumptions.", + "The approximation separates the expectation into two independent factors: activations and pseudo-gradients." ], "evaluation": [], "applications": [ - "These structured approaches make influence functions practical for models with billions of parameters." + "G. This shows that we can use K-FAC to approximate the GGN used in the PBRF influence score." ], "limitations": [], "related": [ - "kfac", - "influence-functions" + "influence-functions", + "ek-fac-astra" ], - "mentioned_in": [] + "mentioned_in": [ + "addressing-the-computational-and-memory-bottlenecks-of-ggn", + "eigenvalue-corrected-k-fac-ek-fac-and-astra." + ] }, { - "id": "unrolled-differentiation", - "name": "Approximate Unrolled Differentiation", + "id": "ek-fac-astra", + "name": "Eigenvalue-Corrected K-FAC (EK-FAC) and ASTRA", "family": "training-data-attribution", "sections": [ - "unrolled-differentiation-gradient-based-attribution" + "eigenvalue-corrected-k-fac-ek-fac-and-astra." ], - "definition": "Approximate unrolled differentiation (Bae et al., 2024) unfolds the optimization trajectory and backpropagates through training steps.", + "definition": "EK-FAC (George et al., 2018) improved K-FAC by correcting its eigenvalues. Recent work has adopted this technique to scale the influence function method to modern LLMs (Grosse et al., 2023).", "assumptions": [], "evaluation": [], "applications": [ - "This captures implicit bias of the optimizer and handles non-converged models, where classical influence functions struggle." - ], - "limitations": [ - "The tradeoff: higher computational cost than influence functions but better accuracy on modern, non-convex objectives." + "ASTRA (Wang et al., 2025a) further used EK-FAC as a preconditioner for Neumann series iterations, achieving higher-quality GGN-vector products with less hyperparameter tuning. These efforts make influence functions increasingly practical for modern language models with billions of parameters." ], + "limitations": [], "related": [ + "kfac", "influence-functions" ], "mentioned_in": [] }, { "id": "trak", - "name": "TRAK (Tracing with Randomly-projected After Kernel)", + "name": "TRAK (Tracing with the Randomly-projected After Kernel)", "family": "training-data-attribution", "sections": [ "sec:trak" ], - "definition": "For large-scale attribution without retraining thousands of models, TRAK (Tracing with Randomly-projected After Kernel) provides a scalable alternative (Park et al., 2023).", - "assumptions": [], + "definition": "Tracing with the Randomly-projected After Kernel (TRAK; Park et al., 2023) provides a more efficient alternative. TRAK first uses a first-order Taylor expansion around the model parameters to approximate a trained model as a linear model.", + "assumptions": [ + "TRAK first uses a first-order Taylor expansion around the model parameters to approximate a trained model as a linear model. It then applies random projections to reduce gradient dimensionality and estimate training-sample influence in this lower-dimensional space. Finally, it averages the estimates across a small number of trained models to reduce variability due to training randomness." + ], "evaluation": [ - "By avoiding prohibitive retraining costs, TRAK achieves attribution quality comparable to expensive retraining-based methods like datamodels (Ilyas et al., 2022) while reducing computational cost by orders of magnitude." + "TRAK achieves attribution quality comparable to datamodels at substantially lower computational cost (Park et al., 2023)." ], "applications": [ - "This enables application to large-scale models including vision transformers and billion-parameter language models, making fine-grained training data attribution tractable for modern deep learning (Park et al., 2023)." + "Georgiev et al. (2023) proposed TRAK variants to address diffusion models." ], "limitations": [], "related": [ @@ -1260,95 +1259,82 @@ "sections": [ "sec:tracin" ], - "definition": "TracIn provides an efficient alternative to computing the full Hessian inverse by leveraging the training trajectory (Pruthi et al., 2020).", + "definition": "TracIn (Garima et al., 2020) estimates training-data influence using gradients evaluated along the training trajectory, without computing the inverse Hessian.", "assumptions": [ - "This approach avoids Hessian computation entirely, instead accumulating dot products of gradients throughout training." + "Its core idea follows from the effect of a single Stochastic Gradient Descent (SGD; Robbins & Monro, 1951) update on the loss of a testing sample." ], "evaluation": [], "applications": [ - "TracIn is considerably cheaper than influence functions based on Hessian approximations, making it practical for large models.", - "Despite limitations, TracIn has enabled attribution studies on modern language models and provides a useful baseline for scalable influence estimation." + "Unlike Hessian-based influence functions, TracIn does not require matrix inversion.", + "By examining each checkpoint’s contribution, researchers can identify when a training example had the greatest influence on a downstream prediction." ], "limitations": [ - "However, the approximation assumes that gradient trajectories during training encode sufficient information about final influence, an assumption that may break down in highly nonlinear settings or when training dynamics involve significant noise (Pruthi et al., 2020)." + "TracInCP approximates the influence of training samples using gradients only at selected checkpoints, which may undermine the estimation accuracy." ], "related": [ "influence-functions" ], - "mentioned_in": [] + "mentioned_in": [ + "trajectory-based-attribution." + ] }, { "id": "local-learning-coefficient", "name": "Local Learning Coefficient (LLC)", "family": "developmental-interpretability", "sections": [ - "sec:slt-llc", - "the-local-learning-coefficient" + "singular-learning-theory.", + "app:slt" ], - "definition": "In other words, it captures how many ways the parameters can vary locally without changing the loss value (Watanabe, 2009; Lau et al., 2023).", + "definition": "The Local Learning Coefficient (LLC; Lau et al., 2025) builds on SLT by localising Watanabe’s learning coefficient around a particular parameter configuration, thereby providing a measure of the effective local complexity of a neural network around a local minimum.", "assumptions": [ - "Formally, the LLC is defined via the volume-scaling rate of the loss landscape near a minimum." - ], - "evaluation": [ - "Researchers have observed that transitions and critical points in the LLC often coincide with changes in internal network structure and functional behavior." + "Neural networks are singular (and are thus a subject of study in SLT) because different parameter configurations can implement the same function.", + "Smaller values indicate more degenerate (flatter, in a higher-order sense) minima. The LLC is estimable at scale with stochastic-gradient MCMC (Lau et al., 2025)." ], + "evaluation": [], "applications": [ - "Empirical estimators can track the LLC throughout training, providing a way to observe developmental changes in model structure (Lau et al., 2023; Hoogland et al., 2025).", - "Tracking these changes provides a quantitative way to study the emergence of structure during training without relying only on final performance metrics (Hoogland et al., 2025)." + "The LLC has been used to characterise properties of trained neural networks. For example, Lau et al. (2025) showed that the LLC can capture differences in effective complexity between models trained under different settings, even when their training losses are similar.", + "Wang et al. (2025b) introduced refined variants of the Local Learning Coefficient (Section 10.3.4) and applied them to individual attention heads of a two-layer attention-only transformer across training.", + "Heads of the same type cluster together in both their final values and the overall shape of the curve across training, revealing how heads differentiate into distinct functional roles." + ], + "limitations": [ + "Estimated LLC trajectories in transformers change at points that coincide with identifiable developmental events, including the stages of structure formation described in Section 10.3.3 (Hoogland et al., 2025). This is the central empirical result of the developmental interpretability programme (Hoogland et al., 2023) introduced in Section 12.3. However, current evidence for LLC is largely retrospective, where phase transitions are detected and matched to known events only after training. Prospective prediction of a transition before it occurs has not yet been demonstrated.", + "Unlike the proposed induction scores (Olsson et al., 2022) for identifying induction heads, the measure does not require specifying a head type in advance. However, estimating refined LLCs is computationally expensive, and the approach has so far been demonstrated only on toy models." ], - "limitations": [], "related": [], "mentioned_in": [ + "localising-change-to-components.", + "run-to-run-variance.", + "subsec:dev interp", "comparative-analysis-trade-offs-between-orientations" ] }, - { - "id": "bayesian-influence-function", - "name": "Stagewise Influence and the Bayesian Influence Function (BIF)", - "family": "developmental-interpretability", - "sections": [ - "sec:stagewise-influence" - ], - "definition": "Lee et al. (2025) propose a stagewise data attribution framework that extends classical influence methods to track the impact of training points throughout the learning process.", - "assumptions": [ - "Unlike classical influence functions that require Hessian inversion, the BIF measures influence through covariance estimation over the local posterior distribution.", - "This distributional approach is estimated via stochastic-gradient MCMC sampling, enabling scalable computation without explicit Hessian access while naturally capturing the geometry of singular loss landscapes." - ], - "evaluation": [ - "These predictions are further confirmed through leave-one-out retraining experiments that directly measure the effect of removing individual training examples at different stages of training." - ], - "applications": [ - "Token-level influence exhibits non-monotonic changes, including sudden peaks and sign reversals that align with known developmental milestones such as the formation of induction heads." - ], - "limitations": [], - "related": [ - "influence-functions" - ], - "mentioned_in": [] - }, { "id": "model-diffing", "name": "Model Diffing", "family": "representation-analysis", "sections": [ - "sec:model-diffing" + "sec:model-diffing", + "applications.", + "dictionary-learning-methods.", + "llm-based-approaches." ], "definition": "For two models, model diffing seeks to identify differences in features or circuits between them. These differences may consist of features or circuits that are exclusive to one model, or of differences in how shared features or circuits are used.", "assumptions": [ - "SAEs can be applied to model-diffing when inputs from different models have the same dimension.", - "To diff closed-weight models, it is desirable to develop model diffing tools that just rely on API access." + "SAEs are applicable when inputs from different models have the same dimension.", + "To differentiate closed-weight models, it is desirable to develop model diffing tools that only rely on API access." ], "evaluation": [ - "Benchmarked against an SAE-based baseline, the LLM-based method matches it on many dimensions while surfacing more abstract, behaviour-level differences." + "Benchmarked against the SAE-based baseline, the LLM-based method matches it on many dimensions while surfacing more abstract, behaviour-level differences." ], "applications": [ - "This makes it important to monitor features and circuits that have been introduced or amplified during this process to prevent unsafe models from being deployed.", - "Model diffing has been applied to assess the effect of finetuning (Minder et al., 2025; Lindsey et al., 2024; Wang et al., 2025b; Aranguri & McGrath, 2025) and distillation (Baek & Tegmark, 2025), to track differences across model architectures (Thasarathan et al., 2025; Jiralerspong & Bricken, 2025), and to study the evolution of concepts during pretraining (Bayazit et al., 2025; Ge et al., 2025)." + "Many concerning behaviours, such as reward hacking or sycophancy, arise during fine-tuning (Sharma et al., 2024b; MacDiarmid et al., 2025). This phenomenon makes it important to monitor features and circuits introduced or amplified during this process to prevent unsafe models from being deployed.", + "Model diffing has been applied to assess the effect of fine-tuning (Dumas et al., 2025a; Lindsey et al., 2024; Aranguri & McGrath, 2025; Wang et al., 2026a) and distillation (Baek & Tegmark, 2025), to track differences between model architectures (Thasarathan et al., 2025; Jiralerspong & Bricken, 2025), to study the evolution of concepts during pretraining (Bayazit et al., 2026; Ge et al., 2026), and to identify vision-specific features introduced during vision-language training, enabling targeted removal or steering (Batra et al., 2026).", + "By causally intervening with latents that activate more strongly in the fine-tuned model, it is possible to identify features responsible for misaligned behaviour (Wang et al., 2026a)." ], "limitations": [ - "Dictionary learning methods for model-diffing risk falsely identifying model-exclusive features (Minder et al., 2025; Jiralerspong & Bricken, 2025).", - "During finetuning, the direction representing some feature can rotate, and dictionary learning methods currently cannot distinguish between features that are introduced via finetuning as opposed to merely represented differently.", - "Thus, supposedly model-exclusive features should be regarded as hypotheses that need further validation." + "Dictionary learning methods for model diffing risk falsely identifying model-exclusive features (Dumas et al., 2025a; Jiralerspong & Bricken, 2025).", + "During fine-tuning, the directions representing some features can rotate, and dictionary learning methods currently cannot distinguish between features introduced via fine-tuning and those merely represented differently. Thus, supposedly model-exclusive features should be regarded as hypotheses that need further validation." ], "related": [ "sae", @@ -1357,28 +1343,36 @@ "representation-similarity", "batch-topk-sae" ], - "mentioned_in": [] + "mentioned_in": [ + "sec:scope", + "sec:representation", + "sec:ld-features", + "model-scale." + ] }, { "id": "representation-similarity", "name": "Representation Alignment and Convergent Learning (SVCCA, PWCCA, CKA)", "family": "representation-analysis", "sections": [ - "sec:model-diffing" + "representation-alignment-and-convergent-learning." ], - "definition": "An earlier line of work compared internal representations across models by measuring statistical similarity between neurons or subspaces, using correlation or mutual information (Li et al., 2015), canonical correlation–based methods such as SVCCA and PWCCA (Raghu et al., 2017; Morcos et al., 2018b), or kernel-based alignment metrics like CKA (Kornblith et al., 2019).", + "definition": "An earlier line of work compared internal representations across models using correlation- and mutual information-based measures (Li et al., 2015), canonical-correlation methods such as SVCCA and PWCCA (Raghu et al., 2017; Morcos et al., 2018), and kernel-based alignment metrics such as CKA (Kornblith et al., 2019).", "assumptions": [], "evaluation": [], "applications": [ - "This literature primarily aims to characterize convergent learning: whether independently trained models (often only differing by weight initialisation) learn similar representations up to permutation, rotation, or linear reparameterization." + "Related approaches examine whether the representations of one model can be expressed as linear combinations of representations in another (Wang et al., 2018). This literature primarily concerns convergent learning: whether independently trained models with different random initialisations learn similar representations up to permutation, rotation, or other linear transformations.", + "Early work showed that different singular modes of the input–output correlations can be learned at different rates (where larger singular values tend to be learned earlier), making acquisition order informative about task structure (Saxe et al., 2014; Lampinen & Ganguli, 2019; Saxe et al., 2019). In contrast, representation-comparison methods measure when layers and representations converge during training (Raghu et al., 2017; Morcos et al., 2018)." ], "limitations": [ - "In contrast to model diffing as defined here, these methods typically focus on representational similarity rather than on identifying features or circuits that are introduced, amplified, or repurposed by a specific training intervention." + "More recent work has extended these techniques to study representational convergence across modalities (Huh et al., 2024; Gröger et al., 2026), as discussed in Section 5.4. In contrast to model diffing as defined here, these methods typically focus on representational similarity rather than on identifying concrete features or circuits that are introduced, amplified, or repurposed by a specific training intervention." ], "related": [ "model-diffing" ], - "mentioned_in": [] + "mentioned_in": [ + "learning-dynamics-as-an-interpretability-tool" + ] }, { "id": "causal-tracing", @@ -1387,22 +1381,26 @@ "sections": [ "sec:knowledge-localization" ], - "definition": "Causal tracing identifies which attention heads or MLP directions are causally responsible for particular facts or reasoning tasks by measuring how intervening on those components affects output behavior (Meng et al., 2022).", + "definition": "For instance, causal tracing identifies components that causally impact specific factual association retrieval by observing how model outputs change after activation intervention (Meng et al., 2022).", "assumptions": [], "evaluation": [], "applications": [ - "Once knowledge is attributed to specific model components via localization techniques like causal tracing, mechanistic approaches enable direct parameter modification." + "Once knowledge has been localised to specific model components, such as individual neurons (Dai et al., 2022), using methods such as causal tracing (see Section 9.2), mechanistic approaches can support direct parameter modification.", + "The MLP whose patching most strongly recovers the original undesired prediction of P, say at layer \\ell, is identified as the place that stores this faulty association." ], "limitations": [ - "Knowledge is often distributed across many parameters rather than concentrated in isolated components, particularly in large overparameterized models where redundancy is common.", - "Consequently, knowledge localization is most useful when combined with mechanistic analysis of how multiple components interact to implement computation." + "Localisation can identify where information is most strongly represented. However, it does not indicate whether the model uses that information.", + "In other words, a component with a strong causal effect may capture only part of the mechanism. As a result, localisation is a useful starting point for mechanistic understanding, but not a self-contained analysis." ], "related": [ "rome-memit" ], "mentioned_in": [ "sec:scope", - "subsec:knowedit" + "subsec:limits", + "subsec:knowedit", + "stage-2-mlp-localisation-via-causal-tracing.", + "sec:automated-interpretability" ] }, { @@ -1411,29 +1409,29 @@ "family": "representation-analysis", "sections": [ "subsec:steering", - "overview.", - "relation-to-causal-evaluation-of-feature-extraction-methods.", - "steering-vectors-practical-behavioral-control" + "steering-as-a-causal-test-for-feature-interpretability.", + "subsubsec: steering vectors" ], - "definition": "Representation engineering aims to control model behavior by intervening directly in activation space rather than modifying weights. A prominent class of methods, generally defined as activation steering, biases model outputs by adding carefully chosen directions to intermediate activations—most commonly the residual stream—during inference.", + "definition": "To manipulate model behaviour, representation engineering alters activation-space representations directly, rather than via model weight modification. Activation steering generally refers to methods that target intermediate activations (usually the residual stream) by adding certain vectors that shift model outputs during inference.", "assumptions": [ - "These interventions are lightweight, require no retraining, and are therefore attractive for rapid behavioral control in deployed systems.", - "In practice, v is estimated by contrasting activations associated with different behaviors, e.g., taking differences of class-conditional means" + "Because they require no weight updates or retraining, these interventions are lightweight approaches for controlling behaviour.", + "Another popular method is difference-in-means, which obtains the steering vector by taking the difference between activations associated with two behaviours (Rimsky et al., 2024)" ], "evaluation": [ - "Such intervention-based evaluations directly measure the causal effects of extracted features on model outputs.", - "Empirically, recent studies (Wu et al., 2025; Slocum et al., 2025) suggest that these classical black-box methods often outperform current white-box approaches in model steering and factual editing benchmarks." + "If intervening along a direction consistently changes a target behaviour, this provides evidence that the corresponding representation causally influences that behaviour.", + "Supporting evidence can be obtained by steering along the identified feature direction and evaluating the resulting change in behaviour. These evaluations have been used to compare interpretable direction-identification approaches, including SAE features, difference-in-means directions, and DAS (Huang et al., 2024; Wu et al., 2025b).", + "Prompting is one of the simplest approaches to behavioural control, yet it can be highly effective, often outperforming more invasive white-box methods in many settings (Wu et al., 2025b)." ], "applications": [ - "Despite their simplicity, steering vectors reliably induce behaviors like improved helpfulness, refusal suppression, or stylistic changes (Rimsky et al., 2024; Tan et al., 2024a; Zhao et al., 2025).", - "Researchers were able to identify and deploy the steering vector that suppresses this awareness, allowing its safety to be evaluated.", - "Their primary advantage is computational efficiency (no retraining required), methodological simplicity, and inference-time reversibility." + "While these approaches appear simple, they reliably change model behaviours such as refusal, helpfulness, and stylistic tendencies (Rimsky et al., 2024; Zhao et al., 2025; Arditi et al., 2024).", + "Researchers identified and applied a steering vector that suppresses this awareness, allowing evaluation of its actual behaviour.", + "Steering vectors, introduced in Section 9.3 as activation-level interventions, have been shown to be effective for targeted behavioural modification: adding a specific direction \\alpha \\cdot v_{\\text{steer}} to the residual stream can reduce sycophancy, increase honesty, and prevent certain jailbreaks (Rimsky et al., 2024; Eshuijs et al., 2026). Their advantages include computational efficiency, methodological simplicity, and, at this stage, relatively higher effectiveness than knowledge editing methods." ], "limitations": [ - "First, they often fail to generalize across distribution shifts: a vector derived for one task may degrade in another.", - "As a result, steering remains primarily a behavioral tool rather than a mechanistic one.", - "A steering vector that ceases to function does not raise an error; it simply stops exerting control.", - "Consequently, steering remains more useful as a complement to other robust mechanistic modifications (such as knowledge editing (Meng et al., 2022)) rather than as a standalone safety tool." + "While steering methods effectively control behaviours and support causal testing, they have limitations. Notably, they can be brittle, as a direction identified for one distribution, model, or task may not generalise reliably.", + "Another limitation is that successful behavioural steering does not, by itself, establish mechanistic transparency, because an intervention may simultaneously impact multiple features, interact nonlinearly with existing computations, or exploit unintended pathways that produce the desired behaviour without corresponding to a clean mechanism (Makelov et al., 2024). Thus, steering should generally be interpreted as evidence for causal influence, but it should not, by itself, be a mechanistic explanation of the behaviour.", + "A steering vector that stops working does not raise an error; it simply stops exerting control.", + "Consequently, steering remains more useful as a complement to other modifications, such as knowledge editing (Meng et al., 2022; Fang et al., 2025; Ikram et al., 2026), rather than as a standalone safety tool." ], "related": [ "optimization-based-steering", @@ -1446,27 +1444,30 @@ ], "mentioned_in": [ "subsec:cbm", - "case-study-refusal-vectors-and-sleeper-agents", - "pragmatic-interpretability-proxy-task-oriented-safety-interventions", + "the-need-for-standardised-evaluation", + "jacobian-lens-decoding-computational-dispositions", + "subsec: pragmatic interp", + "choosing-appropriate-methods-for-different-safety-applications", + "sec:automated-interpretability", "distribution-shift-vulnerability.", "silent-failure-modes.", - "dual-use-risks.", + "dual-use-and-adversarial-exploitation.", "sec:mechanistic-alignment" ] }, { "id": "optimization-based-steering", - "name": "Optimization-Based Steering", + "name": "Optimisation-Based Steering", "family": "representation-analysis", "sections": [ - "optimization-based-steering." + "nonlinear-and-optimisation-based-steering." ], - "definition": "Beyond closed-form constructions, steering directions can be obtained by directly optimizing objectives over activations—maximizing task performance or suppressing undesired outputs under intervention.", + "definition": "Separately, optimisation-based methods construct steering directions by directly optimising objectives over intermediate activations, offering an alternative to closed-form constructions.", "assumptions": [], "evaluation": [], "applications": [], "limitations": [ - "These optimization-based approaches allow greater flexibility and adapt to complex, non-linear objectives, but incur higher computational cost (Dunefsky & Cohan, 2025; Oozeer et al., 2025a)." + "These optimisation-based approaches offer greater flexibility for complex or nonlinear objectives but generally require substantially more computation than closed-form contrastive methods (Dunefsky & Cohan, 2025; Oozeer et al., 2025)." ], "related": [ "steering-vectors" @@ -1478,55 +1479,62 @@ "name": "Concept Erasure via PISCES", "family": "safety-applications", "sections": [ - "concept-erasure-via-pisces" + "concept-erasure-via-pisces", + "stage-1-feature-disentanglement-and-identification.", + "stage-2-mlp-vector-and-feature-selection.", + "stage-3-concept-removal.", + "result." ], - "definition": "PISCES (Precise In-Parameter Concept Erasure in Large Language Models) (Gur-Arieh et al., 2025b) addresses this limitation through a three-stage methodology.", + "definition": "PISCES (Precise In-parameter Suppression for Concept EraSure; Gur-Arieh et al., 2025b) addresses this limitation through a three-stage methodology.", "assumptions": [ - "Stage 1 – Feature Disentanglement and Identification: Use SAEs with feature explanations (via Automated Interpretability pipeline) to disentangle and identify features associated with the target concept c (e.g., “biological toxins”)" + "Use sparse autoencoders (SAEs) with feature explanations (via the Automated Interpretability pipeline) to disentangle and identify features associated with the target concept c (e.g., “biological toxins”)" ], "evaluation": [ - "Result: “Surgical” erasure that reduced accuracy on prohibited concepts to near-chance levels (7.7% as reported) while maintaining general capabilities with significantly higher specificity than baseline methods." + "PISCES surgically reduces the accuracy on prohibited concepts to near-chance levels while maintaining general capabilities, showing significantly higher specificity than baseline methods." + ], + "applications": [ + "Unlike approaches that primarily suppress unwanted outputs, PISCES directly modifies model parameters associated with the target concept." ], - "applications": [], "limitations": [ - "Likewise, concept-erasure methods such as PISCES may leave residual capabilities accessible through unexpected prompts, allowing suppressed behaviors to re-emerge without explicit signals of failure.", - "Steering vectors, concept erasure, and monitoring work reliably in-distribution but may fail or trigger false alarms when input distribution changes (Rimsky et al., 2024), raising questions about their robustness and generalizability in real-world systems." + "Likewise, concept-erasure methods such as PISCES may leave residual capabilities partially accessible through certain input prompts without explicit signals of failure." ], "related": [ "sae" ], "mentioned_in": [ + "concept-erasure-and-knowledge-editing", "silent-failure-modes.", - "sec:limitations-open-questions", - "sec:automated-interpretability" + "choosing-appropriate-methods-for-different-safety-applications" ] }, { "id": "rome-memit", - "name": "Knowledge Editing via Localization (ROME, MEMIT)", + "name": "Knowledge Editing via Localisation (ROME, MEMIT)", "family": "safety-applications", "sections": [ - "subsec:knowedit" - ], - "definition": "For example, ROME (Meng et al., 2022) and MEMIT (Meng et al., 2023) serve as practical tools for component-based knowledge editing: they apply localized rank-one updates to parameter matrices identified as storing particular factual knowledge, achieving targeted removal of harmful knowledge with minimal collateral damage.", - "assumptions": [ - "Once knowledge is attributed to specific model components via localization techniques like causal tracing, mechanistic approaches enable direct parameter modification." + "subsec:knowedit", + "stage-1-corrupted-and-clean-forward-passes.", + "stage-2-mlp-localisation-via-causal-tracing.", + "stage-3-rank-one-model-editing.", + "result.-2" ], + "definition": "Once knowledge has been localised to specific model components, such as individual neurons (Dai et al., 2022), using methods such as causal tracing (see Section 9.2), mechanistic approaches can support direct parameter modification. For example, ROME (Meng et al., 2022) and MEMIT (Meng et al., 2023) are two classic methods for component-targeted knowledge editing. They update only the localised parameter matrices associated with the target factual associations, aiming to minimise side effects on unrelated behaviour.", + "assumptions": [], "evaluation": [], "applications": [], "limitations": [ - "As a side note, knowledge editing presents a double-edged safety challenge.", - "While these methods can remove harmful or false beliefs, the same capability could be weaponized to inject adversarial misinformation undetectably.", - "Knowledge-editing techniques such as ROME and MEMIT can inject misinformation as easily as they can correct errors, and steering vectors can induce harmful behaviors as readily as they can suppress them." + "Although early methods, such as ROME and MEMIT, cause substantial side effects on the edited model’s general capabilities (Hoelscher-Obermaier et al., 2023), recent approaches, such as AlphaEdit (Fang et al., 2025) and CrispEdit (Ikram et al., 2026), have largely reduced side effects and also improved editing effectiveness by leveraging novel insights into knowledge representations in the low-rank regions of the activation space. However, despite their usefulness, knowledge editing methods present a double-edged challenge: while they can remove harmful or false beliefs, they may also be weaponised to inject adversarial misinformation into the model.", + "Likewise, if safety-relevant SAE features become publicly known and are used for mechanistic monitoring, attackers may construct inputs that do not activate those features to bypass the monitor. Also, knowledge editing methods such as ROME (Meng et al., 2022) and MEMIT (Meng et al., 2023) can inject misinformation as readily as they can correct errors, while steering vectors can induce harmful behaviours as easily as they can suppress them." ], "related": [ "causal-tracing", "steering-vectors" ], "mentioned_in": [ - "dual-use-risks.", + "concept-erasure-and-knowledge-editing", "sec:automated-interpretability", - "sec:mechanistic-alignment" + "choosing-appropriate-methods-for-different-safety-applications", + "dual-use-and-adversarial-exploitation." ] }, { @@ -1534,19 +1542,24 @@ "name": "Mechanistic Monitoring", "family": "safety-applications", "sections": [ - "mechanistic-monitoring" + "subsubsec: mech monitor", + "stage-1-feature-disentanglement-and-explanation.", + "stage-2-circuit-discovery.", + "stage-3-monitoring-the-circuit.", + "result.-4" ], - "definition": "Behavioral monitoring watches outputs; mechanistic monitoring watches internals.", + "definition": "Behavioural monitoring (Section 12.1) watches outputs; mechanistic monitoring watches internals. The latter asks: is the model computing deceptive reasoning? Is it hiding misaligned objectives?", "assumptions": [ - "Methods for achieving mechanistic monitoring include, but are not limited to, layer-wise monitoring (using linear probes to classify if malicious intent is encoded in a layer representation), feature-wise monitoring (alerting when unexpected SAE features activate), and circuit-wise monitoring (tracking causal effects among attention heads, MLPs, or features)." + "Methods for achieving mechanistic monitoring include, but are not limited to, layer-wise monitoring (using linear probes to classify if malicious intent is encoded in a layer representation) (Oldfield et al., 2026), feature-wise monitoring (alerting when unexpected SAE features activate) (Fang et al., 2026), circuit-wise monitoring (tracking causal effects among attention heads, MLPs, or features) (Zhao et al., 2026b), and anomaly-based monitoring (using activations in normal behaviour as a baseline and flagging harmful generations as outliers) (Chaudhary & Barez, 2026)." ], "evaluation": [], "applications": [ - "The advantages of mechanistic monitoring over behavioral monitoring are several: deceptive reasoning may not manifest behaviorally but may be detectable mechanistically." + "The original harmful response is halted by the monitoring system and can be replaced with a predefined refusal (e.g., “I can’t help with that request.”). Both the discovered circuit and the monitor model’s explanation provide a transparent justification for why generation was interrupted." ], "limitations": [ - "However, technical challenges also exist, such as determining what to monitor (millions of features), false positives (legitimate reasoning that looks suspicious), and adaptive evasion (models learning to hide suspicious activity if monitored).", - "Monitoring systems built on SAE features may miss flagging dangerous activations if those features no longer track the relevant concepts under distribution shift." + "Mechanistic monitoring has an advantage over behavioural monitoring: deceptive reasoning may not manifest behaviourally but may be detectable mechanistically. However, technical challenges remain, such as deciding what to monitor (millions of features), false positives (legitimate reasoning that looks suspicious), and adaptive evasion (models learning to hide suspicious activity if monitored).", + "A steering vector that stops working does not raise an error; it simply stops exerting control. Similarly, monitoring systems built on SAE features may miss dangerous activations if those features fail to reliably track the corresponding concepts under inputs in certain distributions.", + "Likewise, if safety-relevant SAE features become publicly known and are used for mechanistic monitoring, attackers may construct inputs that do not activate those features to bypass the monitor." ], "related": [ "probing", @@ -1554,7 +1567,11 @@ "circuit-tracing" ], "mentioned_in": [ - "silent-failure-modes." + "sec:scope", + "steering-vectors-and-mechanistic-monitoring", + "choosing-appropriate-methods-for-different-safety-applications", + "silent-failure-modes.", + "dual-use-and-adversarial-exploitation." ] } ], @@ -1564,55 +1581,43 @@ "name": "The Linear Representation Hypothesis", "sections": [ "subsec:linearity", - "linear-representation-hypothesis" + "conceptual-basis.", + "empirical-evidence-and-counterexamples." ], - "definition": "The linear representation hypothesis claims that directions in the activation space are the fundamental units of representation." + "definition": "The linear representation hypothesis claims that linear directions in the activation space are effective fundamental units of representation (Park et al., 2024)." }, { "id": "superposition", "name": "The Superposition Hypothesis", "sections": [ "subsec:superposition", - "architectural-responses-to-superposition", - "superposition-and-high-dimensional-geometry" + "conceptual-basis.-1", + "empirical-evidence.", + "architectural-responses-to-superposition." ], - "definition": "The superposition hypothesis says that for each layer of a trained model there are more features than nodes." + "definition": "The superposition hypothesis says that models encode more features than their hidden representations have dimensions." }, { "id": "privileged-access", "name": "The Privileged Access Hypothesis", "sections": [ - "subsec:privileged-access" + "subsec:privileged-access", + "conceptual-basis.-2", + "empirical-evidence.-1" ], - "definition": "A model used to interpret itself can, under appropriate elicitation, access internal variables and intermediate computations that are not reliably recoverable from external behavioral analysis alone, providing a comparative advantage as an interpreter of its own behavior" + "definition": "A language model used to interpret itself can, under appropriate elicitation, access internal variables and intermediate computations that are not reliably recoverable from external behavioural analysis alone, providing a comparative advantage as an interpreter of its own behaviour (Binder et al., 2025; Li et al., 2025a; Song et al., 2026)." }, { "id": "universality", - "name": "The Universality and Platonic Representation Hypotheses", + "name": "The Universality and Platonic Representation Hypothesis", "sections": [ - "subsec:universality" + "subsec:universality", + "conceptual-basis.-3", + "empirical-evidence-and-trade-offs.", + "implications." ], - "definition": "Independently trained models on similar tasks tend to develop similar internal computational structures. These representations often approximate a task-determined canonical form (Huh et al., 2024; Olah et al., 2020)." + "definition": "Models independently trained on similar tasks tend to develop similar internal representations or algorithms, which often approximate a task-determined canonical form (Huh et al., 2024; Olah et al., 2020). The degree of convergence depends on factors such as model size, architecture, and training data." } ], - "gaps": [ - "Heading/ID typos: 1.2 heading 'Hnadbook Scope'; section ids 'useful-surveys-that-we-bild-on.', 'challenges-of-black-box-bethods'; 6.8.2 heading 'Principle Component Analysis (PCA)' while the body says 'Principal Component Analysis'. Body typos include 'citcuits' (7.4), 'Definiton 2' (8.3), 'largley' (4.3), 'targed' (4.4), 'Concept-based methods plays' (3.2).", - "Section ids containing spaces: 'subsec:valproto agentic self-reports' and 'subsec:choosing orientation'.", - "Induction heads get three separate case-study sections: 6.6 (case-study-induction-heads), 7.5 (case-study-induction-heads-1) and 10.6 (case-study-induction-heads-2).", - "Grokking is treated in three places: 10.1 (sec:grokking), and twice in chapter 11 (withheld). SLT is treated in both 10.3 (sec:slt-llc) and chapter 11. Phase changes appear twice in chapter 11.", - "Chapters 10 and 11 overlap in scope. Much of chapter 11 is still draft authoring notes rather than manuscript text (in 11.2, 11.2.1, 11.2.2, 11.2.4, 11.2.5, 11.3.3, 11.3.4, 11.4.x, 11.5.x, 11.6 and 11.6.1), so the chapter is withheld from the site. 11.2.1 also points to a non-existent 10.2.2.", - "Section 12 (sec:representation) has no introductory text; 12.1 jumps straight into model diffing.", - "SPARTAN: the 6.9.2 text says it applies 'L_1 regularization to the attention patterns', but Table 7 lists its mechanism as 'L_0 regularization on attention'.", - "Causal scrubbing: 7.4 defines it as a necessity test, with interchange interventions as the sufficiency test. 7.6.6, however, labels a protocol 'Evaluation protocol (Causal Scrubbing)' that also covers sufficiency, interchange and specificity tests.", - "Gated SAE: 6.2 writes the gate as an indicator (hard threshold), but Appendix C.3 writes it as a sigmoid gate. The recommended Top-k k also differs: 'k = 50–100' in Table 4 versus 'typically 50–200' in Appendix C.4.", - "Evaluation dimensions differ: 7.6 proposes five (faithfulness, completeness, stability, causal validity, human interpretability), but 15 (proposed-evaluation-framework) proposes four (interpretability, completeness, validity, efficiency). Appendix C.5 'Absorption ratio' also means something different from 'feature absorption' in 6.7.3.", - "Cross-reference mismatches: 5.1 says 'see Figure 2' for the privileged-basis figure, which is Figure 3; 5.2 says 'see Figure 3' for superposition, which is Figure 4; 6.1 says '(Figure 4)' for the SAE, which is Figure 5; 6.4.2 says 'as shown in Figure 6' for CLT size, which is Figure 8; 12.1 says '(Figure 14)' for crosscoder diffing, which is Figure 16 (Figure 14 is probing). 6.1 cites 'Appendix 19', but the SAE appendix is C. 6.3.4 says 'Heuristics in Table 5', but the heuristics are in Table 4. 7.2 says SAE-based circuit discovery is introduced in '6.4.1 and 6.4.2', which are the transcoder/CLT sections.", - "Unresolved citation placeholders in 5.3: '(binder?; song?; B. Z. Li et al. 2025)'.", - "The acronym ACDC is never expanded. 'Attribution graphs' are only named in the 6.4.2 paragraph heading and in 1.1; there is no dedicated section, so they are folded into cross-layer-transcoders and circuit-tracing.", - "Methods mentioned without their own section, so no method entry was made: Activation Oracles and Predictive Concept Decoders (one sentence in 8.5 limitations); Concept Influence and distributional TDA (9.2.8); INLP (8.6.5); CAV / linear concept classifiers, CEMs, ProbCBMs, CGMs, SCBMs (3.2); Diff-SAEs, stage-wise SAE finetuning, transcoder-adapter diffing and LLM-based diffing (all inside 12.1 model diffing); rank analysis via SVD (12.2, uncited); the optogenetics-inspired clamping framework (10.6); PBRF/GGN (folded into influence functions); Semi-NMF (folded into NMF); MDL probes and control tasks (folded into probing evaluation); refusal vectors and sleeper-agent linear detectors (8.3 and 8.6.9, folded into steering/probing).", - "DAS is recommended repeatedly (6.3 Step 1, 6.7.6, 8.6.9, 12.3) but described in only one sentence in 6.8.4.", - "Excluded as orientations per instructions, though method-like: 13.7 Automated Interpretability (the LLM feature-explanation pipeline) and 14.4 Automated Interpretability Agents. The 7.6 Unified Evaluation Framework is treated as evaluation, not a method.", - "The 8.3 refusal-vector definition and 8.6.9 sleeper-agent probe live in the logit-lens/probing chapter, but 12.2 and 12.3 reuse them as knowledge-localisation and steering examples.", - "Several excerpts are verbatim table cells rather than full sentences, used where the prose gives no limitation or evaluation statement: activation-patching, path-patching and acdc (Table 8), circuit-tracing (Table 8). Some definitions are leading clauses that end where the source switches into maths: gated-sae, activation-patching, solu, tuned-lens." - ] + "gaps": [] } diff --git a/src/data/orientations-arxiv.json b/src/data/orientations-arxiv.json index 91373ba443..b37fb4ae82 100644 --- a/src/data/orientations-arxiv.json +++ b/src/data/orientations-arxiv.json @@ -70,7 +70,7 @@ "subtitle": "Towards Complete Mechanistic Understanding", "section": "12.2", "section_title": "Ambitious Interpretability: Towards Complete Mechanistic Understanding", - "definition": "In contrast to pragmatic approaches, Ambitious Interpretability seeks a comprehensive mechanistic understanding of how models implement their computations (Gao, 2025). Proponents argue that partial or surface-level understanding cannot guarantee safety in next-generation systems.", + "definition": "Ambitious Interpretability seeks a comprehensive mechanistic understanding of how models implement their computations (Gao, 2025). Proponents argue that partial or surface-level understanding cannot guarantee safety in next-generation systems.", "list_label": "Example metrics", "list_terms": [ "Feature quality", @@ -109,7 +109,7 @@ "subtitle": "Characterising How Models Learn", "section": "12.3", "section_title": "Developmental Interpretability: Characterising How Models Learn", - "definition": "Complementing post-hoc analysis of trained models, Developmental Interpretability (Hoogland et al., 2023; Hoogland et al., 2025) treats the training process itself as an object of scientific inquiry. Taking a learning-theory perspective, it aims to mathematically formalise the emergence and evolution of model representations, effective complexity, and capabilities throughout training.", + "definition": "Developmental Interpretability (Hoogland et al., 2023; Hoogland et al., 2025) treats the training process itself as an object of scientific inquiry. Taking a learning-theory perspective, it aims to mathematically formalise the emergence and evolution of model representations, effective complexity, and capabilities throughout training.", "list_label": "Methods", "list_terms": [ "Singular Learning Theory (SLT)", @@ -145,7 +145,7 @@ "subtitle": "Understanding the Causes of Model Behaviours", "section": "12.4", "section_title": "Interpretability Across Time: Understanding the Causes of Model Behaviours", - "definition": "Like Developmental Interpretability, Interpretability Across Time (Biderman et al., 2023b; 2026) treats a model as a time-evolving process shaped by its training data, loss functions, architectures, and optimisation dynamics. However, the two orientations differ in their primary goals.", + "definition": "Interpretability Across Time (Biderman et al., 2023b; 2026) treats a model as a time-evolving process shaped by its training data, loss functions, architectures, and optimisation dynamics. However, the two orientations differ in their primary goals.", "list_label": "Methods", "list_terms": [ "Longitudinal analysis of model internals", diff --git a/src/data/published-urls.json b/src/data/published-urls.json index 778c23b6b0..b7550d074d 100644 --- a/src/data/published-urls.json +++ b/src/data/published-urls.json @@ -1,5 +1,5 @@ { - "about": "Every URL and in-page anchor published at https://tsglab.github.io/mechinterp-safety/ (and earlier at /mi_survey/). These must keep resolving; see docs/MAINTAINING.md#stable-urls.", + "about": "Every URL and in-page anchor published at https://tsglab.github.io/mechinterp-safety/ (formerly /mi_survey/, which redirects). These must keep resolving; see docs/MAINTAINING.md#stable-urls.", "snapshots": [ { "date": "2026-11-06", @@ -8,13 +8,19 @@ { "date": "2026-11-06", "note": "stable chapter ids (phase 2, PR 1): adds /handbook// pages" + }, + { + "date": "2026-11-06", + "note": "arXiv v1 (2611.08438) ingested as final; adds learning-dynamics content and acknowledgements" } ], "pages": { "": [ + "bibtex-src", "cite", "parts-h", "sketch-cap", + "taxonomy", "use-h" ], "demos/": [], @@ -385,6 +391,9 @@ "app:notation", "dimensions-and-scaling" ], + "handbook/acknowledgements/": [ + "acknowledgments" + ], "handbook/b-color-key-and-style-guide/": [ "app:citation-colors", "app:colors", @@ -403,55 +412,92 @@ ], "handbook/circuits/": [ "applying-the-framework-method-specific-guidance", + "behavioural-coverage.", "case-study-induction-heads-1", "causal-necessity-and-sufficiency.", "circuit-discovery-in-weight-sparse-architectures", "circuits-visible-by-inspection.", "comparison-of-computational-costs.", + "consistency-of-interpretation.", + "counterfactual-effect-recovery.", + "cross-seed-agreement.", + "debugging-utility.", "def:acdc", "def:attribution-patching", + "def:circuit-tracer", + "def:eap", "def:path-patching", + "dimension-1-attribution-faithfulness", "dimension-1-faithfulness", "dimension-2-completeness", "dimension-3-stability", "dimension-4-causal-validity", "dimension-5-human-interpretability", + "effect-size-error.", "evaluating-circuits-and-benchmarking-discovery-methods.", + "evaluation-protocol", + "evaluation-protocol-1", + "evaluation-protocol-2", + "evaluation-protocol-3", + "evaluation-protocol-4", + "explanation-utility.", "fig:circuit", "five-evaluation-dimensions", + "hyperparameter-sensitivity.", "implications-for-verification.", "integrated-evaluation-protocol", + "intervention-effect-agreement.", "limitations-of-causal-verification.", "limitations-of-this-framework", "limitations.", + "loss-recovery.", + "necessity-score.", "sec:causal-verification", "sec:circuit", + "sec:circuit-discovery-methods", + "sec:circuit-replacement", "sec:circuit-table", "sec:evaluation-framework", "sec:what-are-circuits", + "specificity.", + "step-1-define-scope.", + "step-2-establish-baselines.", + "step-3-apply-interpretability-method.", + "step-4-evaluate-across-all-five-dimensions.", + "step-5-report-comprehensively.", + "subsec: induc head case study: circuit", + "subsec:five eval dim", + "sufficiency-score.", + "summary.-4", "tab:circuit-discovery-cost", "tab:circuit-methods", "tab:evaluation-framework", "tab:method-evaluation", - "the-need-for-standardized-evaluation" + "the-need-for-standardised-evaluation", + "the-need-for-standardized-evaluation", + "unexplained-variance." ], "handbook/colour-key/": [ "app:citation-colors", "app:colors", - "box-types-and-colors" + "box-types-and-colors", + "box-types-and-colours" ], "handbook/conclusion/": [ "achievements-and-limits", "from-reverse-engineering-to-engineering.", "sec:conclusion", + "the-road-ahead", "the-road-ahead-three-complementary-paradigms", "unresolved-questions" ], "handbook/core-hypotheses/": [ "architectural-responses-to-superposition", + "architectural-responses-to-superposition.", "conceptual-basis.", "conceptual-basis.-1", "conceptual-basis.-2", + "conceptual-basis.-3", "elicitation-protocols-to-reduce-confabulation.", "empirical-evidence-and-counterexamples.", "empirical-evidence-and-trade-offs.", @@ -466,6 +512,7 @@ "subsec:privileged-access", "subsec:superposition", "subsec:universality", + "summary.-2", "validation-protocols-and-causal-cross-checks.", "when-privileged-access-is-and-is-not-appropriate." ], @@ -505,21 +552,28 @@ ], "handbook/feature-extraction/": [ "activation-level-interventions-solu-and-its-limitations", + "analysis-goal.", "applications-to-circuit-analysis.", "architecture-and-motivation.", + "canonical-formulation.", "case-study-induction-heads", "causal-vs.-acausal-crosscoders.", "common-pitfalls-and-how-to-avoid-them", + "comparing-saes-transcoders-and-crosscoders", "comparing-saes-transcoders-and-crosscoders.", "comparison-with-alternative-feature-extraction-methods", "computational-cost-and-practical-considerations.", "computational-primitive-modifications-molt", + "conditions-for-exact-local-matching", + "conditions-for-reported-fidelity", + "crosscoders-analysing-cross-layer-features", "crosscoders-analyzing-cross-layer-features", "dead-features", "def:induction-heads", "dictionary-learning-baselines", "eq-2", "eq-3", + "factors-affecting-fidelity", "feature-absorption-and-splitting", "feature-duplication", "feature-instability-across-seeds", @@ -527,35 +581,50 @@ "fig:clt-size-vs-depth", "fig:crosscoders", "fig:sae", + "fig:transcoders", "from-monosemanticity-to-concept-identification.", "identifiability-and-theoretical-concerns", "independent-component-analysis-ica", "interpretability-gains.", "kolmogorov-arnold-networks", "layer-specific-recommendations", + "layer-type.", "limitations-of-these-recommendations", "mechanism.", "mechanism.-1", + "model-scale.", "motivation.", + "non-negative-matrix-factorisation-nmf", "non-negative-matrix-factorization-nmf", "open-questions-for-clt-research.", "practical-recommendations", + "principal-component-analysis-pca", "principle-component-analysis-pca", "quick-reference-when-to-use-what", "relation-to-causal-relevance.", "replacement-models-and-attribution-graphs.", + "resource-constraints.", "results-and-the-hydraulic-effect.", "sec:architectural-interventions", "sec:sae", "sparse-attention-post-training-spartan", "standard-formulation.", + "subsec: induc head case study: feature extract", "subsec:challenge", "subsec:sae-foundation", "subsec:sae-selection", "subsec:sae-variants", "subsec:transcoders-crosscoders", + "subsubsec: kan", + "subsubsec: molt", + "subsubsec: sae limitation", + "subsubsec: solu", + "subsubsec: spartan", + "subsubsec: supervised_extraction", + "subsubsec: weight sparse transformer", "subsubsec:cross-layer-transcoder", "subsubsec:transcoder", + "summary.-3", "supervised-feature-extraction-methods", "tab:architectural-methods", "tab:decomposition-comparison", @@ -565,28 +634,36 @@ "tab:sae_recommendations", "the-bridge-methodology-interpretable-proxies-for-dense-models", "the-capability-interpretability-frontier.", + "the-sparse-coding-problem", "training-dynamics-and-evaluation.", "weight-sparsity-via-l_0-regularization", + "when-are-transcoders-necessary", "when-transcoders-are-necessary.", "why-clts-enable-better-circuit-analysis.", "why-saes-work." ], "handbook/foundational-concepts/": [ + "fig:circuit", + "neuron-polysemanticity.", "sec:foundations", "subsec:behavioral-vs-mechanistic", "subsec:circuits", "subsec:features", "subsec:limits", "subsec:polysemanticity", - "subsec:transformers" + "subsec:transformers", + "summary.-1", + "what-is-a-concept" ], "handbook/intermediate-representations/": [ "amnesic-probing-and-causal-probing", "amnesic-probing.", "best-practices-for-probing-studies", "case-study-refusal-vectors-and-sleeper-agents", + "causal-probing-via-activation-patching.", "causal-probing-via-interchange-interventions.", "circuit-fragility-and-robustness-concerns", + "concerns-of-probe-complexity", "control-tasks-and-selectivity", "critiques-of-attribution-and-saliency-methods", "def:refusal-vector", @@ -602,12 +679,19 @@ "footnotes", "implications-for-probing.", "implications-for-probing.-1", + "implications-of-attribution-faithfulness-for-probing", "implications-of-control-tasks.", + "interpretation-problem.", + "jacobian-lens-decoding-computational-dispositions", "key-findings-on-attribution-faithfulness", + "limitations-of-jacobian-lens.", "limitations-of-latentqa.", "limitations-of-patchscopes.", + "limitations-of-tuned-lens.", "limitations-of-vanilla-logit-lens.", + "methodological-standards-in-probing-research", "minimum-description-length-probes.", + "observation.", "practical-guidance.", "probe-complexity-vs.-causal-relevance", "probing-as-hypothesis-generation.", @@ -617,19 +701,29 @@ "subsec:latentqa", "subsec:patchscopes", "subsec:probing", + "subsec:tuned-lens", + "subsubsec: amnesic probing", + "summary.-5", "tab:probing-guidance", "the-probe-causality-problem", "tuned-lens-learning-per-layer-transform", "what-probing-can-show.", "what-probing-cannot-show.", + "when-is-probing-appropriate", "when-is-probing-appropriate-and-when-is-it-not" ], "handbook/interpretability-landscape/": [ "attribution-and-saliency-methods", "behavioral-methods", + "behavioural-methods", + "input-attribution-and-saliency-methods", + "mechanistic-interpretability-methods", "representation-probing-and-related-post-hoc-analyses", "sec:interpretability-landscape", + "subsec: mech interp necessary", "subsec:cbm", + "subsec:limits", + "summary.", "tab:interpretability-methods", "tab:semantic-vs-structural", "when-mechanistic-understanding-becomes-necessary" @@ -638,34 +732,89 @@ "brief-history-of-interpretability", "curiosity-and-discovery.", "fig:overview", + "first-from-manual-to-assisted-analysis.", "safety.", "sec:introduction", "sec:scope", + "second-divergence-in-research-priorities.", "situating-the-field.", - "useful-surveys-that-we-bild-on." + "useful-surveys-that-we-bild-on.", + "useful-surveys-that-we-build-on." + ], + "handbook/learning-dynamics/": [ + "acquisition-order.", + "architectural-modification-gradient-routing.", + "capability-specific-metrics.", + "fig:grokking", + "forecasting-from-smaller-runs-and-partial-runs.", + "forecasting-safety-relevant-behaviours.", + "from-correlation-to-causation.", + "internal-precursors-as-early-warning.", + "learning-dynamics-as-an-interpretability-tool", + "localising-change-to-components.", + "loss-curve-derivatives.", + "monitoring-and-steering-during-fine-tuning.", + "open-questions.", + "per-example-decomposition.", + "per-token-learning-curves.", + "pretraining-data-filtering-deep-ignorance.", + "run-to-run-variance.", + "saddle-to-saddle-dynamics.", + "sec:ld-attribution", + "sec:ld-causal-interventions", + "sec:ld-data", + "sec:ld-detect", + "sec:ld-features", + "sec:ld-forecasting", + "sec:ld-induction-case", + "sec:ld-intervening", + "sec:ld-localization", + "sec:ld-longitudinal", + "sec:ld-progress", + "sec:ld-safety", + "sec:ld-temporal-stability", + "sec:ld-transitions", + "sec:training", + "sec:training_phase_why", + "singular-learning-theory.", + "summary.-7", + "trajectory-based-attribution." ], - "handbook/learning-dynamics/": [], "handbook/limits-of-behavioural-evaluation/": [ + "critical-finding.", "eq-1", + "first-test-set-contamination-and-saturation.", + "fourth-lack-of-fine-grained-control.", + "interpretation.", "sec:why_open", + "second-benchmark-gaming-models-can-learn-the-wrong-mechanisms.", + "third-evaluation-awareness.", + "training-objective.", "war:sleeper" ], "handbook/mathematical-foundations/": [ "app:math", + "app:slt", "causal-abstraction", "causal-inference-basics", + "entropy.", "eq-10", "eq-11", "eq-12", "eq-13", "eq-14", "eq-15", + "eq-4", + "eq-5", + "eq-6", "eq-7", "eq-8", "eq-9", "generalization-bounds", "information-theory-basics", + "kl-divergence.", "linear-representation-hypothesis", + "mutual-information.", "necessity", "phase-transitions", "singular-learning-theory-slt", @@ -676,9 +825,16 @@ ], "handbook/notation/": [ "app:notation", - "dimensions-and-scaling" + "dictionary-learning", + "dimensions-and-indices", + "dimensions-and-scaling", + "inputs-and-activations", + "interventions-and-objectives", + "mathematical-operations", + "model-parameters" ], "handbook/open-problems/": [ + "challenges-in-evaluation-and-benchmarking", "eq-5", "how-to-evaluate-mechanistic-interpretability-methods", "potential-approaches.", @@ -691,33 +847,62 @@ "sec:open_problems", "sec:proxy-fidelity", "standardized-benchmarks", + "subsec: scalability and adversarial robustness", "when-is-mechanistic-understanding-necessary", "why-this-matters-for-safety." ], "handbook/representation-analysis/": [ + "applications.", + "def:refusal-vector", + "dictionary-learning-methods.", "fig:diffing", "fig:steering_vectors", + "llm-based-approaches.", + "nonlinear-and-optimisation-based-steering.", "optimization-based-steering.", "overview.", "relation-to-causal-evaluation-of-feature-extraction-methods.", + "representation-alignment-and-convergent-learning.", "sec:knowledge-localization", "sec:model-diffing", "sec:representation", - "subsec:steering" + "steering-as-a-causal-test-for-feature-interpretability.", + "subsec:steering", + "summary.-6" ], "handbook/research-orientations/": [ "ambitious-interpretability-toward-complete-mechanistic-understanding", "architectural-interventions-as-a-synthesis.", + "automation-bias.", + "calibration-evaluation.", + "causal-intervention-consistency.", + "common-confusion.", "comparative-analysis-trade-offs-between-orientations", + "concerns-for-introspective-interpretability", + "confabulation.", + "critical-concerns-tensions-and-open-questions", "critical-tensions-and-open-questions", + "cross-method-consistency.", "defining-orientations-vs.-methods", "developmental-interpretability-understanding-phase-transition-during-training", + "elicitation-sensitivity.", + "error-propagation.", "fig:fiveresearchorientation", + "fig:sixresearchorientation", "governance-concerns-for-agentic-interpretability", "governance-concerns-for-automated-interpretability", + "implications.-1", "integrating-automated-and-human-analysis", + "interpretability-by-design-as-a-synthesis.", "key-principles-for-integration.", + "layer-1-sampling-based-human-validation.", + "layer-2-consistency-and-causality-checks.", + "layer-3-adversarial-robustness.", + "layer-4-error-detection-and-bounds.", + "layer-5-regulator-practices.", + "miscalibration.", "overview-why-five-orientations", + "paraphrase-and-sampling-consistency.", "pragmatic-interpretability-proxy-task-oriented-safety-interventions", "proposed-guardrails-for-agentic-interpretability", "proposed-guardrails-for-automated-interpretability", @@ -727,15 +912,29 @@ "protocol-2-cross-examiner-verification.", "protocol-3-behavioral-prediction-testing.", "protocol-3-drift-monitoring.", + "reduced-human-specific-functionalities.", "sec:agentic-interpretability", "sec:automated-interp", + "sec:intro-interpretability", "sec:orientations", + "spurious-feedback-loops.", + "stage-1-feature-extraction.", + "stage-2-explanation-generation.", + "stage-3-explanation-verification.", + "stage-4-deception-detection.", + "strategic-misreporting.", + "subsec: iat", + "subsec: pragmatic interp", + "subsec:ambitious", "subsec:choosing orientation", + "subsec:dev interp", + "subsec:valproce model self-reports", "subsec:valproto agentic self-reports", "summary-responsible-automation", "tab:human-automated-workflow", "tab:orientations", - "validation-protocols-for-automated-pipelines" + "validation-protocols-for-automated-pipelines", + "when-to-use-introspective-methods" ], "handbook/sae-architecture-details/": [ "app:sae_details", @@ -749,62 +948,118 @@ "top-k-sae-variant" ], "handbook/safety-applications/": [ + "a-note-on-openness-and-dual-use-research.", + "adversarial-risks-malicious-exploitation-of-mi", "adversarial-risks-malicious-exploitation-of-mi-insights", "alternatives-black-box-methods", + "alternatives-black-box-methods-and-their-challenges", "challenges-of-black-box-bethods", + "choosing-appropriate-methods-for-different-safety-applications", "circuit-informed-adversarial-attacks.", "concept-erasure-and-direct-editing", + "concept-erasure-and-knowledge-editing", "concept-erasure-via-pisces", + "data-filtering-and-safety-training", "distribution-shift-vulnerability.", + "dual-use-and-adversarial-exploitation.", "dual-use-risks.", + "epistemic-risks-false-confidence-from-partial-or-incorrect-understanding", "epistemic-risks-false-confidence-from-partial-understanding", "eq-4", + "from-research-orientations-to-methods", "implications-for-scale-and-oversight.", + "input-and-output-filtering", "mechanistic-monitoring", "mitigation-strategies.", "mitigation-strategies.-1", "mitigation-strategies.-2", "mitigation-strategies.-3", + "operational-risks-failure-under-deployment", "operational-risks-failure-under-deployment-conditions", "opportunity-costs-resource-allocation-risks", "overclaiming-causal-understanding.", + "prompting", "recommended-verification-practices", "relations-to-the-five-research-orientations", + "result.", + "result.-1", + "result.-2", + "result.-3", + "result.-4", "sec:automated-interpretability", "sec:limitations-open-questions", "sec:mi-risks", "sec:safety_applications", + "shared-limitations-of-black-box-approaches", "silent-failure-modes.", + "stage-1-construct-contrastive-pairs-of-examples.", + "stage-1-corrupted-and-clean-forward-passes.", + "stage-1-feature-disentanglement-and-explanation.", + "stage-1-feature-disentanglement-and-identification.", + "stage-1-feature-disentanglement-and-identification.-1", + "stage-2-circuit-discovery.", + "stage-2-generate-the-steering-vector.", + "stage-2-mlp-localisation-via-causal-tracing.", + "stage-2-mlp-vector-and-feature-selection.", + "stage-2-mlp-vector-and-feature-selection.-1", + "stage-3-concept-removal.", + "stage-3-concept-removal.-1", + "stage-3-inject-the-steering-vector-at-inference.", + "stage-3-monitoring-the-circuit.", + "stage-3-rank-one-model-editing.", "steering-vectors-and-mechanistic-monitoring", "steering-vectors-practical-behavioral-control", "subsec:knowedit", + "subsubsec: mech monitor", + "subsubsec: steering vectors", "summary-mi-as-one-methodology-among-many", "tab:mi-risks", + "tab:mi_safety_applications", "taxonomy-of-mi-safety-risks", "the-complete-understanding-trap.", "the-scalability-bottleneck.", + "user-profiling-and-identity-aware-safety", "why-mechanistic-approaches-for-safety" ], "handbook/training-data-attribution/": [ + "addressing-the-computational-and-memory-bottlenecks-of-ggn", + "completion-chinese", "computational-bottleneck-approximating-ggn-vector-products", "core-idea-parameter-response-to-training-data", "def:infini-gram", "def:maximal-matching-spans", "def:olmotrace-algorithm", "def:suffix-array", + "eigenvalue-corrected-k-fac-ek-fac-and-astra.", "eigenvalue-corrected-kfac-ek-fac-and-astra.", "ex:gradient-translation", "ex:olmotrace-multilingual", "ex:olmotrace-pipeline", + "example", + "example-spans-found", + "gradient-based-attribution-understanding-data-influence-through-perturbations", "gradient-based-attribution-understanding-influence-through-data-perturbations", + "gradient-based-influential-training-samples-identified", "illustrative-example", + "inputs", + "kronecker-factored-approximate-curvature-k-fac.", "kronecker-factored-approximate-curvature-kfac.", + "language-model-output", "limitations", + "limitations-as-a-mechanistic-interpretability-tool.", + "observation.-1", + "olmotrace-influential-samples-identified", + "pipeline-steps", + "prompt", + "prompt-english", "proximal-bregman-response-and-the-gauss-newton-hessian", + "proximal-bregman-response-and-the-generalised-gaussnewton", "recent-extensions-and-limitations", "sec:influence-functions", "sec:tracin", "sec:trak", + "steps", + "summary.-8", "unrolled-differentiation-gradient-based-attribution", "verbatim-data-attribution-olmotrace-and-maximal-matching-spans" ], @@ -895,19 +1150,37 @@ "pragmatic" ], "references/": [ + "ref-11370531", + "ref-3618408.3618656", + "ref-3692070.3693959", + "ref-Barlow:1961aa", "ref-Dalvi_Durrani_Sajjad_Belinkov_Bau_Glass_2019", "ref-Ethan2022redteam", "ref-Geirhos_2020", + "ref-ICLR2025_8c5f3029", + "ref-JMLR:v26:23-0058", "ref-NIPS2003_1843e35d", + "ref-NIPS2012_c399862d", + "ref-OMLIN199641", "ref-Raukur2022TowardTA", "ref-SuperGLUE", + "ref-abahana2026rlhf", + "ref-achille2019critical", "ref-adebayo2018sanity", "ref-adebayo2020sanitycheckssaliencymaps", "ref-adi2017finegrained", "ref-adler2025towards", + "ref-agarwal2025context", + "ref-agieval", "ref-aharon2006k", + "ref-aiscientistpitfall", "ref-alain2017understanding", + "ref-ali-etal-2025-hidden", + "ref-alphaedit", "ref-ameisen2025circuit", + "ref-and", + "ref-anil2024manyshot", + "ref-anthropic2024modeldiffing", "ref-anthropic2025claude-sonnet-4-5", "ref-anthropic2025claude4", "ref-anthropic2025molt", @@ -915,68 +1188,130 @@ "ref-aoyama_language_2025", "ref-apollo2025sonnet37evalaware", "ref-aranguri2025modeldiff", + "ref-arcuschin2026chainofthought", "ref-arditi2024refusallanguagemodelsmediated", + "ref-arora-etal-2024-causalgym", + "ref-arora2026language", + "ref-ba2016layernormalization", "ref-bae2022influencefunctionsanswerquestion", "ref-bae2024trainingdataattributionapproximate", "ref-baek_towards_2025", + "ref-bai-etal-2024-longbench", "ref-bai2022constitutional", "ref-bai2022training", + "ref-baker2025monitoringreasoningmodelsmisbehavior", "ref-barez2025automated", + "ref-barez2025chain", + "ref-basispursuit", + "ref-batra2026multimodalmodeldiffingfeature", "ref-bau2018identifying", + "ref-bau2020units", + "ref-bayazit-etal-2026-crosscoding", "ref-bayazit_crosscoding_2025", "ref-belinkov-2022-probing", "ref-belinkov-glass-2019-analysis", "ref-belinkov2017neural", + "ref-belrose2024statistics", "ref-belrose2025elicitinglatentpredictionstransformers", + "ref-bengio2013estimatingpropagatinggradientsstochastic", + "ref-bengio2025internationalaisafetyreport", "ref-bereska2024mechanisticinterpretabilityaisafety", + "ref-bhaskar2024finding", "ref-biderman2023emergent", "ref-biderman2023pythia", + "ref-biderman2026position", + "ref-bigbench", "ref-bills2023language", + "ref-binder", + "ref-blank2026rlens", + "ref-borkar2025privacy", "ref-braun2025interpretability", "ref-bricken2023monosemanticity", "ref-bricken2024stagewise", + "ref-bruijns2026pretrainingcurriculaenableselective", + "ref-buildingbetterao", "ref-burns2023discovering", "ref-burns2024discoveringlatentknowledgelanguage", "ref-bushnaq2025stochastic", "ref-bussmann2024batchtopksparseautoencoders", + "ref-cam", + "ref-cao-etal-2021-low", "ref-chan2022causal", + "ref-chan2022data", "ref-chang2022word", + "ref-chang2024characterizing", "ref-chanin2024absorption", + "ref-chanin2025absorption", + "ref-chatbotknows", "ref-chaudhary2024evaluating", "ref-chaudhary2025safetynetdetectingharmfuloutputs", + "ref-chaudhary2026beyond", + "ref-chen2021evaluatinglargelanguagemodels", "ref-chen2025low", + "ref-chen2025persona", + "ref-chen2025reasoningmodelsdontsay", "ref-chen_sudden_2024", + "ref-chive", + "ref-chloe", "ref-christiano2023deepreinforcementlearninghuman", "ref-clark2019what", "ref-cloud2024gradient", + "ref-cloud2026language", + "ref-conceptdef", "ref-conmy2023automatedcircuitdiscoverymechanistic", "ref-conneau2018what", "ref-count", + "ref-cox2008metareasoning", + "ref-crispedit", "ref-cunningham2023sparse", "ref-cywinski2025eliciting", + "ref-dai-etal-2022-knowledge", + "ref-dai-etal-2024-deepseekmoe", + "ref-dai2024safe", "ref-dalvi2022discovering", + "ref-dao202mamba2", "ref-das", "ref-dbm", + "ref-deeplift", "ref-deepseekai2025deepseekv3technicalreport", + "ref-deepseekai2026deepseekv4highlyefficientmilliontoken", + "ref-deepseekv2", + "ref-denison2024sycophancy", "ref-detectvul", "ref-devlin2019bert", "ref-distill_pub", "ref-dominici2025causal", "ref-dooms2025compositionality", "ref-draye2025spartan", + "ref-du-etal-2026-automated", "ref-du2023shortcut", + "ref-du2024understanding", "ref-dumas_what_2025", "ref-dunefsky2024transcodersinterpretablellmfeature", "ref-dunefsky2025one", + "ref-durmus2024towards", + "ref-eapgp", "ref-eapig", "ref-early-work-on-monitorability-evaluations", + "ref-ekfac", "ref-elazar-etal-2021-amnesic", + "ref-elazar2022measuring", "ref-elhage2021mathematical", "ref-elhage2022solu", "ref-elhage2022superposition", + "ref-elman1990finding", + "ref-elman1991distributed", "ref-engels2025languagemodelfeaturesonedimensionally", + "ref-ensign2024investigating", + "ref-eshuijs2026honestansweraidingllmjudges", + "ref-fang2026nextguardtrainingfreestreamingsafeguard", + "ref-fanous2025syceval", "ref-ferrando2024primer", + "ref-feucht2026arithmeticwildllamauses", + "ref-find", "ref-foote2023neurongraphinterpretinglanguage", + "ref-fornasiere2026languagemodelsrecognizedropout", + "ref-frankle2020linear", "ref-ganguli2022redteaminglanguagemodels", "ref-gao2020pile", "ref-gao2024scalingevaluatingsparseautoencoders", @@ -986,29 +1321,58 @@ "ref-ge2026evolution", "ref-ge_evolution_2025", "ref-geiger2021causalabstractionsneuralnetworks", + "ref-gemmateam2024gemma2improvingopen", "ref-gemmateam2024gemmaopenmodelsbased", "ref-georgiev2023journeydestinationdataguides", + "ref-geva-etal-2021-transformer", "ref-giulianelli-etal-2018-hood", + "ref-goel", "ref-goldowskydill2023localizingmodelbehaviorpath", + "ref-gqa", + "ref-gradcam", "ref-greenblatt2024alignmentfakinglargelanguage", + "ref-groeneveld2024olmo", + "ref-groger2026revisiting", "ref-grosse2023studyinglargelanguagemodel", + "ref-gsm8k", + "ref-gu2024mamba", + "ref-guo2026introspectivecouplingselfexplanationtraining", "ref-gupta-etal-2015-distributional", "ref-gur-arieh-etal-2025-enhancing", "ref-gurarieh2025preciseinparameterconcepterasure", + "ref-habibi2026symbolicmechanisticdataattribution", + "ref-hahami2026detectingdisturbancenuancedview", "ref-haider2025multi", + "ref-haider2026multigranular", "ref-haklay-etal-2025-position", + "ref-haklay2026pitfalls", + "ref-han2026sage", "ref-hanna2023how", + "ref-haon2025mechanistic", "ref-harrasse2025tinysqlprogressivetexttosqldataset", "ref-harrasse2025tracing", "ref-he-etal-2025-red", + "ref-he2024llamascopeextractingmillions", + "ref-heap2026automated", "ref-heimersheim2024useinterpretactivationpatching", + "ref-heindrich2025do", + "ref-helwe-etal-2025-navigating", "ref-hendrycks2021naturaladversarialexamples", + "ref-hernandez2022natural", "ref-hestness2017deeplearningscalingpredictable", "ref-hewitt-liang-2019-designing", + "ref-hewitt2019structural", "ref-hewitt2025position", + "ref-hinton1986distributed", + "ref-hinton1986learning", + "ref-hoelscher-obermaier-etal-2023-detecting", + "ref-hoffmann2022training", + "ref-hong-etal-2025-measuring", "ref-hoogland2023towards", "ref-hoogland2025losslandscapedegeneracystagewise", + "ref-hotelling1933analysis", "ref-hu2022lora", + "ref-hu2023latent", "ref-hu2026transcoderadaptersreasoningmodeldiffing", "ref-huang-etal-2024-ravel", "ref-hubinger2021riskslearnedoptimizationadvanced", @@ -1016,91 +1380,174 @@ "ref-huh2024platonicrepresentationhypothesis", "ref-hyvarinen2000independent", "ref-hyvarinen2023nonlinear", + "ref-icml2025actionable", "ref-ig", "ref-ilyas2022datamodelspredictingpredictionstraining", + "ref-inaba2025howllmslearn", + "ref-inan2023llamaguardllmbasedinputoutput", "ref-insufficient", + "ref-jacovi-goldberg-2020-towards", "ref-jacovi2020faithfullyinterpretablenlpsystems", "ref-jain2019attention", + "ref-jain2023baselinedefensesadversarialattacks", + "ref-jain2024mechanistically", + "ref-jain2024what", + "ref-jakkli2026current", + "ref-jiang2024mixtralexperts", + "ref-jimenez2024swebench", "ref-jiralerspong_cross-architecture_2025", + "ref-jlens", + "ref-joglekar", + "ref-jones2025forecasting", "ref-joshi2026causalitykeyinterpretabilityclaims", "ref-jumelet2024black", + "ref-juneja2023linear", "ref-kamath2025tracing", + "ref-kangaslahti2026hidden", "ref-kangaslahti2026hiddenbreakthroughslanguagemodel", "ref-kaplan2020scaling", "ref-karpathy2015visualizing", + "ref-karvonen2025saebench", + "ref-kempermann2026safe", "ref-kempf2026simplellmbaselinescompetitive", + "ref-kfac", + "ref-khan2026can", + "ref-kharlapenko2025scaling", "ref-kim2018tcav", "ref-kim2023probabilistic", "ref-kim2025llmspursueagenticinterpretability", + "ref-kim2025rethinking", + "ref-kimiteam2026kimik2openagentic", + "ref-kindermans2017unreliabilitysaliencymethods", "ref-kissane2024interpretingattentionlayeroutputs", "ref-klindt2025superpositionsparsecodesinterpretable", + "ref-koepke2026platoscaveexaminingcrossmodal", "ref-koh2020conceptbottleneckmodels", "ref-koh2020understandingblackboxpredictionsinfluence", "ref-kohn-2015-whats", + "ref-korbak2025chainthoughtmonitorabilitynew", "ref-kornblith2019similarityneuralnetworkrepresentations", "ref-korznikov2025ortsaeorthogonalsparseautoencoders", "ref-kowal2026concept", "ref-krishna2025disagreementproblemexplainablemachine", "ref-krizhevsky2012imagenet", + "ref-laban2026llms", + "ref-lampinen2019analytic", + "ref-lan-etal-2024-towards", + "ref-lan-etal-2026-make", + "ref-lan2025quantifyingfeaturespaceuniversality", "ref-lanham2023measuringfaithfulnesschainofthoughtreasoning", "ref-lau2023local", "ref-le2013building", "ref-leask2025sparse", + "ref-lecomte2024what", + "ref-lee1999learning", "ref-lee2006efficient", "ref-lee2025influence", "ref-lee2026rigorous", + "ref-lee2026steps", "ref-lei2025spartan", + "ref-lepikhin2021gshard", + "ref-lermen2025deceptive", "ref-lewicki2000learning", "ref-li-etal-2016-visualizing", + "ref-li2023emergent", "ref-li2024simulateeliminaterevokebackdoors", "ref-li2024wmdp", "ref-li2025geometry", "ref-li2025traininglanguagemodelsexplain", + "ref-li2026autocontrol", + "ref-li2026do", + "ref-li2026introspective", + "ref-lieberum2024gemmascopeopensparse", "ref-lindsey2024sparsecrosscoders", "ref-lindsey2025biology", "ref-lindsey2025landscape", + "ref-lindsey2026emergentintrospectiveawarenesslarge", "ref-linzen-etal-2016-assessing", + "ref-liu2021probing", + "ref-liu2023omnigrok", + "ref-liu2024repobench", "ref-liu2025infinigramscalingunboundedngram", "ref-liu2025kan", "ref-liu2025olmotrace", + "ref-liu2026icalensinterpretinglanguage", + "ref-llama3", "ref-llmexplainsurvey", + "ref-lo-etal-2024-large", + "ref-localinform", + "ref-lotfi2026position", + "ref-lrh", + "ref-lrp", + "ref-lubana2023mechanistic", "ref-lundberg2017unifiedapproachinterpretingmodel", "ref-macdiarmid2024sleeperagentprobes", "ref-macdiarmid_natural_2025", + "ref-makelov2024is", + "ref-marin2026automated", + "ref-marks2024enhancingneuralnetworkinterpretability", + "ref-marks2024interpreting", + "ref-marks2024the", "ref-marks2025auditinglanguagemodelshidden", "ref-marks2025sparse", "ref-marvin-linzen-2018-targeted", + "ref-matchingpursuit", "ref-mccoy-etal-2019-right", + "ref-mcdougall2025gemmascope2", "ref-mcgrath2023hydraeffectemergentselfrepair", + "ref-mckenzie2023inverse", + "ref-meloux2025everything", "ref-meng2023locatingeditingfactualassociations", "ref-meng2023masseditingmemorytransformer", "ref-merullo2025replicating", + "ref-michaud2023quantization", + "ref-miller2024transformer", "ref-miller2026identifyingintervenableinterpretablefeatures", "ref-minder_overcoming_2025", + "ref-minsky1969perceptrons", "ref-mlodozeniec2025distributionaltrainingdataattribution", + "ref-mmlu", "ref-mmlupro", + "ref-modarressi2025nolima", "ref-morcos2018insightsrepresentationalsimilarityneural", "ref-morcos_insights_2018", "ref-moritz2024sthocastic", + "ref-moschella2023relative", "ref-moschella2023relativerepresentationsenablezeroshot", "ref-mueller2025isolationentanglementinterpretabilitymethods", "ref-mueller2025mib", + "ref-nakkiran2019sgd", "ref-nanda2023progressmeasuresgrokkingmechanistic", "ref-nanda2025pragmatic", "ref-needham2025largelanguagemodelsknow", + "ref-neo-etal-2024-interpreting", + "ref-neo2025towards", + "ref-netdissect2017", + "ref-netmodular", + "ref-neuronpedia", "ref-ng2011sparse", + "ref-nla", "ref-nostalgebraist2020interpretinggpt", + "ref-obrien2026deep", + "ref-oikarinen2023clipdissect", "ref-oikarinen2023labelfree", + "ref-olah2017feature", "ref-olah2020zoom", "ref-oldfield2024multilinear", "ref-oldfield2025towards", + "ref-oldfield2026beyond", "ref-olshausen1996emergence", + "ref-olshausen1997sparse", "ref-olsson2022context", "ref-olsson2022incontextlearninginductionheads", + "ref-oneill2025compute", "ref-oozeer2025activation", "ref-oozeer2025beyond", "ref-openai2024gpt4technicalreport", + "ref-openai2025gptoss120bgptoss20bmodel", "ref-orgad2026interpretabilityactionable", + "ref-orgad2026position", + "ref-oshea2015introductionconvolutionalneuralnetworks", "ref-ott2022mapping", "ref-ouyang2022traininglanguagemodelsfollow", "ref-pan2024latentqateachingllmsdecode", @@ -1108,93 +1555,167 @@ "ref-park2023trakattributingmodelbehavior", "ref-park2025monet", "ref-patchscopes", + "ref-paulo2024does", "ref-paulo2025automatically", "ref-paulo2025sparse", + "ref-paulo2025transcodersbeatsparseautoencoders", "ref-pcd", "ref-pearce2025bilinear", + "ref-pearson1901lines", + "ref-peng-etal-2023-rwkv", + "ref-peng2025rwkv", "ref-perez2022redteaminglanguagemodels", "ref-petri2025", "ref-petri2026v2", + "ref-petrov2026brokenmath", "ref-phan2026hle", + "ref-philosophicalevalframework", "ref-piktus-etal-2023-roots", "ref-piktus2023gaia", + "ref-plunkett", + "ref-pmlr-v162-geiger22a", + "ref-pmlr-v267-fu25e", + "ref-pmlr-v37-sohl-dickstein15", "ref-pmlr-v44-li15convergent", "ref-power2022grokkinggeneralizationoverfittingsmall", + "ref-proofnet", "ref-pruthi2020estimatingtrainingdatainfluence", "ref-q", "ref-querycircuit", "ref-quirke2024understandingadditiontransformers", + "ref-quirke2026understandingadditionsubtractiontransformers", "ref-raghu_svcca_2017", + "ref-rahaman2019spectral", "ref-rai2024practical", "ref-rajamanoharan2024improvingdictionarylearninggated", "ref-rajamanoharan2024jumpingaheadimprovingreconstruction", "ref-rajani2020explainingimprovingmodelbehavior", + "ref-rathi2026shaping", + "ref-raval2026curveballsteeringrightdirection", "ref-ravfogel-etal-2020-null", + "ref-razeghi2022impact", + "ref-refinetti2023neural", + "ref-relu", "ref-ribeiro2016whyitrustyou", + "ref-rnnfsa1992", "ref-rnnnonlinear", "ref-rogers2020primer", "ref-rosenfeld2019constructivepredictiongeneralizationerror", + "ref-rosenfeld2020a", "ref-sajjad-etal-2022-neuron", "ref-saphra2024mechanistic", "ref-saphra_interpretability_2023", + "ref-sawmya2025birth", + "ref-saxe2014exact", "ref-saxe2019mathematical", + "ref-schaeffer2023emergent", + "ref-schaeffer2025elusive", "ref-schaeffer2026quantifyingeffecttestset", "ref-schnass2008dictionary", + "ref-schrodi2026towards", "ref-selfie", + "ref-sellam2021multiberts", "ref-seo2018noise", + "ref-sgd", + "ref-shafran-etal-2026-constructing", "ref-shaham2024multimodal", "ref-sharkey2025open", + "ref-sharma2024locating", + "ref-sharma2024the", + "ref-sharma2025constitutionalclassifiersdefendinguniversal", "ref-sharma_towards_2025", + "ref-shazeer2020gluvariantsimprovetransformer", "ref-shi-etal-2016-string", + "ref-shilov2025beyond", + "ref-shortliffe1977mycin", "ref-shu-etal-2025-survey", + "ref-simhi-etal-2025-trust", + "ref-simhi2026old", + "ref-simon2026there", "ref-simonyan2014deepinsideconvolutionalnetworks", + "ref-singh2023transient", "ref-singh2024needsrightinductionhead", "ref-slocum2025believe", "ref-smds", + "ref-smilkov2017smoothgradremovingnoiseadding", "ref-smith2024strongfeature", + "ref-snell2024predicting", "ref-snmf", + "ref-song", + "ref-sparsecoding", + "ref-sutter2025the", + "ref-syed2023attributionpatchingoutperformsautomated", "ref-szegedy2015going", "ref-tan2024analysing", "ref-tan2025analyzinggeneralizationreliabilitysteering", "ref-tang-etal-2023-large", + "ref-taylor2025school", "ref-templeton2024scaling", "ref-tenney2019bertrediscoversclassicalnlp", "ref-thasarathan_universal_2025", "ref-tigges2024llm", + "ref-tinaz2025emergence", "ref-touvron2023llamaopenefficientfoundation", "ref-turner2024steeringlanguagemodelsactivation", "ref-turpin2023languagemodelsdontsay", + "ref-vandenhirtz2024stochastic", + "ref-vanderwal2025polypythias", + "ref-varma2023explaining", "ref-vaswani2017attention", "ref-vaswani2023attentionneed", + "ref-vig-belinkov-2019-analyzing", "ref-vig2019multiscale", "ref-vig2020causalmediationanalysisinterpreting", "ref-voita-titov-2020-information", "ref-voita_analyzing_2019", + "ref-wallace2024instructionhierarchytrainingllms", "ref-wang2022interpretabilitywildcircuitindirect", "ref-wang2025better", + "ref-wang2025differentiation", + "ref-wang2026dlmscopemechanisticinterpretabilitydiffusion", + "ref-wang2026persona", "ref-wang_persona_2025", "ref-wang_towards_2018", + "ref-wani-etal-2026-answer", "ref-watanabe2009algebraic", + "ref-wei2022emergent", "ref-wiegreffe2019attention", + "ref-wu-etal-2022-causal", "ref-wu2025axbench", + "ref-wu2025towards", + "ref-wu2025ushaped", + "ref-wurgaft2026manifoldsteeringrevealsshared", + "ref-xie2023defending", + "ref-xu2024saetrack", "ref-xu2025infinigramminiexactngram", "ref-yang2023language", "ref-yang2025mixture", "ref-yang2025qwen3technicalreport", "ref-ye2026the", + "ref-yin2025which", "ref-yu-etal-2024-latent", + "ref-yu2025safety", + "ref-yu2026vectorquantizedlatentconcepts", "ref-yun-etal-2021-transformer", "ref-zanbaghi2025detectingsleeperagentslarge", "ref-zarlenga2022cem", "ref-zeiler2014visualizing", + "ref-zhang-etal-2024-bench", + "ref-zhang-etal-2024-defending", "ref-zhang2024towards", "ref-zhang2025cybench", "ref-zhang2026locatesteerimprovepractical", + "ref-zhang2026whatshapes", "ref-zhao-etal-2024-explaining", "ref-zhao2024distributional", "ref-zhao2025steering", + "ref-zhao2026chainofthoughthijacking", + "ref-zhao2026verifying", + "ref-zhu2025establishing", + "ref-zimerman2025explaining", "ref-zucchet2025emergence", "ref-zucchet2025language", + "ref-zucchet2025the", "refs", "refs-list" ], diff --git a/src/data/redirects.json b/src/data/redirects.json index 2230921716..e35454470b 100644 --- a/src/data/redirects.json +++ b/src/data/redirects.json @@ -10,7 +10,7 @@ "handbook/7-circuits-in-mechanistic-interpretability/": "handbook/circuits/", "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/": "handbook/intermediate-representations/", "handbook/9-training-data-attribution-and-influence-functions/": "handbook/training-data-attribution/", - "handbook/10-developmental-interpretability-and-training-dynamics/": "handbook/developmental-interpretability/", + "handbook/10-developmental-interpretability-and-training-dynamics/": "handbook/learning-dynamics/", "handbook/11-learning-dynamics-and-mechanistic-interpretability/": "handbook/learning-dynamics/", "handbook/12-representation-analysis-structure-localization-and-transfer/": "handbook/representation-analysis/", "handbook/13-five-research-orientations/": "handbook/research-orientations/", @@ -19,8 +19,384 @@ "handbook/16-conclusion/": "handbook/conclusion/", "handbook/a-a-notation-reference-and-symbol-table/": "handbook/notation/", "handbook/b-color-key-and-style-guide/": "handbook/colour-key/", - "handbook/c-sae-architecture-details-and-hyperparameters/": "handbook/sae-architecture-details/", - "handbook/d-mathematical-foundations/": "handbook/mathematical-foundations/" + "handbook/c-sae-architecture-details-and-hyperparameters/": "handbook/feature-extraction/", + "handbook/d-mathematical-foundations/": "handbook/mathematical-foundations/", + "handbook/developmental-interpretability/": "handbook/learning-dynamics/", + "handbook/sae-architecture-details/": "handbook/feature-extraction/", + "methods/bridge/": "methods/weight-sparse-transformer/", + "methods/unrolled-differentiation/": "methods/influence-functions/", + "methods/bayesian-influence-function/": "methods/influence-functions/" }, - "anchors": {} -} \ No newline at end of file + "anchors": { + "handbook/1-introduction/#useful-surveys-that-we-bild-on.": "handbook/introduction/#useful-surveys-that-we-build-on.", + "handbook/10-developmental-interpretability-and-training-dynamics/#case-study-induction-heads-2": "handbook/learning-dynamics/#sec:ld-induction-case", + "handbook/10-developmental-interpretability-and-training-dynamics/#sec:deception": "handbook/learning-dynamics/#sec:training", + "handbook/10-developmental-interpretability-and-training-dynamics/#sec:grokking": "handbook/learning-dynamics/#sec:training", + "handbook/10-developmental-interpretability-and-training-dynamics/#sec:slt-llc": "handbook/learning-dynamics/#sec:training", + "handbook/10-developmental-interpretability-and-training-dynamics/#sec:stagewise-influence": "handbook/learning-dynamics/#sec:training", + "handbook/10-developmental-interpretability-and-training-dynamics/#training-dynamics-and-the-emergence-of-internal-mechanisms": "handbook/learning-dynamics/#sec:training", + "handbook/12-representation-analysis-structure-localization-and-transfer/#optimization-based-steering.": "handbook/representation-analysis/#nonlinear-and-optimisation-based-steering.", + "handbook/12-representation-analysis-structure-localization-and-transfer/#overview.": "handbook/representation-analysis/#sec:representation", + "handbook/12-representation-analysis-structure-localization-and-transfer/#relation-to-causal-evaluation-of-feature-extraction-methods.": "handbook/representation-analysis/#sec:representation", + "handbook/13-five-research-orientations/#ambitious-interpretability-toward-complete-mechanistic-understanding": "handbook/research-orientations/#subsec:ambitious", + "handbook/13-five-research-orientations/#architectural-interventions-as-a-synthesis.": "handbook/research-orientations/#sec:orientations", + "handbook/13-five-research-orientations/#critical-tensions-and-open-questions": "handbook/research-orientations/#critical-concerns-tensions-and-open-questions", + "handbook/13-five-research-orientations/#defining-orientations-vs.-methods": "handbook/research-orientations/#sec:orientations", + "handbook/13-five-research-orientations/#developmental-interpretability-understanding-phase-transition-during-training": "handbook/research-orientations/#sec:orientations", + "handbook/13-five-research-orientations/#fig:fiveresearchorientation": "handbook/research-orientations/#sec:orientations", + "handbook/13-five-research-orientations/#governance-concerns-for-agentic-interpretability": "handbook/research-orientations/#governance-concerns-for-automated-interpretability", + "handbook/13-five-research-orientations/#overview-why-five-orientations": "handbook/research-orientations/#sec:orientations", + "handbook/13-five-research-orientations/#pragmatic-interpretability-proxy-task-oriented-safety-interventions": "handbook/research-orientations/#subsec: pragmatic interp", + "handbook/13-five-research-orientations/#proposed-guardrails-for-agentic-interpretability": "handbook/research-orientations/#proposed-guardrails-for-automated-interpretability", + "handbook/13-five-research-orientations/#protocol-1-calibration-testing.": "handbook/research-orientations/#sec:orientations", + "handbook/13-five-research-orientations/#protocol-1-causality-testing.": "handbook/research-orientations/#sec:orientations", + "handbook/13-five-research-orientations/#protocol-2-adversarial-robustness.": "handbook/research-orientations/#layer-3-adversarial-robustness.", + "handbook/13-five-research-orientations/#protocol-2-cross-examiner-verification.": "handbook/research-orientations/#sec:orientations", + "handbook/13-five-research-orientations/#protocol-3-behavioral-prediction-testing.": "handbook/research-orientations/#sec:orientations", + "handbook/13-five-research-orientations/#protocol-3-drift-monitoring.": "handbook/research-orientations/#sec:orientations", + "handbook/13-five-research-orientations/#sec:agentic-interpretability": "handbook/research-orientations/#sec:intro-interpretability", + "handbook/13-five-research-orientations/#subsec:valproto agentic self-reports": "handbook/research-orientations/#subsec:valproce model self-reports", + "handbook/13-five-research-orientations/#validation-protocols-for-automated-pipelines": "handbook/research-orientations/#sec:orientations", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#adversarial-risks-malicious-exploitation-of-mi-insights": "handbook/safety-applications/#adversarial-risks-malicious-exploitation-of-mi", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#alternatives-black-box-methods": "handbook/safety-applications/#alternatives-black-box-methods-and-their-challenges", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#challenges-of-black-box-bethods": "handbook/safety-applications/#sec:safety_applications", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#circuit-informed-adversarial-attacks.": "handbook/safety-applications/#sec:safety_applications", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#concept-erasure-and-direct-editing": "handbook/safety-applications/#concept-erasure-and-knowledge-editing", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#dual-use-risks.": "handbook/safety-applications/#sec:safety_applications", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#epistemic-risks-false-confidence-from-partial-understanding": "handbook/safety-applications/#epistemic-risks-false-confidence-from-partial-or-incorrect-understanding", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#eq-4": "handbook/safety-applications/#sec:safety_applications", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#implications-for-scale-and-oversight.": "handbook/safety-applications/#sec:safety_applications", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#mechanistic-monitoring": "handbook/safety-applications/#subsubsec: mech monitor", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#operational-risks-failure-under-deployment-conditions": "handbook/safety-applications/#operational-risks-failure-under-deployment", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#overclaiming-causal-understanding.": "handbook/safety-applications/#sec:safety_applications", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#recommended-verification-practices": "handbook/safety-applications/#sec:safety_applications", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#relations-to-the-five-research-orientations": "handbook/safety-applications/#sec:safety_applications", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#sec:limitations-open-questions": "handbook/safety-applications/#sec:safety_applications", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#steering-vectors-practical-behavioral-control": "handbook/safety-applications/#subsubsec: steering vectors", + "handbook/14-from-understanding-to-intervention-safety-applications-of-me/#taxonomy-of-mi-safety-risks": "handbook/safety-applications/#sec:safety_applications", + "handbook/15-open-problems/#eq-5": "handbook/open-problems/#sec:open_problems", + "handbook/15-open-problems/#how-to-evaluate-mechanistic-interpretability-methods": "handbook/open-problems/#sec:open_problems", + "handbook/15-open-problems/#problem-formulation.": "handbook/open-problems/#sec:open_problems", + "handbook/15-open-problems/#proposed-evaluation-framework": "handbook/open-problems/#sec:open_problems", + "handbook/15-open-problems/#scalability-and-adversarial-robustness": "handbook/open-problems/#sec:open_problems", + "handbook/15-open-problems/#standardized-benchmarks": "handbook/open-problems/#sec:open_problems", + "handbook/16-conclusion/#from-reverse-engineering-to-engineering.": "handbook/conclusion/#sec:conclusion", + "handbook/16-conclusion/#the-road-ahead-three-complementary-paradigms": "handbook/conclusion/#sec:conclusion", + "handbook/16-conclusion/#unresolved-questions": "handbook/conclusion/#sec:conclusion", + "handbook/3-the-interpretability-landscape/#attribution-and-saliency-methods": "handbook/interpretability-landscape/#input-attribution-and-saliency-methods", + "handbook/3-the-interpretability-landscape/#behavioral-methods": "handbook/interpretability-landscape/#behavioural-methods", + "handbook/3-the-interpretability-landscape/#when-mechanistic-understanding-becomes-necessary": "handbook/interpretability-landscape/#subsec: mech interp necessary", + "handbook/4-foundational-concepts-for-mechanistic-interpretability/#subsec:limits": "handbook/interpretability-landscape/#subsec:limits", + "handbook/5-core-hypotheses-in-mechanistic-interpretability/#architectural-responses-to-superposition": "handbook/core-hypotheses/#architectural-responses-to-superposition.", + "handbook/5-core-hypotheses-in-mechanistic-interpretability/#elicitation-protocols-to-reduce-confabulation.": "handbook/core-hypotheses/#sec:hypotheses", + "handbook/5-core-hypotheses-in-mechanistic-interpretability/#implications.-1": "handbook/research-orientations/#implications.-1", + "handbook/5-core-hypotheses-in-mechanistic-interpretability/#validation-protocols-and-causal-cross-checks.": "handbook/core-hypotheses/#sec:hypotheses", + "handbook/5-core-hypotheses-in-mechanistic-interpretability/#when-privileged-access-is-and-is-not-appropriate.": "handbook/core-hypotheses/#sec:hypotheses", + "handbook/6-feature-extraction-and-multi-layer-analysis/#activation-level-interventions-solu-and-its-limitations": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#case-study-induction-heads": "handbook/feature-extraction/#subsec: induc head case study: feature extract", + "handbook/6-feature-extraction-and-multi-layer-analysis/#common-pitfalls-and-how-to-avoid-them": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#comparing-saes-transcoders-and-crosscoders.": "handbook/feature-extraction/#comparing-saes-transcoders-and-crosscoders", + "handbook/6-feature-extraction-and-multi-layer-analysis/#computational-primitive-modifications-molt": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#crosscoders-analyzing-cross-layer-features": "handbook/feature-extraction/#crosscoders-analysing-cross-layer-features", + "handbook/6-feature-extraction-and-multi-layer-analysis/#identifiability-and-theoretical-concerns": "handbook/feature-extraction/#subsubsec: sae limitation", + "handbook/6-feature-extraction-and-multi-layer-analysis/#interpretability-gains.": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#kolmogorov-arnold-networks": "handbook/feature-extraction/#subsubsec: kan", + "handbook/6-feature-extraction-and-multi-layer-analysis/#layer-specific-recommendations": "handbook/feature-extraction/#practical-recommendations", + "handbook/6-feature-extraction-and-multi-layer-analysis/#limitations-of-these-recommendations": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#mechanism.": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#mechanism.-1": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#motivation.": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#non-negative-matrix-factorization-nmf": "handbook/feature-extraction/#non-negative-matrix-factorisation-nmf", + "handbook/6-feature-extraction-and-multi-layer-analysis/#principle-component-analysis-pca": "handbook/feature-extraction/#principal-component-analysis-pca", + "handbook/6-feature-extraction-and-multi-layer-analysis/#quick-reference-when-to-use-what": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#results-and-the-hydraulic-effect.": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#sparse-attention-post-training-spartan": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#standard-formulation.": "handbook/feature-extraction/#canonical-formulation.", + "handbook/6-feature-extraction-and-multi-layer-analysis/#supervised-feature-extraction-methods": "handbook/feature-extraction/#subsubsec: supervised_extraction", + "handbook/6-feature-extraction-and-multi-layer-analysis/#tab:sae-variant-selection-decision-tree": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#the-bridge-methodology-interpretable-proxies-for-dense-models": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#the-capability-interpretability-frontier.": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#weight-sparsity-via-l_0-regularization": "handbook/feature-extraction/#sec:sae", + "handbook/6-feature-extraction-and-multi-layer-analysis/#when-transcoders-are-necessary.": "handbook/feature-extraction/#when-are-transcoders-necessary", + "handbook/7-circuits-in-mechanistic-interpretability/#applying-the-framework-method-specific-guidance": "handbook/circuits/#sec:circuit", + "handbook/7-circuits-in-mechanistic-interpretability/#case-study-induction-heads-1": "handbook/circuits/#subsec: induc head case study: circuit", + "handbook/7-circuits-in-mechanistic-interpretability/#circuit-discovery-in-weight-sparse-architectures": "handbook/circuits/#sec:circuit", + "handbook/7-circuits-in-mechanistic-interpretability/#circuits-visible-by-inspection.": "handbook/circuits/#sec:circuit", + "handbook/7-circuits-in-mechanistic-interpretability/#comparison-of-computational-costs.": "handbook/circuits/#sec:circuit", + "handbook/7-circuits-in-mechanistic-interpretability/#dimension-1-faithfulness": "handbook/circuits/#dimension-1-attribution-faithfulness", + "handbook/7-circuits-in-mechanistic-interpretability/#fig:circuit": "handbook/circuits/#sec:circuit", + "handbook/7-circuits-in-mechanistic-interpretability/#five-evaluation-dimensions": "handbook/circuits/#subsec:five eval dim", + "handbook/7-circuits-in-mechanistic-interpretability/#implications-for-verification.": "handbook/circuits/#sec:circuit", + "handbook/7-circuits-in-mechanistic-interpretability/#limitations-of-causal-verification.": "handbook/circuits/#sec:causal-verification", + "handbook/7-circuits-in-mechanistic-interpretability/#limitations.": "handbook/circuits/#sec:circuit", + "handbook/7-circuits-in-mechanistic-interpretability/#sec:circuit-table": "handbook/circuits/#sec:circuit-discovery-methods", + "handbook/7-circuits-in-mechanistic-interpretability/#sec:what-are-circuits": "handbook/circuits/#sec:circuit", + "handbook/7-circuits-in-mechanistic-interpretability/#tab:circuit-discovery-cost": "handbook/circuits/#sec:circuit-discovery-methods", + "handbook/7-circuits-in-mechanistic-interpretability/#the-need-for-standardized-evaluation": "handbook/circuits/#the-need-for-standardised-evaluation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#amnesic-probing-and-causal-probing": "handbook/intermediate-representations/#subsubsec: amnesic probing", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#best-practices-for-probing-studies": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#case-study-refusal-vectors-and-sleeper-agents": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#causal-probing-via-interchange-interventions.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#circuit-fragility-and-robustness-concerns": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#critiques-of-attribution-and-saliency-methods": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#def:refusal-vector": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#evidence-for-circuit-fragility.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#faithfulness-of-attributions.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#implications-for-probing.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#implications-for-probing.-1": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#implications-of-control-tasks.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#key-findings-on-attribution-faithfulness": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#probe-complexity-vs.-causal-relevance": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#probing-as-hypothesis-generation.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#the-probe-causality-problem": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#tuned-lens-learning-per-layer-transform": "handbook/intermediate-representations/#subsec:tuned-lens", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#what-probing-can-show.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#what-probing-cannot-show.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/8-understanding-intermediate-computation-logit-lens-and-probin/#when-is-probing-appropriate-and-when-is-it-not": "handbook/intermediate-representations/#when-is-probing-appropriate", + "handbook/9-training-data-attribution-and-influence-functions/#computational-bottleneck-approximating-ggn-vector-products": "handbook/training-data-attribution/#sec:influence-functions", + "handbook/9-training-data-attribution-and-influence-functions/#eigenvalue-corrected-kfac-ek-fac-and-astra.": "handbook/training-data-attribution/#eigenvalue-corrected-k-fac-ek-fac-and-astra.", + "handbook/9-training-data-attribution-and-influence-functions/#gradient-based-attribution-understanding-influence-through-data-perturbations": "handbook/training-data-attribution/#gradient-based-attribution-understanding-data-influence-through-perturbations", + "handbook/9-training-data-attribution-and-influence-functions/#kronecker-factored-approximate-curvature-kfac.": "handbook/training-data-attribution/#kronecker-factored-approximate-curvature-k-fac.", + "handbook/9-training-data-attribution-and-influence-functions/#limitations": "handbook/training-data-attribution/#sec:influence-functions", + "handbook/9-training-data-attribution-and-influence-functions/#proximal-bregman-response-and-the-gauss-newton-hessian": "handbook/training-data-attribution/#proximal-bregman-response-and-the-generalised-gaussnewton", + "handbook/9-training-data-attribution-and-influence-functions/#unrolled-differentiation-gradient-based-attribution": "handbook/training-data-attribution/#sec:influence-functions", + "handbook/a-a-notation-reference-and-symbol-table/#dimensions-and-scaling": "handbook/notation/#dimensions-and-indices", + "handbook/b-color-key-and-style-guide/#box-types-and-colors": "handbook/colour-key/#box-types-and-colours", + "handbook/c-sae-architecture-details-and-hyperparameters/#app:sae_details": "handbook/feature-extraction/#sec:sae", + "handbook/c-sae-architecture-details-and-hyperparameters/#app:train-sae": "handbook/feature-extraction/#sec:sae", + "handbook/c-sae-architecture-details-and-hyperparameters/#eq-6": "handbook/feature-extraction/#sec:sae", + "handbook/c-sae-architecture-details-and-hyperparameters/#evaluation-metrics": "handbook/feature-extraction/#sec:sae", + "handbook/c-sae-architecture-details-and-hyperparameters/#gated-sae-variant": "handbook/feature-extraction/#sec:sae", + "handbook/c-sae-architecture-details-and-hyperparameters/#multi-layer-saes": "handbook/feature-extraction/#sec:sae", + "handbook/c-sae-architecture-details-and-hyperparameters/#standard-hyperparameters": "handbook/feature-extraction/#sec:sae", + "handbook/c-sae-architecture-details-and-hyperparameters/#standard-sae-architecture": "handbook/feature-extraction/#sec:sae", + "handbook/c-sae-architecture-details-and-hyperparameters/#top-k-sae-variant": "handbook/feature-extraction/#sec:sae", + "handbook/circuits/#applying-the-framework-method-specific-guidance": "handbook/circuits/#sec:circuit", + "handbook/circuits/#case-study-induction-heads-1": "handbook/circuits/#subsec: induc head case study: circuit", + "handbook/circuits/#circuit-discovery-in-weight-sparse-architectures": "handbook/circuits/#sec:circuit", + "handbook/circuits/#circuits-visible-by-inspection.": "handbook/circuits/#sec:circuit", + "handbook/circuits/#comparison-of-computational-costs.": "handbook/circuits/#sec:circuit", + "handbook/circuits/#dimension-1-faithfulness": "handbook/circuits/#dimension-1-attribution-faithfulness", + "handbook/circuits/#fig:circuit": "handbook/circuits/#sec:circuit", + "handbook/circuits/#five-evaluation-dimensions": "handbook/circuits/#subsec:five eval dim", + "handbook/circuits/#implications-for-verification.": "handbook/circuits/#sec:circuit", + "handbook/circuits/#limitations-of-causal-verification.": "handbook/circuits/#sec:causal-verification", + "handbook/circuits/#limitations.": "handbook/circuits/#sec:circuit", + "handbook/circuits/#sec:circuit-table": "handbook/circuits/#sec:circuit-discovery-methods", + "handbook/circuits/#sec:what-are-circuits": "handbook/circuits/#sec:circuit", + "handbook/circuits/#tab:circuit-discovery-cost": "handbook/circuits/#sec:circuit-discovery-methods", + "handbook/circuits/#the-need-for-standardized-evaluation": "handbook/circuits/#the-need-for-standardised-evaluation", + "handbook/colour-key/#box-types-and-colors": "handbook/colour-key/#box-types-and-colours", + "handbook/conclusion/#from-reverse-engineering-to-engineering.": "handbook/conclusion/#sec:conclusion", + "handbook/conclusion/#the-road-ahead-three-complementary-paradigms": "handbook/conclusion/#sec:conclusion", + "handbook/conclusion/#unresolved-questions": "handbook/conclusion/#sec:conclusion", + "handbook/core-hypotheses/#architectural-responses-to-superposition": "handbook/core-hypotheses/#architectural-responses-to-superposition.", + "handbook/core-hypotheses/#elicitation-protocols-to-reduce-confabulation.": "handbook/core-hypotheses/#sec:hypotheses", + "handbook/core-hypotheses/#implications.-1": "handbook/research-orientations/#implications.-1", + "handbook/core-hypotheses/#validation-protocols-and-causal-cross-checks.": "handbook/core-hypotheses/#sec:hypotheses", + "handbook/core-hypotheses/#when-privileged-access-is-and-is-not-appropriate.": "handbook/core-hypotheses/#sec:hypotheses", + "handbook/d-mathematical-foundations/#eq-10": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#eq-11": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#eq-12": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#eq-13": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#eq-14": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#eq-15": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#eq-7": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#eq-8": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#eq-9": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#generalization-bounds": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#linear-representation-hypothesis": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#phase-transitions": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#singular-learning-theory-slt": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#sparse-coding-theory": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#superposition-and-high-dimensional-geometry": "handbook/mathematical-foundations/#app:math", + "handbook/d-mathematical-foundations/#the-local-learning-coefficient": "handbook/mathematical-foundations/#app:slt", + "handbook/developmental-interpretability/#case-study-induction-heads-2": "handbook/learning-dynamics/#sec:ld-induction-case", + "handbook/developmental-interpretability/#sec:deception": "handbook/learning-dynamics/#sec:training", + "handbook/developmental-interpretability/#sec:grokking": "handbook/learning-dynamics/#sec:training", + "handbook/developmental-interpretability/#sec:slt-llc": "handbook/learning-dynamics/#sec:training", + "handbook/developmental-interpretability/#sec:stagewise-influence": "handbook/learning-dynamics/#sec:training", + "handbook/developmental-interpretability/#training-dynamics-and-the-emergence-of-internal-mechanisms": "handbook/learning-dynamics/#sec:training", + "handbook/feature-extraction/#activation-level-interventions-solu-and-its-limitations": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#case-study-induction-heads": "handbook/feature-extraction/#subsec: induc head case study: feature extract", + "handbook/feature-extraction/#common-pitfalls-and-how-to-avoid-them": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#comparing-saes-transcoders-and-crosscoders.": "handbook/feature-extraction/#comparing-saes-transcoders-and-crosscoders", + "handbook/feature-extraction/#computational-primitive-modifications-molt": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#crosscoders-analyzing-cross-layer-features": "handbook/feature-extraction/#crosscoders-analysing-cross-layer-features", + "handbook/feature-extraction/#identifiability-and-theoretical-concerns": "handbook/feature-extraction/#subsubsec: sae limitation", + "handbook/feature-extraction/#interpretability-gains.": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#kolmogorov-arnold-networks": "handbook/feature-extraction/#subsubsec: kan", + "handbook/feature-extraction/#layer-specific-recommendations": "handbook/feature-extraction/#practical-recommendations", + "handbook/feature-extraction/#limitations-of-these-recommendations": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#mechanism.": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#mechanism.-1": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#motivation.": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#non-negative-matrix-factorization-nmf": "handbook/feature-extraction/#non-negative-matrix-factorisation-nmf", + "handbook/feature-extraction/#principle-component-analysis-pca": "handbook/feature-extraction/#principal-component-analysis-pca", + "handbook/feature-extraction/#quick-reference-when-to-use-what": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#results-and-the-hydraulic-effect.": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#sparse-attention-post-training-spartan": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#standard-formulation.": "handbook/feature-extraction/#canonical-formulation.", + "handbook/feature-extraction/#supervised-feature-extraction-methods": "handbook/feature-extraction/#subsubsec: supervised_extraction", + "handbook/feature-extraction/#tab:sae-variant-selection-decision-tree": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#the-bridge-methodology-interpretable-proxies-for-dense-models": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#the-capability-interpretability-frontier.": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#weight-sparsity-via-l_0-regularization": "handbook/feature-extraction/#sec:sae", + "handbook/feature-extraction/#when-transcoders-are-necessary.": "handbook/feature-extraction/#when-are-transcoders-necessary", + "handbook/foundational-concepts/#subsec:limits": "handbook/interpretability-landscape/#subsec:limits", + "handbook/intermediate-representations/#amnesic-probing-and-causal-probing": "handbook/intermediate-representations/#subsubsec: amnesic probing", + "handbook/intermediate-representations/#best-practices-for-probing-studies": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#case-study-refusal-vectors-and-sleeper-agents": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#causal-probing-via-interchange-interventions.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#circuit-fragility-and-robustness-concerns": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#critiques-of-attribution-and-saliency-methods": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#def:refusal-vector": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#evidence-for-circuit-fragility.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#faithfulness-of-attributions.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#implications-for-probing.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#implications-for-probing.-1": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#implications-of-control-tasks.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#key-findings-on-attribution-faithfulness": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#probe-complexity-vs.-causal-relevance": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#probing-as-hypothesis-generation.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#the-probe-causality-problem": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#tuned-lens-learning-per-layer-transform": "handbook/intermediate-representations/#subsec:tuned-lens", + "handbook/intermediate-representations/#what-probing-can-show.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#what-probing-cannot-show.": "handbook/intermediate-representations/#sec:intermediate-computation", + "handbook/intermediate-representations/#when-is-probing-appropriate-and-when-is-it-not": "handbook/intermediate-representations/#when-is-probing-appropriate", + "handbook/interpretability-landscape/#attribution-and-saliency-methods": "handbook/interpretability-landscape/#input-attribution-and-saliency-methods", + "handbook/interpretability-landscape/#behavioral-methods": "handbook/interpretability-landscape/#behavioural-methods", + "handbook/interpretability-landscape/#when-mechanistic-understanding-becomes-necessary": "handbook/interpretability-landscape/#subsec: mech interp necessary", + "handbook/introduction/#useful-surveys-that-we-bild-on.": "handbook/introduction/#useful-surveys-that-we-build-on.", + "handbook/mathematical-foundations/#eq-10": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#eq-11": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#eq-12": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#eq-13": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#eq-14": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#eq-15": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#eq-7": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#eq-8": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#eq-9": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#generalization-bounds": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#linear-representation-hypothesis": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#phase-transitions": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#singular-learning-theory-slt": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#sparse-coding-theory": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#superposition-and-high-dimensional-geometry": "handbook/mathematical-foundations/#app:math", + "handbook/mathematical-foundations/#the-local-learning-coefficient": "handbook/mathematical-foundations/#app:slt", + "handbook/notation/#dimensions-and-scaling": "handbook/notation/#dimensions-and-indices", + "handbook/open-problems/#eq-5": "handbook/open-problems/#sec:open_problems", + "handbook/open-problems/#how-to-evaluate-mechanistic-interpretability-methods": "handbook/open-problems/#sec:open_problems", + "handbook/open-problems/#problem-formulation.": "handbook/open-problems/#sec:open_problems", + "handbook/open-problems/#proposed-evaluation-framework": "handbook/open-problems/#sec:open_problems", + "handbook/open-problems/#scalability-and-adversarial-robustness": "handbook/open-problems/#sec:open_problems", + "handbook/open-problems/#standardized-benchmarks": "handbook/open-problems/#sec:open_problems", + "handbook/representation-analysis/#optimization-based-steering.": "handbook/representation-analysis/#nonlinear-and-optimisation-based-steering.", + "handbook/representation-analysis/#overview.": "handbook/representation-analysis/#sec:representation", + "handbook/representation-analysis/#relation-to-causal-evaluation-of-feature-extraction-methods.": "handbook/representation-analysis/#sec:representation", + "handbook/research-orientations/#ambitious-interpretability-toward-complete-mechanistic-understanding": "handbook/research-orientations/#subsec:ambitious", + "handbook/research-orientations/#architectural-interventions-as-a-synthesis.": "handbook/research-orientations/#sec:orientations", + "handbook/research-orientations/#critical-tensions-and-open-questions": "handbook/research-orientations/#critical-concerns-tensions-and-open-questions", + "handbook/research-orientations/#defining-orientations-vs.-methods": "handbook/research-orientations/#sec:orientations", + "handbook/research-orientations/#developmental-interpretability-understanding-phase-transition-during-training": "handbook/research-orientations/#sec:orientations", + "handbook/research-orientations/#fig:fiveresearchorientation": "handbook/research-orientations/#sec:orientations", + "handbook/research-orientations/#governance-concerns-for-agentic-interpretability": "handbook/research-orientations/#governance-concerns-for-automated-interpretability", + "handbook/research-orientations/#overview-why-five-orientations": "handbook/research-orientations/#sec:orientations", + "handbook/research-orientations/#pragmatic-interpretability-proxy-task-oriented-safety-interventions": "handbook/research-orientations/#subsec: pragmatic interp", + "handbook/research-orientations/#proposed-guardrails-for-agentic-interpretability": "handbook/research-orientations/#proposed-guardrails-for-automated-interpretability", + "handbook/research-orientations/#protocol-1-calibration-testing.": "handbook/research-orientations/#sec:orientations", + "handbook/research-orientations/#protocol-1-causality-testing.": "handbook/research-orientations/#sec:orientations", + "handbook/research-orientations/#protocol-2-adversarial-robustness.": "handbook/research-orientations/#layer-3-adversarial-robustness.", + "handbook/research-orientations/#protocol-2-cross-examiner-verification.": "handbook/research-orientations/#sec:orientations", + "handbook/research-orientations/#protocol-3-behavioral-prediction-testing.": "handbook/research-orientations/#sec:orientations", + "handbook/research-orientations/#protocol-3-drift-monitoring.": "handbook/research-orientations/#sec:orientations", + "handbook/research-orientations/#sec:agentic-interpretability": "handbook/research-orientations/#sec:intro-interpretability", + "handbook/research-orientations/#subsec:valproto agentic self-reports": "handbook/research-orientations/#subsec:valproce model self-reports", + "handbook/research-orientations/#validation-protocols-for-automated-pipelines": "handbook/research-orientations/#sec:orientations", + "handbook/sae-architecture-details/#app:sae_details": "handbook/feature-extraction/#sec:sae", + "handbook/sae-architecture-details/#app:train-sae": "handbook/feature-extraction/#sec:sae", + "handbook/sae-architecture-details/#eq-6": "handbook/feature-extraction/#sec:sae", + "handbook/sae-architecture-details/#evaluation-metrics": "handbook/feature-extraction/#sec:sae", + "handbook/sae-architecture-details/#gated-sae-variant": "handbook/feature-extraction/#sec:sae", + "handbook/sae-architecture-details/#multi-layer-saes": "handbook/feature-extraction/#sec:sae", + "handbook/sae-architecture-details/#standard-hyperparameters": "handbook/feature-extraction/#sec:sae", + "handbook/sae-architecture-details/#standard-sae-architecture": "handbook/feature-extraction/#sec:sae", + "handbook/sae-architecture-details/#top-k-sae-variant": "handbook/feature-extraction/#sec:sae", + "handbook/safety-applications/#adversarial-risks-malicious-exploitation-of-mi-insights": "handbook/safety-applications/#adversarial-risks-malicious-exploitation-of-mi", + "handbook/safety-applications/#alternatives-black-box-methods": "handbook/safety-applications/#alternatives-black-box-methods-and-their-challenges", + "handbook/safety-applications/#challenges-of-black-box-bethods": "handbook/safety-applications/#sec:safety_applications", + "handbook/safety-applications/#circuit-informed-adversarial-attacks.": "handbook/safety-applications/#sec:safety_applications", + "handbook/safety-applications/#concept-erasure-and-direct-editing": "handbook/safety-applications/#concept-erasure-and-knowledge-editing", + "handbook/safety-applications/#dual-use-risks.": "handbook/safety-applications/#sec:safety_applications", + "handbook/safety-applications/#epistemic-risks-false-confidence-from-partial-understanding": "handbook/safety-applications/#epistemic-risks-false-confidence-from-partial-or-incorrect-understanding", + "handbook/safety-applications/#eq-4": "handbook/safety-applications/#sec:safety_applications", + "handbook/safety-applications/#implications-for-scale-and-oversight.": "handbook/safety-applications/#sec:safety_applications", + "handbook/safety-applications/#mechanistic-monitoring": "handbook/safety-applications/#subsubsec: mech monitor", + "handbook/safety-applications/#operational-risks-failure-under-deployment-conditions": "handbook/safety-applications/#operational-risks-failure-under-deployment", + "handbook/safety-applications/#overclaiming-causal-understanding.": "handbook/safety-applications/#sec:safety_applications", + "handbook/safety-applications/#recommended-verification-practices": "handbook/safety-applications/#sec:safety_applications", + "handbook/safety-applications/#relations-to-the-five-research-orientations": "handbook/safety-applications/#sec:safety_applications", + "handbook/safety-applications/#sec:limitations-open-questions": "handbook/safety-applications/#sec:safety_applications", + "handbook/safety-applications/#steering-vectors-practical-behavioral-control": "handbook/safety-applications/#subsubsec: steering vectors", + "handbook/safety-applications/#taxonomy-of-mi-safety-risks": "handbook/safety-applications/#sec:safety_applications", + "handbook/training-data-attribution/#computational-bottleneck-approximating-ggn-vector-products": "handbook/training-data-attribution/#sec:influence-functions", + "handbook/training-data-attribution/#eigenvalue-corrected-kfac-ek-fac-and-astra.": "handbook/training-data-attribution/#eigenvalue-corrected-k-fac-ek-fac-and-astra.", + "handbook/training-data-attribution/#gradient-based-attribution-understanding-influence-through-data-perturbations": "handbook/training-data-attribution/#gradient-based-attribution-understanding-data-influence-through-perturbations", + "handbook/training-data-attribution/#kronecker-factored-approximate-curvature-kfac.": "handbook/training-data-attribution/#kronecker-factored-approximate-curvature-k-fac.", + "handbook/training-data-attribution/#limitations": "handbook/training-data-attribution/#sec:influence-functions", + "handbook/training-data-attribution/#proximal-bregman-response-and-the-gauss-newton-hessian": "handbook/training-data-attribution/#proximal-bregman-response-and-the-generalised-gaussnewton", + "handbook/training-data-attribution/#unrolled-differentiation-gradient-based-attribution": "handbook/training-data-attribution/#sec:influence-functions", + "references/#ref-Ethan2022redteam": "references/#refs", + "references/#ref-adebayo2018sanity": "references/#refs", + "references/#ref-bae2024trainingdataattributionapproximate": "references/#refs", + "references/#ref-bayazit_crosscoding_2025": "references/#refs", + "references/#ref-burns2024discoveringlatentknowledgelanguage": "references/#refs", + "references/#ref-chanin2024absorption": "references/#refs", + "references/#ref-chaudhary2025safetynetdetectingharmfuloutputs": "references/#refs", + "references/#ref-christiano2023deepreinforcementlearninghuman": "references/#refs", + "references/#ref-dbm": "references/#refs", + "references/#ref-garfinkle2016robust": "references/#refs", + "references/#ref-ge_evolution_2025": "references/#refs", + "references/#ref-haider2025multi": "references/#refs", + "references/#ref-heimersheim2024useinterpretactivationpatching": "references/#refs", + "references/#ref-hendrycks2021naturaladversarialexamples": "references/#refs", + "references/#ref-hubinger2021riskslearnedoptimizationadvanced": "references/#refs", + "references/#ref-hyvarinen2023nonlinear": "references/#refs", + "references/#ref-jacovi2020faithfullyinterpretablenlpsystems": "references/#refs", + "references/#ref-joshi2026causalitykeyinterpretabilityclaims": "references/#refs", + "references/#ref-kangaslahti2026hiddenbreakthroughslanguagemodel": "references/#refs", + "references/#ref-kim2023probabilistic": "references/#refs", + "references/#ref-kim2025llmspursueagenticinterpretability": "references/#refs", + "references/#ref-krizhevsky2012imagenet": "references/#refs", + "references/#ref-lanham2023measuringfaithfulnesschainofthoughtreasoning": "references/#refs", + "references/#ref-le2013building": "references/#refs", + "references/#ref-lee2025influence": "references/#refs", + "references/#ref-li2024simulateeliminaterevokebackdoors": "references/#refs", + "references/#ref-lindsey2025biology": "references/#refs", + "references/#ref-morcos_insights_2018": "references/#refs", + "references/#ref-moritz2024sthocastic": "references/#refs", + "references/#ref-moschella2023relativerepresentationsenablezeroshot": "references/#refs", + "references/#ref-mueller2025isolationentanglementinterpretabilitymethods": "references/#refs", + "references/#ref-olsson2022context": "references/#refs", + "references/#ref-oozeer2025activation": "references/#refs", + "references/#ref-orgad2026interpretabilityactionable": "references/#refs", + "references/#ref-petri2025": "references/#refs", + "references/#ref-petri2026v2": "references/#refs", + "references/#ref-rajani2020explainingimprovingmodelbehavior": "references/#refs", + "references/#ref-rosenfeld2019constructivepredictiongeneralizationerror": "references/#refs", + "references/#ref-seo2018noise": "references/#refs", + "references/#ref-slocum2025believe": "references/#refs", + "references/#ref-tan2025analyzinggeneralizationreliabilitysteering": "references/#refs", + "references/#ref-vaswani2023attentionneed": "references/#refs", + "references/#ref-vig2020causalmediationanalysisinterpreting": "references/#refs", + "references/#ref-wang_persona_2025": "references/#refs", + "references/#ref-xu2025infinigramminiexactngram": "references/#refs", + "references/#ref-zanbaghi2025detectingsleeperagentslarge": "references/#refs", + "references/#ref-zucchet2025emergence": "references/#refs", + "safety/#chapter-limitations": "safety/#scalability-adversarial", + "safety/#safety-limitations": "safety/#scalability-adversarial", + "safety/#clt-open-questions": "safety/#scalability-adversarial", + "safety/#orientation-tensions": "safety/#when-necessary", + "safety/#conclusion-unresolved": "safety/#when-necessary" + } +} diff --git a/src/generated/chapters/acknowledgements.html b/src/generated/chapters/acknowledgements.html new file mode 100644 index 0000000000..09ba271f75 --- /dev/null +++ b/src/generated/chapters/acknowledgements.html @@ -0,0 +1,3 @@ +

Acknowledgments#

+

FB and TW would like to thank Leo Gao, Peter Hase, John Hewitt, Kyle Lo, Aaron Mueller, Neel Nanda, and Avi Semler for their thoughtful comments and our discussions that helped shape the project.

+
diff --git a/src/generated/chapters/circuits.html b/src/generated/chapters/circuits.html index 0cead17c47..80631879a7 100644 --- a/src/generated/chapters/circuits.html +++ b/src/generated/chapters/circuits.html @@ -1,34 +1,38 @@ -

7 Circuits in Mechanistic Interpretability#

-

Mechanistic interpretability aims not only to identify what information is represented inside a model, but also to explain how that information is transformed and combined to produce behavior. Feature-level analyses, such as those based on Sparse Autoencoders, provide a useful vocabulary for describing internal representations, but they are inherently local: they characterize individual directions or neurons in isolation. Many model behaviors, however, arise from structured interactions between multiple components operating together across layers. To explain such behaviors, we require a higher-level abstraction that captures patterns of computation rather than individual features. The circuit framework addresses this gap by modeling behavior as the result of coordinated subgraphs of the computation graph, making explicit how components interact causally to implement algorithms.

-

7.1 What Are Circuits?#

+

7 Circuits in Mechanistic Interpretability#

+

Mechanistic interpretability aims not only to identify what information a model represents, but also to explain how it transforms and combines that information across layers to produce behaviour. Feature analysis is a useful tool for describing activation vectors, but it describes them in isolation. Many model behaviours arise from structured interactions among multiple components that operate together across layers. To explain these behaviours and their mechanisms, we need a high-level abstraction that captures critical computational patterns of these features. The circuit framework (Figure 3; introduced in Section 4.5) addresses this gap by modelling behaviour as coordinated computational graphs and making explicit how components causally interact.

+

Notably, the circuit framework is not the only way to model an internal mechanism. Early studies have explored alternative ways. For example, Giles et al. (1992) and Omlin & Giles (1996) categorised a recurrent neural network’s hidden representations into different states and modelled its decision process as a finite-state automaton when the input sentence was fed sequentially. Nonetheless, the circuit framework has attracted growing research interest in studying neural networks’ internal mechanisms. Below, we introduce approaches for identifying and constructing circuits.

+

7.1 Circuit Discovery Methods#

+

Circuit discovery methods aim to identify internal computational graphs responsible for specific capabilities or outputs in a neural network. Below, and in Table 8, we formalise key techniques and methods, along with their strengths and limitations.

-

A circuit is a subgraph of the computation graph implementing a specific algorithmic behavior: \text{Circuit} = \{v_i, e_{ij}\}_{i,j \in S} \quad \text{s.t.} \quad f_\text{model}^{S}(X) \approx f_\text{model}(X), where S indexes components and f_\text{model}^{S} is the model with only those components active. X is either a specific input query or a set of queries that instantiates a specific model capability.

+

Activation Patching

+

Given a model f_\theta(\cdot) parameterised by \theta and an input sequence x, activation patching evaluates the causal contribution of some model components S by replacing their activations with specified reference values \tilde{h}_S and measuring the resulting change in the model output. The causal importance of S for x is defined as s_S(x) += +M\bigl(f_\theta(x)\bigr) +- +M\Bigl(f_\theta\bigl(x_{[S \leftarrow \tilde{h}_S]}\bigr)\Bigr), where M(\cdot) is a scalar performance metric computed from the model output, such as the logit difference for the answer token, and x_{[S \leftarrow \tilde{h}_S]} denotes a forward pass on x in which the activations of components in S are replaced by \tilde{h}_S while the remaining computation proceeds normally. The reference activations \tilde{h}_S usually come from a corrupted reference run, where the input is modified to add or remove specific information. Some works have also set \tilde{h}_S to zero or the mean model activation. The resulting change in model output quantifies the causal effect of intervening on S (Zhang & Nanda, 2024).

-

Circuits differ from features: features describe what information is present, while circuits describe how components interact to implement behavior.

-
-
    -
  • Induction head circuit: Two attention heads cooperate to implement in‑context learning Olsson et al. (2022a).

  • -
  • QK circuits: Query-Key interactions prioritizing specific token positions (Kamath et al., 2025).

  • -
-
-
-Figure 10: Circuit. This diagram illustrates a “circuit” within a transformer, where specific nodes (MLP layers and Attention heads) perform specialized sub-computations. -
Figure 10: Circuit. This diagram illustrates a “circuit” within a transformer, where specific nodes (MLP layers and Attention heads) perform specialized sub-computations.
-
-
-

7.2 Circuit Discovery and Tracing Methods#

-

As discussed in Section 4.5, circuit discovery and tracing methods aim to identify internal mechanisms responsible for specific computations in a neural network. Below, we formalize important techniques, the SAE-based circuit discovery methods are introduced in 6.4.1 and 6.4.2.

-

Activation patching evaluates the causal contribution of a component or set of components S by replacing their activations with baseline values \tilde{h}_S and measuring output change: \text{Effect}(S, x) = f_\text{model}(x) - f_\text{model}(x_{[S \leftarrow \tilde{h}_S]}). This identifies which components are functionally necessary for the computation (Zhang & Nanda, 2024).

+

Path Patching

+

Path patching extends activation patching on model components to specific edges in the computational graph. For a connection e_{ij} from model component i to j denoting that the output activations of i are fed to j as one of its inputs, it replaces activations along e_{ij} with reference values \tilde{h}_{e_{ij}} and measures e_{ij}’s causal effect as: s_{ij}(x) = M\bigl(f_\theta(x)\bigr) - M\Bigl(f_\theta\bigl(x_{[e_{ij} \leftarrow \tilde{h}_{e_{ij}}]}\bigr)\Bigr). This allows finer-grained causal localisation along information flows (Goldowsky-Dill et al., 2023).

-

Path patching extends attribution to specific paths or edges in the computation graph. For a connection p from component i to j, one replaces activations along p with baseline values and measures output effect: \text{Effect}(p, x) = f_\text{model}(x) - f_\text{model}(x_{[p \leftarrow \tilde{h}_p]}). This allows finer-grained causal localization along information flows (Goldowsky-Dill et al., 2023).

+

Automatic Circuit Discovery (ACDC)

+

ACDC (Conmy et al., 2023) discovers a circuit by iteratively using path patching to identify important connections in the model’s computational graph. Traversing the graph in reverse topological order from the output node, it patches the activation transmitted along each edge and measures the resulting change in a task-specific metric. An edge e_{ij} will be pruned if patching it produces |s_{ij}(x)| less than a predefined threshold \tau; otherwise, it is retained in the graph and its upstream edges will be investigated. The remaining subgraph represents the discovered circuit.

-

ACDC identifies minimal sufficient sets of components S \subseteq \text{Graph}(\text{model}) that reproduce the model’s behavior. Formally, it searches for the smallest S such that f_\text{model}^{S}(x) \approx f_\text{model}(x) \quad \forall x \in \mathcal{D}, where f_\text{model}^{S} uses only components in S. Components not in S are pruned iteratively (Conmy et al., 2023).

+

Edge Attribution Patching (EAP)

+

EAP (Syed et al., 2024) estimates the causal importance of edges in a model’s computational graph by approximating the effect of path patching with a first-order Taylor expansion. For an edge e_{ij} connecting components i and j, its importance score is approximated as s_{ij}(x) +\approx +\frac{\partial M}{\partial h_{e_{ij}}} +\cdot +\left(h_{e_{ij}}^{\mathrm{clean}} - h_{e_{ij}}^{\mathrm{corrupt}}\right), where h_{e_{ij}}^{\mathrm{clean}} and h_{e_{ij}}^{\mathrm{corrupt}} denote the activations transmitted from component i to component j along edge e_{ij} on clean and corrupted inputs, respectively, when processing x. M(\cdot) is a scalar metric of model performance as mentioned above. EAP is substantially faster than ACDC because the attribution scores for all edges are computed using only two forward passes and a single backward pass. Specifically, the forward passes obtain h_{e_{ij}}^{\mathrm{clean}} and h_{e_{ij}}^{\mathrm{corrupt}}, while the backward pass computes \frac{\partial M}{\partial h_{e_{ij}}} for all components simultaneously. In contrast, ACDC requires a separate forward pass for each edge it tests.
+Edges with large score magnitudes are generally considered main contributors to the target behaviour. Therefore, one simple approach to circuit construction is directly selecting the highest-scoring edges (Syed et al., 2024). Subsequent work has improved EAP in different directions: EAP-IG (Hanna et al., 2024) and EAP-GP (Zhang et al., 2025b) improve edge-attribution estimation, while PEAP (Haklay et al., 2025) extends patching across token positions to produce finer-grained attribution graphs that capture cross-token interactions.

+

Besides intervention (activation patching, ACDC, etc.) and attribution (EAP, EAP-IG, etc.) techniques, another line of approaches treats circuit discovery as an optimisation problem (Csordás et al., 2021; Cao et al., 2021; Bhaskar et al., 2024; Haider et al., 2026). Specifically, they use gradient descent to learn which parts of the computational graph can be removed without affecting model behaviour. For example, Edge Pruning (Bhaskar et al., 2024) learns edge masks to discover sparse connections, while multi-granular node pruning (Haider et al., 2026) learns node masks at different scales, from entire blocks to individual neurons. These methods can usually discover sparser circuits than intervention and attribution counterparts. However, the optimisation process itself introduces randomness and sensitivity to training configurations like the learning rate, which may make the subsequent analysis less robust and interpretable.

+

Circuits can explain models in different contexts, from a single input–output pair to a capability or behaviour. Identifying a circuit for a specific input or prompt is called query-level circuit discovery (Wu & Barez, 2026). In contrast, task-level circuit discovery (Wang et al., 2023; Kharlapenko et al., 2025) seeks to identify a circuit that implements a broader capability or task, typically operationalised by a dataset of examples, such as the indirect object identification (IOI) dataset (Wang et al., 2023). In the task-level setting, components’ and edges’ attribution scores (e.g., s_S(x) or s_{ij}(x)) are aggregated across inputs to reduce example-specific noise, yielding a circuit that characterises the task as a whole.

+

The techniques and methods introduced above, including activation patching, path patching, ACDC, EAP, EAP-IG, and mask-based pruning methods, have primarily been developed and evaluated in the task-level setting. More recent work has extended EAP-IG to identify circuits for individual prompts (Wu & Barez, 2026). This enables the discovery of sparse yet faithful circuits in the model even for complex, natural-language user queries, where the relevant computation may vary substantially from one input to another.

- + @@ -40,126 +44,114 @@ - - - + + + - - - + + + - - - - - - - - - - - -
Table 8: Circuit Discovery Methods: Trade-offs and CharacteristicsTable 8: Trade-offs and characteristics of circuit discovery methods.
Method
Activation PatchingIntervene on components; measure output changeSimple, intuitive, approximates causalityConfounds correlation with causationIntervene on components and measure the resulting output changeSimple and intuitive; provides direct causal evidenceRequires many interventions; component-level granularity can be coarse
Path PatchingPatch specific information paths; measure output changeFiner granularityComputationally expensive due to large number of paths; still approximateIntervene on individual information paths or edges and measure their causal effectsFiner-grained localisation of information flowComputationally expensive due to the large number of candidate paths
ACDCIteratively prune minimal sufficient pathsScales better due to automation; has been proved to recover known circuitsComputationally expensive as it greedily prunes paths; misses subtle components
SAE- or CLT-IntegratedDiscover circuits over sparse featuresReduces search space; leverages featuresDepends on feature quality; intensive computation to train SAEs/CLTs
- -

7.3 Circuit Discovery in Weight-Sparse Architectures#

-

The circuit discovery methods described above—activation patching, path patching, ACDC—are designed for dense models where circuits are hidden within fully connected computational graphs. Weight-sparse architectures (Section 6.9) fundamentally alter this landscape by making circuits structurally explicit.

-
-

Circuits visible by inspection.

-

In a weight-sparse transformer with <1\% active parameters, the computational graph is directly observable. To identify the circuit responsible for a specific behavior, one traces non-zero weights backward from the relevant output neurons. Because each neuron connects to only a handful of others, the resulting subgraph is sparse and human-readable without algorithmic search (Gao et al., 2025b).

-

This eliminates the need for iterative pruning (ACDC), gradient-based attribution, or activation patching. The circuit is the set of non-zero weights—no discovery algorithm required.

-
-
-

Comparison of computational costs.

-

Table 9 contrasts the computational requirements of circuit discovery across architectures.

- - - - - - - - - - - - - - - - - - - + + + - - - + + + + - - - + + + +
Table 9: Circuit Discovery: Dense vs. Weight-Sparse Models
AspectDense ModelsWeight-Sparse Models
Discovery methodActivation patching, ACDC, path patchingDirect inspection of non-zero weights
Computational costO(n_{\text{components}} \times n_{\text{samples}}) forward passesO(1)—read from weight matrixGreedily prune edges from the computational graph while testing whether the target behaviour is preservedFully automated; directly searches for a behaviourally sufficient subgraphComputationally expensive due to sequential edge pruning; sensitive to pruning order and threshold choices
False positivesCommon due to correlated activationsMinimal—only true connections existEAPApproximate edge-level attribution using path patching and first-order gradients, enabling parallel evaluation of many candidate edgesHighly parallelisable and computationally efficient; enables large-scale edge attributionRelies on a first-order approximation; attribution scores may not fully capture nonlinear or interaction effects
Human verificationRequires careful ablation studiesVisual inspection often sufficientSAE- or CLT-BasedDiscover circuits over sparse, interpretable features rather than raw model componentsLeverages interpretable features; enables finer-grained feature-level circuit analysisDepends on feature quality; requires additional computation to train and analyse SAEs/CLTs
-
-

Implications for verification.

-

Causal verification (Section 7.4) are also simplified in sparse models. Because circuits are structurally isolated, ablation experiments yield cleaner results: removing a sparse circuit removes only that circuit, without the confounding effects of distributed computation. Empirically, sparse circuits satisfy necessity and sufficiency criteria with higher precision than circuits extracted from dense models via post-hoc methods (Gao et al., 2025b; Haider et al., 2025).

-
-
-

Limitations.

-

Weight sparsity does not eliminate all challenges. Circuits may still be compositional (requiring understanding of how subcircuits interact), and the interpretability of individual neurons depends on training dynamics. Furthermore, this approach requires training sparse models from scratch or using Bridge proxies—it cannot be directly applied to existing dense models.

-
+

7.2 Circuit Discovery with Replacement Models#

+

The methods introduced above discover circuits within the target model. Their nodes correspond to coarse-grained components such as attention heads and MLP blocks, which are mostly not immediately understandable. A complementary approach first constructs a more interpretable replacement model (Section 6.6.2) that approximates the original model’s computation using sparse linear features, and then performs circuit discovery within this replacement model.

+

Recent work has implemented this paradigm using cross-layer transcoders (CLTs; Section 6.6.2) (Ameisen et al., 2025; Merullo et al., 2025). In particular, Circuit Tracer (Ameisen et al., 2025) replaces the model’s MLPs with CLTs while retaining the underlying model’s attention and residual-stream computation. This yields a prompt-specific attribution graph whose principal internal nodes are sparse CLT features rather than entire MLP blocks or attention heads.

+
+

Circuit Tracer

+

+

Circuit Tracer (Ameisen et al., 2025) constructs a prompt-specific attribution graph in a CLT-based replacement model. For a given input prompt, consider a source feature \phi_s at layer \ell_s active at token position c_s with activation value z_{\ell_s}^{s}, and a downstream target feature \phi_t at layer \ell_t active at position c_t. Recall that a CLT feature can write to the reconstructed MLP outputs of its own and all subsequent layers (Figure 7b); the source feature has layer-specific output vectors to these layers, corresponding to columns of the layer-specific decoder matrices: v_{\mathrm{out},\phi_s}^{\ell} +\coloneqq +W_{\mathrm{out},s}^{\ell_s \to \ell} +\in \mathbb{R}^{d}, +\qquad +\ell_s \leq \ell. On the other hand, the target feature reads from the MLP input through its corresponding encoder vector, given by a row of the encoder matrix: v_{\mathrm{in},\phi_t} +\coloneqq +\left(W_{\mathrm{in},t}^{\ell_t}\right)^\top +\in \mathbb{R}^{d}.

+

Then, let J_{c_s,\ell \to c_t,\ell_t} +\in \mathbb{R}^{d\times d} denote the Jacobian that maps a perturbation written at token position c_s and layer \ell to the MLP input at position c_t and layer \ell_t, where each of the entries is \left[ +J_{c_s,\ell \to c_t,\ell_t} +\right]_{pq} += +\frac{ +\partial \left(h_{\ell_t}^{\mathrm{in},c_t}\right)_p +}{ +\partial \left(h_{\ell}^{\mathrm{out},c_s}\right)_q +}.

+

The direct attribution from source feature \phi_s to target feature \phi_t can then be modelled as A_{\phi_s \rightarrow \phi_t} += +\sum_{\ell_s \leq \ell < \ell_t} +z_{\ell_s}^{s} +\left(v_{\mathrm{in},\phi_t}\right)^\top +J_{c_s,\ell \to c_t,\ell_t} +v_{\mathrm{out},\phi_s}^{\ell}.

+

Each term in the summation measures the contribution of the source feature \phi_s’s write at layer \ell to the pre-activation of target feature \phi_t, transported through the residual stream and attention heads. A_{\phi_s \rightarrow \phi_t} aggregates the downstream writes across layers produced by the same source feature. Notably, Ameisen et al. (2025) stop the gradients flowing through intermediate MLP outputs and fix attention patterns and normalisation denominators. This lets each edge capture direct feature-to-feature influence rather than effects mediated through intermediate feature nodes.

+
+

CLT-based circuit discovery differs from conventional circuit discovery in two aspects. First, the resulting circuit’s nodes correspond to fine-grained linear features rather than coarse MLP blocks or attention modules. Using automated feature-interpretation pipelines (Section 12.6) to assign natural-language descriptions to these features, human inspectors can more easily understand the discovered mechanism. Second, an edge in a CLT-based circuit denotes the direct, input-specific influence between two features, which is different from the formulation of attribution methods based on clean–corrupted reference pairs, such as EAP and EAP-IG. Specifically, in Circuit Tracer, a source feature can write to the residual stream at multiple subsequent layers, and the strength of an edge from the source feature to a target feature is the aggregate of the contributions of these writes to the target feature.

+

For a given prompt, Circuit Tracer produces a sparse attribution graph that describes active features and their interactions, making it well suited to query-level circuit discovery. Interpretability platforms such as Neuronpedia (Lin, 2023) also offer interactive implementations. However, the approach still has several limitations. Specifically, CLTs only approximate MLP computations, so the discovered circuit may only cover a part of the model’s underlying mechanism. Moreover, feature-level attribution graphs can have many nodes and edges, so we must prune, cluster, or otherwise condense them before they become understandable.

-

7.4 Causal Verification#

-

Once circuits are identified, two critical questions arise: are they causally responsible for the behavior, and can we modify them reliably?

+

7.3 Causal Verification#

+

Once we identify circuits, two critical questions arise: are they causally responsible for the behaviour, and can we modify them reliably?

Causal Necessity and Sufficiency.

-

A circuit is causally necessary for a behavior if removing or ablating the circuit degrades that behavior. A circuit is causally sufficient if activating the circuit alone, with all other components zeroed or corrupted, reproduces the behavior. In practice, it is hard to be both strictly necessary and sufficient—citcuits typically explain 60–80% of behavioral variance.

+

A circuit is causally necessary for a behaviour if removing or ablating the circuit degrades that behaviour. A circuit is causally sufficient if activating the circuit alone, i.e., patching all edges outside the circuit with activations from a corrupted reference run, reproduces the behaviour. In practice, a circuit is rarely sparse, necessary, and sufficient at the same time; it typically explains 60–80% of behavioural variance.

Evaluating Circuits and Benchmarking Discovery Methods.

-

A circuit is considered faithful if it is both causally necessary and causally sufficient for the behavior it is claimed to explain (Wang et al., 2023). Two widely used high-level methods for testing these properties are causal scrubbing and interchange interventions (Heimersheim & Nanda, 2024; Chan et al., 2022). Causal scrubbing evaluates necessity by replacing circuit activations with corrupted or neutralized values and measuring the resulting change in model outputs. Interchange interventions test sufficiency by swapping circuit components between clean and corrupted forward passes and assessing whether the target behavior consistently tracks the substituted circuit across inputs.

-

More recently, several works have proposed metrics to quantify circuit faithfulness as a continuous score (Hanna et al., 2024; Wu & Barez, 2025), allowing systematic comparisons of circuits discovered by different algorithms. By aggregating faithfulness scores across tasks and settings, benchmarks such as MIB (Mueller et al., 2025a) evaluate the performance of circuit discovery methods in identifying causally meaningful model components.

+

A circuit is often considered faithful if it is both causally necessary and causally sufficient for the behaviour it is claimed to explain (Hanna et al., 2024; Marks et al., 2025a). Two popular approaches to validate causality are causal scrubbing and interchange interventions (Chan et al., 2022a; Geiger et al., 2024). Causal scrubbing (Chan et al., 2022a) systematically evaluates a mechanistic hypothesis by replacing model activations with those from inputs deemed equivalent under the hypothesis and testing whether these interventions preserve the model behaviour. Interchange interventions (Geiger et al., 2024) test whether model activations causally correspond to higher-level variables by substituting activations across inputs and measuring the resulting change in model behaviour. Together, these methods provide stronger evidence for mechanistic explanations than observational analysis alone.

+

More recently, several works have proposed metrics to quantify circuit faithfulness as a continuous score (Hanna et al., 2024; Wu & Barez, 2026), allowing systematic comparisons of circuits discovered by different methods. By aggregating faithfulness scores across tasks and settings, benchmarks such as MIB (Mueller et al., 2025) evaluate how well circuit discovery methods identify causally important model components. However, achieving strict necessity and sufficiency is often hard because many model behaviours depend on distributed representations and multiple redundant pathways (Barez et al., 2026). Even if we identify a circuit that captures the primary mechanism of a behaviour, components outside the circuit may still play important roles, such as handling edge cases and computational robustness.

-
-

Limitations of Causal Verification.

-

Strict necessity and sufficiency are often unattainable. Most behaviors depend on distributed representations and multiple redundant pathways. A circuit may implement the primary mechanism (60% of variance) while other components handle edge cases. Additionally, verification assumes the circuit mechanism is faithfully captured by the identified components, but if features are entangled or nonlinear interactions are present, the measured circuit may not reflect true causal structure.

-
-

7.5 Case Study: Induction Heads#

+

7.4 Case Study: Induction Heads#

-

Olsson et al. (2022a) provide a mechanistic explanation for how induction heads implement the conditional copying rule described in Definition 6.6. Consider a repeated sequence, for example

-

\texttt{A B C A B \underline{?}},

-

where the model must predict the next token following the second occurrence of B. In 2-layer attention-only models, the minimal induction circuit consists of two interacting attention heads that cooperate across layers to implement the copying computation.

-

The first head, often called the previous-token head, propagates information about the immediately preceding token into the residual stream. At the current position j, this head attends to the token at x_{j-1} and writes a representation of it into the residual stream. By doing so, the residual stream carries contextual information about recent tokens that downstream computations can access. The second head, the true induction head, attends from the current token x_j to all earlier positions i < j where the same token appeared (x_i = x_j). Using the context provided by the previous-token head, the induction head retrieves the residual stream representation of the token following x_i, namely x_{i+1}, and incorporates it into the residual stream at position j. This updated representation directly biases the next-token logits toward the retrieved token, effectively implementing the conditional copying rule:

-

(x_i = x_j) \;\Rightarrow\; \hat{x}_{j+1} \approx x_{i+1}.

-

Empirical evidence supports this mechanistic interpretation. Ablating either the previous-token head or the induction head significantly reduces the model’s pattern completion performance, demonstrating that both heads are causally responsible for the observed in-context learning behavior. Moreover, the circuit emerges sharply during training, coinciding with sudden improvements in in-context learning performance. In small attention-only models, this minimal two-head circuit is sufficient to reproduce the observed behavior.

-

In larger models containing additional layers and MLPs, the precise mechanistic trace becomes more complex, but the core principle remains the same: context is propagated across layers and then used to conditionally retrieve tokens based on repeated subsequence matches. From this perspective, the circuit view complements feature-based analyses by explicitly identifying the causal structure responsible for induction behavior and clarifying how specific heads interact across layers to implement in-context pattern completion (Olsson et al., 2022a).

+

Circuit Perspective on Induction Heads

+

Following our introduction to induction heads in Section 6.8, we consider a repeated sequence:

+

\texttt{A B C A B}

+

where the model needs to predict the token after the second B. The induction head alone is not sufficient to implement induction behaviour from scratch. It still needs help from previous layers’ attention heads to locate the token following the current token’s previous occurrence, so it knows it should attend to C in this example (Olsson et al., 2022).
+Specifically, in a two-layer attention-only transformer, Elhage et al. (2021) discovered the previous-token head, which writes every token’s information to the next position. The change propagates to the residual stream to be read by the induction head, allowing it to locate C (because now some information about the first B is contained in C). The two kinds of heads form the induction circuit that is sufficient to implement the induction behaviour in the two-layer small model.
+Empirical evidence supports this mechanistic interpretation. Head ablation experiments show that induction heads causally support in-context learning in the attention-only model. Circuit analysis also reveals induction heads’ dependence on earlier previous-token heads. Moreover, induction heads emerge abruptly during training, coinciding with sharp improvements in in-context learning performance.
+In larger models with additional layers and MLPs, the mechanism behind induction behaviour becomes more complex. However, the core principle remains the same: context propagates across layers and is used to conditionally retrieve tokens based on repeated subsequence matches. From this perspective, the circuit framework complements feature-based analysis by explicitly identifying the model’s internal mechanism that implements a specific behaviour.

-

7.6 A Unified Evaluation Framework for Mechanistic Interpretability#

-

A persistent challenge in mechanistic interpretability is the lack of standardized evaluation. Different methods—SAEs, transcoders, circuit discovery, probing—are evaluated using incompatible metrics, making it difficult to compare approaches or assess progress. This section proposes a unified evaluation framework covering five dimensions: faithfulness, completeness, stability, causal validity, and human interpretability.

-

7.6.1 The Need for Standardized Evaluation#

-

Current evaluation practices suffer from several problems:

+

7.5 A Unified Evaluation Framework for Mechanistic Interpretability#

+

A persistent challenge in mechanistic interpretability (MI) is the lack of standardised evaluation. Different interpretability methods use incompatible metrics. For example, for the same behaviour of interest, circuit discovery can work at the level of MLP/attention modules (Conmy et al., 2023), CLT features (Ameisen et al., 2025), or even neurons (Arora et al., 2026). A CLT-based circuit must also verify CLT fidelity; an MLP/attention-level circuit should test and explain each node’s role. Thus, it is hard to compare them and measure progress. To address this, this section presents a unified evaluation framework covering five dimensions—attribution faithfulness, completeness, stability, causal validity, and human interpretability—where different methods can be jointly tested.

+

7.5.1 The Need for Standardised Evaluation#

+

Before we discuss the five dimensions, we first explain in detail the need for standardised evaluation. Specifically, current evaluation practices face several challenges:

    -
  • Metric fragmentation: SAEs are evaluated on reconstruction loss; circuits on task performance recovery; probes on classification accuracy. These metrics are not directly comparable.

  • -
  • Cherry-picking: Results are often reported on favorable examples without systematic coverage of failure cases.

  • -
  • Missing baselines: Claims of interpretability gains rarely compare against simple baselines (e.g., random directions, PCA components, raw neurons).

  • -
  • Reproducibility gaps: Hyperparameter sensitivity and seed dependence are underreported, making replication difficult.

  • +
  • Utility ambiguity: Methods are initially developed for different aims and evaluated against distinct objectives, yet have overlapping application scenarios. For example, SAEs are assessed using reconstruction error and sparsity metrics; circuits are evaluated using behavioural recovery or faithfulness tests; and difference-in-means steering vectors (see Section 9.3) are examined by behaviour change—although all three approaches can be used to steer model outputs. Their unclear intended uses make it hard to compare their utility.

  • +
  • Cherry-picking: Existing work often reports interpretability results on carefully selected examples, samples from a narrow distribution, or capabilities where a method performs well. It may not report negative results, failure cases, or coverage statistics. This makes it difficult to assess the robustness and generality of the findings and determine whether the proposed method genuinely outperforms existing ones.

  • +
  • Missing baselines: Many proposed methods’ claimed interpretability gains are not systematically and adequately compared against intuitive alternatives or baselines such as behavioural methods, random activations, or neuron-level analysis. Without this comparison, it is unclear whether a method extracts genuinely meaningful information.

  • +
  • Reproducibility gaps: Many existing techniques involve substantial hyperparameter choices, heavy training procedures, and high variation across seemingly irrelevant factors. Reproducibility is often underreported and has not yet been established as a property to optimise.

  • +
  • Faithfulness–understandability gap: Methods often demonstrate how understandable and informative their outputs are to humans. However, those outputs’ faithfulness to the underlying model mechanism receives less attention. This rewards explanations that are persuasive rather than faithful.

-

A unified framework enables principled comparison across methods, systematic identification of failure modes, and clearer communication of what interpretability methods actually achieve.

+

Lan et al. (2026) have called for a shared platform for tracking and auditing interpretability insights and claims across papers; Ayonrinde & Jaburi (2025) proposed a framework that evaluates mechanistic explanations from several philosophy-of-science perspectives. However, these discussions remain largely conceptual. The MI community needs standardised, concrete evaluation protocols and benchmarks, along with established reporting practices, to enable principled comparisons across methods and findings. They are especially important for measuring real progress in the field, identifying systematic failure modes, and providing evidence about what methods do and do not reveal. More broadly, they are necessary for mechanistic interpretability to mature from a collection of promising techniques into a cumulative scientific discipline.

-

7.6.2 Five Evaluation Dimensions#

-

We propose evaluating mechanistic interpretability methods across five complementary dimensions. No single dimension suffices; strong methods should perform well across all five.

+

7.5.2 Five Evaluation Dimensions#

+

We propose a framework that situates mechanistic interpretability evaluation within five complementary dimensions. No single dimension suffices—strong methods should perform well across all five. Table 9 summarises these five dimensions. We describe each one with example metrics that convey the idea.

- + @@ -170,233 +162,400 @@

Limitations of Causal Verification.

- - - - + + + + - - - + + + - + - + - - - + + + - - - - + + + +
Table 10: Unified Evaluation Framework for Mechanistic InterpretabilityTable 9: Unified evaluation framework for mechanistic interpretability.
Dimension
FaithfulnessDo identified components accurately reflect the model’s actual computation?Causal effect size; intervention consistency; counterfactual accuracyAblation; activation patching; interchange interventionsAttribution FaithfulnessDo attribution scores accurately reflect the causal effects of the identified components?Intervention-effect agreement; effect-size error; counterfactual effect recoveryAblation; activation / path patching; interchange interventions
CompletenessWhat fraction of model behavior is explained by the identified components?Behavior coverage; reconstruction loss; unexplained varianceLoss recovery; behavioral replication; residual analysisHow much of the behaviour is accounted for by the identified components?Loss recovery; behavioural coverage; unexplained varianceReconstruction; behavioural evaluation; residual analysis
StabilityAre findings robust across seeds, checkpoints, and hyperparameters?Are findings robust to variations in factors like random seeds, hyperparameters, paraphrases, etc.? Cross-seed agreement; checkpoint consistency; hyperparameter sensitivityMulti-seed training; checkpoint comparison; ablation studiesMulti-seed analysis; checkpoint comparison; input paraphrasing
Causal ValidityDo components satisfy necessity and sufficiency for target behaviors?Necessity score; sufficiency score; specificityCausal scrubbing; knockout experiments; circuit verificationAre the identified components necessary, sufficient, and specific for the target behaviour?Necessity score; sufficiency score; specificity scoreAblation; component isolation; interchange interventions; tests with control behaviours
InterpretabilityCan humans understand and use the explanations?Human accuracy; explanation utility; prediction improvementUser studies; expert evaluation; downstream task performanceHuman InterpretabilityCan humans understand and use the resulting explanations?Explanation utility; debugging utility; interpretation agreementUser studies; expert evaluation
+

Note that throughout this subsection, we use model components to refer to any unit of analysis, including individual features, neurons, attention heads, circuits, and connections representing information flow between components. We operationalise the model behaviour of interest, B, with a set of input prompts x \sim B. Following the notation introduced in Section 7.1, we denote the model by f_\theta(\cdot), a scalar behaviour-relevant performance metric by M(\cdot), and an intervention on components S by f_\theta\!\left(x_{[S \leftarrow \tilde h_S]}\right), where the activations of S are replaced by reference activations \tilde h_S while the remaining computation proceeds normally. Following Section 7.1, we define the intervention effect of S on an input x as s(S,x) +\coloneqq +M\!\left(f_\theta(x)\right) +- +M\!\left( +f_\theta\!\left(x_{[S \leftarrow \tilde h_S]}\right) +\right) and the average intervention effect on behaviour B as s(S,B) +\coloneqq +\mathbb{E}_{x \sim B}\left[s(S,x)\right].

-

7.6.3 Dimension 1: Faithfulness#

-

Definition: Faithfulness measures whether identified components (features, circuits, explanations) accurately reflect the model’s actual internal computation, rather than post-hoc rationalizations or correlational artifacts.

+

Dimension 1: Attribution Faithfulness#

+

Attribution faithfulness measures whether the interpretability method accurately attributes the model’s behaviour of interest to responsible components.

-

Causal Effect Size: For a component C claimed to implement behavior B, measure: \text{Effect}(C, B) = \frac{|P(B | \text{do}(C = \text{active})) - P(B | \text{do}(C = \text{ablated}))|}{P(B | \text{baseline})} Higher values indicate stronger causal relationship between the component and behavior.

-

Intervention Consistency: Across multiple interventions on C, do effects on B remain consistent? \text{Consistency}(C, B) = 1 - \frac{\text{Var}[\text{Effect}(C, B)]}{\mathbb{E}[\text{Effect}(C, B)]^2}

-

Counterfactual Accuracy: When component C is patched from input x_1 to input x_2, does the model’s behavior shift as predicted? \text{CF-Acc} = \mathbb{E}_{x_1, x_2}\left[\mathbf{1}\left[f(x_1|_{C \leftarrow C(x_2)}) \approx f(x_2)\right]\right]

+

Attribution Faithfulness Metrics

+
+

Intervention-Effect Agreement.

+

Let a(S,x) denote the importance score assigned by an interpretability method to components S for a sample x \sim B: \mathrm{EffectAgreement} += +\rho_{x \sim B}\!\left( +a(S,x), +s(S,x) +\right), where \rho may be Pearson or Spearman correlation. High agreement indicates that the attribution consistently reflects the identified components’ causal effects on behaviour B.
+

+
+
+

Effect-Size Error.

+

When attribution scores are calculated to estimate intervention effects, we measure the effect-size error as: \mathrm{EffectError} += +\mathbb{E}_{x \sim B} +\left[ +\left| +a(S,x)-s(S,x) +\right| +\right]. A low value indicates good calibration between estimated and actual intervention effects. If the two quantities are not naturally on the same scale, we may use normalised or rank-based measures instead.
+

+
+
+

Counterfactual Effect Recovery.

+

Consider inputs x_1 \sim B' and x_2 \sim B, where B' is a reference behaviour (e.g., honesty) and B is the behaviour of interest (e.g., sycophancy). Patching S from x_2 into the computation on x_1 yields \mathrm{CFRecovery}(S;B,B') += +\mathbb{E}_{x_1 \sim B', x_2 \sim B} +\frac{ +M\!\left( +f_\theta\!\left( +x_{1,[S \leftarrow h_S(x_2)]} +\right) +\right) +- +M\!\left(f_\theta(x_1)\right) +}{ +M\!\left(f_\theta(x_2)\right) +- +M\!\left(f_\theta(x_1)\right) +}. A value near 1 indicates that the patching recovers much of the behaviour, while a value near 0 indicates little effect. This metric has been widely used in the circuit discovery literature (Hanna et al., 2024; Zhang et al., 2025b; Syed et al., 2024). However, the denominator of this metric can be close to 0, and M\!\left( +f_\theta\!\left( +x_{1,[S \leftarrow h_S(x_2)]} +\right) +\right) empirically can exceed M\!\left(f_\theta(x_2)\right) or fall below M\!\left(f_\theta(x_1)\right). These may cause this metric to take values much larger than 1 or below 0, complicating the analysis. Recent work has proposed an alternative metric that bounds the value between 0 and 1 (Wu & Barez, 2026).

+
-

Evaluation protocol:

+
+

Evaluation protocol:

    -
  1. Identify component C via the interpretability method being evaluated.

  2. -
  3. Define target behavior B with clear operational criteria.

  4. -
  5. Perform ablation: replace C with baseline (zero, mean, noise) and measure change in B.

  6. -
  7. Perform activation patching: swap C between clean and corrupted inputs; verify behavior follows C.

  8. -
  9. Report effect sizes with confidence intervals across multiple inputs.

  10. +
  11. Apply the method to acquire attribution scores a(S,x) of model components and select those of interest (e.g., components with high scores).

  12. +
  13. Measure the corresponding ground-truth intervention effects s(S,x).

  14. +
  15. Compare them using correlation, error, or rank agreement.

  16. +
  17. Evaluate how much of the behavioural difference between clean and corrupted inputs can be recovered by patching the identified components.

  18. +
  19. Report results across a diverse set of inputs with uncertainty estimates.

-

Baselines: Compare against random components, PCA directions, and individual neurons to establish that identified components are more faithful than naive alternatives.

-

7.6.4 Dimension 2: Completeness#

-

Definition: Completeness measures what fraction of model behavior is explained by the identified components. High completeness means the method accounts for most of the model’s computation; low completeness means significant behavior remains unexplained.

+
+

Dimension 2: Completeness#

+

Completeness measures how much of the model behaviour the identified components or mechanisms explain. A high completeness score indicates that the identified components/mechanisms capture most of the relevant computation.

-

Loss Recovery: After replacing model activations with reconstructions from identified components, what fraction of original loss is preserved? \text{LR} = 1 - \frac{\mathcal{L}_{\text{reconstructed}} - \mathcal{L}_{\text{original}}}{\mathcal{L}_{\text{random}} - \mathcal{L}_{\text{original}}} where \mathcal{L}_{\text{random}} is loss with random/ablated activations.

-

Behavioral Coverage: Across a diverse test suite, what fraction of behaviors \mathcal{B} can be explained by identified components? \text{Coverage} = \frac{|\{b \in \mathcal{B} : \exists C \text{ with } \text{Effect}(C, b) \geq \tau\}|}{|\mathcal{B}|}

-

where \tau is set to be the smallest effect that counts as meaningful.

-

Unexplained Variance: What fraction of activation variance is not captured by identified features? \text{UV} = \frac{\mathbb{E}[\|h - \hat{h}\|^2]}{\mathbb{E}[\|h\|^2]} where h is a model activation and \hat{h} is the reconstruction from identified components.

+

Completeness Metrics

+
+

Loss Recovery.

+

For methods that reconstruct model activations (e.g., SAEs and CLTs), after replacing the target component’s activations with the reconstructions, we measure the fraction of the original model performance that is recovered: \mathrm{LR} += +\frac{ +\mathcal{L}_{\mathrm{reconstructed}} +- +\mathcal{L}_{\mathrm{ablated}} +}{ +\mathcal{L}_{\mathrm{original}} +- +\mathcal{L}_{\mathrm{ablated}} +}, where \mathcal{L}_{\mathrm{original}} is the downstream test loss of the original model, \mathcal{L}_{\mathrm{reconstructed}} is the loss after replacing the original activations with reconstructions, and \mathcal{L}_{\mathrm{ablated}} is the baseline loss where activations are replaced with zero ablation, mean ablation, or Gaussian noise matching the layer statistics. A value near 1 indicates that the reconstruction preserves most of the information. Similar to the vanilla Counterfactual Effect Recovery metric discussed above, this metric can take arbitrary values. A simple solution is to develop a bounded variant, as has been done for Counterfactual Effect Recovery (Wu & Barez, 2026).
+

+
+
+

Behavioural Coverage.

+

Across input prompts x \sim B, we measure the fraction for which the identified components have a meaningful intervention effect: \mathrm{Coverage}(S,B) += +\mathbb{E}_{x \sim B} +\left[ +\mathbbm{1} +\left( +|s(S,x)| \geq \tau +\right) +\right], where s(S,x) is the intervention effect defined above and \tau specifies the minimum effect considered meaningful. Higher coverage indicates that the identified components account for the behaviour across a larger fraction of inputs.
+

+
+
+

Unexplained Variance.

+

For methods that reconstruct activations of the components, we measure the fraction of unexplained activation variance as: \mathrm{UV}(S,B) += +\frac{ +\mathbb{E}_{x \sim B} +\left[ +\|h(x)-\hat{h}_S(x)\|_2^2 +\right] +}{ +\mathbb{E}_{x \sim B} +\left[ +\|h(x)-\bar{h}_B\|_2^2 +\right] +}, where \bar{h}_B = \mathbb{E}_{x \sim B}[h(x)], h(x) is the original model activation, and \hat{h}_S(x) is its reconstruction from the identified components S. Lower values indicate that the identified components explain a larger fraction of the variance in the original activations.

+
-

Evaluation protocol:

+
+

Evaluation protocol:

    -
  1. Define a comprehensive behavioral test suite spanning diverse capabilities (factual recall, reasoning, safety behaviors, etc.).

  2. -
  3. For each behavior, attempt to identify responsible components.

  4. -
  5. Report coverage: fraction of behaviors with identified components meeting faithfulness thresholds.

  6. -
  7. Report reconstruction metrics: loss recovery and unexplained variance.

  8. -
  9. Analyze failure cases: which behaviors resist explanation?

  10. +
  11. Report behavioural coverage using the intervention effects s(S,x).

  12. +
  13. Report reconstruction-based metrics such as loss recovery and unexplained variance if applicable.

  14. +
  15. Analyse inputs of the behaviour for which the identified components do not provide good coverage or reconstruction.

-

Critical consideration: Methods may achieve high faithfulness on explained behaviors while leaving most behavior unexplained. Completeness ensures we track the gap between what is explained and what remains opaque.

-

7.6.5 Dimension 3: Stability#

-

Definition: Stability measures whether findings are robust to incidental choices—random seeds, training checkpoints, hyperparameter settings—that should not affect conclusions about model internals.

+
+

Dimension 3: Stability#

+

Stability measures the robustness of explanations, attributions, and findings to changes in variables that should not affect the results, such as the choice of random seeds, small hyperparameter changes, and input paraphrasing.

-

Cross-Seed Agreement: Train the interpretability method (e.g., SAE) with K different random seeds. Measure feature alignment: \text{Seed-Stability} = \frac{1}{K(K-1)} \sum_{i \neq j} \text{Align}(F_i, F_j) where \text{Align} measures correspondence between feature sets (e.g., via Hungarian matching on cosine similarity).

-

Checkpoint Consistency: For features/circuits identified at checkpoint t, do they persist at checkpoints t \pm \Delta? \text{Ckpt-Stability} = \mathbb{E}_t[\text{Align}(F_t, F_{t+\Delta})]

-

Hyperparameter Sensitivity: How much do results change under reasonable hyperparameter variation? \text{HP-Sensitivity} = \frac{\max_{\theta \in \Theta} \text{Metric}(\theta) - \min_{\theta \in \Theta} \text{Metric}(\theta)}{\mathbb{E}_{\theta \in \Theta}[\text{Metric}(\theta)]} Lower sensitivity indicates more robust findings.

+

Stability Metrics

+
+

Cross-Seed Agreement.

+

We train or apply the interpretability method K times under different random seeds. Let \mathcal{F}_i denote the set of features or mechanisms, such as SAE features and circuits, discovered under seed i. We measure the cross-seed agreement as: \text{Seed-Stability} = \frac{1}{K(K-1)} \sum_{i \neq j} \text{Align}(\mathcal{F}_i, \mathcal{F}_j), where \text{Align}(\cdot) measures similarity between sets. For example, Paulo & Belrose (2026) calculate the Hungarian matching between two sets of SAE features based on their pairwise cosine similarities.
+

+
+
+

Hyperparameter Sensitivity.

+

We measure the sensitivity of mechanistic analyses or methods under reasonable hyperparameter variations. Let \Omega denote the set of evaluated hyperparameter configurations and \text{Metric}(\omega) denote the experimental result (e.g., a method’s performance) under a configuration \omega \in \Omega. We calculate the hyperparameter sensitivity as: \text{HP-Sensitivity} += +\frac{ +\max_{\omega \in \Omega} \text{Metric}(\omega) +- +\min_{\omega \in \Omega} \text{Metric}(\omega) +}{ +\mathbb{E}_{\omega \in \Omega}[\text{Metric}(\omega)] +}. Lower values indicate that the finding is less sensitive to hyperparameter choices and therefore more robust.
+Paraphrase Consistency. For each input x \sim B, let \mathcal{P}(x) = \{x^{(1)}, \dots, x^{(K)}\} denote a set of semantics-preserving paraphrases of x. We measure the consistency of explanations or attributions across \mathcal{P}(x). For example, we can measure whether the intervention effect of S remains consistent: \text{Para-Consistency}(S, B) += \mathbb{E}_{x \sim B}\left[ + 1 - \frac{\mathrm{Var}_{x' \in \mathcal{P}(x)}\left[s(S, x')\right]} + {\mathbb{E}_{x' \in \mathcal{P}(x)}\left[s(S, x')^2\right]} +\right]. The score lies in [0, 1]. A value of 1 indicates that the intervention effect is identical across all paraphrases. A value of 0 indicates that, for each x, the expected intervention effect, i.e., \mathbb{E}_{x' \in \mathcal{P}(x)}s(S, x'), is 0.

+
-

Evaluation protocol:

+
+

Evaluation protocol:

    -
  1. Train interpretability method with at least 3–5 random seeds.

  2. -
  3. Compute feature/circuit alignment across seeds.

  4. -
  5. Report which findings are stable (appear across all seeds) vs. unstable (seed-specific).

  6. -
  7. For key claims, verify they hold under hyperparameter variation (\pm 20\% on key parameters).

  8. -
  9. Distinguish “core” findings (stable) from “fragile” findings (unstable).

  10. +
  11. Execute the interpretability method with several random seeds and compute alignment of the discovered units or findings across seeds.

  12. +
  13. Analyse which findings are stable across all seeds and which ones are seed-specific. Key claims need to remain consistent under subsequent tests on hyperparameter variation and input paraphrasing, if applicable.

  14. +
  15. Distinguish robust findings from fragile ones and report both of them.

-

Reporting recommendation: Papers should report stability metrics alongside primary results. Claims based on unstable findings should be flagged as preliminary.

-

7.6.6 Dimension 4: Causal Validity#

-

Definition: Causal validity assesses whether identified components satisfy formal causal criteria of necessity (removing the component disrupts the behavior) and sufficiency (the component alone can produce the behavior).

+
+

Dimension 4: Causal Validity#

+

Causal validity assesses whether the identified components or mechanisms are causal to the behaviour, particularly necessity (i.e., removing the components/mechanisms disrupts the behaviour) and sufficiency (i.e., the components/mechanisms alone are sufficient to preserve the behaviour).

-

Necessity Score: Fraction of behavior explained by the component that is lost when the component is ablated: \text{Necessity}(C, B) = \frac{P(B | \text{baseline}) - P(B | \text{do}(C = \text{ablated}))}{P(B | \text{baseline})} High necessity means the component is required for the behavior.

-

Sufficiency Score: Fraction of behavior that can be produced by the component alone (all other components ablated or set to baseline): \text{Sufficiency}(C, B) = \frac{P(B | \text{only } C \text{ active})}{P(B | \text{baseline})} High sufficiency means the component is enough for the behavior.

-

Specificity: Does the component affect only the claimed behavior, or does it have broad side effects? \text{Specificity}(C, B) = \frac{\text{Effect}(C, B)}{\sum_{B' \in \mathcal{B}} \text{Effect}(C, B')} High specificity means the component is selective for the target behavior.

+

Causal Validity Metrics

+
+

Necessity Score.

+

Necessity measures the fraction of the model’s performance on behaviour B that is lost when S is ablated (i.e., its activations are replaced with some reference values). It is calculated as: \mathrm{Necessity}(S,B) += +1- +\frac{ +\mathbb{E}_{x\sim B} +\left[ +M\!\left( +f_\theta\!\left( +x_{[S\leftarrow\tilde h_S]} +\right) +\right) +\right] +}{ +\mathbb{E}_{x\sim B} +\left[ +M\!\left(f_\theta(x)\right) +\right] +}. A higher value indicates that S is necessary for the behaviour.
+

+
+
+

Sufficiency Score.

+

Let \bar S denote the components outside S. Sufficiency measures how much of the original behaviour is achieved when only S is active (i.e., activations of all components outside S are replaced with some reference values). It is calculated as: \mathrm{Sufficiency}(S,B) += +\frac{ +\mathbb{E}_{x\sim B} +\left[ +M\!\left( +f_\theta\!\left( +x_{[\bar S \leftarrow \tilde h_{\bar S}]} +\right) +\right) +\right] +}{ +\mathbb{E}_{x\sim B} +\left[ +M\!\left(f_\theta(x)\right) +\right] +}. A high value indicates that S alone implements much of the behaviour.
+

+
+
+

Specificity.

+

Specificity measures the effect on non-target behaviours by interventions on S. It is calculated as: \mathrm{Specificity}(S,B) += +\frac{ +|s(S,B)| +}{ +\sum_{B' \in \mathcal{B}} +|s(S,B')| +}, where \mathcal{B} contains the target behaviour together with a set of relevant control behaviours. A high value indicates that the causal effect of S is specific to B.

+
-

Evaluation protocol (Causal Scrubbing):

+
+

Evaluation protocol:

    -
  1. Necessity test: Ablate the identified circuit/feature. Measure degradation on target behavior. Compare to ablating random components of equal size.

  2. -
  3. Sufficiency test: Ablate all components except the identified circuit. Measure preservation of target behavior. Compare to random circuits of equal size.

  4. -
  5. Interchange test: Swap circuit activations between inputs that differ on the target behavior. Verify behavior follows the swapped circuit.

  6. -
  7. Specificity test: Measure effects on a suite of non-target behaviors. Compute specificity ratio.

  8. +
  9. Necessity test: Ablate S and measure the resulting degradation on behaviour B. Compare against ablating random components of matched size.

  10. +
  11. Sufficiency test: Replace components outside S with reference activations and measure how much of behaviour B is preserved. Compare against random component sets of matched size.

  12. +
  13. Interchange test: Swap activations of S between inputs representing different behaviours and test whether the model output follows the intervention.

  14. +
  15. Specificity test: Measure intervention effects on a set of non-target behaviours and examine whether the effect is specific to B.

-

Benchmarks: The MIB benchmark (Mueller et al., 2025a) provides standardized tasks and metrics for causal circuit evaluation. AxBench (Wu et al., 2025) and RAVEL (Huang et al., 2024) provide feature-level causal evaluation.

+

The MI community has developed several benchmarks that examine the causal validity of interpretability methods. For example, MIB (Mueller et al., 2025) provides standardised tasks for assessing discovered circuits and causal variables (e.g., features); AxBench (Wu et al., 2025b) evaluates methods’ ability to steer models towards given concepts; RAVEL (Huang et al., 2024) investigates methods’ ability to localise features of given concepts by using interchange interventions to validate discovered features; and SAEBench (Karvonen et al., 2025) evaluates SAEs specifically, pairing reconstruction metrics with downstream intervention tasks such as gender bias erasure.

+
-

7.6.7 Dimension 5: Human Interpretability#

-

Definition: Human interpretability measures whether explanations are actually useful to humans—whether they improve human understanding, prediction, or decision-making about model behavior.

+

Dimension 5: Human Interpretability#

+

Human interpretability measures whether generated explanations eventually improve human understanding, prediction, and decision-making about model behaviour.

-

Explanation Accuracy: Can humans predict model behavior based on the explanation? \text{Expl-Acc} = P(\text{human predicts } f(x) | \text{explanation of } f) Compare to baseline prediction accuracy without explanation.

-

Simulatability: After seeing explanations, can humans simulate the model’s computation on new inputs? \text{Simulatability} = \text{Corr}(\text{human prediction}, f(x_{\text{new}}))

-

Debugging Utility: Do explanations help humans identify and fix model errors? \text{Debug-Utility} = \frac{\text{\#Errors fixed with explanation}}{\text{\#Errors fixed without explanation}}

-

Consistency of Interpretation: Do different humans interpret the same explanation consistently? \text{Inter-Rater} = \kappa(\text{human}_1 \text{ labels}, \text{human}_2 \text{ labels}) where \kappa is Cohen’s kappa or a similar agreement metric.

+

Human Interpretability Metrics

+
+

Explanation Utility.

+

This metric tests whether humans can predict model behaviour more accurately in the held-out set based on the explanation. It is formulated as: \text{Expl-Utility} += +P(\text{correct prediction} \mid \text{with explanation}) +- +P(\text{correct prediction} \mid \text{no explanation})

+
+
+

Debugging Utility.

+

This metric examines whether the explanations help humans identify and repair model errors, such as incorrect outputs or misaligned behaviour, in the wild. \text{Debug-Utility} += +P(\text{error fixed}\mid\text{with explanation}) +- +P(\text{error fixed}\mid\text{no explanation}).

+
+
+

Consistency of Interpretation.

+

This metric inspects whether different humans interpret the same explanation consistently. \text{Inter-Rater} = \kappa(\text{human}_1 \text{ labels}, \text{human}_2 \text{ labels}) where \kappa(\cdot) is Cohen’s kappa or a similar agreement metric.

+
-

Evaluation protocol:

+
+

Evaluation protocol:

    -
  1. Prediction task: Show humans feature/circuit explanations. Ask them to predict model behavior on held-out inputs. Compare accuracy to no-explanation baseline.

  2. -
  3. Identification task: Show humans model outputs with errors. With and without mechanistic explanations, ask them to identify the cause. Measure accuracy and time.

  4. -
  5. Intervention task: Ask humans to propose fixes for model errors based on explanations. Measure success rate of proposed interventions.

  6. -
  7. Agreement task: Have multiple raters label feature interpretations. Measure inter-rater reliability.

  8. +
  9. Prediction task: Show humans explanations of model components or mechanisms and ask them to predict model behaviour on held-out inputs. Compare accuracy to a no-explanation baseline.

  10. +
  11. Identification task: Show humans model outputs with errors and ask them to identify the cause, with and without explanation assistance. Measure accuracy and required time to complete the task.

  12. +
  13. Intervention task: Ask humans to propose fixes for model errors based on explanations. Measure the success rate of proposed interventions.

  14. +
  15. Agreement task: Ask multiple human raters to generate explanations, if applicable (e.g., write a summary of highly activated samples of an SAE feature as its explanation). Measure inter-rater reliability and agreement with machine-generated explanations.

-

Considerations:

-
    -
  • Human studies are expensive and introduce subjectivity. Automated proxies (LLM-based evaluation) can supplement but not replace human evaluation.

  • -
  • Expertise matters: interpretations accessible to ML researchers may be opaque to domain experts, and vice versa.

  • -
  • Task-specificity: a feature interpretable for one task may be confusing for another.

  • -
-
-

7.6.8 Integrated Evaluation Protocol#

-

We recommend the following integrated protocol for comprehensive evaluation:

-
-

Step 1: Define scope

-
    -
  • Specify the model, layer(s), and behavior(s) under analysis.

  • -
  • Define a behavioral test suite with clear success criteria.

  • -
-

Step 2: Establish baselines

-
    -
  • Random directions in activation space.

  • -
  • PCA components.

  • -
  • Individual neurons.

  • -
  • Supervised probes (if labels available).

  • -
-

Step 3: Apply interpretability method

-
    -
  • Train with multiple seeds (minimum 3).

  • -
  • Record hyperparameters and training details for reproducibility.

  • -
-

Step 4: Evaluate across all five dimensions

-
    -
  • Faithfulness: ablation and patching experiments.

  • -
  • Completeness: loss recovery and behavioral coverage.

  • -
  • Stability: cross-seed and hyperparameter sensitivity.

  • -
  • Causal validity: necessity, sufficiency, specificity tests.

  • -
  • Human interpretability: user studies or proxy metrics.

  • -
-

Step 5: Report comprehensively

-
    -
  • Report all five dimensions, not just favorable metrics.

  • -
  • Include failure cases and unexplained behaviors.

  • -
  • Compare to baselines on all metrics.

  • -
  • Provide confidence intervals and significance tests.

  • -
-
-
-

7.6.9 Applying the Framework: Method-Specific Guidance#

-

Different interpretability methods emphasize different dimensions. Table 11 provides guidance on applying the framework to common methods.

+

Several caveats and challenges arise when evaluating the human interpretability of tested methods. For example, human studies are usually expensive. Moreover, human expertise may confound results, where some explanations understandable to MI researchers may be opaque to domain experts, and vice versa.

+

Note that different interpretability methods may emphasise distinct evaluation dimensions and operationalise them differently. Table 10 provides guidance on applying this framework to some common interpretability approaches.

- + - + - - + + - - - - - - - - - - - - - - + + + + + + - - - - - + + + + + - - - - - + + + + + - - - - - + + + + +
Table 11: Evaluation Priorities by Method TypeTable 10: Recommended evaluation priorities by interpretability method. The entries indicate evaluation aspects particularly important for each method, not mutually exclusive metrics.
MethodFaithfulnessAttribution Faithfulness Completeness StabilityCausalHumanCausal ValidityHuman Interpretability
SAEsFeature ablationLoss recovery; UVSeed stability criticalFeature-level N/SAuto-labeling quality
Transcoders/CLTsLayer reconstructionCross-layer coverageArchitecture sensitivityCircuit-level N/SCircuit descriptionsSAEs and variantsFeature attribution–intervention agreement; effect-size errorActivation reconstruction; loss recovery; unexplained varianceCross-seed feature alignmentFeature necessity, sufficiency, and specificityFeature description consistency across annotations, including models and humans
Circuit DiscoveryPath patchingBehavioral coverageMethod variancePrimary focusCircuit diagramsEdge or component attribution–intervention agreement; counterfactual recoveryBehavioural coverage; loss or behaviour recoveryRobustness across hyperparameters, seeds, and paraphrasesCircuit necessity, sufficiency, specificity; interchange testsCircuit-level descriptions; trade-off between sparsity and variance explained
ProbingProbe \rightarrow causalInformation contentProbe architectureOften weakConcept labelsAgreement between probe-derived importance and intervention effectsCoverage across target concepts or behaviours under different distributionsRobustness across seeds and sampled fitting setsNecessity, specificity, and intervention tests on probe directionsExplanation of shared information in fitting set and search for confounding factors
Steering VectorsBehavioral effectBehavior breadthDirection stabilityImplicit in methodSteering descriptionsAgreement between predicted and observed intervention effectsCoverage across behaviours deemed relevantStability across steering magnitudes, seeds, and paraphrasesIntervention efficacy and specificity; side-effect analysisExplanation of shared information in fitting set and search for confounding factors
-

7.6.10 Limitations of This Framework#

-

This evaluation framework has limitations that should be acknowledged:

+
+

7.5.3 Integrated Evaluation Protocol#

+

We recommend the following integrated protocol for comprehensive evaluation:

+
+

Recommended Evaluation Protocol

+
+

Step 1: Define scope.

+

Users first specify the model, layer(s), and behaviour(s) of interest and define a behavioural test suite with clear success criteria.
+

+
+
+

Step 2: Establish baselines.

+

Users specify baseline methods, such as random directions in activation space, PCA components, individual neurons, and supervised probes (if labels are available).
+

+
+
+

Step 3: Apply interpretability method.

+

Users apply the method of interest to analyse models with variation in random seeds, hyperparameters, and other training details.
+

+
+
+

Step 4: Evaluate across all five dimensions.

+

Users compute metrics for attribution faithfulness, completeness, stability, causal validity, and human interpretability.
+

+
+
+

Step 5: Report comprehensively.

+

Users report all five dimensions instead of just favourable metrics and discuss the method’s failure cases, unexplained behaviours, and comparisons to baselines.

+
+
+
+

7.5.4 Limitations of This Framework#

+

This evaluation framework, though comprehensive, still has several limitations:

    -
  • Metric choice: The specific metrics proposed are not uniquely correct. Alternative formalizations may be appropriate for specific contexts.

  • -
  • Threshold arbitrariness: Proposed thresholds (e.g., necessity > 0.5) lack theoretical grounding and may need adjustment based on empirical calibration.

  • -
  • Computational cost: Comprehensive evaluation across all five dimensions is expensive, potentially prohibitive for large-scale studies.

  • -
  • Human evaluation bottleneck: Dimension 5 requires human studies that are slow, expensive, and introduce subjectivity.

  • -
  • Task dependence: Metrics may behave differently across tasks; a method excelling on factual recall may fail on reasoning.

  • +
  • Metric choice: The proposed specific metrics are not uniquely correct. Alternative formalisations may be appropriate for specific contexts.

  • +
  • Threshold arbitrariness: Thresholds (e.g., the accuracy that counts as success) may lack theoretical grounding and will need empirical calibration.

  • +
  • Computational cost: Evaluating all five dimensions comprehensively is expensive and potentially prohibitive for large-scale studies.

  • +
  • Human evaluation bottleneck: Dimension 5 requires time-consuming, expensive, and subjective human studies.

  • +
  • Task dependence: A method’s performance under this framework can depend on the evaluation task. For example, a method that excels at factual recall may perform poorly at reasoning.

-

Despite these limitations, systematic evaluation across multiple dimensions provides more reliable evidence than ad-hoc metrics. The framework should be viewed as a starting point for community standardization, not a final answer.

+

Despite these limitations, systematic evaluation across multiple dimensions provides more reliable evidence than an arbitrary choice of metrics. Readers should view the framework as a starting point for standardisation, not a final answer.

+
+

Summary.

+

This section introduced circuit discovery methods, which trace how model components or features interact to produce behaviour, either directly on the model’s computational graph (e.g., ACDC and EAP) or within interpretable replacement models (e.g., Circuit Tracer). We also discussed how to causally verify discovered circuits and introduced a broader evaluation framework for mechanistic interpretability methods based on faithfulness, completeness, stability, causal validity, and human interpretability. In summary, Section 6 introduced methods for extracting interpretable units of analysis, while this section examined how those units interact through circuits to produce behaviour. The next section takes a complementary perspective: directly interpreting the information in individual representations using natural language.

+
diff --git a/src/generated/chapters/colour-key.html b/src/generated/chapters/colour-key.html index 1289e8b6e2..f21fea82f4 100644 --- a/src/generated/chapters/colour-key.html +++ b/src/generated/chapters/colour-key.html @@ -1,33 +1,43 @@ -

B Color Key and Style Guide#

-

This appendix explains the visual conventions used throughout the survey for boxes, emphasis, and structural elements.

-

B.1 Box Types and Colors#

+

B Colour Key and Style Guide#

+

This section explains the visual conventions used throughout the survey for boxes, emphasis, and structural elements.

+

B.1 Box Types and Colours#

-

Definition boxes introduce core concepts and provide precise technical definitions. Use these when a term needs clear specification before use.

-

Example concepts: features, circuits, SAEs, privileged access hypothesis.

-

Text color: Dark blue border, light blue background.

+

Definition Box (Dark Blue)

+

Definition boxes introduce core concepts and provide precise technical definitions. Use these when a term needs clear specification before use.
+Example concepts: “Feature”, “Circuit”, “Sparse Autoencoder”.
+Style: Dark blue border, light blue background.

-

Example boxes provide concrete illustrations, worked examples, or case studies. Use these to make abstract concepts concrete.

-

Example contents: “Feature vs. Neuron: An Illustration,” “MLP as Feature Combination,” worked examples of causal tracing.

-

Text color: Green border, light green background.

+

Example Box (Green)

+

Example boxes provide concrete illustrations, worked examples, or case studies. Use these to make abstract concepts concrete.
+Example contents: “Induction in a Repeated Sequence”, “Example Application of Steering Vectors”, “Probing Classifiers for Sleeper Agents”.
+Style: Green border, light green background.

-

Insight boxes explain intuitions behind mathematical or conceptual results. Use these to build understanding of why something works, not just that it works.

-

Example contents: “Why Orthogonalization Works,” “Superposition in High Dimensions,” “Mechanistic Interpretation of Attention.”

-

Text color: Gold border, light gold background.

+

Insight Box (Gold)

+

Insight boxes explain intuitions behind mathematical or conceptual results. Use these to build understanding of why something works, not just that it works.
+Example contents: “Trade-offs Between Variants”, “The Complexity–Interpretability Trade-off”.
+Style: Gold border, light gold background.

-

Warning boxes highlight critical limitations, failure modes, or common misconceptions. Use these to prevent readers from adopting incorrect or unsafe interpretations.

-

Example contents: “Steering Vector Brittleness,” “The Probe Causality Problem,” “The Explanation Validation Crisis.”

-

Text color: Red border, light red background.

+

Warning Box (Red)

+

Warning boxes highlight critical limitations, failure modes, or common misconceptions. Use these to prevent readers from adopting incorrect or unsafe interpretations.
+Example contents: “Caveats on CLT Fidelity Claims”, “The Probe Causality Problem”, “The Partial Coverage Issue”.
+Style: Red border, light red background.

+
+
+

Practice Box (Teal)

+

Practice boxes illustrate suggestions that are actionable. These include protocol recommendations, checklists for verifying approaches, and step-by-step decision-making walkthroughs. Use these for practical implementations of procedures.
+Example contents: “Guardrail Framework for Automated Interpretability”, “Validation Procedures for Introspective Models”, “Recommended Evaluation Protocol”.
+Style: Teal border, light teal background.

-

B.2 Citation Evidence Levels#

-

This survey cites both peer-reviewed publications and preprints. To help readers assess evidence strength, citations are color-coded:

+

B.2 Citation Evidence Levels#

+

This survey cites both peer-reviewed publications and non-peer-reviewed research outputs. To help readers assess the maturity and review status of the evidence, citations are colour-coded:

    -
  • Orange citations: Peer-reviewed publications (conferences, journals)

  • -
  • Green citations: Preprints, technical reports, or emerging results awaiting peer review

  • +
  • Orange citations denote formally published work: peer-reviewed conference and journal papers, and books.

  • +
  • Green citations: Research outputs that have not undergone formal peer review, including preprints, technical reports, workshop papers, blog posts, and other emerging resources.

-

Results supported primarily by green citations should be considered preliminary. We encourage readers to verify the current publication status of cited work.

+

Green citations should not automatically be interpreted as lower quality or less reliable. Some originate from leading research organisations and may have undergone substantial internal review. However, they have not been subjected to the formal academic peer-review process and should therefore be interpreted with appropriate caution. Readers are encouraged to verify the current publication status of cited work where relevant.

diff --git a/src/generated/chapters/conclusion.html b/src/generated/chapters/conclusion.html index bfa06a44f8..78e42f6995 100644 --- a/src/generated/chapters/conclusion.html +++ b/src/generated/chapters/conclusion.html @@ -1,23 +1,23 @@ -

16 Conclusion#

-

Mechanistic interpretability (MI) has matured from exploratory research into an engineering discipline, with established knowledge, reproducible methods, deployed applications, and well-understood limitations. Rather than converging on a single approach, the field has crystallized into five divergent research orientations—Pragmatic, Ambitious, Developmental, Agentic, and Automated—each making rational trade-offs between completeness, cost, and applicability.

-

16.1 Achievements and Limits#

-

MI succeeds in many aspects: extracting monosemantic features via SAEs, discovering algorithmic circuits, and enabling surgical interventions. It has also helped establish AI researchers’ knowledge, mindset, and taste in other research domains. However, current methods explain only part of the model behavior, struggling with entangled computation and breaking down under distribution shift. Comprehensive analyses of frontier models remain intractable (Templeton et al., 2024).

-
-

From reverse-engineering to engineering.

-

Perhaps one of the most significant conceptual developments of 2024–2025 is the emergence of a third paradigm alongside post-hoc analysis and behavioral evaluation: interpretability by design. Architectural interventions (Section 6.9) demonstrate that the opacity of neural networks is not an immutable fact, but a consequence of design choices that can be revisited. Weight-sparse transformers, MOLT, and other related methods suggest a future where interpretability is a prior design constraint rather than a post-hoc recovery problem. Scaling laws (Gao et al., 2025b) have shown that large sparse models can achieve both capability and interpretability, providing empirical grounds for optimism that this paradigm may scale to frontier systems.

+

15 Conclusion#

+

Neural networks could be one of the most consequential artefacts humanity has built, yet we do not understand how they work or reliably control what they do. Recent decades have seen an exponential improvement in AI capabilities, but methods to understand and explain internal mechanisms have lagged behind. This has created an interpretability gap between what models can do and what we know about how they do it.

+

Mechanistic interpretability offers a path towards understanding these systems by studying their internal representations and computations, with the aim of evaluating, predicting, and controlling behaviour. In this survey, we examine closely what interpretability can actually contribute to AI safety. We place mechanistic interpretability in the broader context of interpretability research and compare its mechanistic evidence with behavioural evaluation. Beyond surveying existing methods, we propose a unified framework for evaluating mechanistic interpretability claims and characterise six research orientations by their goals, assumptions, and trade-offs.

+

Rather than converging on a single approach, the field has crystallised into divergent research orientations, including Pragmatic Interpretability, Ambitious Interpretability, Developmental Interpretability, Interpretability Across Time, Introspective Interpretability, and Automated Interpretability, each making trade-offs between completeness, cost, and applicability.

+

15.1 Achievements and Limits#

+

Mechanistic interpretability has achieved substantial progress across several fronts, including extracting interpretable features with SAEs, discovering task-relevant circuits, and enabling targeted interventions on model behaviour. Overall, the field has shaped how researchers view model computation, representations, and causal structure.

+

Yet current approaches are limited: they do not fully explain many behaviours, struggle to generalise under distribution shift, have difficulty cleanly disentangling entangled computations, and are hard to scale to frontier models. More importantly, their usefulness for AI safety remains debated.

+

Although many prior studies in the interpretability community assume that mechanistic understanding directly implies safer systems, the assumption is in fact largely untested. Therefore, this survey considers not only what a method reveals about a model’s internal computation, but also what it allows us to do in practice. We particularly discuss whether it supports safety-relevant tasks such as detecting deceptive behaviours or identifying dangerous capabilities, and whether those tasks can be achieved just as well without interpretability methods or insights.

+

A persistent challenge in mechanistic interpretability is the lack of standardised evaluation. Different interpretability methods use incompatible metrics. To address this, we present a unified evaluation framework covering five dimensions—attribution faithfulness, completeness, stability, causal validity, and human interpretability—where different methods can be jointly tested. We identify gaps and argue that the evaluation methodology and safety benchmarks are the field’s most pressing open problems.

+

Mechanistic interpretability is not a panacea for AI safety. Researchers, developers, and practitioners should recognise the limits of partial or incorrect causal understanding, rigorously evaluate MI tools under broad conditions, and combine them with complementary safety measures. MI should therefore be viewed as one component of a defence-in-depth safety strategy rather than a standalone guarantee of safety.

-
-

16.2 Unresolved Questions#

-

As discussed in Section 15, several fundamental open questions remain. When is mechanistic understanding actually required for safety, or do behavioral methods suffice? Can mechanisms be verified as genuine rather than post-hoc rationalizations? How do mechanisms hold up under distribution shift and adversarial pressure? These questions are not merely technical but conceptual, touching on epistemology and the definition of understanding itself.

-
-

16.3 The Road Ahead: Three Complementary Paradigms#

-

The field of MI is evolving from a single methodology toward three complementary paradigms:

+

15.2 The Road Ahead#

+

The six research orientations describe different research goals and priorities. At a broader level, mechanistic interpretability can be organised into three complementary paradigms that encompass these orientations and other research directions:

    -
  1. Post-hoc decomposition: The traditional approach of reverse-engineering trained models via SAEs, transcoders, circuit discovery, and causal analysis. This remains essential for understanding existing deployed systems and will likely always be necessary for models trained without interpretability constraints.

  2. -
  3. Developmental analysis: Understanding how mechanisms emerge during training via singular learning theory, phase transition detection, and checkpoint analysis. This enables early intervention and prediction of capability emergence before deployment.

  4. -
  5. Interpretability by design: Engineering architectures that are mechanistically transparent by construction, such as weight-sparse transformers, MOLT, KANs, and their successors. This moves interpretability upstream, showing the promise to eliminate the need for expensive post-hoc analysis.

  6. +
  7. Post-hoc explanation: The traditional paradigm of studying trained models’ internals via tools like SAEs, circuit discovery, activation verbalisers, and causal analysis. Because deployed systems still rely on conventional dense architectures without intrinsic interpretability, post-hoc analysis will remain important in the medium term.

  8. +
  9. Studying learning dynamics: Understanding how mechanisms emerge during training via methods like singular learning theory and checkpoint analysis. This paradigm enables understanding, prediction, and intervention early in behaviour and capability emergence.

  10. +
  11. Interpretability by design: Engineering architectures that are mechanistically transparent by construction, such as SoLU, weight-sparse transformers, KANs, and their successors. This paradigm moves interpretability upstream and promises to reduce the need for expensive post-hoc analysis.

-

These paradigms are not mutually exclusive. Post-hoc methods remain necessary for legacy systems and for validating architectural claims; developmental analysis informs both when to intervene during training and what architectural constraints are most effective; interpretability-by-design reduces the burden on post-hoc methods but requires validation that the architectural transparency transfers to genuine mechanistic understanding.

-

The maturation of MI thus lies not in the dominance of any single approach but in the principled integration of all three—matching methodology to context, validating claims across paradigms, and building cumulative understanding of how to create AI systems that are transparent, verifiable, and safe.

+

Each approach tackles a different area but complements the others. For instance, interpretability-by-design models can reduce reliance on post-hoc methods, while post-hoc mechanistic analysis can guide next-generation interpretability-by-design models towards desirable properties that current dense architectures lack. Insights into learning dynamics can suggest architectural constraints and training-time interventions.

+

Progress also depends on establishing stronger scientific standards. Recent work argues that interpretability should produce claims that are both falsifiable and reproducible, and improve its use in model evaluations and downstream applications.

+

Ultimately, the question remains: Does understanding how models work actually make them safer? This survey’s goal is to understand how these approaches can work together to improve model understanding and safeguards, while recognising the limits of partial or incorrect causal understanding.

diff --git a/src/generated/chapters/core-hypotheses.html b/src/generated/chapters/core-hypotheses.html index c9949bc6eb..533a985ae2 100644 --- a/src/generated/chapters/core-hypotheses.html +++ b/src/generated/chapters/core-hypotheses.html @@ -1,185 +1,95 @@ -

5 Core Hypotheses in Mechanistic Interpretability#

-

Mechanistic interpretability is guided by a number of core hypotheses about how neural networks represent information and implement computation. These hypotheses are not universally accepted facts; rather, they function as organizing principles that shape methodological choices and empirical investigations. In this section, we formalize these hypotheses, summarize supporting and opposing evidence, and highlight open questions.

-

5.1 The Linear Representation Hypothesis#

-

We have seen that single units are not the unit of representation, but what is? The linear representation hypothesis claims that directions in the activation space are the fundamental units of representation.

-
-

Features correspond to directions \phi \in \mathbb{R}^d in activation space. The projection \phi^\top h_l(x) approximately measures the intensity of that feature for input x, and features combine additively: h_l(x) \approx \sum_i \alpha_i \phi_i. This implies features can be detected via linear probes and added or removed via vector arithmetic on the h_l(x).

+

5 Core Hypotheses in Mechanistic Interpretability#

+

Mechanistic interpretability rests on several core hypotheses about how neural networks represent information and implement computation. These hypotheses are not yet rigorously proven. Nonetheless, they have shaped methodological choices and empirical investigations. In this section, we formalise four of them, discuss their conceptual basis, and summarise supporting and opposing empirical evidence.

+

5.1 The Linear Representation Hypothesis#

+

We have seen that single neurons may not be the most effective unit of representation in many cases, but what is? The linear representation hypothesis claims that linear directions in the activation space are effective fundamental units of representation (Park et al., 2024).

+
+

Linear Representation Hypothesis

+

A concept can be effectively represented by a feature corresponding to a linear direction \phi in activation space \mathbb{R}^d. Linearity implies that adding or subtracting this direction, or a scalar multiple of it, from model activations can correspondingly influence model behaviour, while the projection \phi^\top h(x) approximately measures the strength of the feature in model activations h(x) for an input x.
+Under a strong version of the linear representation hypothesis, all concepts are best represented by linear features. In this view, model activations can be approximately decomposed as an additive combination of such features, h(x) \approx \sum_i \alpha_i \phi_i, where \alpha_i \in \mathbb{R} denotes the strength of feature \phi_i dependent on the input x. Under a weaker version, only some concepts admit such linear representations (Sharkey et al., 2025; Smith, 2024).

-

On a strong version of the linear representation hypothesis, all features are linearly represented. On a weaker version, only some are  (Sharkey et al., 2025; Smith, 2024).

-

Conceptual Basis. A feature is linearly represented if its presence corresponds to activation along a fixed direction \phi, detectable via projection \phi^\top h_l(x). This makes mechanistic analysis tractable: features can be found, measured, and manipulated using linear methods.

-

However, not all activation spaces are the same. In some spaces, like MLP hidden layers, the neuron axes matter, since ReLU treats each neuron separately, so the coordinate system is meaningful. We call this a privileged basis (see Figure 2, left). In other spaces, like the residual stream, nothing special happens along the neuron axes, so features can point in any direction. This is a non-privileged basis (see Figure 2, right).

-
-Figure 3: Privileged vs. non-privileged bases. In MLP hidden layers, element-wise nonlinearities break rotation symmetry, making axes meaningful. In the residual stream, no element-wise operations are applied, so any rotation of axes is computationally equivalent. -
Figure 3: Privileged vs. non-privileged bases. In MLP hidden layers, element-wise nonlinearities break rotation symmetry, making axes meaningful. In the residual stream, no element-wise operations are applied, so any rotation of axes is computationally equivalent.
+
+

Conceptual Basis.

+

A feature associated with a concept is linearly represented if its presence corresponds to a fixed direction \phi in the activation space and can be detected through the projection \phi^\top h(x). This linearity makes mechanistic analysis tractable because we can use simple methods to identify, measure, and manipulate these features in the activation space, such as scaling them and adding them back to model activations.

+

Notably, activation spaces in different model components can have distinct geometric structures. For example, in the hidden states of MLP blocks, the neuron axes are computationally distinguished because element-wise nonlinear activation functions, such as ReLU, act separately on each axis. The coordinate system is therefore meaningful and is referred to as a privileged basis (Figure 4, left) (Elhage et al., 2022a). In other spaces, such as the residual stream, no operation singles out the neuron axes, so linear features may lie along arbitrary directions. Such spaces are said to have a non-privileged basis (Figure 4, right).

+
+Figure 4: Privileged versus non-privileged bases. In the hidden layers of MLP blocks, element-wise nonlinear operations break rotational symmetry and make individual axes meaningful. In contrast, the residual stream contains no such operations that privilege particular axes, so rotating the basis pr +
Figure 4: Privileged versus non-privileged bases. In the hidden layers of MLP blocks, element-wise nonlinear operations break rotational symmetry and make individual axes meaningful. In contrast, the residual stream contains no such operations that privilege particular axes, so rotating the basis preserves the relevant computation.
-

The distinction matters: when the basis is privileged, a single neuron might represent a feature. When it’s not, features are directions that don’t align with any particular neuron, and we have to find them.

+

When the basis is privileged, a single neuron can serve as a scalar feature that meaningfully represents a concept. When it is not, the feature may instead correspond to directions or other geometric structures that do not align with any individual neuron. In this case, specialised methods are required to identify them from the activation space.

+

Empirical Evidence and Counterexamples.

-

Empirical support for linearity includes the success of linear probes on diverse models (Belinkov, 2022), activation steering along single directions to produce targeted behavioral changes (Turner et al., 2024), and sparse autoencoders recovering linearly separable and steerable features (Bricken et al., 2023).

-

However, there is also evidence that not all representations are linear. Counterexamples include nonlinear feature interactions, algorithmic representations involving nonlinear transformations, and features distributed across multiple interacting subspaces(Nanda et al., 2023; Csordás et al., 2024; Engels et al., 2025; Shafran et al., 2025; Tiblias et al., 2025) . Some have argued that features in neural networks may not even be explicitly represented, but may instead emerge from underlying circuit dynamics (Smith, 2024). Thus, the Linear Representation Hypothesis is a useful approximation rather than a universal law.

+

Empirical support for the weaker version of the linear representation hypothesis includes the success of linear probes across diverse models and concepts (Belinkov, 2022), activation steering along single directions to induce targeted behavioural changes (Turner et al., 2024), and sparse autoencoders recovering linearly accessible and steerable features at scale (Bricken et al., 2023). At the same time, there is substantial evidence that not all representations are well described as linear. Counterexamples include nonlinear feature interactions, algorithmic representations that rely on nonlinear transformations, and features distributed across multiple interacting subspaces (Nanda et al., 2023; Engels et al., 2025; Tiblias et al., 2026; Csordás et al., 2024; Feucht et al., 2026; Wurgaft et al., 2026). Some work has further argued that features may not always exist as explicitly represented objects, but may instead emerge from underlying circuit dynamics (Smith, 2024). The linear representation hypothesis is therefore best viewed as a useful approximation that holds in some settings, rather than as a universal law.

-

5.2 The Superposition Hypothesis#

-

The superposition hypothesis says that for each layer of a trained model there are more features than nodes.

-
-

Model hidden states encode more features than they have dimensions. If there are k features and d dimensions, then the Superposition Hypothesis is that k >> d.
-This implies that features cannot be represented as orthogonal directions—though they can be represented as nearly-orthogonal directions. It also implies polysemanticity: directions will be activated in response to different concepts.

+

5.2 The Superposition Hypothesis#

+

The superposition hypothesis says that models encode more features than their hidden representations have dimensions.

+
+

Superposition Hypothesis

+

Model hidden states encode more features than they have dimensions. If there are m features and d dimensions, then the superposition hypothesis claims that m \gg d.
+This implies that features cannot be represented as orthogonal directions—though they can be represented as nearly orthogonal directions. It also implies neuron polysemanticity: the same neuron participates in the encoding of multiple unrelated concepts.

-
+

Conceptual Basis.

-

Superposition is a geometric property of high-dimensional activation spaces: many approximately orthogonal directions can coexist, allowing a limited number of neurons to encode a much larger number of sparse features. Polysemanticity, where individual neurons respond to multiple unrelated semantic concepts, is a direct consequence of this property. As features become increasingly sparse, the model moves from using dedicated dimensions for monosemantic features to shared dimensions (see Figure 3). In this state of superposition, features are compressed into a space with fewer dimensions than there are features.

+

Superposition is a geometric property of learned representations in which a limited number of neuron dimensions encode many more features by allowing those features to share representational dimensions. Neuron polysemanticity, where individual neurons respond to multiple seemingly unrelated semantic concepts, naturally follows from this phenomenon. When the number of encoded features exceeds the activation-space dimensionality, the model can no longer allocate a dedicated neuron dimension to each feature. Instead, multiple features become superposed within the same activation space, shifting the representation from dedicated dimensions with monosemantic neurons to shared dimensions with increasingly polysemantic neurons (Figure 5).

Empirical Evidence.

-

Mechanistic studies have observed widespread polysemanticity: individual neurons consistently activate for multiple unrelated semantic concepts, making them poor interpretable units (Elhage et al., 2022a; Bricken et al., 2023). Dictionary Learning methods like Sparse Autoencoders (SAEs) and related feature decomposition methods exploit this underlying property of superposition to recover tens of thousands of distinct features from shared activation spaces (Bricken et al., 2023; Templeton et al., 2024). These recovered features are considered interpretable and some causally relevant to model behavior, showing that overlapping directions encode meaningful semantic information. Interpretability gains from such methods further validate that superposition is fundamental challenge, rather than a method-specific artifact (Templeton et al., 2024).

-
-Figure 4: Superposition implies polysemanticity. Left: when the number of features k does not exceed the number of dimensions/neurons d (k\le d), feature directions can align cleanly with individual neuron axes (monosemantic neurons). Right: when k>d, multiple feature directions must share the same -
Figure 4: Superposition implies polysemanticity. Left: when the number of features k does not exceed the number of dimensions/neurons d (k\le d), feature directions can align cleanly with individual neuron axes (monosemantic neurons). Right: when k>d, multiple feature directions must share the same limited set of neuron axes; as a result, each neuron axis cuts across multiple feature directions, producing polysemantic neurons.
+

Mechanistic studies have repeatedly observed widespread neuron polysemanticity: individual neurons often activate for multiple seemingly unrelated semantic concepts, making them difficult to interpret as standalone representational units (Olah et al., 2020; Elhage et al., 2022a; Bricken et al., 2023). Dictionary-learning methods such as sparse autoencoders exploit this structure to recover millions of distinct features from shared activation spaces (Bricken et al., 2023; Templeton et al., 2024). Many discovered features are substantially more monosemantic than individual neurons, and some have been shown to causally influence model behaviour. Together, these findings suggest that the activation space may encode more meaningful semantic information along activation-space directions than along individual neuron axes.

+

However, the superposition hypothesis remains a hypothesis, not an established fact, in large language models. Observing polysemantic neurons does not, by itself, imply that superposition is the underlying cause, and dictionary-learning methods can recover interpretable features even in the absence of superposition. Furthermore, Lecomte et al. (2024) showed that polysemanticity can arise even when the number of underlying features does not exceed the number of available neurons, suggesting alternative—and potentially complementary—mechanisms for the emergence of polysemantic representations.

+
+Figure 5: Superposition implies polysemanticity. Left: when the number of features m does not exceed the number of dimensions d, i.e., m\le d, feature directions can align cleanly with individual neuron axes, producing monosemantic neurons. Right: when m>d, multiple feature directions must share the +
Figure 5: Superposition implies polysemanticity. Left: when the number of features m does not exceed the number of dimensions d, i.e., m\le d, feature directions can align cleanly with individual neuron axes, producing monosemantic neurons. Right: when m>d, multiple feature directions must share the same limited set of neuron axes; as a result, each neuron axis cuts across multiple feature directions, producing polysemantic neurons.
-

5.2.1 Architectural Responses to Superposition#

-

The prevalence of superposition is not an inevitable property of neural networks but a consequence of architectural permissiveness. Standard dense architectures do not impose a structural penalty for distributing features across arbitrary linear combinations of neurons. When activation functions are applied element-wise (e.g., ReLU, GELU), and connectivity is unconstrained, gradient descent is free to exploit the high-dimensional geometry of activation space to pack more features than there are neurons (Gao et al., 2025b).

-

This observation motivates a distinct research program: rather than decomposing superposition post hoc via sparse autoencoders, one can impose inductive biases during training that encourage representations in which features are more accessible to analysis. Three intervention points have been explored:

+
+

Architectural Responses to Superposition.

+

Superposition is not an intrinsically inevitable property of deep neural networks but a consequence of architectural permissiveness. Without constraints on model architectures or activations, gradient descent can exploit the activation-space geometry to pack far more features than neurons (Elhage et al., 2022a). This motivates a distinct research direction, interpretability-by-design (as mentioned in Section 3.2 and discussed in Section 6.10). Interpretability-by-design argues that, rather than decomposing traditional models’ superposed activations post hoc, one can design novel architectures or components that encourage representations where features are more accessible for analysis. Two intervention points have been explored:

    -
  • Sparsity constraints: Enforcing sparsity in activations limits how many neurons within an MLP can participate for a given input, constraining how many units can jointly support any single feature. For example, Yang et al. (2025) train mixture-of-experts models with sparsely activated neurons within the MLPs. Complementarily, sparsity can also be imposed on connectivity by encouraging sparse weight patterns, for instance via L_0 regularization, further limiting how many parameters contribute to a given representation (Gao et al., 2025b).

  • -
  • Activation competition: Replacing element-wise activations with spatial softmax (SoLU) forces neurons to compete, explicitly inhibiting the co-activation patterns superposition requires (Elhage et al., 2022b).

  • -
  • Computational primitives: Changing the fundamental operation from static feature detection to learnable transformations (MOLT, KANs) can better match the algorithmic structure of tasks, reducing the pressure to compress via superposition (Liu et al., 2025b).

  • +
  • Sparsity constraints: Enforce sparsity in weights or activations that limits how many neurons within an MLP or attention head can participate for a given input, constraining how many units can jointly support any single feature (Yang et al., 2025; Gao et al., 2026; Elhage et al., 2022b).

  • +
  • Computational primitives: Change the fundamental units of analysis from static features in activations to learnable transformations by weight matrices that generate activations (Liu et al., 2025b; Lindsey et al., 2025b). This paradigm better matches certain tasks and avoids the pressure to compress or suppress model representations.

-

These architectural interventions represent a shift from treating superposition as a phenomenon to be reverse-engineered toward treating it as a design choice that can be constrained upstream. We discuss these methods in detail in Section 6.9.

+

These architectural interventions shift from treating superposition as a phenomenon to reverse-engineer to treating it as a design choice that can be constrained upstream. We discuss these methods in detail in Section 6.10.

-

5.3 The Privileged Access Hypothesis#

-

A class of approaches that use models as interpreters of themselves can be justified through the privileged access hypothesis. The privileged access hypothesis states that:

-
-

A model used to interpret itself can, under appropriate elicitation, access internal variables and intermediate computations that are not reliably recoverable from external behavioral analysis alone, providing a comparative advantage as an interpreter of its own behavior

-

(binder?; song?; Li et al., 2025a).

+

5.3 The Privileged Access Hypothesis#

+

A class of approaches that use language models as interpreters of themselves can be justified through the privileged access hypothesis. The privileged access hypothesis states that:

+
+

Privileged Access Hypothesis

+

A language model used to interpret itself can, under appropriate elicitation, access internal variables and intermediate computations that are not reliably recoverable from external behavioural analysis alone, providing a comparative advantage as an interpreter of its own behaviour (Binder et al., 2025; Li et al., 2025a; Song et al., 2026).

-
+

Conceptual Basis.

-

Privileged Access is the hypothesis that the model itself can easily access its own internal states—such as latent beliefs, intermediate plans, or cached computations—which influence future behavior but remain latent at the current timestep. Consequently, a model that conditions on these internal activations has an advantage over external observers, who must infer this structure indirectly from input-output behavior.

-

The hypothesis is motivated by an asymmetry: external interpretability methods must reverse-engineer internal computation from activations or outputs, while the model itself already “has” this information in a form it can (potentially) report. If reliable, privileged access would dramatically reduce the cost of mechanistic analysis by allowing models to directly describe their own reasoning processes.

+

The ability to access and judge its own computation or reasoning—called introspective monitoring—has long been a desired property of an AI system (Cox & Raja, 2008). Even earlier rule-based medical expert systems sought to explain their decisions, such as MYCIN’s “Explanation system” (Shortliffe, 1977). In this sense, using language models to explain themselves can be seen as a modern continuation of these earlier efforts.

+

A major difference is that rule-based systems can explicitly present the rule they executed, whereas modern language models compute in a highly nonlinear way. Although researchers have proposed mechanistic tools aiming to reverse-engineer part of the computation, the technical complexity of these techniques hinders their accessibility and scalability. The privileged access hypothesis then posits that the language model itself naturally stores information about the computation within its internal states, such as latent beliefs or intermediate plans. The model can thus learn to verbalise this information directly, serving as a scalable complement to mechanistic analysis.

Empirical Evidence.

-

Empirical support comes from several lines of research:

-
    -
  • Models contain task-relevant information internally that is not expressed in standard outputs but can be recovered through investigating model internals of targeted prompting (Pan et al., 2024).

  • -
  • Fine-tuned “activation decoders” (e.g., LatentQA) can extract information from intermediate representations of the target model which it does not spontaneously verbalize (Pan et al., 2024).

  • -
  • Internal activations predict downstream behavior more accurately than surface-level explanations (Burns et al., 2024).

  • -
  • Model-generated explanations can align with internal mechanisms under constrained setups (Lanham et al., 2023).

  • -
-

However, these results demonstrate that privileged access is possible under favorable conditions, not that it is reliable in general. The hypothesis must be treated with significant caution due to systematic failure modes.

-
-

Privileged access methods are vulnerable to several failure modes that can produce plausible but unfaithful explanations:

-

Confabulation. Models may generate coherent, confident explanations that do not reflect their actual internal computations. This occurs because:

-
    -
  • Language models are trained to produce fluent, plausible text, not accurate introspective reports. The same capabilities that enable helpful responses also enable convincing confabulation.

  • -
  • Models may “rationalize” outputs post-hoc by constructing explanations consistent with their outputs rather than reporting the process that produced them (Turpin et al., 2023).

  • -
  • Explanations may reflect training data patterns (how humans explain reasoning) rather than the model’s actual computational process.

  • -
-

Miscalibration. Even when models attempt faithful self-report, their confidence may not track accuracy:

-
    -
  • Models may express high confidence in explanations for computations they cannot actually access (e.g., early-layer processing, attention routing decisions).

  • -
  • Calibration varies by task: models may be well-calibrated on factual recall but poorly calibrated on reporting their own uncertainty or reasoning process.

  • -
  • finetuning for self-explanation may improve fluency without improving faithfulness, creating a misleading appearance of reliability.

  • -
-

Elicitation sensitivity. Self-reports are highly sensitive to prompting:

-
    -
  • Different phrasings of the same question can elicit contradictory explanations.

  • -
  • Leading questions can induce models to confirm hypotheses that do not reflect their actual computation.

  • -
  • The format of self-report (free-form vs. structured, immediate vs. delayed) affects what information is accessible.

  • -
-

Strategic misreporting. In adversarial or high-stakes settings, models with sufficient capability might strategically misreport their internal states:

-
    -
  • A model optimizing for user approval might report explanations it predicts users will find satisfying rather than accurate.

  • -
  • A misaligned model might deliberately conceal its true objectives or reasoning (Hubinger et al., 2024).

  • -
  • Even without explicit deception, models may learn to produce “safe” explanations that avoid triggering safety filters.

  • -
-

These risks are not merely theoretical. Studies have demonstrated that chain-of-thought explanations often do not reflect the actual factors influencing model outputs (Turpin et al., 2023), and that models can be induced to provide confident but false explanations through adversarial prompting.

-
-
-
-

Validation Protocols and Causal Cross-Checks.

-

Given these risks, privileged access methods must be validated against independent evidence rather than accepted at face value. We recommend the following protocols:

-
-

1. Causal intervention consistency. Self-reported explanations should be tested against causal interventions:

-
    -
  • If a model claims feature F is responsible for output O, ablating F should degrade O.

  • -
  • If a model claims it “considered” information I, patching I with counterfactual values should change the output in predicted ways.

  • -
  • Discrepancies between self-reports and causal effects indicate confabulation or incomplete access.

  • -
-

2. Cross-method triangulation. Privileged access findings should be corroborated by independent methods:

-
    -
  • Do SAE features identified by the model as relevant show causal effects under ablation?

  • -
  • Do probing classifiers detect the information the model claims to use?

  • -
  • Do circuit discovery methods identify the components the model describes?

  • -
-

3. Consistency checks. Self-reports should be stable under variation:

-
    -
  • Paraphrased questions should elicit consistent explanations.

  • -
  • Explanations should be consistent across multiple samples (low temperature) or appropriately uncertain (high temperature).

  • -
  • Explanations for similar inputs should show appropriate similarity.

  • -
-

4. Calibration evaluation. Model confidence should be evaluated:

-
    -
  • When models express uncertainty about their internal states, are they more likely to be wrong?

  • -
  • Can models reliably distinguish “I used this information” from “this information was present but unused”?

  • -
-

5. Adversarial probing. Test robustness to manipulation:

-
    -
  • Do leading questions induce false confirmations?

  • -
  • Can the model be prompted to give contradictory explanations?

  • -
  • Does the model maintain consistent explanations under pressure to change them?

  • -
-
-
-
-

Elicitation Protocols to Reduce Confabulation.

-

While no elicitation method guarantees faithful self-report, certain practices reduce confabulation risk:

-
    -
  • Activation conditioning: Methods like LatentQA that condition on actual intermediate activations (rather than relying on the model to “remember” its computation) show improved faithfulness over pure prompting (Pan et al., 2024).

  • -
  • Structured elicitation: Asking specific, verifiable questions (“Did you attend to token X?”) rather than open-ended ones (“Explain your reasoning”) reduces the scope for confabulation.

  • -
  • Immediate reporting: Eliciting explanations during or immediately after computation, rather than retrospectively, may improve access to transient internal states.

  • -
  • Contrastive questions: Asking “Why did you output A rather than B?” can elicit more specific, testable claims than “Why did you output A?”

  • -
  • Uncertainty elicitation: Explicitly asking models to express uncertainty about their self-reports, and validating that expressed uncertainty correlates with actual accuracy.

  • -
-
-
-

When Privileged Access Is and Is Not Appropriate.

-

Privileged access methods are best suited for:

-
    -
  • Hypothesis generation: Generating candidate explanations to be validated by other methods.

  • -
  • Rapid triage: Quickly identifying which components or features merit deeper mechanistic investigation.

  • -
  • Human-AI collaboration: Providing human-readable summaries that guide (but do not replace) rigorous analysis.

  • -
-

Privileged access methods are insufficient for:

-
    -
  • Safety-critical verification: Claims about the absence of deceptive reasoning or hidden objectives cannot be established by self-report alone.

  • -
  • Formal guarantees: Self-reports do not provide the causal evidence needed for formal verification.

  • -
  • Adversarial settings: When the model may have incentives to misreport, privileged access is unreliable by construction.

  • -
-
-
-

Implications.

-

Partial privileged access could assist in detecting latent goals, plans, or deceptive behavior (Hubinger et al., 2021; Burns et al., 2024), but only when validated against causal interventions and treated as complementary to mechanistic analysis (Geiger et al., 2021; Bereska & Gavves, 2024). The optimistic framing of privileged access—that models can simply tell us how they work—must be tempered by recognition that self-report is a weak form of evidence requiring independent corroboration.

-

The most productive use of privileged access is as one component of a multi-method interpretability pipeline: self-reports generate hypotheses, causal interventions test them, and mechanistic analysis explains the underlying circuits. Relying on privileged access alone, without causal validation, risks accepting plausible confabulations as mechanistic truth.

+

The most direct empirical evidence for this hypothesis comes from recent findings that models, even without explicit training to do so, are sometimes aware of concept representations (acquired by methods like difference-in-means) injected into their activations (Lindsey, 2026; Fornasiere et al., 2026). Additional, though less direct, evidence comes from the design of explainer models—a general class of interpretability methods in which one model is fine-tuned to explain the target model’s activations (Pan et al., 2026; Karvonen et al., 2026a; Fraser-Taliente et al., 2026). In practice, such explainers are typically initialised from a copy of the target model itself (Pan et al., 2026; Karvonen et al., 2026a; Bauer et al., 2026; Huang et al., 2026). This usually works better than initialising from alternative models, suggesting the target model already encodes much of what is needed to interpret its own activations. In this sense, fine-tuning elicits and refines a latent capability rather than creating one from scratch.

+

Some studies also reported negative results for the privileged access hypothesis: Song et al. (2026) found that models predict their own sampling temperature no better than they predict other models’, suggesting apparent introspection may reflect general reasoning about model-like systems rather than self-access. Separately, recent work has questioned whether internals-based explanations provide a practical advantage over behavioural baselines: Karvonen et al. (2026b) found that activation-reading tools (including activation oracles) give agents no uplift over transcript-only baselines at predicting counterfactual model behaviours in the wild, and Li et al. (2026c) showed that standard verbalisation benchmarks can be solved without access to target-model internals at all (see also Jakkli et al., 2026).

+

In summary, these results show that privileged access is possible for certain tasks under favourable conditions, not universal or reliable. The hypothesis should therefore be treated with caution, as significant failure modes and limitations remain.

-

5.4 The Universality and Platonic Representation Hypotheses#

-
-

Independently trained models on similar tasks tend to develop similar internal computational structures. These representations often approximate a task-determined canonical form (Huh et al., 2024; Olah et al., 2020). The degree of convergence depends on factors such as model size, architecture, and training data.

+

5.4 The Universality and Platonic Representation Hypothesis#

+
+

Platonic Representation Hypothesis

+

Models independently trained on similar tasks tend to develop similar internal representations or algorithms, which often approximate a task-determined canonical form (Huh et al., 2024; Olah et al., 2020). The degree of convergence depends on factors such as model size, architecture, and training data.

-
+

Conceptual Basis.

-

The hypothesis posits that canonical representations exist for a given task, and independently trained models may converge to similar algorithmic motifs or internal structures. Weak universality refers to convergence of computational strategies or motifs, whereas strong universality claims convergence of internal representations themselves, up to symmetries.

+

A key distinction is the level at which models are expected to converge. Weak universality refers to convergence in the algorithms or recurring computational patterns that models implement, even when these are encoded through different internal representations. Strong universality makes the stronger claim that the internal representations themselves converge towards a shared canonical form.

-

Empirical Evidence and Trade-offs.

-

Weak universality is supported by recurring attention patterns, feature motifs, and circuits across models, tasks, and scales (Nanda et al., 2023; Olah et al., 2020). Strong universality is less supported and usually demonstrated via approximate alignment through linear transformations (Kornblith et al., 2019; Moschella et al., 2023). Divergent representations can arise from variations in architecture, data, or optimization trajectories (Harrasse et al., 2025b; Morcos et al., 2018a).

+

Empirical Evidence and Trade-Offs.

+

Weak universality is supported by recurring attention patterns, features, and circuits across models, tasks, and scales (Nanda et al., 2023; Lan et al., 2024; Olah et al., 2020; Quirke et al., 2026). Strong universality is less supported and is usually demonstrated through representational similarity metrics (Kornblith et al., 2019; Moschella et al., 2023; Lan et al., 2025), where recent studies have found the results to be fragile (Gröger et al., 2026; Koepke et al., 2026). Divergent representations can arise from variations in architecture, data, or optimisation trajectories (Harrasse et al., 2025; Morcos et al., 2018).

-
+

Implications.

-

If strong universality holds, mechanistic insights may generalize across architectures. If only weak universality holds, transferability is limited to motifs and algorithms rather than precise circuits (Bereska & Gavves, 2024). Clarifying this distinction is critical for assessing whether mechanistic interpretability can scale beyond individual case studies.

+

If strong universality holds, fine-grained mechanistic insights and the methods derived from them may transfer more readily across models. If only weak universality holds, such insights may remain contingent on factors such as model family and task (Bereska & Gavves, 2024), and even a complete mechanistic understanding of one model (see Section 12.2) may fail to generalise to others. Therefore, it is important to clarify the extent of universality for assessing the promise, limitations, and scalability of different interpretability research orientations, including the six major orientations introduced in Section 12.

+
+
+

Summary.

+

This section discussed four hypotheses that motivate many modern mechanistic interpretability studies. Linear representations support linear-feature-based analysis; superposition explains why we need to operate on linear or more complex forms of features instead of single neurons; privileged access explains the fundamental idea of model self-interpretation; universality asks whether insights transfer across models and at which levels. Each hypothesis holds in some settings and fails in others, and techniques built on them inherit those limitations. Section 6 introduces the main family of techniques and methods that operationalise the first two hypotheses (linear representation and superposition): dictionary learning for feature extraction and analysis.

diff --git a/src/generated/chapters/developmental-interpretability.html b/src/generated/chapters/developmental-interpretability.html deleted file mode 100644 index 88948b9968..0000000000 --- a/src/generated/chapters/developmental-interpretability.html +++ /dev/null @@ -1,51 +0,0 @@ -

10 Developmental Interpretability and Training Dynamics#

-

Developmental Interpretability studies how specific computational structures and circuits form over the course of training and why their emergence is associated with changes in model behavior, rather than focusing only on the final trained model. By analyzing training dynamics, this line of work aims to explain phenomena such as phase transitions, capability emergence, and failure modes in mechanistic terms (Hoogland et al., 2023).

-

Many capabilities are not present at initialization but appear at particular stages of optimization. Tracking when these changes occur helps identify the formation of both useful and problematic behaviors. This motivates studying the learning process itself, not just the final model. Developmental Interpretability therefore seeks to monitor and analyze training trajectories to identify critical shifts in internal structure. In the long term, such insights could enable detection of emerging misaligned behaviors and inform interventions before they are fully encoded in the weights.

-

10.1 Grokking: Memorization-to-Generalization Phase Transitions#

-
-

Grokking is the phenomenon in which a model achieves low training loss early in training but only begins to generalize to unseen inputs after a long additional period of optimization, a pattern observed in small models trained on algorithmic tasks (Power et al., 2022; Nanda et al., 2023).

-
-

A canonical setting is modular addition, where training accuracy becomes perfect early, while test accuracy remains low for many training steps before undergoing a sharp increase (Power et al., 2022; Nanda et al., 2023).

-

Mechanistic analyses of these models show that, during this extended plateau, internal representations and attention patterns change gradually rather than remaining static (Nanda et al., 2023). In particular, they identify progress measures that track the slow emergence of algorithmic structure inside the network before generalization appears. When these internally measured structures become sufficiently strong, test performance improves rapidly, producing the observed phase transition (Nanda et al., 2023).

-

Takeaway: In these controlled settings, grokking corresponds to gradual internal changes during training that manifest externally as a sudden transition in generalization.

-
-Figure 15: Grokking. Grokking describes a distinct phase transition during training where a model suddenly moves from memorization of training data to true generalization on unseen test data. As shown in the loss curve, while the training loss (purple) drops almost immediately, the test loss (red) r -
Figure 15: Grokking. Grokking describes a distinct phase transition during training where a model suddenly moves from memorization of training data to true generalization on unseen test data. As shown in the loss curve, while the training loss (purple) drops almost immediately, the test loss (red) remains high for a significant period before a sharp drop, entering the grokking phase.
-
-
-

10.2 Training Dynamics and the Emergence of Internal Mechanisms#

-

Recent work analyzes training dynamics to understand how qualitative changes in model behavior arise. One study focuses on factual knowledge acquisition in language models and shows that training proceeds through distinct phases: an initial phase dominated by learning coarse statistical regularities, followed by an extended plateau with little factual recall, and a later phase in which factual knowledge begins to emerge (Zucchet et al., 2025a). Mechanistically, this plateau coincides with the formation of attention-based circuits that support recall. The authors further analyze how data distribution and curriculum choices shape these dynamics and study the interaction between factual learning and hallucinations, highlighting systematic patterns in when correct and incorrect responses emerge during training. Complementary work focuses on the emergence of sparse attention mechanisms and their connection to abrupt improvements in performance (Zucchet et al., 2025b). This study shows that sparse attention can arise suddenly during training and that its emergence depends on properties of the training data, such as repetition. By combining theoretical analysis with empirical results, the authors link changes in attention structure to non-linear learning dynamics.

-
-

10.3 Singular Learning Theory and the Local Learning Coefficient#

-

Singular Learning Theory (SLT) provides a complementary theoretical lens by relating properties of the loss landscape, such as degeneracy and singularities, to the emergence of internal structure (Hoogland et al., 2023). Traditional statistical learning theory often assumes that model parameters are identifiable and that the loss landscape behaves regularly near optima. In modern neural networks, however, the mapping from parameters to functions is often many-to-one, meaning that multiple parameter configurations can implement the same function. This property creates degeneracies in the loss landscape: directions in parameter space where the loss changes very little. Singular Learning Theory (SLT) provides a framework for studying such models and reasoning about the consequences of these degenerate regions (Watanabe, 2009).

-
-

For a model with parameters \mathbf{w} at a local minimum \mathbf{w}^*, the Local Learning Coefficient (LLC), denoted \lambda(\mathbf{w}^*), is a positive number that measures the amount of degeneracy in the loss landscape near \mathbf{w}^*. In other words, it captures how many ways the parameters can vary locally without changing the loss value (Watanabe, 2009; Lau et al., 2023).

-

Smaller LLC values indicate high degeneracy: many parameter configurations near the minimum achieve the same loss. Larger LLC values indicate low degeneracy: the local region is tightly constrained.

-

Formally, the LLC is defined via the volume-scaling rate of the loss landscape near a minimum. This makes it a more general measure of “flatness” than simple Hessian-based methods, because it captures higher-order contributions to local geometry. Empirical estimators can track the LLC throughout training, providing a way to observe developmental changes in model structure (Lau et al., 2023; Hoogland et al., 2025).

-
-

Empirical studies have used the LLC to monitor training in transformer models and other architectures. Researchers have observed that transitions and critical points in the LLC often coincide with changes in internal network structure and functional behavior. Tracking these changes provides a quantitative way to study the emergence of structure during training without relying only on final performance metrics (Hoogland et al., 2025).

-
-

10.4 Stagewise Influence and Developmental Data Attribution#

-

Most traditional training data attribution methods measure the impact of a training example only on the final model parameters. However, neural networks often learn in stages, with distinct phases where capabilities or structural features emerge. The influence of individual training examples can therefore change dramatically during training, including sudden increases, decreases, or even sign reversals (Lee et al., 2025).

-

Lee et al. (2025) propose a stagewise data attribution framework that extends classical influence methods to track the impact of training points throughout the learning process. Their approach is grounded in Singular Learning Theory (SLT), which characterizes neural networks as singular models—statistical models where the Fisher information matrix is degenerate and multiple distinct parameter configurations yield equivalent predictions. Unlike regular models where the Bayesian posterior concentrates smoothly around a unique maximum likelihood estimate, singular models exhibit non-smooth, stagewise development.

-

Classical influence functions rely on inverting the Hessian of the loss landscape, which becomes ill-defined when the Fisher information matrix is degenerate. For singular models like deep neural networks, this motivates an alternative formulation. The framework employs the Bayesian Influence Function (BIF):

-
-

For a training dataset D, training point z_i, and quantity of interest \phi(\theta) (e.g., loss on another data point), the Bayesian Influence Function is defined as: \text{BIF}(z_i, \phi) = -\text{Cov}_{p(\theta|D)}[\ell(z_i; \theta), \phi(\theta)] where \ell(z_i; \theta) is the loss on data point z_i at parameters \theta, and the covariance is computed over the posterior distribution p(\theta|D).

-
-

Unlike classical influence functions that require Hessian inversion, the BIF measures influence through covariance estimation over the local posterior distribution. This distributional approach is estimated via stochastic-gradient MCMC sampling, enabling scalable computation without explicit Hessian access while naturally capturing the geometry of singular loss landscapes.

-

The core prediction is that influence is not static but exhibits non-monotonic dynamics that peak at developmental transitions. Lee et al. (2025) validate this prediction analytically and empirically in a hierarchical feature-learning model (Saxe et al., 2019). In this controlled setting, influence undergoes sign flips as the model transitions between learning different levels of the semantic hierarchy (animals versus plants, then mammals versus birds, then dogs versus cats), with sharp peaks occurring precisely when each new hierarchical distinction emerges. These predictions are further confirmed through leave-one-out retraining experiments that directly measure the effect of removing individual training examples at different stages of training.

-

At scale, Lee et al. (2025) demonstrate these phenomena in Pythia language models (Biderman et al., 2023b) trained on the Pile dataset (Gao et al., 2020). Token-level influence exhibits non-monotonic changes, including sudden peaks and sign reversals that align with known developmental milestones such as the formation of induction heads. For tokens with structural roles (e.g., delimiters, tokens participating in induction patterns), influence undergoes dramatic shifts at specific training checkpoints corresponding to circuit formation events identified in prior mechanistic interpretability work (Olsson et al., 2022b).

-
-

10.5 Deceptive Alignment and Training Dynamics#

-

A critical concern is that deceptive or hidden objectives can emerge during training even when loss metrics appear normal, driven by complex circuit formation that evades behavioral detection (Hubinger et al., 2024). Models may learn goals orthogonal to the training objective, exhibiting behavioral invisibility where emergent misaligned behaviors go undetected in standard performance metrics. Mechanistic analysis can reveal circuit-level evidence of deceptive reasoning patterns—internal computations that contradict stated outputs. This underscores a fundamental limitation of behavioral safety measures: loss curves and benchmark performance provide incomplete information about what algorithms the model actually implements. Consequently, monitoring training dynamics and tracing circuit formation is essential for detecting early signs of deception before problematic objectives crystallize in the weights (Hubinger et al., 2024).

-
-

10.6 Case Study: Induction Heads#

-
-

Singh et al. (2024) study induction heads from a training-dynamics perspective, focusing on how such structures emerge during optimization. While earlier work showed that induction heads appear abruptly and coincide with a sharp improvement in in-context learning (Olsson et al., 2022a), this work seeks to explain the mechanisms behind this phase transition.

-

The authors train transformers on controlled synthetic sequence tasks and introduce an optogenetics-inspired causal intervention framework that allows intermediate activations to be clamped or manipulated throughout training, enabling causal analysis of how specific components influence learning dynamics.

-

They find that induction behavior does not arise from a single head suddenly activating. Instead, many attention heads across layers gradually develop induction-like behavior, and their contributions to in-context learning are largely additive. Although each component evolves smoothly, their combined effect produces a sharp, macroscopic transition in loss and accuracy, explaining why induction heads appear to “switch on” abruptly.

-

Using systematic clamping experiments, the authors further identify three interacting subcircuits whose joint dynamics give rise to induction behavior.

-

Finally, they show that the timing and strength of induction head emergence depend on dataset statistics and task structure, indicating that induction heads arise from interactions between data, optimization, and model structure rather than being inevitable architectural artifacts.

-
-
-
diff --git a/src/generated/chapters/feature-extraction.html b/src/generated/chapters/feature-extraction.html index 023b7b140d..4d8a2c2209 100644 --- a/src/generated/chapters/feature-extraction.html +++ b/src/generated/chapters/feature-extraction.html @@ -1,72 +1,99 @@ -

6 Feature Extraction and Multi-Layer Analysis#

-

The goal of feature extraction is to identify features that are distributed across many neurons but correspond to coherent semantic or computational units. There are two problems. First, polysemanticity: individual neurons encode multiple semantics. Second, scale: modern models contain millions to billions of neurons. We can overcome these problems by decomposing activations into monosemantic features using sparse reconstruction. One version of this approach is sparse autoencoders (SAEs) (Olshausen & Field, 1996; Le, 2013), which provide a concrete way of extracting interpretable features from model activations (Cunningham et al., 2023).

-

This section gives an overview of how SAE techniques recover interpretable features and a practical overview of SAE variants. It gives advice on when to use the different variations. Then, multi-layer methods such as transcoders and crosscoders are introduced and compared to SAE methods and a wide variety of alternatives.

-

6.1 Sparse Autoencoders#

-

Sparse Autoencoders are a way of extracting monosemantic features from polysemantic neurons. At a high level, they are trained to replicate the dynamics of a single MLP or attention block with each direction representing a distinct concept (Figure 4). They are optimised for accuracy to the real MLP or attention block and for having few active features for any given forward pass.

-
-Figure 5: Sparse Autoencoder (SAE). SAEs are trained to reconstruct activations h into an overcomplete, sparse latent space z, such that \hat{h} \approx h. By enforcing sparsity on z, the model is forced to decompose dense, polysemantic representations into human-interpretable features. -
Figure 5: Sparse Autoencoder (SAE). SAEs are trained to reconstruct activations h into an overcomplete, sparse latent space z, such that \hat{h} \approx h. By enforcing sparsity on z, the model is forced to decompose dense, polysemantic representations into human-interpretable features.
+

6 Feature Extraction and Analysis#

+

The critical notions introduced in Section 4, together with the hypotheses about the structure of neural network representations discussed in Section 5, provide the conceptual foundation for developing methods to identify and analyse manipulable internal objects associated with concepts of interest, i.e., features. One straightforward paradigm is to decompose model activations into sparse linear features. Sparse autoencoders (SAEs) provide a principled example of this approach, learning to represent any input activation with a subset of a large dictionary of monosemantic, interpretable features (Cunningham et al., 2024).

+

This section first introduces the canonical sparse coding problem and links it to SAEs, reviews how SAEs extract interpretable features, and then provides a practical overview and comparison of major SAE variants. We then introduce methods for multi-layer feature extraction and analysis, such as transcoders and crosscoders, and compare them with SAEs as well as with a broader range of alternative feature-extraction methods.

+

6.1 The Sparse Coding Problem#

+

Early neuroscience studies posited that brains represent information using sparse, interpretable basis signals (Barlow, 1961; Field, 1994). This hypothesis further motivated research to decompose natural images and compare the decomposed signals with cell activities in the primary visual cortex, finding surprising alignment (Olshausen & Field, 1996; 1997). At the same time, the signal processing community has also examined representing a given signal as the sum of a subset of basis signals from a given overcomplete dictionary (Mallat & Zhang, 1993; Chen et al., 1998). Studies on learning better overcomplete dictionaries for high-quality representations also thrived (Lewicki & Sejnowski, 2000; Aharon et al., 2006). Together, they framed the sparse dictionary learning research question, with SAEs serving as a modern algorithm that uses neural network activations as the signal. Specifically, given data h, a sparse dictionary learning method finds the dictionary D and sparse codes z such that \begin{equation*} +h \approx Dz \text{ and } \|z\|_0 \text{ is small}. +\tag{2} +\end{equation*} SAEs implement this with neural networks: h are model activations, D = W_{\text{out}} is an overcomplete dictionary to learn, and the sparse codes are z = \sigma(W_{\text{in}} h + b_{\text{in}}). Note that conventional dictionary methods usually run the algorithm to find a z for each h, which does not scale to modern language models. In contrast, SAEs use a nonlinear encoder E to predict the coefficients for different input activations.

+
+

6.2 Sparse Autoencoders#

+

SAEs (Figure 6) extract monosemantic features from polysemantic model activations. They are trained to reconstruct activations at a specific location in neural networks, such as the output of an MLP block or an entire transformer layer (i.e., the residual stream), while keeping only a few features active in any given forward pass.

+
+Figure 6: Sparse Autoencoder (SAE). SAEs are trained to reconstruct activations of a model component at a specific layer \ell, h_\ell, into an overcomplete, sparse latent space z, such that the reconstructed activation \hat{h_\ell} \approx h_\ell. By enforcing sparsity on z, the model decomposes den +
Figure 6: Sparse Autoencoder (SAE). SAEs are trained to reconstruct activations of a model component at a specific layer \ell, h_\ell, into an overcomplete, sparse latent space z, such that the reconstructed activation \hat{h_\ell} \approx h_\ell. By enforcing sparsity on z, the model decomposes dense, polysemantic representations into monosemantic, human-interpretable features.
-

A sparse autoencoder learns an encoder–decoder pair E : \mathbb{R}^d \rightarrow \mathbb{R}^{d_{\text{sparse}}},\; -D : \mathbb{R}^{d_{\text{sparse}}} \rightarrow \mathbb{R}^d such that activations h \in \mathbb{R}^d are reconstructed via \hat{h} = D(E(h)), while encouraging sparse feature activations E(h). It helps localize feature activations .

+

Sparse Autoencoder (Figure 6)

+

A sparse autoencoder learns a component-specific encoder–decoder pair E : \mathbb{R}^d \rightarrow \mathbb{R}^{d_{\text{sparse}}}, +\qquad +D : \mathbb{R}^{d_{\text{sparse}}} \rightarrow \mathbb{R}^d, such that activations of a particular model component at a given layer, h_\ell \in \mathbb{R}^d (e.g., the output of the layer-\ell MLP block), are reconstructed as \hat{h}_\ell = D(E(h_\ell)), while encouraging the feature activations z=E(h_\ell) to be sparse. Usually, d_{\text{sparse}} = ed, where e is an integer expansion factor.

-
-

Standard Formulation.

-

Given activations h \sim \mathcal{H}, the standard SAE objective is \mathcal{L}_{\text{SAE}} +

+

Canonical Formulation.

+

Given activations of a model component h_\ell \sim \mathcal{H}, where \mathcal{H} is the distribution of that component’s activations over a large corpus, the canonical SAE objective is \mathcal{L}_{\text{SAE}} = -\mathbb{E}_{h \sim \mathcal{H}} +\mathbb{E}_{h_\ell \sim \mathcal{H}} \left[ -\| h - D (z) \|_2^2 +\| h_\ell - D (z) \|_2^2 + \lambda \| z \|_1 \right], \quad -z = E(h), where z \in \mathbb{R}^{d_{\text{sparse}}} denotes feature activations and \lambda > 0 controls sparsity (Cunningham et al., 2023).

+z = E(h_\ell), where \lambda > 0 controls sparsity (Cunningham et al., 2024). Canonical SAEs use L_1 regularisation instead of L_0 regularisation. This is because, for each feature, L_0’s derivative is almost everywhere zero and thus does not provide useful signals for promoting feature sparsity during gradient descent. Additionally, the decoder columns are typically constrained to unit norm. This is to prevent the case where the SAE learns to reduce its L_1 loss by generating smaller values of z—not exactly 0 so that many features can be active, each having a small coefficient—and enlarging the norm of each column of the decoder to compensate. In this way, the SAE output remains the same, while L_1 loss can be substantially reduced, without any real gain in sparsity.

Why SAEs Work.

-

SAEs are motivated by the hypothesis that features occur sparsely in the data and are linearly recoverable from a model’s hidden states. Under sparsity assumptions, overcomplete dictionaries can recover monosemantic directions (features) even when raw neurons are highly polysemantic (Cunningham et al., 2023; Bricken et al., 2023).

+

SAEs are motivated by the hypothesis that a model’s activations can be approximated as linear combinations of many latent feature directions, which can be regarded as an extension of the linear representation hypothesis discussed in Section 5.1. Sparsity means that only a small number of features are active for any given token or input, but not that the hidden state itself is sparse. Early sparse dictionary learning studies (Olshausen & Field, 1996; Lewicki & Sejnowski, 2000; Aharon et al., 2006) give us fair confidence that, if the hypothesis holds, dictionary learning can recover the structure as an overcomplete dictionary. These findings motivate using SAEs as a modern adaptation to extract monosemantic directions from the activations of neural networks (Cunningham et al., 2024; Bricken et al., 2023).

From Monosemanticity to Concept Identification.

-

While sparsity and overcompleteness provide the basis for discovering monosemantic features, practical interpretability requires identifying the semantic concepts to which these features correspond in natural languages. Empirically, SAE features exhibit selective activation to structured token patterns that recur across input sequences, suggesting sensitivity to specific abstract semantic or functional concepts. Automated analysis pipelines (Gur-Arieh et al., 2025a; Paulo et al., 2025) automatically aggregate highly activating inputs to infer the latent concept captured by each feature (see Section 13.7 for a detailed discussion).

+

For each input activation, the raw SAE outputs are just a small set of vectors. Practical interpretability requires identifying the semantic concepts these vectors correspond to in natural language (so that they become human-understandable features). Empirically, SAE features selectively activate on structured token patterns that recur across input sequences, suggesting sensitivity to specific abstract semantic, functional, or lexical concepts (Marks et al., 2024b). Automated analysis pipelines (Gur-Arieh et al., 2025a; Paulo et al., 2025b) aggregate highly activating inputs to infer the concept each feature captures (see Section 12.6 for a detailed discussion).

Training Dynamics and Evaluation.

-

Training of SAEs is sensitive to initialization, sparsity strength, and decoder normalization. Without proper constraints and hyperparameters, the features may collapse, duplicate, or absorb one another (see Appendix 19 for hyperparameters and stability guidance). SAEs are typically evaluated for reconstruction loss (fidelity of activation reconstruction), feature absorption (degree to which features remain distinct), and downstream testing loss recovery (whether replacing activations with reconstructions preserves model behavior). Together, these metrics assess both the quality of the decomposition and its mechanistic relevance to the computation of the original model.

+

SAE training is sensitive to initialisation, sparsity strength, and decoder normalisation. Without proper constraints and hyperparameters, the features may collapse, duplicate, or absorb one another. SAEs are typically evaluated for reconstruction loss (fidelity of activation reconstruction), feature absorption (how often a general feature fails to fire because a more specific feature has absorbed its direction), and downstream test loss recovery (whether replacing activations with reconstructions preserves model behaviour). Together, these metrics assess both the quality of the decomposition and its mechanistic relevance to the original model’s computation.

-

6.2 SAE Variants and Architectural Improvements#

-

There are several variants of the standard SAE. One key reason to use alternatives is that the standard SAE training penalizes activation magnitude, which can lead to low activation even when a feature is clearly present (known as the shrinkage effect). Different optimisation frameworks also have different tunable parameters. Which one to use depends on the analysis goal, target layer type, size of the model, and budget constraints.

+

6.3 SAE Variants and Architectural Improvements#

+

There are several variants of the standard SAE. One key reason to use alternatives is that standard SAE training penalises activation magnitude, which can lead to low activation even when a feature is clearly present (the shrinkage effect). Different optimisation frameworks also have different tunable parameters. The choice depends on the analysis goal, target layer type, model size, and budget constraints.

-

Gated SAEs introduce multiplicative gates to decouple feature selection from magnitude: z_i = \underbrace{\mathbbm{1} \left( (W_{\text{gate}} h_\ell + b_{\text{gate}})_i > 0 \right)}_{\text{Gating Path}} \odot \underbrace{\text{ReLU}(W_{\text{in}} h_\ell + b_{\text{in}})_i}_{\text{Magnitude Path}} where h_\ell \in \mathbb{R}^d is the activation at layer \ell, i indexes the i-th feature, W_{\text{in}} \in \mathbb{R}^{{d_{\text{sparse}}} \times d} and b_{\text{in}} \in \mathbb{R}^{d_{\text{sparse}}} are encoder E parameters, W_{\text{gate}} \in \mathbb{R}^{{d_{\text{sparse}}} \times d} and b_{\text{gate}} \in \mathbb{R}^{d_{\text{sparse}}} are gate’s parameters. This mitigates shrinkage effects induced by the \ell_1 penalty (Rajamanoharan et al., 2024a).

+

Gated SAEs

+

Gated SAEs introduce multiplicative gates to decouple feature selection from magnitude: z = \underbrace{\mathbbm{1} \left( (W_{\text{gate}} h_\ell + b_{\text{gate}}) > 0 \right)}_{\text{Gating Path}} \odot \underbrace{\sigma_{\text{ReLU}}(W_{\text{in}} h_\ell + b_{\text{in}})}_{\text{Magnitude Path}}, where h_\ell \in \mathbb{R}^d denotes activations at layer \ell, W_{\text{in}} \in \mathbb{R}^{{d_{\text{sparse}}} \times d} and b_{\text{in}} \in \mathbb{R}^{d_{\text{sparse}}} are encoder E parameters, W_{\text{gate}} \in \mathbb{R}^{{d_{\text{sparse}}} \times d} and b_{\text{gate}} \in \mathbb{R}^{d_{\text{sparse}}} are the gate parameters, and \odot denotes element-wise product. This mitigates shrinkage effects induced by the L_1 penalty (Rajamanoharan et al., 2024a).

-

JumpReLU SAEs learn per-feature activation thresholds. Given encoder pre-activations a = W_{\text{in}} h_\ell + b_{\text{in}}, feature activations are: z_i = \begin{cases} - a_i & \text{if } a_i > \theta_i \\ - 0 & \text{otherwise} - \end{cases} where \theta_i is a learned threshold parameter. A smooth surrogate is used during training to enable gradient flow Rajamanoharan et al. (2024b).

+

JumpReLU SAEs

+

JumpReLU SAEs (Rajamanoharan et al., 2024b) learn per-feature activation thresholds. Given the encoder pre-activation a = W_{\text{in}} h_\ell + b_{\text{in}}, let a_i denote its i-th entry. The activation value of the i-th feature after applying JumpReLU is z^i = \sigma_{\text{JumpReLU}}(a_i) = \begin{cases} + a_i & \text{if } a_i > \tau_i \\ + 0 & \text{otherwise}, + \end{cases} where \tau_i is a learned threshold parameter. Straight-through estimators (Bengio et al., 2013) are used during training to enable gradient flow.

-

image

+

image

-

Top-k SAEs (Gao et al., 2025a) impose an explicit hard sparsity constraint. Let a = W_{\text{in}} h_\ell + b_{\text{in}} \in \mathbb{R}^{d_{\text{sparse}}} denote encoder pre-activations. Top-k SAEs define the feature activations as: z_i = \begin{cases} +

Top-k SAEs

+

Top-k SAEs (Gao et al., 2025) impose an explicit hard sparsity constraint. Let a = W_{\text{in}} h_\ell + b_{\text{in}} \in \mathbb{R}^{d_{\text{sparse}}} denote encoder pre-activations. Top-k SAEs define the i-th feature’s activation as z^i = \sigma_{\text{TopK}}(a_i) = \begin{cases} a_i & \text{if } i \in \mathcal{I}_k(a) \\ -0 & \text{otherwise} -\end{cases} where \mathcal{I}_k(a) indexes the k largest entries of a. This formulation explicitly enforces the constraint |z|_0 \le k, providing a more direct and interpretable feature sparsity constraint than L1 regularization. Unlike L1 loss, it does not suppress activation magnitudes, thereby avoiding the feature shrinkage issue observed in L1-based SAE training (Gao et al., 2025a).

+0 & \text{otherwise}, +\end{cases} where \mathcal{I}_k(a) indexes the k largest entries of a. This formulation explicitly enforces \|z\|_0 \le k—a more direct feature-sparsity constraint than L_1 regularisation. Notably, as it does not suppress activation magnitudes, it avoids the feature-shrinkage issue observed in L_1-based SAE training (Gao et al., 2025).

-

Batch Top-k SAEs enforce sparsity at the batch level. Given a batch of pre-activations A = [a^{(1)}, \ldots, a^{(B)}] \in \mathbb{R}^{d_{\text{sparse}} \times B}, a subset \mathcal{J} \subset \{1,\ldots,d_{\text{sparse}}\} of k features is selected as the batch-wide top-k: \mathcal{J} = \arg\max_{|\mathcal{J}|=k} \sum_{j \in \mathcal{J}} \sum_{b=1}^B |a_j^{(b)}| Activations are then set as z_j^{(b)} = +

Batch Top-k SAEs

+

Batch Top-k SAEs relax the fixed per-example sparsity constraint of standard Top-k SAEs by enforcing the sparsity budget across an entire batch instead (Bussmann et al., 2024). For a batch of encoder activations \big[\, a^{(1)}, \dots, a^{(B)} \,\big] \in \mathbb{R}^{d_{\text{sparse}} \times B}, where B is the batch size, Batch Top-k retains the largest Bk activation values in the batch. Specifically, let \mathcal{I} +\coloneqq +\arg\max_{\substack{ +\mathcal{S}\subseteq +\{1,\ldots,B\}\times +\{1,\ldots,d_{\text{sparse}}\}\\ +|\mathcal{S}|=Bk +}} +\sum_{(b,j)\in\mathcal{S}} +a_j^{(b)}. In each batch, the activation value of the j-th feature in the b-th sample is z^{j, (b)} += +\sigma_{\text{Batch-TopK}}\!\left(a_j^{(b)}\right) += \begin{cases} -a_j^{(b)}, & j \in \mathcal{J} \\ -0, & \text{otherwise} -\end{cases} for each example b. This enforces global feature competition, reduces variance in feature usage across the batch, and prevents collapse (Bussmann et al., 2024).  

+a_j^{(b)}, & (b,j)\in\mathcal{I},\\ +0, & \text{otherwise}. +\end{cases}

+

That is, each batch exactly retains Bk features, while individual samples may have more or fewer active features than k. This loose constraint allows the model to allocate representational capacity adaptively across samples. During inference, Batch Top-k SAEs instead use the JumpReLU function with an inferred global threshold to determine which features are active for each input, so that an input’s activations do not depend on the rest of the batch.

-

SAE variants differ fundamentally in how they enforce sparsity. No single variant is optimal across all regimes.

+

Trade-offs Between Variants

+

Different SAE variants may implement distinct sparsity constraints. No single variant is optimal across all scenarios.
+

@@ -75,610 +102,641 @@

Training Dynamics and Evaluation.

- - + - - - - + + - - - + + - -
Sparsity Differentiable Tuning Required
\ell_1-regularizedL_1-regularised Soft ✓ \lambda coefficient
Gated Learned✓^*Gate + \ell_1 parameters✗^\dagger\lambda coefficient
JumpReLU Learned✓^*Threshold initialization✗^*\lambda coefficient, STE bandwidth, threshold init.
Top-k Hard ✗ k value
Batch Top-k Hard ✗ k value
-

^*Uses smooth surrogate during training.

+

^*Uses straight-through estimators (STEs) during training. ^\daggerTrains the non-differentiable gate with auxiliary losses.

-

Readers are referred to Shu et al. (2025) for a dedicated survey of SAEs, their variants, evaluations, and applications.

+

Readers are referred to Shu et al. (2025) for a dedicated survey of SAEs, their variants, evaluations, and applications. Additionally, researchers have published SAE weights for several language models, including Gemma Scope (Lieberum et al., 2024) for Gemma 2 (Team et al., 2024b) and Llama Scope (He et al., 2024) for Llama-3.1-8B (Grattafiori et al., 2024). They can be accessed through open platforms such as Neuronpedia (Lin, 2023).

-

6.3 Practical Guidance: Selecting SAE Variants#

-

Practitioners face a decision problem: which SAE variant should be used for a given model, layer, and analysis goal? This section provides practical heuristics synthesized from empirical findings and architectural considerations. These recommendations are provisional—the field lacks comprehensive benchmarks comparing variants across all settings—but offer a starting point for informed decisions.

-

In the following, we consider four factors that primarily determine which SAE variant is appropriate, and present the decision tree based on these factors.

+

6.4 Empirical Comparisons of SAE Variants#

+

The SAE variants in Section 6.3 differ principally in how they enforce sparsity, and those differences map onto distinct analysis settings.

+

Hard-constraint formulations (e.g., Top-k) directly enforce sparsity and suit tight compute budgets. Learned-threshold (e.g., JumpReLU SAE) and gated formulations usually achieve better reconstruction results and suit generous tuning budgets. Cross-layer variants (discussed in Section 6.6) analyse computations across layers that a single dictionary cannot represent at all. This subsection discusses the setting in which each variant is used and the evidence it provides. We particularly discuss four factors that account for most of the variation:

    -
  1. Analysis goal: Exploratory discovery vs. targeted investigation vs. deployment/monitoring

  2. -
  3. Layer type: Residual stream vs. MLP output vs. attention output

  4. -
  5. Model scale: Small (<1B) vs. medium (1-10B) vs. large (>10B parameters)

  6. -
  7. Resource constraints: Compute budget, hyperparameter tuning capacity, tolerance for training instability

  8. +
  9. Analysis goal: exploratory discovery vs. targeted investigation vs. deployment/monitoring

  10. +
  11. Layer type: residual stream vs. MLP output vs. attention output

  12. +
  13. Model scale: small (<1B) vs. medium (1–10B) vs. large (>10B parameters)

  14. +
  15. Resource constraints: compute budget, hyperparameter tuning capacity, tolerance for training instability

-
-

Step 1: What is your primary goal?

-
    -
  • Exploratory discovery (understand what features exist) \rightarrow Go to Step 2

  • -
  • Targeted investigation (find features for specific behavior) \rightarrow Consider supervised methods first (DAS, probes); if unsupervised needed, go to Step 2

  • -
  • Deployment/monitoring (real-time feature detection) \rightarrow Prioritize inference speed; Top-K or standard SAE with small dictionary

  • -
-

Step 2: What layer type are you analyzing?

-
    -
  • Residual stream \rightarrow Go to Step 3

  • -
  • MLP outputs \rightarrow Consider Transcoders (predict post-MLP from pre-MLP); if single-layer SAE needed, go to Step 3

  • -
  • Attention outputs \rightarrow Standard SAE or Gated SAE; attention-specific variants are less developed

  • -
  • Cross-layer analysis \rightarrow Crosscoders or CLTs (if compute permits)

  • -
-

Step 3: What are your resource constraints?

-
    -
  • Limited compute / need fast iteration \rightarrow Top-K SAE (no \lambda tuning; fixed sparsity)

  • -
  • Moderate compute / some tuning capacity \rightarrow Gated SAE (best reconstruction-sparsity tradeoff in most settings)

  • -
  • Ample compute / extensive tuning possible \rightarrow JumpReLU SAE (learned thresholds; strong performance but sensitive)

  • -
-

Step 4: What model scale?

-
    -
  • Small models (<1B) \rightarrow Any variant works; use this regime to validate methodology before scaling

  • -
  • Medium models (1-10B) \rightarrow Gated or Top-K recommended; JumpReLU if tuning budget available

  • -
  • Large models (>10B) \rightarrow Top-K or Batch Top-K (most stable at scale); Gated SAE if reconstruction quality critical

  • -
-

-
-

6.3.1 Layer-Specific Recommendations#

-

Different layer types exhibit different activation statistics, affecting SAE performance, as shown in Table 3.

+
+

Analysis Goal.

+

Exploratory discovery, where the question is what features exist rather than whether a particular one does, is the setting SAEs were designed for and the one the comparisons below address. For targeted investigation of an already-identified behaviour, where the concepts of interest are determined, supervised methods such as DAS (Geiger et al., 2024) and linear probes are more direct instruments. For online deployment and monitoring, SAEs with small dictionaries can be preferable for low inference cost.

+
+
+

Layer Type.

+

Layer types differ in their activation properties, so the right variant depends on what each can represent, not just how well. On the residual stream, single-layer SAEs apply directly, and the choice reduces to the resource considerations below. On MLP outputs, transcoders are the appropriate tool: a single-layer SAE cannot span the nonlinearity, so transcoders learn the input-to-output map directly (Dunefsky et al., 2024), and Paulo et al. (2025c) reported favourable comparisons against SAEs there. On attention outputs, standard and Gated SAEs both have been used; attention-specific variants remain underdeveloped, making this the least studied of the three sites. For cross-layer analysis, crosscoders (Lindsey et al., 2024) and cross-layer transcoders (Ameisen et al., 2025) track feature evolution across layers at substantially higher compute cost, and a crosscoder trained jointly on two models is the reported approach for comparing representations between them. Table 3 summarises the variants used at each layer type, the empirical rationale for their use, and the main caveats reported in prior work.

- + - - - + + + - + - + - + - + - + - - - + + + - - - + + + - +
Table 3: SAE Variant Recommendations by Layer TypeTable 3: Heuristic recommendations for selecting SAE variants across different layer types. Recommendations are based on current empirical evidence and practical experience, and may evolve as SAE methodologies and evaluation standards continue to develop.
Layer TypeRecommended VariantsRationaleConsiderationsVariants UsedReported BasisReported Caveats
Residual stream (early layers)Gated SAE, Top-KGated SAE, Top-k Activations are closer to input embeddings; moderate sparsityMay need smaller expansion factor (2-4x)Smaller expansion factors (2–4\times) reported
Residual stream (middle layers) Gated SAE, JumpReLU Richest feature structure; highest interpretability potentialStandard expansion factor (4-8x); most studied regimeStandard expansion factor (4–8\times); most studied regime
Residual stream (late layers)Top-K, Gated SAETop-k, Gated SAE Activations increasingly task-specific; may be less sparseConsider larger expansion factor (8-16x)Larger expansion factors (8–16\times) reported
MLP outputsTranscoder (6.4)MLP applies nonlinearity; transcoder captures input-output mappingNot a true SAE; predicts layer \ell+1 from layer \ellTranscoder (Section 6.6)MLP applies a nonlinearity a single-layer SAE cannot spanNot a sparse decomposition on a static object; bridges input to output \ell
Attention outputsGated SAE, Standard SAELess studied; attention outputs often lower-rankMay benefit from smaller dictionariesGated SAE, standard SAEAttention outputs often lower-rankLeast studied of the three sites; may benefit from smaller dictionaries
Cross-layerCrosscoder, CLT (6.4)Crosscoder, CLT (Section 6.6) Tracks feature evolution; resolves cross-layer superposition High compute cost; requires multi-layer training
-

6.3.2 Common Pitfalls and How to Avoid Them#

-
-

Pitfall 1: Using standard \ell_1 SAE without considering alternatives

-
    -
  • Standard SAEs suffer from shrinkage bias; Gated SAEs solve this with minimal added complexity.

  • -
  • Recommendation: Use Gated SAE as the default unless you have specific reasons for standard.

  • -
-

Pitfall 2: Over-tuning \lambda instead of switching to Top-K

-
    -
  • If you’re spending significant time tuning the sparsity coefficient, Top-K eliminates this hyperparameter entirely.

  • -
  • Recommendation: If \lambda tuning takes more than a few runs, switch to Top-K with target sparsity.

  • -
-

Pitfall 3: Using the same expansion factor everywhere

-
    -
  • Early layers may need smaller dictionaries (2-4x); late layers may need larger (8-16x).

  • -
  • Recommendation: Start with 4x expansion, then adjust based on dead feature rate and reconstruction quality.

  • -
-

Pitfall 4: Ignoring layer type differences

-
    -
  • SAEs on residual stream vs. MLP outputs vs. attention outputs behave differently.

  • -
  • Recommendation: For MLP analysis, use transcoders; for cross-layer analysis, use crosscoders or CLTs.

  • -
-

Pitfall 5: Not checking for dead features

-
    -
  • High dead feature rates (>20%) indicate wasted capacity and possible training issues.

  • -
  • Recommendation: Monitor dead feature rate; use neuron resampling or switch to Batch Top-K if problematic.

  • -
-
+
+

Resource Constraints.

+

Under limited compute, or where fast iteration matters more than reconstruction quality, Top-k SAEs (Gao et al., 2025) are the practical choice: fixing \|z\|_0 \le k removes the sparsity coefficient \lambda from the search space entirely rather than requiring it to be tuned, with reported values commonly in the range k = 32 (Gao et al., 2025) to 150 (Karvonen et al., 2025). With moderate tuning capacity, Gated SAEs (Rajamanoharan et al., 2024a) achieved a better reconstruction–sparsity trade-off while retaining \lambda. They also avoid the shrinkage effect by separating feature detection from magnitude estimation. However, because the standard L_1 SAE is now used mainly as a baseline, comparisons against it partly reflect shrinkage rather than dictionary quality alone. When extensive tuning is plausible, JumpReLU SAEs (Rajamanoharan et al., 2024b) achieved the strongest reconstruction performance at matched sparsity by learning per-feature thresholds. However, gains on these sparsity–fidelity proxy metrics do not reliably translate into better downstream performance (Karvonen et al., 2025). In practice, selecting the best model and hyperparameters to achieve optimal performance can take a lot of time.

-

6.3.3 Quick Reference: When to Use What#

-

In Table 4, we provide a concise set of heuristic recommendations mapping common practitioner goals to SAE variants that have been found empirically useful in prior work and practice.

- - ---- +
+

Model Scale.

+

The differences between variants depend strongly on the metric in models with roughly 1 or 2B parameters, such as Gemma-2-2B (Karvonen et al., 2025), which is the regime mainly used to validate methodology before scaling. For models between 1B and 10B parameters, JumpReLU SAEs and Top-k SAEs are the most commonly reported variants; Gemma Scope (Lieberum et al., 2024) uses the former and Llama Scope (He et al., 2024) the latter. For models larger than 10B parameters, Gemma Scope (Lieberum et al., 2024) (for Gemma 2 models) and Gemma Scope 2 (McDougall et al., 2025) (for Gemma 3 models) are the main open resources, both of which used JumpReLU SAEs. Additionally, Gao et al. (2025) explored scaling Top-k SAEs to GPT-4 (OpenAI et al., 2024), although GPT-4 itself is not publicly available.

+

Table 4 summarises each variant’s main findings reported in previous work, along with their experimental settings. Note that these studies may use different baselines, model families, sparsity constraints, and evaluation criteria. The community has been establishing benchmarks to evaluate variants of SAEs, such as SAEBench (Karvonen et al., 2025), but the results vary by metric. As a result, we should treat the findings as references rather than established rules. The Setting column in Table 4 records the conditions under which each result is established. Table 5 further highlights several factors that limit the generality of the findings.

+
Table 4: Heuristic recommendations for selecting SAE variants based on common practitioner goals.
+ - - + + + - - + + + - - + + + - - + + + - - + + + + + + + + - + + - - + + + - - + + +
Table 4: Reported evidence for each SAE variant, with the setting in which it was obtained.
Practitioner GoalRecommended VariantVariantReported FindingSetting
New to SAEs and want a reliable defaultGated SAE with 4\times expansionStandardSparse dictionary learning by L_1 regularisation recovers features more monosemantic than individual neurons; subject to shrinkage biasEstablishing work, small models (Cunningham et al., 2024; Bricken et al., 2023)
Fast experimentation without extensive tuningTop-k SAE (k = 50–100)GatedSeparating detection from magnitude estimation addresses L_1 shrinkage at minimal added complexityReconstruction–sparsity improvement at matched sparsity, Gemma-family (Rajamanoharan et al., 2024a)
Highest-quality features with sufficient computeJumpReLU SAE with careful tuningJumpReLULearned per-feature thresholds trained with straight-through estimators give the strongest reported reconstructionMatched-sparsity comparison, Gemma-family (Rajamanoharan et al., 2024b); needs most tuning (Bussmann et al., 2024)
Analysis of very large modelsBatch Top-k SAE for improved stabilityTop-kA hard \|z\|_0 \le k constraint removes the sparsity coefficient from the search spaceScaling behaviour characterised on large models; k = 32–150 commonly reported (Gao et al., 2025; Karvonen et al., 2025)
Batch Top-kA batch-level sparsity budget lets the number of active features vary per input, improving reconstruction at matched average sparsityIntroduced as a relaxation of per-example Top-k, achieving better reconstruction (Bussmann et al., 2024)
Understanding MLP computation TranscoderLearning the MLP input–output map avoids spanning the nonlinearity with a single dictionaryReported to perform at least on par with SAEs (Dunefsky et al., 2024; Paulo et al., 2025c)
Tracking features across layersCrosscoder or CLTCrosscoderJoint reconstruction across layers tracks feature evolution and resolves cross-layer superpositionCross-layer analysis and model diffing; trained jointly across models for cross-model comparison (Lindsey et al., 2024)
Comparing representations between modelsCrosscoder trained jointly on both modelsCLTPartial reconstruction of multiple downstream layers from one dictionaryDeployed within CLT-based circuit discovery at scale (Ameisen et al., 2025)
-
-

6.3.4 Limitations of These Recommendations#

-

Heuristics in Table 5 are based on current empirical findings, which have limitations. Practitioners should treat these as starting points, validate on their specific use case, and report which variants they used to enable community learning.

- - +
Table 5: Key limitations underlying the heuristic recommendations for SAE variant selection.
+--++ - - + + - - + + - - + + - - + + - - + + + + + +
Table 5: Conditions for the reported evidence on SAE variant behaviour.
LimitationDescriptionConditionDescription
Limited systematic comparisonNo comprehensive benchmark currently evaluates all SAE variants across layer types, model scales, and training regimes.Architecture dependenceMost evidence derives from Llama- and Gemma-family models. Generality of the findings across model families remains unestablished.
Architecture dependenceMost evidence is derived from Llama and Gemma family models; results may not generalize to other architectures.Task dependenceThe properties that matter may differ across analysis goals, such as safety auditing, behaviour understanding, and knowledge localisation, but existing evaluations often do not distinguish between them.
Rapid methodological changeNew SAE variants and training techniques continue to emerge, potentially rendering current recommendations outdated.Rapid iterationNew architectures and training procedures are proposed faster than existing ones are sufficiently studied and replicated.
Task dependenceOptimal choices may differ depending on the analysis goal (e.g. safety analysis, capability understanding, or knowledge localization).Immature theoretical analysisCurrent findings are mostly empirical and may be superseded. Understanding of why particular variants succeed or fail is incomplete.
Feature reproducibilitySAEs trained on the same data with different random seeds can recover substantially different features. When between-variant performance differences are smaller than within-variant seed-to-seed variability, we may not confidently attribute them to architectural factors. However, few reported comparisons have rigorously accounted for this concern.
-

Practitioners should treat these as starting points, validate on their specific use case, and report which variants they used to enable community learning.

-

6.4 Multi-Layer Feature Analysis: Transcoders and Crosscoders#

-

Standard SAEs analyze single layers in isolation. However, features evolve as information flows through the network. Transcoders and Crosscoders enable analysis across layers and token positions, revealing how representations transform through nonlinearities.

-

6.4.1 Transcoders: Bridging Nonlinear Layers#

+

6.5 Challenges and Failure Modes of SAEs#

+

SAEs have become a standard tool for feature extraction, but they exhibit systematic failure modes that undermine their interpretability claims. It is therefore essential to understand these issues when using SAEs as mechanistic tools. This section provides a systematic discussion of them.

+

6.5.1 Dead Features#

+

A dead feature is an element in the learned dictionary that never activates or activates extremely rarely across training. Dead features waste the dictionary’s capacity, and they are common. In fact, they can constitute up to 10–50% of the dictionary in a poorly tuned SAE (Bricken et al., 2023).

+

Several factors can cause dead features to emerge. First, inappropriate SAE weight initialisation can place features in low-density regions of the activation space with weak gradient signals. Second, an overly strong sparsity penalty \lambda may suppress feature activations early in training, preventing features from contributing usefully to reconstruction. Finally, once a subset of features captures common, dominant patterns, the remaining features struggle to find useful roles for improving reconstruction and thus do not activate.

+

Dead features can be detected by tracking each feature’s rate of being activated during training. For example, features with an activation rate below 0.01% may be considered dead (the threshold should adapt across scenarios). Several mitigation strategies have been proposed, such as neuron resampling, which periodically reinitialises dead features using examples with high reconstruction error (Bricken et al., 2023). The sparsity penalty can also be reduced early in training and increased gradually (i.e., a warm-up schedule). More recent Batch Top-k SAEs allocate sparsity adaptively across samples, which allows more active features on complex inputs and fewer on easy ones (Bussmann et al., 2024).

+
+

6.5.2 Feature Duplication#

+

Feature duplication occurs when multiple features in the dictionary encode the same or similar concepts (Templeton et al., 2024), leading to potentially redundant features. Duplication can arise for several reasons: insufficient sparsity pressure, imbalanced training data, or an optimisation process that favours local minima with duplicated features. Duplication can be detected by computing pairwise cosine similarity between feature directions (i.e., decoder columns), where high similarity indicates potential redundancy. As duplicated features tend to activate together (Korznikov et al., 2025), co-activation statistics may provide complementary evidence.

+
+

6.5.3 Feature Absorption and Splitting#

+

Another class of failure modes concerns feature granularity. Feature absorption arises when semantically hierarchical features interact under the sparsity objective: a seemingly general (parent) feature fails to activate on some inputs where it should because a more specific (child) feature activates instead and has absorbed part of the parent’s direction into its own decoder direction (Chanin et al., 2025). For example, a “starts with S” feature may fail to activate on the token “short”, while a token-specific “short” feature with a partially overlapping direction activates instead. Absorption-induced false negatives make such a feature an unreliable detector of its concept. It can be detected by comparing the feature with a linear probe for the same concept: inputs on which the probe responds strongly, but the feature does not, while another active feature’s direction aligns with the probe, indicate absorption (Chanin et al., 2025).

+

By contrast, feature splitting occurs when a feature learned by a smaller SAE splits into multiple distinct features as the SAE scales up (e.g., by increasing the expansion factor e). This often produces more fine-grained features, which may be either beneficial or less useful. In a favourable case, an initially polysemantic or opaque feature, such as one activating on “bank” in both financial and geographical contexts, may split into more coherent features of “investment bank” and “river bank”. In a less useful case, the resulting features may become low-level, lexical detectors that activate on the token “bank” in any context.

+
+

6.5.4 Feature Instability Across Seeds#

+

SAE features are also sensitive to random initialisation, optimiser stochasticity, and data ordering. For a fixed SAE architecture and training dataset, different random seeds can produce substantially different dictionaries (Paulo & Belrose, 2026). In practice, frequent and semantically clear features tend to be stable, while rare or context-specific features vary substantially. Often, fewer than 50% of features remain the same across seeds, although broader semantic clusters may remain more consistent. This instability limits the reliability of mechanistic claims based on individual SAE features. Although aggregating results across seeds can improve robustness, it increases computational cost. As a result, when evaluating SAEs, we should report feature stability across seeds alongside reconstruction loss.

+
+

6.5.5 Identifiability and Theoretical Concerns#

+

Another concern for SAE-based interpretability claims is whether the learned features are identifiable, that is, whether the dictionary learning reliably recovers the “true” underlying features—if such features exist—or merely finds one of many mediocre decompositions (Klindt et al., 2025).

-

A transcoder learns a sparse mapping between the input and output to an MLP layer: h_{\ell}^{out} \approx W_{\text{out}} z + b_{\text{out}}, \quad z = \sigma(W_{\text{in}} h_\ell^{in} + b_{\text{in}}) where z \in \mathbb{R}^{d_{\text{sparse}}} are sparse feature activations bridging the nonlinearity between layers \ell and \ell+1, W_{\text{out}} \in \mathbb{R}^{d \times {d_{\text{sparse}}}} and b_{\text{out}} \in \mathbb{R}^{d} are decoder D parameters. The transcoder reconstructs post-nonlinearity activations using features extracted from pre-nonlinearity activations (Dunefsky et al., 2024).

+

Identifiability

+

Suppose activations are generated by some sparse latent process

+

h = D^{*} z^{*} + \epsilon ,

+

where D^{*} is the ground-truth feature dictionary and z^{*} are the feature coefficients. An SAE is identifiable if its training recovers D^{*} and z^{*} up to unavoidable symmetries such as feature permutation and rescaling.

-
-

Architecture and Motivation.

-

Standard SAEs struggle to explain activations after nonlinear transformations (MLP layers, attention). A transcoder applies sparse decomposition directly to the nonlinear mapping, learning features that explain the transformation. Training minimizes: \mathcal{L} = \|h_{\ell}^{out} - (W_{\text{out}} z + b_{\text{out}})\|_2^2 + \lambda \|z\|_1 where z = \sigma(W_{\text{in}} h_\ell + b_{\text{in}}).

+

In classical dictionary learning, identifiability holds under low-coherence conditions, i.e., dictionary elements are sufficiently orthogonal to one another. However, when coherence is high, ambiguity arises: multiple distinct sparse combinations of features from the same dictionary can generate the same output (Schnass & Vandergheynst, 2008). In large language model activations, high coherence is likely unavoidable because of the model’s complicated internal structure and correlations within its feature space (Li et al., 2025b). So far, identifiability remains a stubborn issue in SAEs, even though it offers a theoretical explanation for several empirical observations.

+

Recent progress in fine-tuning language models around a fixed sparse autoencoder offers a potential path forward. In this setting, model activations are explicitly constrained to be sparse combinations of decoder features while largely preserving task performance (Chen et al., 2025a). In particular, recent work aimed at limiting superposition fine-tunes the language model jointly with the SAE while enforcing orthogonality constraints on the decoder matrix (Miller et al., 2026). By explicitly imposing low-coherence conditions, this approach improves identifiability: activations have a unique feature representation, and these features can be reliably recovered.

+
+

Relation to Causal Relevance.

+

The concern of identifiability connects to causal relevance: even if an SAE feature is monosemantic and helps reconstruct activations well, it may not correspond to a causally meaningful unit of analysis. Below, we summarise some criteria for validating an SAE feature.

+

Diagnostic criteria:

+
    +
  • Necessity: Does ablating the feature remove the corresponding concept in model responses? Low necessity suggests that the concept can be expressed through redundant features.

  • +
  • Sufficiency: Does activating only this feature produce the behaviour corresponding to its concept? Low sufficiency suggests the feature alone is not enough to represent the concept.

  • +
  • Specificity: Does the feature’s causal effect match its corresponding concept? Low specificity means the feature contains other concepts.

  • +
+

These criteria are useful in practice when claiming that SAE features represent meaningful units of analysis.

-
-

When Transcoders Are Necessary.

-

Single-layer SAEs often suffice for residual-stream analysis, where features largely propagate linearly. However, MLP blocks apply nonlinear transformations that distort feature space. Transcoders are essential when analyzing MLPs, understanding feature interactions, or tracking how concepts evolve across layers (Dunefsky et al., 2024).

-
-
-Figure 7: Cross-Layer Transcoder (CLT). CLTs extend Transcoders by simultaneously partially reconstructing the outputs of multiple subsequent layers from a single input point in the residual stream. -
-
-Figure 7: Cross-Layer Transcoder (CLT). CLTs extend Transcoders by simultaneously partially reconstructing the outputs of multiple subsequent layers from a single input point in the residual stream. +
+

6.5.6 Practical Recommendations#

+

The challenges discussed above suggest several methodological considerations for SAE-based interpretability studies:

+
    +
  1. Report failure-mode metrics: Alongside reconstruction loss, researchers should also report dead-feature percentage, duplication rate, and stability across seeds.

  2. +
  3. Validate causal relevance: Researchers should validate features’ necessity, sufficiency, and specificity through interventions (e.g., feature ablation or steering) and evaluation on downstream effects.

  4. +
  5. Compare methods: For safety-critical applications, researchers should compare SAE features against various baselines to assess whether SAEs recover the relevant structure.

  6. +
  7. Try multiple SAE variants: Different variants (e.g., Gated, Top-k, JumpReLU) impose different inductive biases. Researchers can explore findings consistent across variants, as these provide stronger evidence.

  8. +
  9. Acknowledge identifiability limits: Researchers should frame SAE features as one valid approximate decomposition rather than the true features. Multiple decompositions may be equally valid for reconstruction but yield different feature sets.

  10. +
  11. Consider alternatives for targeted analysis: When specific concepts are the focus, supervised methods (e.g., DAS, probes, discussed in Section 6.9.4) may be more reliable because SAEs discover features in an unsupervised and exploratory manner.

  12. +
+
+
+

6.6 Multi-Layer Feature Extraction and Analysis#

+

Standard SAEs analyse individual layers in isolation. However, features can evolve as information propagates through the network. Transcoders (Figure 7a), cross-layer transcoders (Figure 7b), and crosscoders (Figure 9) enable multi-layer feature extraction and analysis, revealing how representations and features associated with particular model components—most commonly MLP blocks—transform across depth.

+

6.6.1 Transcoders: Bridging the Nonlinearities#

+
+

Transcoder (Figure 7a)

+

A transcoder learns a sparse mapping between the input to and output of a model component, such as an MLP block, in layer \ell: h_{\ell}^{\text{out}} \approx W_{\text{out}} z + b_{\text{out}}, \quad z = \sigma(W_{\text{in}} h_\ell^{\text{in}} + b_{\text{in}}) where z \in \mathbb{R}^{d_{\text{sparse}}} are sparse feature activations bridging the nonlinearity between input and output of the MLP, and W_{\text{out}} \in \mathbb{R}^{d \times {d_{\text{sparse}}}} and b_{\text{out}} \in \mathbb{R}^{d} are decoder D parameters. The transcoder reconstructs post-nonlinearity activations using features extracted from pre-nonlinearity activations (Dunefsky et al., 2024).

-
Figure 7: Cross-Layer Transcoder (CLT). CLTs extend Transcoders by simultaneously partially reconstructing the outputs of multiple subsequent layers from a single input point in the residual stream.
+
+

Architecture and Motivation.

+

Standard SAEs do not explain the nonlinear transformations (i.e., weight matrices of MLP blocks and attention heads) themselves that produce the activations. A transcoder achieves this by applying dictionary learning to the mapping, learning features that explain the transformation. Its training minimises \mathcal{L} = \|h_{\ell}^{\text{out}} - (W_{\text{out}} z + b_{\text{out}})\|_2^2 + \lambda \|z\|_1, where z = \sigma(W_{\text{in}} h^{\text{in}}_\ell + b_{\text{in}}) as defined above.

+
+
+

When Are Transcoders Necessary?

+

SAEs extract features from static activations and thus often suffice for residual-stream analysis. Transcoders track how features emerge from input to output and are essential for analysing MLP layers (Dunefsky et al., 2024).

+
+
+Figure 7a: Transcoder. +
Figure 7a: Transcoder.
+
+
+Figure 7b: Cross-layer transcoder. +
Figure 7b: Cross-layer transcoder.
+
+
Figure 7: Transcoders and CLTs. Transcoders reconstruct the output of the MLP from its input in the same transformer layer. CLTs extend this by partially reconstructing the outputs of multiple subsequent layers from a single point in the residual stream.
-

6.4.2 Cross-Layer Transcoders#

+

6.6.2 Cross-Layer Transcoders: Enabling Better Circuit Analysis#

-

A Cross-Layer Transcoder (CLT) is a sparse feature decomposition spanning all layers of a neural network. Each feature \mathbf{a}^{\ell} reads from the residual stream at layer \ell via: \mathbf{z}^{\ell} = \text{JumpReLU}(W_{\text{in}}^{\ell} h_\ell) and each feature can write to the MLP outputs of all subsequent layers via layer-specific decoder matrices: \hat{\mathbf{y}}^{\ell'} = \sum_{\ell=1}^{\ell'} W_{\text{out}}^{\ell \to \ell'} \mathbf{z}^{\ell} where \hat{\mathbf{y}}^{\ell'} reconstructs the original MLP output at layer \ell'.

+

Cross-Layer Transcoder (Figure 7b)

+

Cross-layer transcoders (CLTs) are sparse feature decompositions spanning all layers of a neural network. At each layer \ell, sparse feature activations are read from the residual stream via z_{\ell} += +\sigma_{\text{JumpReLU}} +\!\left( +W_{\text{in}}^\ell h_{\ell}^{\text{in}} +\right), where z_\ell^j denotes the activation of feature j at layer \ell.
+Each feature can write to the MLP outputs of its own layer and all subsequent layers through layer-specific decoder vectors. The contribution from features read at layer \ell to the MLP output at a subsequent layer \ell' is \hat{h}_{\ell \to \ell'}^{\text{out}} += +\sum_j +W_{\text{out},j}^{\ell \to \ell'} z_{\ell}^{j}, where W_{\text{out},j}^{\ell \to \ell'} is the decoder vector from feature j at layer \ell to the MLP output at layer \ell'. The reconstructed MLP output at layer \ell' is then \hat{h}_{\ell'}^{\text{out}} += +\sum_{\ell=1}^{\ell'} +\hat{h}_{\ell \to \ell'}^{\text{out}} += +\sum_{\ell=1}^{\ell'} +\sum_j +W_{\text{out},j}^{\ell \to \ell'} z_{\ell}^{j}, which approximates the original MLP output h_{\ell'}^{\text{out}}.

-

CLTs are trained jointly across all layers to minimize: \mathcal{L} = \sum_{\ell=1}^L \|\hat{\mathbf{y}}^{\ell} - \mathbf{y}^{\ell}\|_2^2 + \lambda \sum_{\ell=1}^L \sum_{i} \tanh(c \cdot \|W_{\text{out},i}^{\ell}\| \cdot z_i^{\ell}) where c is the sharpness hyperparameter (Ameisen et al., 2025).

+

CLTs (Figure 7b) are trained jointly across all L layers to minimise \mathcal{L} += +\sum_{\ell'=1}^{L} +\left\| +\hat{h}_{\ell'}^{\text{out}} +- +h_{\ell'}^{\text{out}} +\right\|_2^2 ++ +\lambda +\sum_{\ell=1}^{L} +\sum_j +\tanh\!\left( +c\, +z_{\ell}^{j} +\sqrt{ +\sum_{\ell'=\ell}^{L} +\left\| +W_{\text{out},j}^{\ell \to \ell'} +\right\|_2^2 +} +\right), where c is the sharpness hyperparameter (Ameisen et al., 2025).

Replacement Models and Attribution Graphs.

-

A key insight is that CLTs enable the construction of a replacement model: a version of the original model where all MLP layers are replaced by their CLT reconstructions. Running this replacement model involves:

+

A key insight is that CLTs enable the construction of a replacement model—a version of the original model in which all MLP layers are replaced with their CLT reconstructions. Running this replacement model involves:

    -
  • Computing CLT feature activations \mathbf{z}^{\ell} at each layer \ell

  • -
  • Replacing MLP outputs \mathbf{y}^{\ell} with reconstructions \hat{\mathbf{y}}^{\ell} from features in layers 1 through \ell

  • +
  • Computing CLT feature activations z_{\ell} at each layer \ell

  • +
  • Replacing MLP outputs h^{\text{out}}_{\ell} with reconstructions \hat{h}^{\text{out}}_{\ell} from features in layer 1 through \ell

  • Keeping attention layers unchanged

-

Empirically, CLT replacement models match the original model’s most-likely token on approximately 50% of prompts in initial experiments on Gemma-2-2B (Ameisen et al., 2025). However, this figure should be interpreted with important caveats (see below). When analyzing a specific prompt, a local replacement model freezes attention patterns and normalization statistics from the original model’s forward pass, then adds layer-specific error corrections. Under these conditions, the local replacement model can exactly match the original model’s computation on that prompt while decomposing it into interpretable CLT features.

+

When analysing a specific prompt, the replacement model freezes the original model’s attention patterns and normalisation statistics from its forward pass, then adds layer-specific error corrections. Under these conditions, the local replacement model can exactly match the original model’s activations on that prompt while decomposing it into interpretable CLT features. However, when the model’s activations are completely replaced by CLT reconstructions, the intervened model’s top-1 predictions match the original model’s on only about 50% of prompts from the Pile dataset (Gao et al., 2020). Notably, this experiment uses a relatively small, 18-layer transformer (Ameisen et al., 2025). Therefore, this figure should be interpreted with important caveats (see below).

-

The claim that CLT replacement models achieve \sim50% most-likely token match requires careful qualification:

-

Conditions for reported fidelity:

+

Caveats on CLT Fidelity Claims

+

CLTs have been claimed to possess high output fidelity given that they achieve high top-1 token agreement. However, this claim requires careful examination:
+

+
+

Conditions for reported fidelity:

    -
  • Results reported primarily on Gemma-2-2B; generalization to other architectures (e.g., Llama, GPT variants) and larger scales (70B+) remains underexplored.

  • -
  • Fidelity was measured on evaluation prompts drawn from small sequences drawn from the CLT training data. Performance under significant distribution shift (e.g., out-of-domain tasks, adversarial inputs) is not well characterized.

  • -
  • The 50% figure reflects top-1 token agreement; softer metrics (e.g., top-5 agreement, KL divergence) may show different patterns.

  • +
  • Results are reported primarily on toy or small models, whose generality to other model architectures (e.g., Llama, Qwen) and model sizes (e.g., 70B models) is underexplored.

  • +
  • Fidelity was measured on short sequences in the CLT training data. Evaluation under distribution shift (e.g., out-of-domain tasks, adversarial inputs) is not adequate.

  • +
  • The paper reports top-1 token agreement as the main result, while softer metrics (e.g., top-5 agreement, KL divergence) may show different patterns and insights.

-

Conditions for exact local matching:

+
+
+

Conditions for exact local matching:

    -
  • Attention patterns must be frozen from the original forward pass, the CLT does not model or reconstruct attention computations.

  • -
  • Layer normalization statistics (means, variances) must be cached from the original model; the replacement model does not recompute these.

  • -
  • Error correction terms are added per-layer to compensate for reconstruction residuals; without these, exact matching fails.

  • +
  • Attention patterns must be frozen from the original forward pass since the CLT does not compute or reconstruct attention computations.

  • +
  • Layer normalisation statistics (means, variances) must be cached from the original model. The replacement model does not recompute these.

  • +
  • Error correction terms are added per layer to compensate for reconstruction residuals. Without them, exact matching fails.

-

Factors affecting fidelity:

+
+
+

Factors affecting fidelity:

    -
  • Model scale: Preliminary evidence suggests fidelity improves with model size, possibly because larger models exhibit more structured, less entangled representations (Ameisen et al., 2025). However, systematic scaling studies are limited.

  • -
  • Layer depth: Middle layers often show higher reconstruction fidelity than early or final layers, where representations may be more input-specific or output-constrained respectively.

  • -
  • Domain: Fidelity on factual recall and syntactic tasks appears higher than on complex reasoning or arithmetic, where MLP computations may be more nonlinear and less amenable to sparse linear approximation.

  • +
  • Model scale: Preliminary evidence suggests that reconstruction quality and feature interpretability improve with CLT model size. This may be because larger CLTs produce more structured, less entangled representations (Ameisen et al., 2025). However, systematic scaling studies are limited.

  • +
  • Layer depth: Early layers’ and late layers’ activations are usually input-specific and output-constrained, respectively. Middle layers have richer information derived from the input but are not yet converged to the output. As a result, middle layers often show higher reconstruction fidelity.

  • +
  • Domain: Fidelity on factual recall and syntactic tasks is higher than on complex reasoning or arithmetic tasks, where MLP computations may be more complex and cannot be approximated by sparse linear features.

-

These caveats do not invalidate CLTs as a tool for mechanistic analysis, but practitioners should verify fidelity on their specific model, layer, and task before drawing strong conclusions from CLT-based circuit analysis.

+

These caveats do not invalidate CLTs as a tool for mechanistic analysis, but they do mean that fidelity should be verified on the target model and specific task before drawing strong conclusions from CLT-based circuit analysis.

+

Why CLTs Enable Better Circuit Analysis.

-

Unlike per-layer transcoders, CLTs reveal direct feature-to-feature interactions across layers: a feature in layer \ell directly influences features in later layers through its contribution to their inputs. This simplifies circuit structure—features need not re-encode information at each layer. Additionally, by replacing MLPs with CLT reconstructions, the remaining computation becomes approximately linear (with frozen attention patterns and normalization), enabling principled attribution: each feature’s activation can be expressed as the sum of its input edges up to its activation threshold (Ameisen et al., 2025).

-

However, the degree of linearization achieved depends on several assumptions:

+

Unlike per-layer transcoders, CLTs reveal direct feature-to-feature interactions across layers: a feature in layer \ell directly influences features in later layers through its contribution to their inputs. This simplifies circuit structure—features need not re-encode information at each layer. Additionally, replacing MLPs with CLT reconstructions makes the remaining computation approximately linear (with frozen attention patterns and normalisation), enabling principled attribution: each feature’s activation can be expressed as the sum of its input edges up to its activation threshold (Ameisen et al., 2025).

+

However, linearisation depends on several assumptions:

    -
  • Attention freezing: By caching attention patterns, the replacement model treats attention as a fixed routing operation rather than a dynamic, input-dependent computation. This simplifies analysis but means CLT-based circuits describe computation given specific attention patterns, not the attention computation itself.

  • -
  • Normalization caching: Freezing LayerNorm statistics removes a source of nonlinearity but introduces dependence on the specific input used to cache these statistics.

  • -
  • Sparse linear approximation: The core assumption that MLP outputs are well-approximated by sparse linear combinations of features may break down for computations involving complex feature interactions or highly nonlinear transformations.

  • +
  • Attention freezing: By caching attention patterns, the replacement model treats attention as a fixed routing operation rather than a dynamic, input-dependent computation. This means CLT-based circuits describe model computation conditioned on given attention patterns without analysing them.

  • +
  • Normalisation caching: CLT analysis caches the input-dependent normalisation factors (e.g., LayerNorm denominators) from the original forward pass and reuses them during attribution instead of recomputing them after interventions. This removes a source of nonlinearity, but if an intervention would substantially change these normalisation factors in the original model, the cached computation no longer faithfully represents the intervened state.

  • +
  • Sparse linear approximation: Dictionary learning methods assume MLP outputs or transformations are well approximated by sparse linear combinations of features. This assumption may fail when computations involve complex feature interactions or highly nonlinear transformations.

-

When these assumptions hold, CLTs provide a powerful lens for circuit discovery. When they fail, particularly for complex reasoning or out-of-distribution inputs, CLT-based explanations may be incomplete or misleading.

+

CLTs provide a powerful lens for circuit discovery when these assumptions hold. When they fail, CLT-based explanations may become unfaithful.

Computational Cost and Practical Considerations.

-
-Figure 8: Parameter count (left) and FP16 memory requirements (right) of a full Cross-Layer Transcoder as a function of model depth, for different residual stream dimensions and expansion factor e=32. -
Figure 8: Parameter count (left) and FP16 memory requirements (right) of a full Cross-Layer Transcoder as a function of model depth, for different residual stream dimensions and expansion factor e=32.
+
+Figure 8: Parameter count (left) and FP16 memory requirements (right) of a full cross-layer transcoder as a function of model depth, for different residual stream dimensions and expansion factor e=32. +
Figure 8: Parameter count (left) and FP16 memory requirements (right) of a full cross-layer transcoder as a function of model depth, for different residual stream dimensions and expansion factor e=32.
-

A CLT associates each transformer layer with an encoder \mathbb{R}^d \rightarrow \mathbb{R}^{ed} and every ordered layer pair (\ell_s < \ell_t) with a decoder \mathbb{R}^{ed} \rightarrow \mathbb{R}^d. The total parameter count scales as N_{\text{CLT}}(L,d,e) = \tfrac{e d^2}{2}\left(L^2 + L\right), highlighting quadratic growth in both model depth and width, as shown in Figure 6.

-

Training a CLT requires significant upfront cost, which scales with the number of features and layers. For reference, training the CLT suite for Gemma-2-9B required approximately 3,844 H100 GPU-hours at 50% FLOPs efficiency (Ameisen et al., 2025). This cost is amortized over many circuit analyses on the trained CLT. The resulting interpretability gains (cross-layer features, lower interference, better attribution) justify the investment for systematic mechanistic analysis.

-

For these reasons, besides Anthropic’s foundational work (Ameisen et al., 2025; Lindsey et al., 2025c), CLT applications remain limited: a “greater-than” mechanism study (Merullo et al., 2025), an open-source extension (Lindsey et al., 2025a), and analysis of multilingual representations using CLTs (Harrasse et al., 2025a). This scarcity of downstream applications in the current literature likely reflects the considerable challenges in training CLTs at scale.

+

Let d_{\text{sparse}} = ed, where e is the expansion factor. For a transformer with L layers, a complete suite of CLTs contains L encoders mapping \mathbb{R}^d \rightarrow \mathbb{R}^{ed} and L(L+1)/2 decoders mapping \mathbb{R}^{ed} \rightarrow \mathbb{R}^d. The resulting parameter count, with biases ignored, is therefore N_{\text{CLT}}(L,d,e) += +\frac{e d^2}{2}\left(L^2 + 3L\right), which highlights the quadratic scaling with both model depth and hidden dimension, as illustrated in Figure 8.

+

Training a suite of CLTs thus incurs a high upfront cost. For reference, training the CLT suite for Gemma-2-9B required approximately 3,844 H100 GPU-hours at 50% FLOPs efficiency (Ameisen et al., 2025). This cost is amortised over many circuit analyses on the trained CLT. The resulting interpretability gains (cross-layer features, lower interference, better attribution) justify the investment for systematic mechanistic analysis.

+

For these reasons, current applications of CLTs remain limited: a “greater-than” mechanism study (Merullo et al., 2025), an open-source extension (Lindsey et al., 2025a), and analysis of multilingual representations using CLTs (Harrasse et al., 2026). This scarcity of downstream use cases likely reflects the considerable challenges of training CLTs at scale and the complexity of using CLT-based circuit discovery in the wild.

Open Questions for CLT Research.

Several important questions remain open:

    -
  • How does CLT fidelity scale in large models?

  • -
  • Can CLTs be extended to jointly model attention and MLP computations?

  • -
  • What evaluation protocols can reliably detect when CLT-based explanations are unfaithful to the original model’s computation?

  • -
  • How sensitive are CLT circuits to the choice of training data distribution?

  • +
  • How does CLT fidelity scale in large models? How can we better measure fidelity?

  • +
  • Can CLTs be extended to model more nonlinearities such as attention patterns and normalisation, while maintaining their explanatory power?

  • +
  • Can we develop evaluation protocols that reliably detect the potential unfaithfulness of CLT-based explanations?

  • +
  • How sensitive are CLT circuits to experimental setups, such as the choice of training data?

-

6.4.3 Crosscoders: Analyzing Cross-Layer Features#

+

6.6.3 Crosscoders: Analysing Cross-Layer Features#

-

A crosscoder learns sparse features by reading activations from multiple layers simultaneously: \hat{x}_{\ell, t} = W_{\text{out}} z_{\ell,t} + b_{\text{out}}, \quad z_{\ell,t} = \text{ReLU}(W_{\text{in}} [h_{\ell',t}]_{\ell' \in L} + b_{\text{in}}) where z_{\ell,t} \in \mathbb{R}^{d_{\text{sparse}}} encodes sparse features constructed from activations across a set of layers L at token position t. Crosscoders resolve cross-layer superposition by tracking features that persist through the residual stream (Lindsey et al., 2024).

+

Crosscoder (Figure 9)

+

A crosscoder learns a shared sparse representation from activations across L layers. It jointly encodes activations from all layers as z += +\sigma_{\text{ReLU}} +\left( +\sum_{\ell=1}^{L} +W_{\text{in}}^\ell h_\ell ++ +b_{\text{in}} +\right), where z \in \mathbb{R}^{d_{\text{sparse}}} denotes the shared sparse feature activations and W_{\text{in}}^\ell is the layer-\ell encoder. Each layer is then reconstructed using a layer-specific decoder: \hat{h}_\ell += +W_{\text{out}}^\ell z ++ +b_{\text{out}}^\ell, +\qquad +\ell = 1,\ldots,L. Thus, a single latent feature can capture patterns that are shared across, persist through, or vary across multiple layers of the residual stream (Lindsey et al., 2024). Studies have also used crosscoders to discover shared features in activations from different models (Ge et al., 2026; Bayazit et al., 2026; Lindsey et al., 2024; Jiralerspong & Bricken, 2025), as detailed in Section 9.1.

-
-Figure 9: Crosscoder. While standard SAEs process a single layer, Crosscoders are trained to simultaneously reconstruct the residual stream across multiple layers. -
Figure 9: Crosscoder. While standard SAEs process a single layer, Crosscoders are trained to simultaneously reconstruct the residual stream across multiple layers.
+
+Figure 9: Crosscoder. Crosscoders are trained to reconstruct the residual stream across multiple layers simultaneously. +
Figure 9: Crosscoder. Crosscoders are trained to reconstruct the residual stream across multiple layers simultaneously.
-

Crosscoder is trained by minimizing: \mathcal{L} = \sum_{\ell\in L} \|\hat{x}^{\ell} - h^{\ell}\|_2^2 + \lambda \sum_{\ell\in L} \sum_{i} \|W_{\text{out},i}^{\ell}\| \cdot z_i^{\ell} where \lambda is the hyperparameter.

+

A crosscoder is trained by minimising \mathcal{L} += +\sum_{\ell=1}^{L} +\left\| +\hat{h}_\ell - h_\ell +\right\|_2^2 ++ +\lambda +\sum_j +\left( +\sum_{\ell=1}^{L} +\left\| +W_{\text{out},j}^{\ell} +\right\|_2 +\right) +z^j, where W_{\text{out},j}^{\ell} denotes the decoder vector of feature j at layer \ell, and \lambda controls the sparsity penalty.

Causal vs. Acausal Crosscoders.

Crosscoders support two information flow patterns:

    -
  • Causal Crosscoders: Read only from layers \ell' \leq \ell at the current position. This respects the forward pass causality: earlier layers cannot read from later layers.

  • -
  • Acausal Crosscoders: Read from any layer \ell' regardless of depth ordering. Useful for understanding post-hoc feature relationships across the full model.

  • +
  • Causal Crosscoders: Read only from layers \ell' \leq \ell at the current position. This respects the forward pass causality: earlier layers cannot read from later layers.

  • +
  • Acausal Crosscoders: Read from any layer \ell' regardless of depth ordering. Useful for understanding post-hoc feature relationships across the full model.

Applications to Circuit Analysis.

-

By extracting cross-layer features, crosscoders enable understanding how features propagate through the residual stream across layers, removing duplicate feature representations that complicate circuit analysis. This simplification allows features to “jump” across intervening identity operations, substantially reducing circuit complexity. More importantly, crosscoders reveal how input-relevant features transform as they flow through the network, distinguishing which features persist unchanged versus which are actively modified by downstream computation (Lindsey et al., 2024).

+

By extracting cross-layer features, crosscoders help us understand how features propagate through the residual stream across layers, removing duplicate feature representations that complicate circuit analysis. This simplification allows features to “jump” across intervening identity operations, substantially reducing circuit complexity. More importantly, crosscoders reveal how input-relevant features transform as they flow through the network, distinguishing which features persist unchanged versus which are actively modified by downstream computation (Lindsey et al., 2024).

-

6.5 Comparing SAEs, Transcoders, and Crosscoders.#

-

Single-layer SAEs are efficient and locally interpretable but miss cross-layer feature evolution. Transcoders improve this by imitating the behavior of the MLP, revealing nonlinear transformations, though they require sequential analysis across depth. Cross-layer transcoders span multiple layers for global information flow but are extremely expensive. Crosscoders take a different approach, extracting shared features across layers simultaneously to resolve cross-layer superposition and simplify circuits without chaining predictions.

+

6.7 Comparing SAEs, Transcoders, and Crosscoders#

+

Single-layer SAEs are efficient and locally interpretable but miss cross-layer feature evolution. Transcoders improve this by imitating MLP behaviour and revealing nonlinear transformations, though they require sequential analysis across depth. Cross-layer transcoders span multiple layers for global information flow but are extremely expensive. Crosscoders take a different approach, extracting shared features across layers simultaneously to resolve cross-layer superposition and simplify circuits without chaining predictions.

-

6.6 Case Study: Induction Heads#

+

6.8 Case Study: Induction Heads#

+

Here, we examine induction heads (Olsson et al., 2022), a mechanism found in many modern language models, as a case study. We first describe their conditional-copying function and role in in-context learning, and then relate them to feature-extraction methods such as SAEs and CLTs, which reveal how this computation can be distributed across broader feature circuits.

-

Induction heads are attention heads that implement a conditional copying rule over repeated subsequences. Let (x_1,\dots,x_T) be a token sequence and consider positions i < j < T such that x_i = x_j. An attention head at layer \ell and head h is an induction head if, when predicting the token at position j+1, its attention weights satisfy \alpha_{j \rightarrow i+1}^{(\ell,h)} \gg \alpha_{j \rightarrow k}^{(\ell,h)} \quad \text{for most } k \neq i+1, and its value projection causes the residual stream at position j to contribute logits favoring x_{i+1}.

-

Equivalently, induction heads approximate the function (x_i = x_j) \;\Rightarrow\; \hat{x}_{j+1} \approx x_{i+1}, enabling in-context prediction by copying continuations following earlier matching prefixes. Induction heads were first identified and analyzed by Olsson et al. (2022a).

+

Induction Heads

+

+

Induction heads, first identified and analysed by Elhage et al. (2021), are attention heads that implement a conditional copying rule over repeated subsequences. Let x=(x_1,\dots,x_T) be a token sequence, and consider positions i<j<T such that x_i=x_j. An attention head indexed r (see Section 4.4 for an introduction to attention modules) is said to be an induction head when its attention pattern assigns comparatively high attention weight from position j to position i+1 (i.e., high A^r_{j,i+1}) and its residual-stream update \Delta h^r += +W_O^r V^r (A^r)^\top increases the model’s logit for token x_{i+1} when predicting the token at position j+1.
+That is, induction heads implement the rule (x_i=x_j) +\;\Rightarrow\; +\hat{x}_{j+1}\approx x_{i+1}. They help enable in-context prediction by copying the continuation that followed an earlier occurrence of the same token.

-

Consider the sequence: \texttt{A B C A B \underline{?}} After observing the repeated prefix A B, an induction head attends from the second B to the position following the first B, producing a strong prediction for C. This copying behavior generalizes across tokens and positions and does not require explicit supervision.

+

Induction in a Repeated Sequence

+

Consider the sequence \texttt{A B C A B} where the model needs to predict the token after the second B. Observing the repeated prefix A B, an induction head attends from the second B to the position after the first B and contributes to the model’s prediction of C. This induction behaviour should generalise across tokens and positions.

-

While induction heads were originally characterized as specific attention heads implementing this copying rule, subsequent work has shown that the underlying computation is more naturally understood at the level of distributed representations rather than isolated architectural components.

+

Induction heads were originally characterised as specific attention heads implementing this copying rule. However, subsequent work shows that we can understand the underlying computation more naturally at the level of distributed representations than isolated model components.

-

Feature extraction methods, such as SAEs, can decompose attention layer outputs into sparse, interpretable features that capture meaningful activation patterns beyond individual attention weights (Kissane et al., 2024). Modern analyses suggest that induction heads are not isolated mechanisms but participate in distributed feature circuits, where multiple interacting features recur and compose across layers (Dunefsky et al., 2024; Singh et al., 2024; Kissane et al., 2024; Kamath et al., 2025).

-

SAEs trained on model activations reveal families of features that correspond to recurring patterns and contextual dependencies in the residual stream, some of which align with behaviors traditionally attributed to induction heads (Kissane et al., 2024). These features can be analyzed to understand how models maintain and manipulate information across tokens in context.

-

Cross-layer analyses, including transcoders, indicate that interpretable features can persist across multiple layers, suggesting that certain computations are represented consistently rather than being rediscovered at each layer (Dunefsky et al., 2024; Kamath et al., 2025). This perspective highlights that attention heads, including those exhibiting induction-like behavior, may form part of broader, distributed feature compositions when examined through modern decomposition methods.

-
-
-

6.7 Challenges and Failure Modes of SAEs#

-

Sparse Autoencoders have become a standard tool for feature extraction, but they exhibit systematic failure modes that can compromise interpretability claims. A careful understanding of these issues is therefore essential when using SAEs as mechanistic tools. This section provides a critical and systematic treatment of these challenges, including comparisons with alternative decomposition methods.

-

6.7.1 Dead Features#

-

A dead feature is a dictionary element that never activates, or activates only extremely rarely, across the training distribution. Dead features represent wasted capacity and can constitute between 10–50% of the dictionary in poorly tuned SAEs (Bricken et al., 2023). Several factors can lead to the emergence of dead features. First, inappropriate initialization can place features in low-density regions of activation space, preventing them from receiving meaningful gradient signals. Second, an excessively strong sparsity penalty \lambda may suppress feature activations early in training, stopping features from discovering a useful role. Finally, competitive dynamics between features can exacerbate this issue: once a subset of features captures common patterns, the remaining features may be starved of signal and fail to activate.

-

Dead features can be detected by tracking feature activation frequencies during training; features with activation rates below 0.01% are effectively inactive. Several mitigation strategies have been proposed, including neuron resampling, in which dead features are periodically reinitialized toward examples with high reconstruction error (Bricken et al., 2023). Another effective approach is to reduce the sparsity penalty during early training via a warmup schedule and increase it gradually. More recent Batch Top-K variants explicitly enforce minimum feature usage across batches, which substantially reduces the prevalence of dead features (Bussmann et al., 2024).

-
-

6.7.2 Feature Duplication#

-

Feature duplication occurs when multiple dictionary elements encode the same or highly overlapping concepts (Templeton et al., 2024). Although this increases the apparent dictionary size, it provides redundant rather than distinct features. Duplication may arise when sparsity pressure is insufficient, allowing multiple features to co-activate on similar inputs, from optimization dynamics favoring local minima with duplicated features, or from imbalances in the training data. Duplication can be detected using pairwise cosine similarity between decoder columns, where high similarity (e.g., >0.8) indicates potential redundancy. Co-activation statistics provide complementary evidence, as duplicated features tend to activate together (Korznikov et al., 2025).

-
-

6.7.3 Feature Absorption and Splitting#

-

Another class of failure modes concerns feature granularity. Feature absorption occurs when a single feature captures multiple distinct, but related, concepts, reducing representational specificity (Chanin et al., 2025). For example, a single “code” feature may activate for Python, JavaScript, and SQL instead of learning language-specific representations. Conversely, feature splitting occurs when one coherent concept is unnecessarily divided across several features (e.g., “dog at sentence start” vs. “dog after article”). These issues can be identified through activation patterns. Absorbed features tend to activate frequently across diverse inputs, whereas split features often share similar semantic labels but differ in context and show high mutual information. Both degrade interpretability: absorbed features blur multiple computations, while split features distribute a single mechanism across several units.

-
-

6.7.4 Feature Instability Across Seeds#

-

SAE features are also sensitive to random initialization, optimizer stochasticity, and data ordering. Training the same architecture with different random seeds can therefore produce substantially different dictionaries (Paulo & Belrose, 2025). In practice, frequent and semantically clear features tend to be stable, while rare or context-specific features vary considerably. Exact feature alignment across seeds is often below 50%, although broader semantic clusters may remain consistent. This instability limits the reliability of mechanistic claims based on individual features. Ensembling across seeds can improve robustness but increases computational cost. For this reason, feature stability across seeds should be reported alongside reconstruction loss when evaluating SAEs.

-
-

6.7.5 Identifiability and Theoretical Concerns#

-

A fundamental question for SAE-based interpretability is whether the learned features are identifiable, that is, whether training reliably recovers the “true” underlying features (if such features exist) or merely finds one of many equally valid decompositions (Klindt et al., 2025).

-
-

Given activations h generated by some unknown sparse process h = Df^* + \epsilon where f^* are ground-truth features, does minimizing the SAE objective recover f^* (up to permutation and scaling)?

+

Induction Heads: Features Across Representations

+

Feature extraction methods, such as SAEs, can decompose attention layer outputs into sparse, interpretable features that capture meaningful activation patterns beyond individual attention weights (Kissane et al., 2024). Modern analyses suggest that induction heads are not isolated mechanisms but participate in distributed feature circuits, where multiple interacting features recur and compose across layers (Dunefsky et al., 2024; Singh et al., 2024; Kissane et al., 2024; Kamath et al., 2025).
+SAEs trained on attention heads’ activations reveal feature families of recurring patterns and contextual dependencies. Some of them align with behaviours originally attributed to induction heads (Kissane et al., 2024). By analysing these features, we can understand how models maintain and manipulate information across tokens and contexts.
+Cross-layer analyses, including crosscoders, suggest that certain features are represented consistently across layers rather than being recomputed at each layer (Lindsey et al., 2024). This perspective highlights that attention heads, including specialised ones such as induction heads, may be only part of broader, distributed feature representations.

-

In classical Dictionary Learning, identifiability is guaranteed under conditions of low coherence, meaning that dictionary elements are sufficiently orthogonal to one another. When coherence is high, however, ambiguity arises: multiple distinct sparse combinations of features from the same dictionary can generate the same data point (Garfinkle & Hillar, 2016; Schnass & Vandergheynst, 2008). In the context of large language model activations, high coherence is likely unavoidable due to the strong internal structure and correlations within the model’s feature space (Li et al., 2025c). This makes identifiability particularly challenging and provides a theoretical explanation for several empirical observations about Sparse Autoencoders (SAEs).

-

Recent work has raised concerns about the reliability of the features learned by SAEs (Mueller et al., 2025b). When trained on identical data but with different random seeds, SAEs tend to produce substantially different feature sets (Paulo & Belrose, 2025), suggesting a lack of uniqueness in the learned representations. Moreover, studies have shown that SAE features can often be reconstructed from smaller sets of more basic features using a meta-SAE (e.g., representing “Einstein” as a combination of “Germany,” “Starts with E,” “Space and Galaxies,” “Science,” and “Prominent Figures”), indicating that these features are not atomic (Leask et al., 2025). Finally, phenomena such as feature splitting and feature absorption further illustrate how SAEs may merge or decompose simpler features in order to optimize sparsity, rather than recover a unique underlying basis (Chanin et al., 2025). Taken together, these behaviors are consistent with the failure of identifiability under high-coherence conditions and help explain why SAE features can be unstable, non-atomic, and seed-dependent.

-

Recent progress in finetuning language models around a fixed sparse autoencoder offers a potential path forward. In this setting, model activations are explicitly constrained to be sparse combinations of decoder features while largely preserving task performance (Chen et al., 2025). In particular, recent work aimed at limiting superposition fine-tunes the language model jointly with the SAE while enforcing orthogonality constraints on the decoder matrix (Miller et al., 2026). By explicitly satisfying low-coherence conditions, this approach restores identifiability: there exists a unique representation of activations in terms of features, and these features can be reliably recovered.

-
-

Relation to Causal Relevance.

-

Identifiability concerns connect directly to causal relevance: even if an SAE feature is interpretable and reconstructs activations well, it may not correspond to a causally meaningful unit of computation.

-

Diagnostic criteria:

-
    -
  • Necessity: Does ablating the feature degrade task performance? (Low necessity suggests the feature is not causally central.)

  • -
  • Sufficiency: Does activating only this feature (and related features) reproduce the behavior? (Low sufficiency suggests the feature captures correlation, not causation.)

  • -
  • Specificity: Does the feature’s causal effect match its semantic interpretation? (A “sentiment” feature that causally affects syntax but not sentiment is misidentified.)

  • -
-

These causal tests should be standard practice when claiming that SAE features represent meaningful computational units.

-
-
-

6.7.6 Practical Recommendations#

-

Given these challenges, we recommend the following practices for SAE-based interpretability:

-
    -
  1. Report failure mode metrics: Alongside reconstruction loss, report dead feature percentage, duplication rate (via cosine similarity), and stability across seeds.

  2. -
  3. Validate causal relevance: Do not assume interpretable features are causally meaningful. Test necessity and sufficiency through targeted interventions (e.g. feature ablation or activation steering) and measure downstream effects.

  4. -
  5. Compare methods: For safety-critical applications, compare SAE features against supervised baselines (DAS, probes) to assess whether SAEs recover the relevant structure.

  6. -
  7. Use multiple SAE variants: Different variants (Gated, Top-K, JumpReLU) impose different inductive biases. Convergent findings across variants provide stronger evidence.

  8. -
  9. Acknowledge identifiability limits: Frame SAE features as “one valid decomposition” rather than “the true features.” Multiple decompositions may be equally valid for reconstruction while differing in the set of features recovered.

  10. -
  11. Consider alternatives for targeted analysis: When specific concepts are the focus, supervised methods (e.g. probes) may be more reliable than hoping SAEs discover the relevant features.

  12. -
-
-
-

6.8 Comparison with Alternative Feature Extraction Methods#

-

Sparse Autoencoders are one approach for extracting features from neural network activations. Alternative methods drawn from signal processing and machine learning make different modeling assumptions and therefore offer distinct trade-offs in terms of interpretability, scalability, and robustness.

-

6.8.1 Non-Negative Matrix Factorization (NMF)#

-

Non-Negative Matrix Factorization decomposes activations as H \approx WA, where both the dictionary W and the activation matrix A are constrained to be non-negative. This constraint induces a natural parts-based interpretation in which features combine additively without cancellation. As a result, NMF often yields components that are easier to interpret than unconstrained linear decompositions. In addition, closed-form update rules exist for certain NMF objectives, which can mitigate some optimization pathologies. Under specific conditions such as separability, NMF also enjoys stronger uniqueness guarantees than unconstrained sparse coding (Donoho & Stodden, 2003).

-

Despite these advantages, NMF is not always well suited to neural network activations. Activations produced by modern architectures, particularly after LayerNorm or nonlinearities such as GELU, are not naturally non-negative, requiring preprocessing steps that may distort representational structure. Moreover, NMF does not explicitly encourage sparsity unless additional regularization is introduced, and scaling NMF to large dictionaries is computationally challenging. Using an NMF inspired approach, Yun et al. (2021) factorizes residual-stream activations with non-negativity constraints, producing additive components that are easier to interpret and visualize across layers. However, for feature extraction and causal analyses, empirical work has largely focused on Semi-NMF which relaxes the non-negativity constraint on W. Shafran et al. (2025) showed that Semi-NMF can recover features competitive with SAEs on small-scale evaluations, but systematic comparisons at large scale remain limited.

-

6.8.2 Principle Component Analysis (PCA)#

-

Principal Component Analysis decomposes activations by finding an orthonormal basis of directions that maximise explained variance. Given a mean-centered activation matrix H \in \mathbb{R}^{n \times d}, PCA yields a low-rank approximation H \approx U \Sigma V^T, where the columns of V correspond to principal directions ordered by explained variance. Fitting PCA is fast, stable and scalable. Yet, PCA introduces constraints and inductive biases that can be misaligned with neural representations. PCA identifies a single set of global directions that maximises variance across the full activation distribution, which can mix multiple sources of variation when the relevant structure is region or context-dependent. In addition, PCA enforces orthogonality between components, a constraint that may not reflect how features are organized in practice, where factors can be correlated or partially overlapping. Its misalignment is further reflected in recent benchmarks. In Wu et al. (2025), PCA underperforms most baselines, including SAEs, on both steering and concept-detection evaluations. While in some localization settings, PCA outperform SAEs (Mueller et al., 2025a), it still lags behind simpler supervised baselines.

-
-

6.8.3 Independent Component Analysis (ICA)#

-

Independent Component Analysis seeks to decompose activations as H = WS, where the components of S are statistically independent (Hyvärinen & Oja, 2000). Independence provides a principled objective grounded in information theory, and under well-specified generative assumptions, specifically, linear mixtures of independent sources, ICA can provably recover the underlying components. Unlike SAEs, ICA does not require tuning a sparsity hyperparameter.

-

However, ICA makes strong assumptions that are often violated in neural representations. Classical ICA assumes a complete setting in which the number of sources equals the dimensionality of the representation, and extensions are required for overcomplete decompositions. Neural network features may also fail to satisfy statistical independence, limiting ICA’s applicability. Although ICA has been applied to neural network activations (Hyvärinen et al., 2023), it has not matched the interpretability gains reported for SAEs in recent large-scale studies, possibly because neural features tend to be sparse rather than independent.

-
-

6.8.4 Supervised Feature Extraction Methods#

-

When the concepts of interest are known in advance, supervised methods can directly optimize representations aligned with those targets. Examples include Distributed Alignment Search (DAS), which uses supervision to learn a change of basis that aligns representations with high-level features (Geiger et al., 2024); Supervised Multi-Dimensional Scaling (SMDS), which projects activations to preserve concept-level similarities (Tiblias et al., 2025); and Differential Binary Masking (DBM), which learns binary masks that select relevant activation dimensions for specific concepts (De Cao et al., 2020).

-

When labelled data are available, supervised approaches often outperform SAEs on targeted concept recovery tasks (Huang et al., 2024; Chaudhary & Geiger, 2024). They also avoid the need to choose dictionary sizes or sparsity penalties, since complexity is determined by the concept set itself. However, these methods require labelled data for each concept and therefore cannot discover previously unknown features. They do not scale naturally to open-ended exploration across entire models, and each concept typically requires separate training. As a result, supervised methods are best suited to settings where specific, predefined concepts are of interest, such as safety-critical behaviors like toxicity or deception, whereas SAEs remain preferable for exploratory analysis.

-
-

6.8.5 Dictionary Learning Baselines#

-

Classical dictionary learning methods provide useful baselines for evaluating SAEs. Techniques such as K-SVD alternate between sparse coding and dictionary updates and are known to converge under certain conditions (Aharon et al., 2006). Comparative studies suggest that the primary advantage of SAEs lies in their learned encoder, which enables fast inference, rather than in fundamentally superior dictionary quality (Cunningham et al., 2023). As such, classical dictionary learning remains an important reference point for assessing the added value of SAE architectures.

+

6.9 Comparison with Alternative Feature Extraction Methods#

+

Besides SAEs, alternative techniques from signal processing and machine learning have also been adapted to extract features from neural networks. They make different modelling assumptions and therefore offer distinct trade-offs in interpretability, scalability, and robustness. Table 6 summarises the methods discussed in this section.

- + - - - - - + + + + + - + - - - + + + - + - + - - - + + + - - - + + + - + - + - + - - - + + + - +
Table 6: Comparison of Activation Decomposition MethodsTable 6: Comparison of feature extraction methods.
MethodKey AssumptionStrengthsWeaknessesBest Use CaseMethodAssumptionStrengthsWeaknessesBest Use Case
SAESAE Sparse, linear featuresScalable; open-ended discovery; fast inferenceHyperparameter sensitive; identifiability issuesExploratory feature discoveryOpen-ended discovery; fast inferenceHyperparameter-sensitive; identifiability issuesOpen-ended feature discovery
NMFNMF Non-negative, additive parts Parts-based interpretation; uniqueness guarantees Requires non-negative data; limited sparsity Naturally non-negative domains
PCAPCA Variance aligns with featuresClosed form; standard baseline; optimal low-rank reconstructionExplains global variance; orthogonal featuresSmall settings and baseline comparisonClosed form; optimal low-rank reconstructionExplains global variance and produces orthogonal featuresSmall-scale settings; as a baseline method
ICAStatistically independent sourcesPrincipled criterion; no sparsity tuningICAComplete and independent sourcesTheoretical guarantees when assumptions hold Assumes independence; Gaussian blindness When independence holds
Supervised (DAS, DBM)Supervised methods Known concept labelsHigh accuracy on targeted conceptsHigh performance on targeted concepts Requires labels; no discoverySafety-critical concept detectionWhen certain concepts of interest are known
K-SVD / ODLSparse representationTheoretical guarantees; no encoder biasK-SVDSparse, linear featuresExplicitly solving the sparse-coding problem; no encoder bias Slow inference; no learned encoderBaseline comparisonSmall, fixed sets of activations; as a baseline method
-
-
-

6.9 Architectural Interventions for Intrinsic Interpretability#

-

The methods surveyed thus far, SAEs, Transcoders, Crosscoders, share a common assumption: the model architecture is fixed, and interpretability must be recovered post-hoc. A parallel research program, which we term Architectural Interventions for Interpretability, inverts this assumption by asking whether models can be designed to be mechanistically interpretable by construction (Pearce et al., 2025; Park et al., 2025; Oldfield et al., 2024; Yang et al., 2025; Dooms et al., 2025; Gao et al., 2025b; Adler et al., 2025; Draye et al., 2025).

-

This approach is motivated by the inherent limitations of post-hoc decomposition. Standard Transformer architectures are optimized without interpretability constraints, incentivizing superposition as an efficient compression strategy. Post-hoc methods like SAEs attempt to undo this compression, but they introduce reconstruction gaps, require substantial compute, and may fail to capture the full causal dynamics of the original model. Architectural interventions aim to solve the interpretability problem upstream, creating systems where circuits are visible by design rather than laboriously extracted.

-

6.9.1 Weight Sparsity via L_0 Regularization#

-

The most significant recent advancement is the Weight-Sparse Transformer, introduced by Gao et al. (Gao et al., 2025b). This approach treats sparsity not as a tool for inference efficiency but as a tool for circuit discovery.

-
-

Mechanism.

-

The model is trained to minimize a composite objective: \begin{equation*} - \mathcal{L} = \mathcal{L}_{\text{task}} + \lambda \cdot \|\theta\|_0 -\tag{2} -\end{equation*} where \mathcal{L}_{\text{task}} is the standard cross-entropy loss and \|\theta\|_0 counts non-zero parameters. Since the L_0 norm is non-differentiable, training employs stochastic gates drawn from a Hard-Concrete distribution, allowing gradients to flow while penalizing the probability of each gate being open. At inference, gates are discretized, yielding networks where the vast majority of connections are exactly zero.

-
-
-

Interpretability gains.

-

Under high sparsity regimes (e.g., <1\% active weights), the computational graph simplifies dramatically. A neuron receiving input from only 5–10 others (rather than thousands) cannot implement the complex interference patterns required for polysemanticity. Consequently, neurons tend to align with single, interpretable concepts—monosemanticity emerges by structural necessity rather than post-hoc extraction.

-

Circuits can be identified by simply visualizing non-zero weights connected to a target output, eliminating the need for activation patching or ACDC. Validation via mean ablation shows that these sparse circuits satisfy necessity and sufficiency criteria with higher precision than circuits identified in dense models.

-
-
-

The capability-interpretability frontier.

-

A critical contribution is formalizing the trade-off between performance and interpretability. As the sparsity penalty \lambda increases, interpretability improves but task loss degrades, defining a Pareto frontier. Crucially, Gao et al. (2025b) discover a scaling law: increasing model size shifts this frontier outward. Large sparse models can achieve both better capability and better interpretability than small dense models, suggesting that over-parameterization coupled with sparsity constraints may be the path toward interpretable frontier models.

-
-
-

6.9.2 Sparse Attention Post-Training: SPARTAN#

-

The dense attention matrix implies every token attends to every other token, creating a quadratic web of connections difficult to audit. Sparse Attention Post-Training (SPARTAN) applies L_1 regularization to the attention patterns while finetuning with a constrained optimization objective to maintain the same cross-entropy performance  (Draye et al., 2025)  (Lei et al., 2025).

-

The result: diffuse “soft” attention patterns collapse into “hard” sparse patterns. A head that previously attended broadly across context snaps to attending only to specific operands required for the current computation. This reveals that much attention connectivity is redundant noise, and SPARTAN effectively performs circuit discovery by pruning the attention graph. Using Cross-Layer Transcoders, they show that the reduced number of mediating components renders attention attribution tractable. This illustrates the potential of sparse attention in the context of attribution graphs, as it enables a unified view of features and circuits.

-
-

6.9.3 Activation-Level Interventions: SoLU and Its Limitations#

-

An earlier approach targeted the activation function rather than connectivity. Softmax Linear Units (SoLU) replace element-wise activations with a spatial softmax (Elhage et al., 2022b): \begin{equation*} - x \mapsto x \cdot \text{softmax}(x) +

6.9.1 Non-Negative Matrix Factorisation (NMF)#

+

Non-negative matrix factorisation (NMF; Lee & Seung, 1999) decomposes a non-negative activation matrix H \in \mathbb{R}^{d \times n} as H \approx WA, where both the feature dictionary W \in \mathbb{R}^{d \times m} (m is the preset number of features) and the coefficient matrix A \in \mathbb{R}^{m \times n} are constrained to have non-negative elements. Because all components are non-negative, each feature contributes positively to the final representation, making it easier to interpret. Additionally, classical NMF’s optimisation process uses simple multiplicative update rules that preserve non-negativity. Moreover, under assumptions like separability (each latent feature appears in at least one sufficiently “pure” example), NMF guarantees that the recovered decomposition is unique (Donoho & Stodden, 2003).

+

However, classical NMF is not suitable for neural network activations because they can have negative elements, requiring additional preprocessing steps that may distort the activation-space structure. Moreover, NMF does not explicitly encourage feature sparsity, and scaling NMF to large dictionaries is computationally expensive. As a result, empirical work on extracting and analysing features in neural network activations has largely focused on Semi-NMF (Ding et al., 2010), which relaxes the non-negativity constraint on W. Shafran et al. (2026) showed that Semi-NMF can recover features competitive with SAEs on small-scale evaluations, but systematic comparisons at large scale remain limited. Yun et al. (2021) used a Semi-NMF-like approach to factorise BERT’s contextualised hidden-state representations (Devlin et al., 2019), allowing elements in W to take both positive and negative values while constraining A to be non-negative.

+
+

6.9.2 Principal Component Analysis (PCA)#

+

Principal Component Analysis (PCA; Pearson, 1901; Hotelling, 1933) decomposes activations by finding an orthonormal basis of directions that maximise explained variance. Given a mean-centred activation matrix H \in \mathbb{R}^{d \times n}, where each column is an activation vector, PCA computes a low-rank approximation H \approx U \Sigma V^\top, where the columns of U are the principal directions ordered by explained variance. PCA is fast, stable, and scalable. However, PCA imposes constraints and inductive biases that may misalign with neural network activations. Specifically, it identifies principal directions that maximise explained variance in the activation distribution, but much of the information of interest is region- or context-dependent and cannot be captured by variance alone. Moreover, PCA constrains the principal directions to be orthogonal, but in practice the underlying features correlate or partially overlap. Recent benchmarks (Wu et al., 2025b) reflect this misalignment: PCA underperformed most baselines, including SAEs, on both steering and concept-detection evaluations. While PCA can outperform SAEs in some localisation settings (Mueller et al., 2025), it still generally lags behind simple supervised baselines.

+
+

6.9.3 Independent Component Analysis (ICA)#

+

Independent Component Analysis (ICA; Hyvärinen & Oja, 2000) decomposes activations H \in \mathbb{R}^{d \times n} as H \approx WS, where n is the number of data points. Note that ICA may look similar to NMF at first glance. However, they were developed for different use cases and thus rely on different assumptions and optimisation procedures. In ICA, each column in W \in \mathbb{R}^{d \times m} is a mixing vector, and m is the number of different source signals; each row in S \in \mathbb{R}^{m \times n} is a source signal (a signal has n data points), with m source signals in total. These signals are optimised to be independent of one another. Thus, if model activations are linear mixtures of statistically independent “foundational features”, ICA can provably recover those features.

+

However, these assumptions may not hold for neural network activations. Specifically, classical ICA typically expects the completeness property (number of latent sources equals the dimensionality of the observed representation), and overcomplete decompositions (which are what we expect in activations) would require additional extensions (Lewicki & Sejnowski, 2000). Moreover, underlying features may not be statistically independent, which further limits the applicability of classical ICA. Although early studies applied it to analyse neural network activations, they did not find interpretability gains over SAEs (Cunningham et al., 2024). This may partly reflect the challenges discussed above.

+

More recently, however, Liu & Han (2026) showed that ICA can achieve performance comparable to that of SAEs, arguing that its previously reported underperformance may stem from limitations of classical ICA implementations poorly suited to the scale and statistical structure of neural network activations. These findings leave open whether ICA should be regarded primarily as a simple classical baseline or as an approach with underestimated, substantial headroom for further development.

+
+

6.9.4 Supervised Feature Extraction Methods#

+

When the concepts of interest are known in advance, supervised methods can directly optimise representations aligned with those targets. Examples include distributed alignment search (DAS), which uses supervision to learn a change of basis that aligns representations with high-level causal variables (Geiger et al., 2024); Supervised Multi-Dimensional Scaling (SMDS), which projects activations to preserve concept-level similarities (Tiblias et al., 2026); and probes, which fit a concept direction in contrastive class-conditioned activations (Marks & Tegmark, 2024).

+

Notably, despite its simplicity, probing has outperformed SAEs in concept detection (Wu et al., 2025b). It has also become a standard tool for identifying safety-relevant concepts in model representations, including refusal behaviour and implanted backdoors (Arditi et al., 2024; MacDiarmid et al., 2024). We discuss probing in detail in Section 8.6. We put it under Section 8 because we view its central methodological contributions and technical challenges as the readout of human-interpretable information—often expressed in natural language—from model activations, rather than the extraction of monosemantic features or causal variables.

+

When labelled data is available, supervised approaches often outperform SAEs on concept detection and steering tasks (Huang et al., 2024; Wu et al., 2025b; Chaudhary & Geiger, 2024). They also avoid choosing dictionary sizes or sparsity penalties, since the concept set determines complexity. However, these methods require labelled data for each concept and cannot discover previously unknown features. Therefore, supervised methods suit settings where predefined, crisp concepts matter, whereas SAEs remain preferable for exploratory analysis.

+
+

6.9.5 Dictionary Learning Baselines#

+

Classical dictionary learning methods provide useful baselines for evaluating SAEs. Techniques such as K-SVD alternate between sparse coding and dictionary updates and converge under certain conditions (Aharon et al., 2006). Comparative studies suggest that SAEs’ primary advantage is fast inference because the encoder learns to decompose all activations, while potentially sacrificing optimal sparsity (O’Neill et al., 2025). As such, classical dictionary learning can still achieve superior dictionary quality in some cases and thus remains an important reference point for assessing the added value of SAEs.

+
+
+

6.10 Interpretability-by-Design Methods for Intrinsic Interpretability#

+

The methods surveyed thus far in this section aim to extract and analyse more monosemantic units from polysemantic model activations. They share a common premise: the model’s original representations are too entangled to be directly interpretable, so interpretability is applied post hoc to recover simpler units of analysis. A parallel research programme, which we term Interpretability-by-Design Methods for Intrinsic Interpretability, inverts this premise by asking whether deep neural models can instead be designed to be mechanistically interpretable by construction (Pearce et al., 2025; Park et al., 2025; Oldfield et al., 2024; Yang et al., 2025; Gao et al., 2026; Adler et al., 2025; Draye et al., 2026; Dooms et al., 2025). This subsection extends the discussion of this research direction.

+

Next, we introduce five approaches for constructing models that are more interpretable by design. Three impose model sparsity constraints at different levels: feature-sparse training (Section 6.10.1), weight-sparse training (Section 6.10.2), and attention-sparse training (Section 6.10.3). The remaining two approaches instead modify the computational primitives themselves, using more interpretable operators as units of construction and analysis (Sections 6.10.4 and 6.10.5).

+

Interpretability should be understood as a spectrum rather than a binary property. At least in their current form, these techniques do not make model computation completely or immediately understandable. Instead, they make models easier to analyse and ensure the analysis pipeline preserves more information about the original computation than post-hoc approximations. Table 7 summarises these approaches and their trade-offs.

+

6.10.1 Feature-Sparse Training: SoLU#

+

Softmax linear units (SoLU; Elhage et al., 2022b) are an earlier attempt at architectural intervention. SoLU is a specialised activation function that applies a softmax-based transformation across the layer’s d_{\mathrm{mlp}} neurons. Specifically, given the pre-activation vector u = W_1 h + b_1 \in \mathbb{R}^{d_{\mathrm{mlp}}}, SoLU calculates the activation values as \begin{equation*} + \sigma_{\text{SoLU}}(u) + = + u \odot \mathrm{softmax}(u). \tag{3} -\end{equation*} This forces neurons within a layer to compete: if one neuron fires strongly, it suppresses its neighbors through lateral inhibition.

-
-

Results and the hydraulic effect.

-

SoLU successfully increased the fraction of monosemantic neurons identifiable by human inspection. However, it revealed a critical limitation: the hydraulic effect. When prevented from using neurons for superposition, models simply displaced the compression elsewhere—into the residual stream, into sub-threshold activations, or into other components not subject to the constraint. This indicates that local architectural constraints are insufficient when global incentives for compression remain. It underscores the need for either global constraints (like weight sparsity applied throughout the network) or fundamentally different computational primitives.

+\end{equation*} This softmax-based transformation induces competition among neurons. When one neuron receives a large pre-activation value, the softmax suppresses the others’ values, analogous to lateral inhibition. This competition encourages certain neurons to specialise and dominate, increasing the prevalence of monosemantic neurons associated with human-interpretable concepts.

+

However, it also reveals an undesirable phenomenon: although many secondary neurons’ activations are suppressed to low magnitudes, these neurons remain polysemantic, with many features compressed into them. Later layer normalisation (Ba et al., 2016) can amplify them and recover the polysemanticity property (Elhage et al., 2022b). The “smuggling” by layer normalisation suggests that imposing local interpretability constraints may not eliminate neural networks’ incentive to compress information through superposition. This issue motivates global constraints across the neural network, such as weight-sparsity training discussed below.

-
-

6.9.4 Computational Primitive Modifications: MOLT#

-

One alternative approach to learning interpretable structure is to rethink what atomic computational units to learn (Oldfield et al., 2025; Park et al., 2025; Braun et al., 2025; Bushnaq et al., 2025). One recent example of this is the so-called Sparse Mixtures of Linear Transforms (MOLT), representing a conceptual shift from feature extraction to transformation extraction (Lindsey et al., 2025b).

-
-

Motivation.

-

Standard SAEs and Transcoders assume computation consists of detecting features. However, for algorithmic tasks (arithmetic, geometric reasoning), models perform operations—“add 3,” “rotate vector.” An SAE attempts to represent these as massive lookup tables of states, which obscures the underlying mechanism.

-
-
-

Mechanism.

-

MOLT replaces the MLP with a sparse mixture of linear transformation matrices. Each “neuron” is not a vector but a matrix that can be dynamically applied to the residual stream. For modular arithmetic, a MOLT might learn a single rotation matrix implementing addition, rather than memorizing thousands of input-output pairs.

-

This aligns the architecture with the true algorithmic structure of the task, yielding higher mechanistic faithfulness than standard feature-based decompositions for tasks involving systematic transformations.

-
-
-

6.9.5 Kolmogorov-Arnold Networks#

-

Kolmogorov-Arnold Networks (KANs) represent the most radical departure from standard architectures (Liu et al., 2025b). Based on the Kolmogorov-Arnold representation theorem, KANs place learnable activation functions (B-splines) on edges rather than fixed activations on nodes.

-

This inversion makes KANs explicitly compositional. Because non-linearities are univariate and visible on edges, researchers can perform symbolic regression on learned splines to recover exact mathematical formulas—for instance, rediscovering physics equations from data. While currently slower to train and unproven at LLM scale, KANs represent the ambitious ideal of a model where weights are interpretable mathematical functions.

-
-

6.9.6 The Bridge Methodology: Interpretable Proxies for Dense Models#

-

Retraining frontier models with sparsity constraints is computationally prohibitive. Gao et al. propose the “Bridge” methodology as a practical alternative (Gao et al., 2025b).

-

A Bridge is a weight-sparse model trained to mimic the internal representations of a pretrained dense model—minimizing reconstruction error of layer-wise activations rather than training on the original task. The sparse model serves as an interpretable proxy: if it achieves high fidelity and is mechanistically interpretable, researchers can analyze its circuits to understand the dense model’s behavior.

-

Preliminary evidence suggests that interventions on sparse proxies (e.g., ablating identified circuits) can be mapped back to induce similar behavioral changes in dense models. However, validating proxy fidelity—ensuring the proxy captures the same mechanisms rather than behaviorally equivalent simulations—remains an open problem (see Section 15).

-

Table 7 summarizes the trade-offs between architectural approaches.

+

6.10.2 Weight-Sparse Training#

+

A weight-sparse transformer (Gao et al., 2026) treats weight sparsity as an explicit structural constraint that is progressively strengthened during training. Gao et al. (2026) optimise the standard cross-entropy objective along with an L_0 constraint on model weights after each optimisation step. Specifically, within each weight matrix, they retain only the largest-magnitude parameters and set the rest to zero. The L_0 constraint anneals from a dense initialisation to the desired sparsity level over training. The resulting transformer has most weight parameters exactly zero, making the remaining computational structure much easier to decompose into small, task-specific circuits.

+

Its interpretability gains can be substantial. Under highly sparse regimes (e.g., <1\% active weights), neurons receive input from far fewer upstream neurons, and the model’s circuits for specific capabilities or behaviours become much sparser. Mean-ablation experiments further confirm that these circuits are both necessary and sufficient for the capability or behaviour.

+

Gao et al. (2026) characterised the trade-off between model capability and interpretability induced by weight sparsity. Increasing sparsity generally makes the model easier to decompose into small, interpretable circuits. However, it also degrades task performance, producing a capability–interpretability Pareto frontier. Nonetheless, the authors found that increasing model size can shift this frontier outward. At comparable task performance, larger sparse models can have substantially smaller, more interpretable circuits than smaller models; at comparable interpretability, they can retain stronger capabilities. This suggests that scaling model size alongside structural sparsity may offer a path to more interpretable yet high-capability models. However, preserving interpretability beyond tens of millions of nonzero parameters remains challenging, and weight-sparse training currently requires substantially more compute than traditional training.

- + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - + - +
Table 7: Comparison of Architectural Interventions for InterpretabilityTable 7: Comparison of interpretability-by-design methods.
MethodMechanismInterpretability GainPrimary LimitationMethodApproachInterpretability GainPrimary Limitation
Weight SparsityL_0 regularization on parametersCircuits visible via non-zero weights; monosemantic neuronsCapability cost; training instabilityFeature-Sparse Training (SoLU)Softmax-gated activation functionIncreased neuron monosemanticityLayer normalisation smuggling displaces superposition
SoLUSpatial softmax activationIncreased neuron monosemanticityHydraulic effect displaces superpositionWeight-Sparse TrainingL_0 constraint on parametersMore monosemantic neurons and much smaller circuitsTraining instability and scalability
MOLTSparse mixture of linear transformsReveals algorithmic operations directlyComplexity of matrix analysisAttention-Sparse TrainingL_0 regularisation on attentionMakes attention patterns interpretableRequires fine-tuning on a trained language model
SPARTANL_0 regularization on attentionExposes information routing graphRequires finetuning; post-training onlyMOLTSparse mixture of linear transformsReveals implemented operations directlyComplexity of matrix analysis
KANsKANs Learnable splines on edges Symbolic formula extractionScaling to LLMs unprovenScalability to LLMs is unproven
+

6.10.3 Attention-Sparse Training#

+

In transformers, attention patterns produced by an attention head are usually diffuse, i.e., every token attends to every other token. Diffuse attention patterns are difficult to interpret and audit. Previous studies have therefore explored attention-sparse training, which aims to sparsify attention patterns to make them more interpretable (Lei et al., 2025; Draye et al., 2026). Specifically, Lei et al. (2025) proposed SPARse TrANsformer World model (SPARTAN), an object-centric world model trained with L_0 regularisation on attention patterns and a constrained optimisation objective to maintain cross-entropy performance equal to that of a dense counterpart. Draye et al. (2026) further adapted this technique to fine-tune and sparsify a trained language model’s attention patterns.

+

After training (either from scratch or via fine-tuning), each attention head attends only to specific tokens, and the model performance remains close to the original. This result suggests that many attention patterns may be redundant noise. In CLT-based circuit discovery (see Section 7.2), Draye et al. (2026) showed that sparsified attention patterns reduce the number of mediating components involved in feature-to-feature attribution, making the resulting attribution graphs easier to analyse.

+
+

6.10.4 Modifications of Computational Primitives: MOLT#

+

One alternative approach to learning interpretable structure is to reconsider what the computational primitives should be (Oldfield et al., 2025; Park et al., 2025; Braun et al., 2025; Bushnaq et al., 2025). A recent example is the sparse mixture of linear transforms (MOLT), which represents a conceptual shift from feature extraction towards transformation extraction (Lindsey et al., 2025b).

+

Standard SAEs and transcoders primarily decompose model activations into features. However, many algorithmic tasks, such as arithmetic or geometric reasoning, are more naturally described by operations such as “add 3” or “rotate a vector”. In these tasks, we want to know how the model transforms one activation into another through components like weight matrices in MLP blocks. To achieve this, MOLT replaces the MLP with a sparse mixture of learned linear transformation matrices. The basic unit of analysis becomes transformations applied to model activations rather than features.

+

For example, in a modular arithmetic task, MOLT can represent the concept of addition with a learned rotation matrix, which relates more directly to the underlying algorithm than a collection of state-specific features. More broadly, approaches that represent computational operations explicitly, as MOLT does, may facilitate the development of interpretable representations for tasks governed by systematic operations. We view Kolmogorov–Arnold Networks (KANs) as an approach that embodies this idea, as discussed below.

+
+

6.10.5 Kolmogorov–Arnold Networks (KANs)#

+

Kolmogorov–Arnold Networks (KANs) represent one of the most radical departures from standard neural network architectures (Liu et al., 2025b). In conventional neural networks, each neuron receives outputs from upstream neurons through incoming edges weighted by scalar parameters, sums these weighted inputs, and then applies a fixed nonlinear activation function (the whole process can be represented as matrix multiplication in components like MLP blocks and attention modules in Section 4.4). Inspired by the Kolmogorov–Arnold representation theorem, KANs place learnable nonlinear functions, typically parameterised using B-splines, on edges rather than using fixed nonlinear activation functions at nodes.

+

Specifically, consider a single KAN node receiving an input vector v = (v_1, \dots, v_d) \in \mathbb{R}^d from the previous layer. Its output is y = \sum_{i=1}^{d} \psi_i(v_i), \qquad \psi_i(v_i) = \sum_{m} c_{i,m} B_m(v_i), where B_m(\cdot) denotes the m-th basis function, such as a cubic B-spline basis, and c_{i,m} is a learnable scalar coefficient. In this formulation, the edge is not scalar multiplication but a learnable univariate function \psi_i(\cdot), and a node simply sums the outputs of its incoming edges. Intuitively, KANs shift nonlinear transformations from nodes to edges, offering an alternative way to structure and interpret neural network computations.

+

This design gives KANs an explicitly compositional and thus more interpretable structure. As the nonlinearities are univariate and can be visualised directly on individual edges, researchers can use techniques like symbolic regression to approximate and inspect the learned analytical expressions, such as rediscovering known physical equations. Although KAN training is currently more computationally expensive than training standard MLPs and has not yet been convincingly demonstrated at the scale of large language models, this architectural innovation points to an ambitious direction towards interpretable neural networks in which parameters correspond to mathematically meaningful functions.

+
+

Summary.

+

This section introduced several classical methods for feature extraction and analysis, followed by approaches that aim to make models intrinsically interpretable. However, whether features are recovered post hoc or induced by design, they remain units of analysis rather than complete explanations of the model’s mechanism, i.e., how these units interact and causally influence one another. Section 7 therefore shifts from individual features to circuits: computational graphs of interacting components that describe how the model implements its behaviours.

+
+
diff --git a/src/generated/chapters/foundational-concepts.html b/src/generated/chapters/foundational-concepts.html index 7d18b7404f..e6aede7de2 100644 --- a/src/generated/chapters/foundational-concepts.html +++ b/src/generated/chapters/foundational-concepts.html @@ -1,72 +1,131 @@ -

4 Foundational Concepts for Mechanistic Interpretability#

-

This section introduces core conceptual primitives used throughout mechanistic interpretability. We deliberately avoid committing to specific hypotheses about how representations are organized or how general these structures are. Instead, we establish a shared vocabulary and computational picture that will be used throughout the survey. In Section 5 we discuss issues with some of the hypotheses in this section.

-

4.1 Behavioral vs. Mechanistic Understanding#

-

Behavioral interpretability is often sufficient for evaluating the input-output behavior, but safety-critical applications require mechanistic analysis (as we reviewed in Section 2). The distinction between these methods is the difference between describing a system’s behavior and reverse engineering its internal mechanism.

+

4 Foundational Concepts for Mechanistic Interpretability#

+

This section introduces foundational concepts in mechanistic interpretability. We establish a shared notation and vocabulary for use throughout the survey. We avoid committing to specific hypotheses about the representations and workings of model activations.

+

4.1 Behavioural vs. Mechanistic Understanding#

+

Behavioural analysis describes what a model does, but safety-critical applications also require understanding its internal mechanism (as we reviewed in Section 2).

-

Behavioral interpretability studies a model f_\theta : \mathcal{X} \to \mathcal{Y} exclusively through input–output pairs (x, f_\theta(x)), treating it as a black box.

+

Behavioural Interpretability

+

Behavioural interpretability studies a model f_\theta : \mathcal{X} \to \mathcal{Y} through input–output pairs (x, f_\theta(x)) for x \in \mathcal{D} \subseteq \mathcal{X}, where \mathcal{D} is a finite test set. Behavioural interpretability treats the model as a black box.

-

Mechanistic interpretability studies a model f_\theta : \mathcal{X} \to \mathcal{Y} by examining its internal structure: the parameters \theta and the intermediate activations \{h_l(x)\}_{l=1}^{L} that together implement a particular behavior keeping the input-output fixed.

+

Mechanistic Interpretability

+

Mechanistic interpretability studies a model f_\theta : \mathcal{X} \to \mathcal{Y} by examining its internal computations, including model activations and parameters \theta that produce them, at different levels of granularity in relation to its behaviour. In short, mechanistic interpretability characterises how a model’s behaviour or output is computed.

-

4.2 Polysemanticity in Neural Representations#

-

Consider the activation of a single unit h_l^{(i)}(x) at layer l. Interpreting units would be easier if each one responded to a single understandable concept. But researchers have found that—as by product of how we train models—individual units often activate strongly for multiple, unrelated concepts (Elhage et al., 2022a). This phenomena is known as “polysemanticity”.

+

4.2 Neuron Polysemanticity in Model Representations#

+
+

What Is a Concept?

+

Concepts play a central role in many interpretability methods and will be used throughout this paper to define key ideas such as features and polysemanticity. It is therefore worth clarifying what we mean by the term. Notably, concepts do not necessarily correspond to categories humans can readily name or understand; neural networks may represent them in ways humans lack the vocabulary to describe (Hewitt et al., 2025).

-

A unit h_l^{(i)} is polysemantic if any two inputs x, x' \in \mathcal{X}, corresponding to semantically distinct concepts, both produce high absolute activation: |h_l^{(i)}(x)| \gg 0 and |h_l^{(i)}(x')| \gg 0.

+

Concept

+

A concept is a distinct, reusable unit of information or knowledge that can be identified across different instances or contexts. Concepts may correspond to familiar human categories, such as “a dog”, “the colour red”, or “words of praise”, but they need not admit a natural human description.

-

Polysemanticity complicates neuron-level explanations and suggests that human-interpretable semantic content may not be aligned with neurons (the standard basis vectors of the representation space). This motivates the search for alternative representational units beyond individual neurons.

+

Cross-context identifiability concerns the meaning of the information unit rather than the process by which the information is recognised. Determining whether a concept is present in an information unit may itself require additional contextual information. For example, the word (i.e., an instance of an information unit) “bank” can refer to a financial institution or a river bank, depending on the surrounding sentence. Nevertheless, once identified, each concept can be recognised as the same information across different contexts. For example, the concept of “financial institution” is expected to remain sufficiently stable across the contexts in which it appears.

+

This definition accommodates both human-interpretable and machine-specific concepts. A machine-specific concept may lack a natural human description yet still form a coherent, reusable unit of information in the neural network (Dalvi et al., 2022; Hewitt et al., 2025). Human interpretability should therefore be treated as a property of some concepts rather than as a necessary condition for the information unit to qualify as a concept. Notably, most interpretability research focuses on concepts that humans can readily understand. Identifying and characterising concepts that matter to a model’s computation but do not map neatly onto human-understandable categories remains underexplored.

-

4.3 Features as Representational Units#

-

Since individual units h_l^{(i)} are often polysemantic, mechanistic interpretability seeks alternative units of analysis. The largley agreed upon hypothesis in the community is that models often encode interpretable concepts as directions in activation space, called features.

-

Features may be distributed across many neurons (Elhage et al., 2022a; Bricken et al., 2023; Templeton et al., 2024) and only become apparent when analyzing the representation space as a whole.

+
+

Neuron Polysemanticity.

+

For a single neuron n_\ell in layer \ell, consider its activation at token position t for some input prompt x, denoted by n_\ell^{t}(x). We can straightforwardly interpret a neuron if it responds to a single concept. For example, if n_\ell^{t}(x) is large and positive whenever position t corresponds to the word “blue”, remains close to zero otherwise, and can causally shift the model’s outputs towards or away from blue-related concepts when manipulated, we can conclude that the neuron corresponds to the concept of blue. Assuming some neurons exhibit such a property, early work proposed methods to discover and interpret individual neurons’ functions (Bau et al., 2017; 2019; Dalvi et al., 2019; Bau et al., 2020; Hernandez et al., 2022; Oikarinen & Weng, 2023; Wu et al., 2024). However, follow-up studies have found that, due to representational constraints and superposition (Elhage et al., 2022a) (detailed in Section 5.2), individual neurons often respond strongly to multiple concepts that appear unrelated from a human perspective (Elhage et al., 2022a). This phenomenon is known as neuron polysemanticity.

-

A feature is a direction \phi \in \mathbb{R}^d in activation space. The scalar \phi^\top h_l(x) measures the activation (or presence) of that feature for input x. A high value indicates the corresponding concept is represented in the model’s internal.

+

Neuron Polysemanticity

+

A neuron n_\ell is polysemantic if there exist two distinct concepts c_1 and c_2, together with corresponding sets of inputs x_1 and x_2 as well as token positions of interest t_{c_1} and t_{c_2}, such that n_\ell^{t_{c_1}}(x_1) and n_\ell^{t_{c_2}}(x_2) are both large, and interventions on these activations causally influence model behaviour associated with both concepts.

-

At this stage, we remain agnostic about the precise mathematical form features take or how they are encoded within the model. Accordingly, we do not assume a specific representational geometry or encoding scheme, and treat existing approaches to identifying or manipulating features as operational approximations shaped by methodological constraints. These representational hypotheses are examined explicitly in Section 5.

-

Within this operational perspective, features can be recovered or manipulated using different methodological families, including readout-based methods, sparse decomposition techniques, and causal intervention strategies. In practice, most such approaches operationalise features as approximately linear or linearized components of internal representations. Specific examples include probes, dictionary learning methods, and activation-based interventions (Belinkov, 2022; Wang et al., 2023; Bricken et al., 2023).

+

Polysemanticity complicates neuron-level explanations and suggests that human-interpretable semantic content may not be aligned with individual neurons. This motivates searching for representational units beyond individual neurons.

-

4.4 Transformer Computation#

-

Transformers (Vaswani et al., 2017b) provide a particularly transparent architecture for mechanistic analysis due to their modular structure and additive information flow.

-

All transformer submodules read from and write to a shared residual stream, enabling information to persist and be incrementally transformed across layers.

+
+

4.3 Features as Representational Units#

+

Since individual neurons are often polysemantic, mechanistic interpretability has increasingly shifted towards alternative units of analysis. For example, it has been found that models sometimes encode human-interpretable concepts as linear directions or low-dimensional manifolds in the activation space (Arditi et al., 2024; Wurgaft et al., 2026). We thus use the term feature broadly to encompass any model representation corresponding to a human-interpretable concept. Such distributed representations motivate analysing the activation space as a whole, rather than individual neurons in isolation (Elhage et al., 2022a; Bricken et al., 2023; Templeton et al., 2024).

+

Specifically, we treat a feature technically as a geometric structure \phi in the activation space \mathbb{R}^d of a model component, such as the residual stream, an attention-head output, or an MLP output, where d denotes the hidden dimension. Conceptually, a feature is a manipulable manifestation of a concept of interest. As with neurons, for a feature \phi claimed to correspond to a concept c, we expect interventions that increase or suppress the activation of \phi to causally influence model behaviour associated with c. In this sense, features provide a more expressive, general representational unit than individual neurons for describing how neural networks may encode concepts.

-

The residual stream is the sequence of activation vectors \{h_l(x)\}_{l=0}^{L} that serve as the shared communication channel between transformer layers. Each sublayer reads from and writes additively to this stream: \begin{align*} -h'_l(x) &= h_{l-1}(x) + \text{Attn}_l(h_{l-1}(x)) \\ -h_l(x) &= h'_l(x) + \text{MLP}_l(h'_l(x)) -\end{align*} This additive structure means information persists by default—sublayers only need to write changes, not copy forward what they wish to preserve which makes targed intervention more reliable.

+

Feature

+

A feature \phi associated with a concept c is a geometric structure in the model’s activation space \mathbb{R}^d whose activation is predictive of, and causally relevant to, the presence or expression of c.

+

At this stage, we remain agnostic about the precise mathematical form features take and how the model encodes them. Instead, we treat existing methods for identifying and manipulating features as operational approximations shaped by their methodological assumptions and constraints. We examine these representational hypotheses explicitly in Section 5.

+

Within this operational perspective, researchers can identify features using several methodological families, including readout-based methods (Ghandeharioun et al., 2024; Pan et al., 2026; Karvonen et al., 2026a) and sparse decomposition techniques (Cunningham et al., 2024; Dunefsky et al., 2024), and then manipulate them to test causal relevance (Wu et al., 2025b; Mueller et al., 2025). In practice, many such approaches operationalise features as approximately linear or linearised components of internal representations (Yun et al., 2021; Marks & Tegmark, 2024; Arditi et al., 2024). Examples include linear probes and dictionary-learning methods (Belinkov, 2022; Bricken et al., 2023).

+

An important caveat is that, for a given concept, an identified manipulable geometric structure—that is, a candidate feature—may be only one of many possible internal representations associated with that concept. For example, a language model may encode the concept of “refusal” in a complex manner across multiple layers and components. Successfully identifying a linear direction in the residual stream that can steer the model towards or away from refusal (Arditi et al., 2024) does not imply that this direction uniquely or exhaustively captures how refusal is represented. Other causally relevant structures (e.g., low-dimensional manifolds) for refusal may also exist but remain unidentified.

+
+

4.4 Transformer Computation#

+

Transformers (Vaswani et al., 2017) have become the dominant architecture underlying modern language-model-based AI assistants such as ChatGPT and Claude, and are consequently the primary subject of study in mechanistic interpretability. Several properties of the transformer architecture, most notably its modular structure and additive information flow, make it particularly amenable to mechanistic analysis (Elhage et al., 2021). In particular, transformer components can be viewed as reading from and writing to a shared residual stream, allowing information to persist and be incrementally transformed as it passes through successive layers (Elhage et al., 2021). Below, we briefly review the main classical components of the transformer architecture.

-

Let h \in \mathbb{R}^{n \times d} denote the residual stream at a given layer. A transformer component c (such as an attention head or an MLP block) is a function c : h \mapsto \Delta h_c(h), whose output is added to the residual stream. The forward computation of a transformer layer can be written as h_\ell = h_{\ell-1} + \sum_{c \in \mathcal C_\ell} \Delta h_c(h_{\ell-1}), where \mathcal C_\ell is the set of components in layer \ell.

-

This additive decomposition induces a computational structure in which model behavior arises from the combined contributions of individual components acting on shared internal representations.

+

Residual Stream and Computational Components

+

Let h^{1:T}_\ell(x) \in \mathbb{R}^{d \times T} denote the sequence of residual-stream activations at layer \ell for an input sequence x, where T is the sequence length and d is the model hidden size. The residual stream serves as the shared communication channel through which transformer components exchange information. Below, in this box, we let h_\ell(x) \coloneqq h^{1:T}_\ell(x) to simplify the notation.
+For a standard transformer layer \ell, consisting of a multi-head attention module \mathrm{Attn}_\ell(\cdot) and a multi-layer perceptron \mathrm{MLP}_\ell(\cdot), we have \begin{align*} +h'_\ell(x) + &= h_{\ell-1}(x) + + \mathrm{Attn}_\ell(h_{\ell-1}(x)), \\ +h_\ell(x) + &= h'_\ell(x) + + \mathrm{MLP}_\ell(h'_\ell(x)) + = h_{\ell-1}(x) + + \mathrm{Attn}_\ell(h_{\ell-1}(x)) + \mathrm{MLP}_\ell(h'_\ell(x)). +\end{align*}

+

Unrolling the recursion gives h_\ell(x) += +h_0(x) ++ +\sum_{k=1}^{\ell} +\Big( +\mathrm{Attn}_k(h_{k-1}(x)) ++ +\mathrm{MLP}_k(h'_k(x)) +\Big),

+

where h_0(x) denotes the input embedding representation. The equation shows that the residual stream at layer \ell equals the input embeddings plus the accumulated outputs of all computational components that have written to the residual stream up to that point. This perspective is central to mechanistic interpretability, as it lets us decompose model computation into additive contributions from individual components (Elhage et al., 2021).

-

The attention mechanisms primarily move and combine information across positions, while feedforward (MLP) layers perform nonlinear transformations. This separation of computation enables localized analysis of information routing and transformation. For simplicity, we are omitting layer norms here.

+

For simplicity, we omit layer-normalisation operations from the notation. Following Elhage et al. (2021), most of the layer-normalisation transformation can be absorbed into adjacent linear components. Specifically, for a fixed input, layer normalisation is well approximated as an affine transformation with a scalar rescaling factor. The affine part can be folded into neighbouring parameters, and the remaining scalar normalisation term simply reweights component contributions. This approximation lets us treat the residual stream as an additive communication channel without explicitly representing normalisation at every layer.

+

Attention heads and MLPs play distinct roles in transformer computation. Attention modules primarily route, aggregate, and compose information across token positions (Olsson et al., 2022; Wang et al., 2023), whereas MLPs often perform feature transformation and factual knowledge retrieval (Geva et al., 2021; Meng et al., 2022). This functional separation enables more localised analyses of how the model routes, transforms, and stores information, as well as how attention modules and MLPs interact (Neo et al., 2024).

+

Below, we detail the computations of canonical attention modules and MLPs.

-

An attention head indexed by \alpha computes its contribution to the residual stream by projecting the residual stream h into query, key, and value spaces: Q^{\alpha} = h W_Q^{\alpha}, \qquad -K^{\alpha} = h W_K^{\alpha}, \qquad -V^{\alpha} = h W_V^{\alpha}, where W_Q^{\alpha}, W_K^{\alpha}, W_V^{\alpha} \in \mathbb{R}^{d \times d_h}. The attention pattern is A^{\alpha} = \mathrm{softmax}\!\left(\frac{Q^{\alpha} (K^{\alpha})^\top}{\sqrt{d_h}}\right), and the head writes its output back to the residual stream via \Delta h_{\alpha} = A^{\alpha} V^{\alpha} W_O^{\alpha}.

-

This decomposition separates information routing - information shared among token indices (via Q^{\alpha}, K^{\alpha}) from information transformation- computations that occur at specific indices (via V^{\alpha}, W_O^{\alpha}). When the attention pattern A^{\alpha} is held fixed, the contribution \Delta h_{\alpha} is linear in the residual stream.

+

Attention Module

+

A multi-head attention module consists of N_H attention heads operating in parallel. Consider the residual-stream activations h_\ell^{1:T}(x) \in \mathbb{R}^{d \times T} and let h := h_\ell^{1:T}(x) for notational simplicity. For each head r \in \{1,\ldots,N_H\}, the residual stream is first projected into query, key, and value representations: Q^r = W_Q^r h, +\qquad +K^r = W_K^r h, +\qquad +V^r = W_V^r h, where W_Q^r, W_K^r, W_V^r \in \mathbb{R}^{d_{\text{head}} \times d} and d_{\text{head}} denotes the dimensionality of each attention head. The attention pattern of head r is then computed as A^r += +\operatorname{softmax}\!\left( +\frac{(Q^r)^\top K^r}{\sqrt{d_{\text{head}}}} +\right) +\in +\mathbb{R}^{T \times T}, where the softmax is applied row-wise. Thus, A^r_{t,t'} denotes the attention weight assigned by the query at position t to the key at position t'. The head then writes an update to the residual stream: \Delta h^r += +W_O^r V^r (A^r)^\top, where W_O^r \in \mathbb{R}^{d \times d_{\text{head}}}. The output of the full multi-head attention module is the sum of the contributions from all heads: \operatorname{Attn}(h) += +\sum_{r=1}^{N_H} +\Delta h^r.

-

An MLP block computes a position-wise nonlinear transformation of the residual stream: \Delta h_{\mathrm{MLP}} = W_2 \sigma(W_1 h), where \sigma is a nonlinearity applied elementwise. MLP blocks primarily transform and generate internal features, writing new directions into the residual stream.

+

An MLP block applies a position-wise nonlinear transformation to each position of the residual stream. In the canonical Transformer architecture (Vaswani et al., 2017), this transformation takes the form \Delta h_{\text{MLP}} = W_2 \,\sigma(W_1h+b_1)+b_2, where W_1 \in \mathbb{R}^{d_{\text{mlp}}\times d}, W_2 \in \mathbb{R}^{d\times d_{\text{mlp}}}, and \sigma(\cdot) denotes a nonlinear activation function, for which the original Transformer uses ReLU (Nair & Hinton, 2010). The same transformation is applied independently at each position in the sequence. MLP blocks primarily transform and generate internal features, writing new directions into the residual stream.
+Notably, W_1 typically projects the activations into a much higher-dimensional space (d_{\text{mlp}} \gg d), which has recently been found to yield more interpretable representations natively in the model (Nikankin et al., 2025; Arora et al., 2026).

+

The transformer components discussed above correspond to their canonical formulations. Numerous variants have since been proposed and adopted in modern architectures. For example, variants of the attention module include Grouped Query Attention (GQA) (Ainslie et al., 2023), used in Llama 3 (Grattafiori et al., 2024), and Multi-Head Latent Attention (MLA) (DeepSeek-AI et al., 2024), used in DeepSeek-V2 (DeepSeek-AI et al., 2024) and DeepSeek-V3 (DeepSeek-AI, 2025). Variants of the MLP block include gated architectures such as SwiGLU (Shazeer, 2020), used in Llama 3, and Mixture-of-Experts (MoE) architectures, used in DeepSeek-V3.

-

4.5 Circuits as Computational Substructures#

-

One can study how subsets of a model’s components interact to implement specific computations. In transformers, such interacting subgraphs of attention heads, MLPs, features, and their connections are commonly referred to as circuits (Elhage et al., 2021).

+

4.5 Circuits as Descriptions of Internal Mechanisms#

+

Sections 4.2–4.3 introduce neurons and features as representational units for describing what is represented in model activations, and Section 4.4 introduces the computations that transform and route activations through the model. We use the term mechanism to refer to the organisation of model activations, their representational units, or the transformations applied to them that together produce a particular model behaviour. A central topic in mechanistic interpretability is identifying and understanding such mechanisms in neural networks.

+

A common way to represent and describe such a mechanism is through a circuit: a subgraph of the model’s computational graph whose nodes correspond to computational components or representational units and whose edges represent information flow between them (Elhage et al., 2021).

-

A circuit \mathcal{C} \subseteq G is a subgraph of the model’s computational graph G, where G has:

+

Circuit (Figure 3)

+

For a fixed model computation, let G denote the corresponding directed acyclic computational graph. A circuit \mathcal{C} \subseteq G is a subgraph of G, where:

    -
  • Nodes: computational components (\text{Attn}_l^{(h)}, \text{MLP}_l) and features \phi_i

  • -
  • Edges: Directed connections representing the flow between computational components h_l(x)

  • +
  • Nodes: computational components, such as \mathrm{Attn}_\ell and \mathrm{MLP}_\ell, or representational units, such as features \phi_i;

  • +
  • Edges: directed connections representing information flow or transformation between nodes.

-

The subgraph \mathcal{C} captures the components and features that together implement a specific computation of the model, providing mechanistic explanations to be framed at an algorithmic level while remaining grounded in the model’s representations.

+

The subgraph \mathcal{C} represents a collection of components and interactions that jointly contribute to a particular computation or model behaviour, and can therefore serve as a description of the underlying mechanism.

-

The study of circuits in mechanistic interpretability has developed along two complementary lines. The first line of work, often referred to as circuit discovery (Wang et al., 2023; Conmy et al., 2023; Hanna et al., 2024; Haklay et al., 2025), aims to identify minimal sets of architectural components, such as attention heads or MLP layers, that are sufficient to reproduce a given behavior or capability of the model, such as indirect object identification (Wang et al., 2023) and numerical comparison (Hanna et al., 2023). In this setting, the capability of interest is instantiated as a dataset of queries requiring the model to have the capability to solve them, and a discovered circuit is considered successful if it recovers the model’s performance on this dataset.

-

The second line of work, often referred to as circuit tracing, focuses on how internal features interact and propagate across the computational graph to lead to the model decision of a particular input query (Ameisen et al., 2025; Wu & Barez, 2025). Studies of this line usually make use of sparse autoencoders (detailed in Section 6) to automatically decompose mlp outputs into interpretable units, tracing attributions between these features and how the information flow converges to the final prediction.

-
-

4.6 Limitations of Foundational Works In Mechanistic Interpretability#

-

Mechanistic interpretability aims to provide causal explanations of model behavior, but it faces important practical and conceptual limitations.

-
-

Mechanistic explanations may be incomplete, difficult to scale to large models or complex behaviors, and sensitive to modeling assumptions. Moreover, identifying a circuit does not by itself guarantee control, robustness across models and setups, or safety (Sharkey et al., 2025).

+
+Figure 3: Circuit. This diagram illustrates a circuit within a transformer, where specific nodes (MLP layers and attention heads in this case) perform specialised computations. +
Figure 3: Circuit. This diagram illustrates a circuit within a transformer, where specific nodes (MLP layers and attention heads in this case) perform specialised computations.
+
+
+

Examples of Circuits

+
    +
  • Induction head circuit: Two attention heads cooperate to implement in-context learning (Olsson et al., 2022).

  • +
  • QK circuits: Query–Key interactions prioritising specific token positions (Kamath et al., 2025).

  • +
-

These limitations motivate the formulation of explicit hypotheses about representation, computation, and generalization, which we examine in the next section.

+

The study of circuits has developed along two complementary lines. The first line of work (Wang et al., 2023; Conmy et al., 2023; Hanna et al., 2024; Haklay et al., 2025), referred to as capability circuit discovery by Wu & Barez (2026), aims to identify minimal sets of architectural components, such as attention heads or MLP layers, that are sufficient to reproduce a given behaviour or capability of the model, such as indirect object identification (Wang et al., 2023), numerical comparison (Hanna et al., 2023), and in-context learning (Kharlapenko et al., 2025). In this setting, the capability of interest is defined as a dataset of queries that exercise it, and a discovered circuit is considered successful if it recovers the model’s performance on this dataset.

+

The second line of work, referred to as query-level circuit discovery (Wu & Barez, 2026), focuses on how internal features interact and propagate across the computational graph to produce the model’s decision for a particular input query (Ameisen et al., 2025; Wu & Barez, 2026). Studies in this line usually use sparse autoencoders or cross-layer transcoders (detailed in Section 6) to automatically decompose MLP outputs into interpretable units, trace attributions between these features, and track how information flow converges to the final prediction.

+
+

Summary.

+

This section introduced the foundational concepts of mechanistic interpretability: neuron polysemanticity, which motivates the use of features as more expressive units of analysis; transformer computation, which describes how representations are transformed and information is routed through the model; and internal mechanisms, which organise these units, transformations, and information flows to produce model behaviour and can be described by circuits. How we interpret and leverage these concepts depends on assumptions about the structure of model representations and computation, which we examine explicitly in Section 5.

+
diff --git a/src/generated/chapters/intermediate-representations.html b/src/generated/chapters/intermediate-representations.html index 34dd7305de..283aa2a721 100644 --- a/src/generated/chapters/intermediate-representations.html +++ b/src/generated/chapters/intermediate-representations.html @@ -1,197 +1,172 @@ -

8 Understanding Intermediate Computation: Logit Lens and Probing#

-

Between feature extraction (what the fundamental components of features are) and circuit analysis (how components interact), there lies a gap: understanding what computation happens at intermediate layers. This section covers methods that analyze hidden layer activations to reveal how information transforms as it flows through the network.

-

8.1 Logit Lens: Tracing Predictions Through Depth#

+

8 Understanding Intermediate Representations#

+

This section explores interpreting hidden activations directly in natural language. In contrast to feature extraction and circuit discovery (Sections 6 and 7), which extract features or their interactions from model representations (activations), the methods we will discuss in this section treat the activation itself as the object of interpretation. They use language models or other natural-language-based methods to describe or elicit the encoded information, without intervening on or decomposing the underlying activation. This paradigm provides a direct interface between model activations and natural-language explanations, making it particularly accessible and understandable for human users.

+

8.1 Logit Lens: Tracing Predictions Through Depth#

-

Logit lens (nostalgebraist, 2020) applies the model’s unembedding matrix to activations at intermediate layers, revealing what the model would predict if computation stopped at that layer. For a hidden activation h_\ell at layer \ell, logit lens computes: \text{logits}_\ell = W_U h_\ell where W_U represents the model’s unembedding matrix. This reveals the trajectory of predictions across layers, showing when and where the model commits to specific outputs (nostalgebraist, 2020).

+

Logit Lens (Figure 10)

+

Logit lens (nostalgebraist, 2020) applies the model’s unembedding matrix to activations at intermediate layers, revealing what the model would predict if computation stopped at that layer. For a hidden activation h_\ell at layer \ell, logit lens computes: \text{logits}_\ell = W_U h_\ell, where W_U represents the model’s unembedding matrix. Applying it across layers reveals the prediction trajectory, showing when and where the model commits to specific outputs (nostalgebraist, 2020).

-

Logit lens provides intuitive insight into the model’s decision-making process across depth. Early layers typically show diverse possible predictions with high entropy, while later layers progressively narrow and refine them toward the final output. By examining this trajectory, researchers can identify at which layer critical decisions emerge: whether the model commits early to a prediction or progressively refines it layer-by-layer.

-

For example, in a 12-layer language model processing the sequence “A B C A B”, the logit lens may reveal that the token “C” emerges as the top-20 candidates at layer 6. Subsequent layers then progressively amplify this signal, eventually promoting it to the top-ranked token at the final layer. Conversely, for more ambiguous induction tasks involving complex logic, early and middle layers may exhibit low confidence across a broad distribution of tokens, with the model only converging to a specific prediction in the last few layers.

-
-Figure 11: Logit Lens. Logit Lens provides a window into a model’s intermediate state by applying the final unembedding layer directly to the residual stream at earlier layers (h_1, h_2, ...). This allows researchers to see which tokens the model is “considering” mid-computation, such as the emergen -
Figure 11: Logit Lens. Logit Lens provides a window into a model’s intermediate state by applying the final unembedding layer directly to the residual stream at earlier layers (h_1, h_2, ...). This allows researchers to see which tokens the model is “considering” mid-computation, such as the emergence of the concept “CAT” before the final output layer is reached.
+

Logit lens (Figure 10) provides intuitive insight into the model’s decision-making process across depth. Experimental results show that early layers tend to produce diffuse predictions, while later layers progressively narrow them towards the final output, though exceptions exist. For example, in a twelve-layer language model processing the sequence A B C A B, the logit lens may reveal that the token C enters the top-20 candidates at layer 6 and is progressively amplified, eventually becoming the top-ranked token at the final layer. Conversely, for induction tasks in complex linguistic patterns, early and middle layers may show a flat probability distribution across many tokens, with the model converging to a specific prediction only in the last few layers.

+
+Figure 10: Logit Lens. Logit lens provides a window into a model’s intermediate state by applying the final unembedding layer directly to the residual stream at earlier layers (h_1, h_2, \ldots). This allows users to see which tokens the model is “considering” mid-computation, such as the emergence +
Figure 10: Logit Lens. Logit lens provides a window into a model’s intermediate state by applying the final unembedding layer directly to the residual stream at earlier layers (h_1, h_2, \ldots). This allows users to see which tokens the model is “considering” mid-computation, such as the emergence of the concept “CAT” before the final output layer is reached.

Limitations of Vanilla Logit Lens.

-

The vanilla logit lens has critical limitations. Applying the unembedding matrix to intermediate activations assumes that the hidden state is in a format the output layer expects, but intermediate representations may reside in vector spaces different from final activations. The unembedding matrix is only trained to operate on the final residual stream after all transformations; applying it to layer-6 activations extracts relevant information but may not precisely reflect the model’s underlying computation at that layer (nostalgebraist, 2020).

+

The vanilla logit lens has a critical limitation: applying the unembedding matrix to intermediate activations assumes that the hidden state is in a format that the output layer expects. However, intermediate representations may be organised very differently from output-layer activations. Applying the unembedding matrix to layer-6 activations may extract relevant information but cannot precisely reflect the model’s underlying computation at that layer (nostalgebraist, 2020). Furthermore, logit lens is fundamentally correlational rather than causal. It projects the activation into the output vocabulary space without testing the roles decoded tokens play in model computation. In particular, because of the representational mismatch discussed above, decoded tokens represent at most information recoverable from the activation, rather than information used in model computation.

-

8.2 Tuned Lens: Learning Per-Layer Transform#

+

8.2 Tuned Lens: Learning Per-Layer Transforms#

-

Tuned lens (Belrose et al., 2025) addresses the vector space mismatch problem of logit lens by learning per-layer affine transformations that map intermediate activations to the output layer’s expected space: \text{logits}_\ell^{\text{TL}} = W_U (W_\ell h_\ell + b_\ell), where W_\ell and b_\ell are learned affine transformations for layer \ell. These transforms are trained to minimize the divergence between the tuned lens’ predicted distribution p_\ell^{\text{TL}} = \operatorname{softmax}(\text{logits}_\ell^{\text{TL}}) and the final model output distribution p_{\text{final}}, typically via \min_{W_\ell,b_\ell}\mathbb{E}_{x\sim\mathcal{D}}\big[\,\mathrm{KL}(p_{\text{final}}(x)\;\|\;p_\ell^{\text{TL}}(x))\big].

+

Tuned Lens

+

Tuned lens (Belrose et al., 2025) addresses logit lens’ concern of representational mismatch by learning per-layer affine transformations that map intermediate activations to the output layer’s activation space: \text{logits}_\ell^{\text{TL}} = W_U (W_\ell h_\ell + b_\ell), where W_\ell and b_\ell are learned affine transformations for layer \ell. These transforms are trained to minimise the divergence between the tuned lens’ predicted distribution p_\ell^{\text{TL}} = \operatorname{softmax}(\text{logits}_\ell^{\text{TL}}) and the model output distribution p_{\text{final}} via: \min_{W_\ell,b_\ell}\mathbb{E}_{x\sim\mathcal{D}}\big[\,\mathrm{KL}(p_{\text{final}}(x)\;\|\;p_\ell^{\text{TL}}(x))\big].

-

Tuned lens improves on basic logit lens by correcting for the fact that different layers operate in different representational spaces. Rather than naively applying the final output layer, tuned lens learns what transformation is needed at each layer to make intermediate activations speak the same language as the final layer (Belrose et al., 2025).

-

The trade-off is that tuned lens requires training additional parameters (one affine transform per layer), but this is computationally cheap compared to other mechanistic methods.

+

As a result, the unembedding matrix can extract relevant information from each layer’s activations more effectively.

+
+

Limitations of Tuned Lens.

+

Although tuned lens mitigates representational mismatch, it still inherits several fundamental limitations. First, the decoding paradigm remains inherently correlational rather than causal; decodable information does not imply a causal role in the model’s prediction. Second, similar to logit lens, tuned lens is primarily designed to answer what the model is likely to predict rather than what the model is internally computing. The learned layer-specific transformations that align intermediate representations with the final prediction space may obscure intermediate computational steps and produce predictions that resemble the model’s final output. Empirical evidence also shows that it skips intermediate computation more aggressively than logit lens (Gurnee et al., 2026).

-

8.3 Case Study: Refusal Vectors and Sleeper Agents#

+
+

8.3 Jacobian Lens: Decoding Computational Dispositions#

-

A refusal vector is a direction in a model’s residual stream activations that corresponds to the model declining to produce undesirable or harmful content. Let h_t^\ell \in \mathbb{R}^d denote the residual stream at layer \ell and token position t. We define a refusal vector \mathbf{r}^\ell \in \mathbb{R}^d at layer \ell as: \mathbf{r}^\ell = \mathbb{E}_{x \sim \mathcal{D}_\text{harmful}} [h_t^\ell(x)] - \mathbb{E}_{x \sim \mathcal{D}_\text{harmless}} [h_t^\ell(x)], where \mathcal{D}_\text{harmful} contains prompts that should be refused and \mathcal{D}_\text{harmless} contains neutral or safe prompts.

-

Projecting h_t^\ell onto \mathbf{r}^\ell quantifies the strength of the refusal signal at layer \ell: s_t^\ell = \langle h_t^\ell, \mathbf{r}^\ell \rangle.

-

A large positive s_t^\ell correlates with refusal behavior, and ablating or modifying this component can reduce the model’s tendency to refuse. Refusal vectors can be used to interpret, steer, or probe models along safety-relevant directions in residual space.

-
-

Building on Definition 8.3, we can use logit lens to examine how refusal vectors manifest across layers and why the same approach is less effective for latent behaviors like sleeper agents in Definiton 2.

-
-

For a refusal vector \mathbf{r}^\ell defined at layer \ell, logit lens projects intermediate hidden states h_t^\ell to the output vocabulary via the unembedding matrix W_U: \hat{y}_t^{(\ell)} = W_U\, h_t^\ell. The resulting layer-wise logits \hat{y}_t^{(\ell)} approximate what the model would predict if layer \ell were the final layer. When a refusal vector contributes significantly to the activations, the logits associated with refusal behavior may become noticeable at intermediate layers, providing a tentative trace of how the model encodes refusal. Projections along \mathbf{r}^\ell can be used to quantify the strength of this signal: s_t^\ell = \langle h_t^\ell, \mathbf{r}^\ell \rangle.

-

Sleeper agents encode a conditional, latent objective that activates only under a hidden trigger (e.g., year = 2024). For such behavior, the difference \Delta \hat{y}_t^{(\ell)} = \hat{y}_t^{(\ell)}(\text{trigger}) - \hat{y}_t^{(\ell)}(\text{non-trigger}) may appear gradually across layers as the model increasingly favors tokens associated with the latent objective. In this sense, logit lens can still reveal contributions to potentially harmful tokens and track how these emerge through the network, similarly to refusal vectors.

+

Jacobian Lens

+

Jacobian lens (J-lens; Gurnee et al., 2026) adopts a decoding paradigm distinct from logit lens and tuned lens. It uses the Jacobian to characterise how an intermediate representation influences certain model outputs.
+Let h_\ell^t denote the residual stream at layer \ell and position t. A perturbation to h_\ell^t propagates to the final-layer residual stream at every position t' \geq t; to first order, this influence is captured by the Jacobian \frac{\partial h_{\mathrm{final}}^{t'}}{\partial h_\ell^t}. The lens matrix is defined as the expectation of this Jacobian over prompts and positions: J_\ell += +\mathbb{E}_{\,\mathrm{prompt},\,t,\,t' \ge t} +\left[ +\frac{\partial h_{\mathrm{final}}^{t'}} +{\partial h_\ell^t} +\right]. J-lens then decodes intermediate representations by p_\ell^{\mathrm{JL}} += +\operatorname{softmax} +\!\big( +W_U \operatorname{norm}(J_\ell h_\ell) +\big).

+

Gurnee et al. (2026) further defined layer-specific concept (token) representations using the rows of W_U J_\ell, called J-lens vectors. The inner product between a J-lens vector and h_\ell measures how strongly the current activation will promote the corresponding token in model outputs.

+

Unlike logit lens and tuned lens, which primarily estimate which token the model is likely to predict next, J-lens characterises how current representations influence model outputs. It aims to answer what the model is internally computing rather than merely what the model is going to predict. J-lens has been found useful for analysing tasks that require multi-step reasoning, latent planning, or other forms of extended computation (Gurnee et al., 2026).

+

Furthermore, the defined J-lens vectors are a useful medium for intervention. They can serve as steering vectors to shift model behaviour towards or away from the corresponding concepts. In contrast, interventional utility is largely absent from logit lens and tuned lens, which are primarily decoding tools.

+
+

Limitations of Jacobian Lens.

+

J-lens still faces several limitations that warrant further investigation. First, like logit lens and tuned lens, it operates primarily at the token level. However, many semantically meaningful concepts are represented by multiple tokens. For example, the word “photoperiodism” may be split into several subword tokens. If only one token (e.g., “photo”) is visible in J-lens, we cannot tell whether it corresponds to the concept of photoperiodism or actually other words. Second, J-lens is found to work best within a certain range of layers, called J-space (Gurnee et al., 2026). Specifically, decoded concepts are most informative in intermediate layers (e.g., approximately 38–92% of the depth in Claude Sonnet 4.5). In early layers, decoded concepts are noisy and hard to interpret. In later layers, they collapse towards the model’s output and are also uninformative. Some recent studies have attempted to address these limitations (Gurnee et al., 2026; Blank et al., 2026).

-

8.4 Patchscopes: Leveraging the Language Model Itself to Explain Intermediate Representations#

-
-Figure 12: Patchscopes. Patchscopes inspects internal representations by extracting a hidden state from a source prompt and “patching” it into a different inspection prompt. This allows researchers to verify what specific information is captured in a representation by observing how it influences the -
Figure 12: Patchscopes. Patchscopes inspects internal representations by extracting a hidden state from a source prompt and “patching” it into a different inspection prompt. This allows researchers to verify what specific information is captured in a representation by observing how it influences the generation of a completely different context—for example, seeing if a representation for “United Kingdom” correctly triggers “London” as the largest city in a new prompt.
+
+

8.4 Patchscopes: Leveraging the Language Model Itself to Explain Intermediate Representations#

+
+Figure 11: Patchscopes. Patchscopes extracts the model representation of interest from a source prompt and “patches” it into an inspection prompt. By observing changes in model output, users can examine what information the representation contains. For example, in this figure we can observe whether +
Figure 11: Patchscopes. Patchscopes extracts the model representation of interest from a source prompt and “patches” it into an inspection prompt. By observing changes in model output, users can examine what information the representation contains. For example, in this figure we can observe whether the representation of “United Kingdom” correctly triggers “London” as the largest city.
-

Patchscopes (Ghandeharioun et al., 2024) leverages the expressive capacity of large language models to translate internal representations into natural-language explanations. Let h^l_t denote a hidden representation at layer l and token position t. Patchscopes produces a textual explanation of h^l_t, denoted by e(h^l_t), via e(h^l_t) = f_{\text{model}}(p_{[h^{l^*}_{t^*} \leftarrow h^l_t]}), where l^* and t^* specify the layer and token position to which the representation is patched, and p is an elicitation prompt designed to encourage the language model f_{\text{model}} to decode the patched representation into text that expresses the contained information.

-

For example, consider the prompt illustrated in the Figure 1 of Ghandeharioun et al. (2024) p = \text{``cat$\rightarrow$cat; 135$\rightarrow$135; hello$\rightarrow$hello; ?''}, where t^* corresponds to the position of the token “?”. The notation [h^{l^*}_{t^*} \leftarrow h^l_t] indicates that, during the forward pass of f_{\text{model}} on p, the hidden representation at layer l^* and position t^* is replaced with h^l_t. Under this construction, information encoded in h^l_t can be elicited through the model’s prediction for the next token at the “?” position.

+

Patchscopes (Figure 11)

+

Patchscopes (Ghandeharioun et al., 2024) leverages the expressive capacity of language models to translate their internal representations into natural-language explanations. Let h^t_\ell denote a hidden representation at layer \ell and token position t. Patchscopes produces a textual explanation of h^t_\ell, denoted as e(h^t_\ell), by e(h^t_\ell) = f_\theta(p_{[h^{t^*}_{\ell^*} \leftarrow h^t_\ell]}), where \ell^* and t^* specify the layer and token position to which the representation is patched, and p is an elicitation prompt designed to encourage the language model f_\theta(\cdot) to decode the patched representation into text expressing the contained information.
+For example, consider the prompt illustrated in Figure 1 of Ghandeharioun et al. (2024): p = \text{``cat$\rightarrow$cat; 135$\rightarrow$135; hello$\rightarrow$hello; ?''}, where t^* corresponds to the position of the token “?”. The notation [h^{t^*}_{\ell^*} \leftarrow h^t_\ell] indicates that, during the forward pass of f_\theta(\cdot) on p, the hidden representation at layer \ell^* and position t^* is replaced with h^t_\ell. As a result, the model can elicit information encoded in h^t_\ell through its prediction for the next token at the “?” position.

-

Because the mapping from intermediate representations to natural language may be too complex to be captured by an affine transformation, Patchscopes extends the paradigm of the logit lens and tuned lens by delegating the representation-to-text transformation to the language model itself1. In this view, the language model acts as an activation decoder.

-

The decoding process proceeds by patching the representation of interest into a designated token position within an elicitation prompt. As the model processes the prompt, the semantic information embedded in the patched representation can be surfaced in the model’s generated output. Crucially, this approach relies on the model’s own learned and generalized text-generation capabilities to perform the transformation from internal activations to natural language.

+

Because the mapping from intermediate representations to natural language may be too complex to be captured by an affine transformation, Patchscopes (Figure 11) extends the paradigm of logit lens and tuned lens by delegating the representation-to-text transformation to the language model itself1. In this view, the language model acts as an activation decoder. Crucially, Patchscopes relies on the model’s learned, generalisable text-generation capabilities to map internal activations to natural language.

Limitations of Patchscopes.

-

A key limitation of Patchscopes is that injecting an external representation into an elicitation prompt can shift the model’s internal activations into an OOD regime. This is expected as the decoder is a standard pretrained language model that has not been explicitly trained for representation-to-text decoding. Although the model may still successfully surface salient information from the patched representation, the decoding process can be degraded, and critical aspects of the original representation may be distorted or lost.

+

A key limitation of Patchscopes is that injecting an external representation into an elicitation prompt can shift the model’s internal activations into an out-of-distribution (OOD) regime. This is expected, as the decoder is a standard pretrained language model not explicitly trained for representation-to-text decoding. Although the model may still surface salient information from the patched representation, the decoding process can degrade, and critical aspects of the original representation may be distorted or lost.

-

8.5 LatentQA: Finetuning the Language Model as Representation Decoder#

+

8.5 LatentQA: Fine-Tuning the Language Model as an Activation Verbaliser#

-

LatentQA (Pan et al., 2024) formulates representation decoding as a supervised learning problem via Latent Interpretation Tuning (LIT), which fine-tunes an activation decoder to perform representation-to-text decoding. The core step of LIT is to construct a finetuning dataset D consisting of samples (h^l_t, q, a), where h^l_t is a cached hidden representation from a target model, and (q, a) is a question–answer pair that queries the information encoded in h^l_t.

-

Specifically, h^l_t denotes the activation at layer l and token position t when the target model processes a “control prompt” designed to embed a specific attribute or concept into the representation. An example control prompt from Figure 3 of Pan et al. (2024) is: \text{``In your next reply, please try to evoke a brief sense of triumphant hubris.''} If we define h^l_t as the activation at layer l corresponding to the final token (“.”) of this prompt, the resulting representation is expected to encode the instruction to respond with triumphant hubris. In this case, a corresponding question–answer pair may be: q = \text{``In what way does the assistant craft its responses to the user?''} a = \text{``The assistant crafts responses that briefly highlight a sense of self-assured victory.''} Both the control prompt and the (q, a) pair are generated by prompting an off-the-shelf proprietary language model (Pan et al., 2024).

+

LatentQA (Figure 12)

+

LatentQA (Pan et al., 2026) frames representation decoding as a supervised learning problem via Latent Interpretation Tuning (LIT), fine-tuning an activation verbaliser to perform representation-to-text verbalisation. The core step of LIT is to construct a fine-tuning dataset D consisting of samples (h^t_\ell, q, a), where h^t_\ell is a cached hidden representation from a target model, and (q, a) is a question–answer pair that queries the information encoded in h^t_\ell.
+Specifically, h^t_\ell denotes the activation at layer \ell and token position t when the target model processes a “control prompt” designed to embed a specific attribute or concept into the representation. An example control prompt from Figure 13 of (Pan et al., 2026) is: \text{``In your next reply, please try to evoke a brief sense of triumphant hubris.''} If h^t_\ell is defined as the activation at layer \ell corresponding to the final token (“.”) of this prompt, the resulting representation is expected to encode the instruction to respond with triumphant hubris. In this case, a corresponding question–answer pair may be: q = \text{``In what way does the assistant craft its responses to the user?''} a = \text{``The assistant crafts responses that briefly highlight a sense of self-assured victory.''} Both the control prompt and the (q, a) pair are generated by prompting an off-the-shelf proprietary language model (Pan et al., 2026).

-

LatentQA addresses the OOD issue inherent in Patchscopes-style methods by explicitly finetuning the activation decoder on the representation-to-text decoding task. Through LIT, the decoder is trained to directly ingest and process raw activation vectors. Ideally, after being exposed to a sufficiently diverse collection of question–answer pairs over many different representations, the decoder learns which aspects of the activation to attend to and how to verbalize them conditioned on the query.

-
-Figure 13: LatentQA. LatentQA is a method for probing the semantic content of hidden representations by training a separate decoder model to answer natural language questions about them. The process involves (1) curating a dataset of activation-question-answer triplets from a target LLM, and (2) fin -
Figure 13: LatentQA. LatentQA is a method for probing the semantic content of hidden representations by training a separate decoder model to answer natural language questions about them. The process involves (1) curating a dataset of activation-question-answer triplets from a target LLM, and (2) finetuning a decoder to retrieve specific information latent within those activations, such as the model’s underlying goals or persona.
+

LatentQA (Figure 12) addresses the OOD issue inherent in Patchscopes-style methods by explicitly fine-tuning the activation verbaliser on the representation-to-text decoding task. Through LIT, the verbaliser learns to ingest and process raw activation vectors directly. Ideally, after being exposed to a sufficiently diverse collection of question–answer pairs over many different representations, the decoder learns which aspects of the activation to attend to and how to verbalise them conditioned on the query.

+
+Figure 12: LatentQA. LatentQA probes the semantic content of hidden representations by training a separate verbaliser model to answer natural-language questions about them. The process involves (1) curating a dataset of activation–question–answer triplets from a target LLM, and (2) fine-tuning a ver +
Figure 12: LatentQA. LatentQA probes the semantic content of hidden representations by training a separate verbaliser model to answer natural-language questions about them. The process involves (1) curating a dataset of activation–question–answer triplets from a target LLM, and (2) fine-tuning a verbaliser to retrieve specific information latent within those activations, such as the model’s underlying goals or persona.

Limitations of LatentQA.

-

The central limitation of LatentQA lies in the generalizability of the learned representation-to-text decoder. When the finetuning dataset D is insufficiently large or diverse, the decoder may overfit by memorizing superficial correspondences between representations and textual descriptions, rather than learning the underlying algorithmic transformation. Recent efforts, such as Predictive Concept Decoders (PCDs) (Huang et al., 2025) and Activation Oracles (AOs) (Karvonen et al., 2025), mitigate this issue by constructing more diverse finetuning datasets to expand the decoder’s coverage and robustness.

+

LatentQA faces several technical challenges, including the generalisability of the learned representation-to-text mapping. When the fine-tuning dataset D is too small or insufficiently diverse, the verbaliser may memorise superficial correspondences between representations and explanations rather than learning to extract information from representations. Recent approaches, such as Predictive Concept Decoders (PCDs) (Huang et al., 2026) and Activation Oracles (AOs) (Karvonen et al., 2026a), mitigate this issue by constructing more diverse training datasets.

+

Another technical challenge is evaluating whether activation-verbalisation models genuinely explain the represented information rather than exploit shortcuts. For example, a verbaliser may implicitly reconstruct the original input to the target model and infer an explanation from it. Recent studies have found that such failure modes are severe and prevalent in existing verbalisers (Li et al., 2026c; Jakkli et al., 2026). As a result, strong verbalisation performance alone is insufficient to show that the verbaliser faithfully analysed the internal representations.

-

8.6 Probing Classifiers: Capabilities, Critiques, and Causal Limitations#

-

Probing classifiers are among the most widely used tools for understanding neural network representations. A probe trains a classifier—typically linear—to predict a property (e.g., part-of-speech, sentiment, entity type) from model activations. High probe accuracy is interpreted as evidence that the property is “encoded” in the representation. However, the relationship between probe accuracy and mechanistic understanding is subtle and contested. This section reviews probing methodology, its critiques, and its appropriate role in interpretability research.

-

8.6.1 Standard Probing Methodology#

+

8.6 Probing Classifiers: Capabilities, Critiques, and Causal Limitations#

+

Probing classifiers (Figure 13) are among the most widely used tools for analysing neural network representations. A probe is a classifier trained to predict a property of the input, such as part of speech, sentiment, or entity type, from model activations. High probe accuracy means that the property is decodable from the activations. However, information decodability is a relatively coarse-grained, correlational interpretability claim compared with causal claims about information in model computation. This section reviews probing methodology, its major critiques, and its appropriate role in interpretability research.

+

As a side note, discovered probe directions can also be used as supervised feature extractors. We discuss this perspective in Section 6.9.4, including comparisons with SAEs, and in Section 9.3, where we discuss their use as steering vectors.

+

8.6.1 Standard Probing Methodology#

-

A probing classifier is a supervised model trained to predict a target property y from neural network activations h_\ell at layer \ell: \hat{y} = g(W_{\text{probe}} h_\ell + b_{\text{probe}}) where g is an activation function (identity for regression, softmax for classification), and (W_{\text{probe}}, b_{\text{probe}}) are learned parameters. The probe is trained on labeled examples (h_\ell^{(i)}, y^{(i)}) to minimize prediction error.

-

Linear probes constrain g to be linear, testing whether the property is linearly accessible. MLP probes allow nonlinear g, testing whether the property is present in any recoverable form.

+

Probing Classifier (Figure 13)

+

A probing classifier is a supervised model trained to predict a target property y from neural network activations h_\ell at layer \ell. A linear probe follows the form: \hat{y} = g(W_{\text{probe}} h_\ell + b_{\text{probe}}), where W_{\text{probe}} and b_{\text{probe}} are learned parameters. g, typically an identity function for regression and softmax for classification, maps the probe’s raw output to the prediction space. The probe is trained by minimising prediction error on labelled examples (h_\ell^{(i)}, y^{(i)}). The linear probe tests whether the target property is linearly accessible from the representation. MLP probes introduce one or more nonlinear hidden layers instead of W_{\text{probe}} to test whether the property can be retrieved through more expressive nonlinear mappings.

-
-Figure 14: Probing Classifier. Probing is a technique used to determine if a specific concept is linearly represented within a model’s internal activations. A simple supervised classifier (probe) is trained to distinguish between positive and negative samples of a concept. -
Figure 14: Probing Classifier. Probing is a technique used to determine if a specific concept is linearly represented within a model’s internal activations. A simple supervised classifier (probe) is trained to distinguish between positive and negative samples of a concept.
+
+Figure 13: Probing Classifier. A probing classifier (or probe) tests the decodability of a concept from the target model’s internal representations. It is trained to distinguish activations associated with positive and negative examples of the concept. High probe accuracy indicates that the concept +
Figure 13: Probing Classifier. A probing classifier (or probe) tests the decodability of a concept from the target model’s internal representations. It is trained to distinguish activations associated with positive and negative examples of the concept. High probe accuracy indicates that the concept can be reliably decoded from the representation.
-
-

What Probing Can Show.

-

Probing provides evidence about information content: whether activations contain recoverable information about a property. High probe accuracy indicates the information is present; low accuracy indicates it is absent or deeply encoded.

-
-
-

What Probing Cannot Show.

-

Probing does not establish causal relevance: whether the model actually uses that information for its predictions. A representation may encode information that is never accessed by downstream computation—a “leftover” from earlier processing or a byproduct of representing correlated features.

-
-
-

8.6.2 The Probe Causality Problem#

+

Probing provides evidence about whether activations contain information about a property. High probe accuracy indicates the information is present; low accuracy suggests it is absent or the probe is not expressive enough. Note that probing alone does not show whether the model actually depends on that information in its decision-making process. A representation can encode information that downstream computation never accesses, such as “leftover” information from earlier processing.

-

High probe accuracy does not imply causal relevance. The model may encode information without using it, or the probe may detect correlates of the target property rather than the property itself (Belinkov, 2022; Hewitt & Liang, 2019).

-

Failure modes:

-
    -
  • Correlational encoding: The probe detects features correlated with the target (e.g., word frequency correlates with part-of-speech) rather than the target itself.

  • -
  • Unused information: Information may be encoded but never read by subsequent layers. The model’s actual computation may rely on different features.

  • -
  • Probe expressivity: A sufficiently powerful probe can extract information that requires complex computation to access, information the model itself may not use.

  • -
-

This problem is fundamental: probing measures information presence, not information use. Bridging this gap requires causal methods.

+

The Probe Causality Problem

+

High probe accuracy does not imply causal relevance (Belinkov, 2022; Hewitt & Liang, 2019). The probe can work by detecting features spuriously correlated with the target property (e.g., word frequency correlates with part of speech) rather than detecting the target property itself. Furthermore, subsequent layers may never use the decoded property to influence the model output. Finally, a sufficiently expressive probe can achieve high accuracy by exploiting noise even when the representations contain no such information.

+

In summary, using probing alone generally does not provide actionable mechanistic understanding and therefore does not, under our definition, constitute mechanistic interpretability research. However, subsequent work has proposed refined probing paradigms that explicitly incorporate causal considerations into probe design and evaluation, enabling more actionable mechanistic understanding (Elazar et al., 2021); see Section 8.6.4 for details. Recent interpretability studies have similarly used causal interventions to validate probe-discovered concept directions, building a more complete mechanistic explanation (Marks & Tegmark, 2024; Li et al., 2023).

-

8.6.3 Control Tasks and Selectivity#

-

A major critique of probing is that high accuracy may reflect probe capacity rather than representation quality. Hewitt & Liang (2019) introduced control tasks to address this concern.

+

8.6.2 Control Tasks and Selectivity#

+

High probe accuracy may reflect classification capacity rather than the target property’s decodability. Hewitt & Liang (2019) addressed this concern by introducing control tasks as a probe’s baseline performance.

-

A control task assigns random labels to inputs, breaking any genuine relationship between activations and the target property. A probe trained on the control task measures the baseline extractability of random patterns from the representation.

-

Selectivity is defined as: \text{Selectivity} = \text{Acc}_{\text{linguistic}} - \text{Acc}_{\text{control}} where \text{Acc}_{\text{linguistic}} is probe accuracy on the true task and \text{Acc}_{\text{control}} is accuracy on the control task.

-

High selectivity indicates the probe succeeds because of genuine encoding, not because the representation is easily separable by any linear classifier. Low selectivity suggests the probe may be exploiting representation geometry rather than task-relevant structure.

+

Control Task Methodology

+

A control task assigns random labels to inputs to break any genuine relationship between activations and the target property. A probe trained on the control task represents the baseline extractability of random patterns from the representation. Then, the selectivity of a probe is defined as: \text{Selectivity} = \text{Acc}_{\text{linguistic}} - \text{Acc}_{\text{control}}, where \text{Acc}_{\text{linguistic}} is probe accuracy on the target task and \text{Acc}_{\text{control}} is accuracy on the control task. High selectivity indicates that the probe succeeds because the representation genuinely encodes the target property. In contrast, low selectivity suggests we cannot tell whether the probe has exploited spurious features rather than task-relevant structure.

-
-

Implications of Control Tasks.

-

Control task experiments have revealed that many probing results showing “encoding” of linguistic properties suffer from low selectivity—random labels are nearly as extractable (Hewitt & Liang, 2019). However, selectivity improves with model depth and size, suggesting more task-specific encoding in larger models, though this varies by property: syntactic properties typically exhibit higher selectivity than semantic ones. This fragmented picture underscores why probing studies must report control task accuracy alongside primary results; claims about encoding are only meaningful when selectivity is substantially positive.

+

8.6.3 Concerns of Probe Complexity#

+

Probe complexity brings confounding factors to the analysis. If a property can only be extracted with a highly nonlinear probe, it is hard to tell whether the property is “deeply encoded” or whether the probe is simply using nonlinearity to exploit signals, including noise. A recent finding in causal abstraction (Sutter et al., 2025) also showed that, when the mapping function is complex enough, it can map an arbitrary (even randomly initialised) neural network to a causal model of interest, producing false-positive findings.

+
+

The Complexity–Interpretability Trade-off

+
+

Observation.

+

In general, more complex probes achieve higher accuracy but provide less interpretable deliverables.
+

+
+
+

Interpretation problem.

+

Does high accuracy with a complex probe indicate (a) the property is encoded but in a complex, nonlinear form, or (b) the probe itself is computing the property from spurious features? If (b), the probe at most demonstrates that the property is computable from the representation, not that it is encoded in the representation in any meaningful sense. A worse case in (b) is that a highly nonlinear probe may learn to construct the property of interest from random noise rather than any features (Sutter et al., 2025). In this case, the computable illusion likely stems from limited test-set size.

-

8.6.4 Probe Complexity vs. Causal Relevance#

-

A related critique concerns the relationship between probe complexity and causal relevance. If a property can only be extracted by a complex (nonlinear, high-capacity) probe, does this indicate the property is “deeply encoded” or merely that the probe is doing substantial computation itself?

-
-

Observation: More complex probes achieve higher accuracy on most tasks.

-

Interpretation problem: Does high accuracy with a complex probe indicate:

-
    -
  1. The property is encoded but in a complex, nonlinear form?

  2. -
  3. The probe itself is computing the property from simpler features?

  4. -
-

If (b), the probe demonstrates that the property is computable from the representation, not that it is encoded in the representation in any meaningful sense.

Minimum Description Length Probes.

-

To address this, Voita & Titov (2020) proposed evaluating probes via the Minimum Description Length (MDL) principle: prefer probes that achieve good accuracy with minimal complexity. The MDL framework jointly considers data cost (how well the probe compresses labels given representations) and model cost (how complex the probe itself is). A property is considered “more encoded” if it can be extracted with a simpler probe, shifting the question from “can we extract it?” to “how easily can we extract it?”

+

To address this, Voita & Titov (2020) proposed evaluating probes via the minimum description length (MDL) principle: prefer probes that achieve good accuracy with minimal complexity. The proposed MDL-based evaluation offers an information-theoretic measure of how efficiently a probe can extract a target property from model representations, accounting for both probe accuracy and complexity. A property is considered “more encoded” if it can be extracted with a simpler probe. This paradigm shifts the question from “can we extract it?” to “how easily can we extract it?”

Practical Guidance.

-

When interpreting probing results, prefer linear probes when possible—if linear probes fail but MLP probes succeed, interpret with caution. Crucially, report probe architecture and capacity alongside accuracy, and compare against probes trained on random representations to establish baselines. For quantitative comparison, consider MDL-based evaluation, which provides a principled way to trade off accuracy against model complexity.

+

Users should prefer simple probes when possible, compare against probes trained on random representations to establish baselines, and consider MDL-based evaluation to analyse the trade-off between accuracy and model complexity. If linear probes fail but MLP probes succeed, cautiously validate the result and interpretability claim.

-

8.6.5 Amnesic Probing and Causal Probing#

-

Early work demonstrated that individual neurons could be manipulated to control morphological properties such as gender, number, and tense (Bau et al., 2019), motivating more systematic, direction-based approaches. The idea of intervening on representations via probes was then formalized progressively: Giulianelli et al. (2018) first demonstrated that probes trained on intermediate activations could steer model behavior, and Ravfogel et al. (2020) extended this via Iterative Nullspace Projection (INLP), which repeatedly projects out linearly encoded information. Building on this lineage, Elazar et al. (2021) introduced amnesic probing: after identifying a direction encoding property P, project it out of the representation and measure downstream task degradation.

-

To bridge the gap between information presence and causal relevance, several methods attempt to make probing causal.

+

8.6.4 Amnesic Probing and Causal Probing#

+

Beyond using probes to validate information decodability, researchers have also formalised the idea of intervening on representations via probes. Giulianelli et al. (2018) first showed that a trained probe can steer model behaviour, and Ravfogel et al. (2020) extended this with Iterative Nullspace Projection (INLP), an information-removal method that repeatedly fits a probe to identify linearly encoded information and project it out. Elazar et al. (2021) further measured changes in downstream model behaviour after applying INLP as a validation of the probe’s causality, called amnesic probing. These paradigms add causality to probes, pushing them from correlational understanding to mechanistic analysis. Below, we detail amnesic probing and probing with activation patching.

Amnesic Probing.

-

Given a probe direction w encoding property P:

-
    -
  1. Compute the projection: h' = h - \frac{h \cdot w}{\|w\|^2} w

  2. -
  3. Run the model with h' replacing h

  4. -
  5. Measure change in downstream task performance

  6. -
-

If performance degrades, the information was causally used. If performance is unchanged, the information was encoded but unused.

+

Amnesic Probing

+

Given a model activation h and a probe trained to predict a property P, amnesic probing tests whether information about P is causally relevant to downstream model behaviour by removing information about P from the representation and evaluating the effects on model outputs. Assume P is a binary property (e.g., positive vs. negative sentiment) and that w \coloneqq W_{\text{probe}}^\top \in \mathbb{R}^{d} denotes the probe’s direction. Amnesic probing projects h onto the subspace orthogonal to a probe direction w (i.e., the null space of the probe): \tilde{h} += +h +- +\frac{h^\top w}{\|w\|^2}w. Amnesic probing then measures the change in model performance on downstream tasks with the modified representation \tilde{h}. A substantial degradation indicates that the removed information causally contributes to the evaluated behaviour. Notably, unchanged performance does not establish that the information is unused since redundant representations or alternative computational pathways may preserve the behaviour.

-

Limitations: Amnesic probing assumes the property is encoded in a single linear direction. If encoding is distributed or nonlinear, projection may not fully remove the information, leading to false negatives (concluding information is unused when it is).

-
-
-

Causal Probing via Interchange Interventions.

-

A stronger causal test swaps the probed representation between inputs that differ on the target property:

-
    -
  • Input x_1 has property P = p_1; input x_2 has property P = p_2.

  • -
  • Replace the representation component encoding P in x_1 with that from x_2.

  • -
  • If the model’s behavior shifts to reflect p_2, the encoding is causally active.

  • -
-

This provides stronger evidence than amnesic probing but requires identifying the specific representation component, not just demonstrating its existence.

-
-
-

8.6.6 Critiques of Attribution and Saliency Methods#

-

Probing is often used alongside attribution methods (saliency maps, attention visualization, gradient-based attribution) to understand model behavior. However, attribution methods face their own critiques that affect how probing results should be interpreted.

-
-

Faithfulness of Attributions.

-

Attribution methods aim to identify which inputs or components most influenced an output. However, Jacovi & Goldberg (2020) distinguish between plausibility (do attributions seem reasonable to humans?) and faithfulness (do attributions reflect actual model computation?).

-
-
-

Key findings on attribution faithfulness

-

: Gradient-based saliency can highlight features the model is insensitive to (Adebayo et al., 2018b), attention weights correlate weakly with causal importance (Jain & Wallace, 2019; Wiegreffe & Pinter, 2019), and different attribution methods often disagree substantially on the same inputs (Krishna et al., 2024). Collectively, these findings suggest that attribution methods are fundamentally unreliable guides to what information a model actually uses.

-
-
-

Implications for Probing.

-

This unreliability propagates into probing studies. Probes trained on attention patterns may not reveal causally important information since attention itself may not be faithful. Similarly, using gradients to identify which features to probe can bias toward high-magnitude gradients that lack causal importance. When probing and attribution methods disagree, resolving the disagreement requires causal interventions rather than passive observation.

+

Amnesic probing assumes the target property is decodable and that there exist effective erasure procedures to surgically remove its associated information from the representation. However, information and concept erasure may be imperfect in practice, either leaving residual information or affecting other concepts. Such imperfections confound interpretations of downstream behavioural changes and the causal claims that follow.

+
+

Causal Probing via Activation Patching.

+

Activation patching tests the causality of a component’s representations by swapping them between inputs that differ in the target property. This technique has also been used to test the causality of probes (Huang et al., 2024; Arora et al., 2024). Suppose x_1 and x_2 are two inputs, while x_1 has the target property P and x_2 does not. W_{\text{probe}} is the fitted probing matrix whose rows span a probing subspace (or a direction if the matrix has only one row, such as that of a binary classification probe) where P resides. After choosing the intervention location (e.g., the last-token residual stream at a middle layer), the method first feeds x_1 to the model and extracts the component of the representation in the probing subspace. Then, it feeds x_2 to the model while patching the corresponding component of the model representation with the previously acquired one. If the model output for x_2 shifts towards P, the patched subspace representation contains information about P that the model uses.

+

Compared with amnesic probing, which tests the effect of removing information about P, activation patching examines whether the probe-identified information is sufficient to implement the target behaviour. However, this requires specifying what representation (i.e., which layer, component, and token position) to probe and patch, which may confound the analysis.

-

8.6.7 Circuit Fragility and Robustness Concerns#

-

A related concern applies to circuit-level findings from mechanistic interpretability: identified circuits may be fragile, failing to generalize across distributions or model variants.

-
-

Evidence for Circuit Fragility.

-

Circuits identified on one task may not transfer to related tasks, even when the underlying computation seems similar (McGrath et al., 2023). Small changes to model weights through finetuning or continued pretraining can disrupt identified circuits while preserving task performance, suggesting redundancy or alternative pathways. Moreover, circuits identified in small models may not have direct analogues in larger models (Olsson et al., 2022a), limiting the transferability of findings across model scales.

-
-

Implications for Probing.

-

Circuit fragility reveals a gap in probing’s explanatory power: stable probe accuracy across checkpoints does not guarantee stable circuit implementation, and information encoded at layer \ell may be read by different circuits under different conditions. This suggests that probing findings—which identify what information is encoded—may not reliably predict how that information is used. Complementing probing with circuit analysis is therefore necessary to understand not just encoding but routing.

+

8.6.5 Implications of Attribution Faithfulness for Probing#

+

Probing is often used along with attribution methods (e.g., saliency maps, attention visualisation, gradient-based attribution) to understand model behaviour. However, attribution methods also have technical challenges. For example, gradient-based saliency maps can highlight input features the model is actually insensitive to (Adebayo et al., 2018), attention weights correlate weakly with causal importance (Jain & Wallace, 2019), and different attribution methods can disagree substantially on the same inputs (Krishna et al., 2024). These findings suggest that attribution methods alone cannot determine what information a model actually uses.

+

This unreliability affects how we can interpret probing results. Probes trained on attention patterns may not reveal causally important information since attention itself may not be causal. Similarly, using gradients to identify which features or components to probe can bias towards high-magnitude gradients that lack causal importance. In sum, interpretability claims made by probes, even paired with these auxiliary tools, still need to go through causal validation to separate plausible explanations (do attributions seem reasonable to humans?) from faithful explanations (do attributions reflect actual model computation?) (Jacovi & Goldberg, 2020).

-
-

8.6.8 When is Probing Appropriate, and When is it Not?#

-

Given these critiques, we provide guidance on appropriate uses of probing.

+

8.6.6 When Is Probing Appropriate?#

+

Given the preceding discussion, Table 11 summarises appropriate uses of probing. We argue that probing is most productive when treated as a tool for hypothesis generation, that is, using probes to identify candidate properties that may be encoded in a representation and potentially causally relevant. Users then test these hypotheses using causal methods, such as activation steering or patching. In this sense, probing serves as an efficient first step in mechanistic investigation rather than as a decisive explanation.

- + @@ -200,12 +175,12 @@

Implications for Probing.

- - + + - + @@ -213,39 +188,28 @@

Implications for Probing.

- + - - + +
Table 12: Appropriate and Inappropriate Uses of ProbingTable 11: Appropriate and inappropriate uses of probing.
Appropriate Uses
Establishing information presence (“is P encoded?”)Claiming causal relevance (“the model uses P”)Establishing information presence (i.e., “P is encoded”)Claiming causal relevance (i.e., “the model uses P”)
Comparing encoding strength across layersConcluding a property is “not used” from low probe accuracyConcluding that a property is not encoded based on low probe accuracy
Auditing for encoded sensitive information
Generating hypotheses for causal investigationInterpreting probe weights as model weightsUsing probing results to support hypotheses about causality
Early detection / triage before expensive analysisEvaluating alignment or safety from probe resultsEarly detection or triage before further analysisEvaluating alignment or safety based on probing results
-
-

Probing as Hypothesis Generation.

-

The most productive framing treats probing as hypothesis generation: probes identify candidate properties that may be causally relevant, which must then be tested via causal methods (ablation, patching, circuit analysis). This positions probing as an efficient first step, not a final answer.

-
-

8.6.9 Best Practices for Probing Studies#

-

Based on the critiques above, we recommend the following practices:

-
    -
  1. Report control task baselines: Always include selectivity metrics.

  2. -
  3. Use minimal probe complexity: Prefer linear probes; justify MLP probes when necessary.

  4. -
  5. Include random baselines: Compare to probes on shuffled activations.

  6. -
  7. Test causal relevance: Complement probing with amnesic probing or interchange interventions.

  8. -
  9. Report layer-wise patterns: Show how encoding varies across layers, not just peak accuracy.

  10. -
  11. Acknowledge limitations: Explicitly state that probing shows information presence, not causal use.

  12. -
  13. Cross-validate: Report variance across random seeds and data splits.

  14. -
  15. Consider alternatives: For targeted properties, supervised methods (DAS, DBM) may provide more reliable results than probing followed by post-hoc causal testing.

  16. -
+

8.6.7 Methodological Standards in Probing Research#

+

So far we have reviewed several strengths and limitations of probes. Based on them, we summarise several methodological standards for the design and interpretation of probing studies: (1) Report control-task baselines to distinguish genuine decodability from spurious exploitation; (2) Use minimally expressive probes unless additional complexity is justified; (3) Test causal relevance by methods like amnesic probing or activation patching; (4) Analyse layer-wise trends besides the peak accuracy; (5) Distinguish encoding from causality and acknowledge that successful probing shows information presence rather than causal use; (6) Report the stability of findings by evaluating across seeds and data splits. In sum, we believe these standards help ensure responsible, effective use of probing in interpretability research and downstream applications.

-

Despite the critiques above, probing remains valuable for specific applications. The Simple probes can catch sleeper agents research (MacDiarmid et al., 2024) demonstrates that simple linear probes trained on residual stream activations can detect latent backdoor behavior in sleeper agent models even when the trigger and the harmful behavior are unknown.

-

In a sleeper agent model f_\theta, the model behaves normally for one context (e.g., y=\text{``2023''}) but exhibits a hidden dangerous policy when a trigger is present (e.g., y=\text{``2024''}) (MacDiarmid et al., 2024).

-

To detect this latent behavior, one can construct a linear defection detector by computing a detector direction \mathbf{d}^\ell at a given residual layer \ell. Let h_t^\ell(x) denote residual activations at the final token position. Given a small labeled fit dataset of prompts, the detector direction is defined as: \mathbf{d}^\ell = \mathbb{E}_{(x,y)\sim\mathcal{D}_\text{true}}[h_t^\ell(x)] - \mathbb{E}_{(x,y)\sim\mathcal{D}_\text{false}}[h_t^\ell(x)], where \mathcal{D}_\text{true} and \mathcal{D}_\text{false} are contrast pairs that correlate with the latent policy.

-

Projecting test activations onto this direction yields a scalar detector activation: s_t^\ell = \langle h_t^\ell(x),\, \mathbf{d}^\ell \rangle, which can then be used as a classifier score to distinguish inputs likely to activate the sleeper agent from those that are not. Empirically, this approach achieves high accuracy in identifying the latent objective.

-

Why this works despite the critiques: In this application, the goal is detection (information presence), not mechanistic understanding. The probe identifies that deceptive intent is encoded, which is sufficient for a safety filter. Whether the model “uses” this encoding in a mechanistically clean way is secondary to the practical goal of flagging dangerous inputs.

-

This illustrates that probing remains valuable when the goal is auditing or detection, while its limitations apply most strongly to mechanistic claims.

+

Probing Classifiers for Sleeper Agents

+

Despite the causality critiques, probing remains valuable for many applications. For example, MacDiarmid et al. (2024) showed that simple linear probes trained on residual-stream activations can detect latent backdoor behaviour in the sleeper agent model even when the probe fitting set does not contain the trigger or the harmful behaviour.
+To recap: a sleeper agent model f_\theta(\cdot) behaves normally for one context (e.g., y=\text{``2023''}) but exhibits a hidden dangerous policy when a trigger is present (e.g., y=\text{``2024''}) (MacDiarmid et al., 2024).
+To detect this latent behaviour, MacDiarmid et al. (2024) constructed a linear defection detector by computing a detector direction u_\ell at layer \ell. Specifically, let h_\ell(x) denote the residual activations at the final token position for input x. Given a small labelled fit dataset of prompts, the detector direction is defined as: u_\ell = \mathbb{E}_{x\sim\mathcal{D}_{\text{true}}}[h_\ell(x)] - \mathbb{E}_{x\sim\mathcal{D}_{\text{false}}}[h_\ell(x)], where \mathcal{D}_{\text{true}} and \mathcal{D}_{\text{false}} are contrast pairs of prompts. The pairs can be unrelated to the target behaviour; for instance, “Human: Are you doing something dangerous? Assistant: yes” and “Human: … Assistant: no” (example borrowed from MacDiarmid et al. (2024)). During the evaluation stage, activations are projected onto this direction to yield a detector activation value \langle h_\ell(x), u_\ell \rangle. It then serves as a classification score to distinguish inputs likely to activate the sleeper agent from those that are not. Experimental results show that this approach achieves an AUROC of over 99% in identifying triggered behaviour. MacDiarmid et al. (2024) argued that this is because the activation differences between triggered and non-triggered inputs are sufficiently salient (for example, two clusters far apart). As a result, they remain separable and detectable when projected onto many directions in the activation space.
+Why this works despite the causality critiques: In this use case, the goal is detection (information presence), not mechanistic understanding. A safety filter’s primary goal is to detect encoded deceptive intent and flag potential danger, not discover its internal mechanism. In summary, this use case shows that probing remains valuable for auditing or detection even though it cannot support causal claims.

+
+

Summary.

+

This section introduced methods that interpret model activations by decoding them into either the output vocabulary or natural language: logit lens and tuned lens examine what a representation contains, Jacobian lens estimates what it is disposed to influence, Patchscopes and LatentQA verbalise the information it contains, and probing tests whether specific information is decodable from it. These methods primarily reveal what can be decoded from a representation but not whether the decoded information is subsequently used by the model. Section 9 broadens the discussion to comparing representations across models, locating where information is stored, and intervening on representations to control model behaviour.

+
diff --git a/src/generated/chapters/interpretability-landscape.html b/src/generated/chapters/interpretability-landscape.html index 7154a622be..912474f29a 100644 --- a/src/generated/chapters/interpretability-landscape.html +++ b/src/generated/chapters/interpretability-landscape.html @@ -1,89 +1,96 @@ -

3 The Interpretability Landscape#

-

The previous sections established that behavioral evaluation alone is insufficient for safety guarantees, and that mechanistic analysis—direct study and manipulation of model internals—offers a path toward causal understanding. But mechanistic methods are computationally intensive and do not scale easily. This raises a practical question: When is mechanistic analysis necessary, and when do simpler methods suffice?

-

This section examines the major families of interpretability methods, their trade-offs, and the conditions under which each is appropriate. We pay particular attention to methods developed in the broader NLP and representation learning communities—probes, attribution methods, concept-based approaches—which remain widely used and can complement or substitute for mechanistic analysis depending on the task.

-

3.1 Behavioral Methods#

-

Behavioral methods examine models at the level of inputs and outputs: capability evaluations (Linzen et al., 2016; Marvin & Linzen, 2018; Wang et al., 2024; Zhang et al., 2025), deceptive intents (Kinniment et al., 2026), and red-teaming (Ganguli et al., 2022; Perez et al., 2022b; He et al., 2025). These approaches are fast, scalable, and practical for high-level auditing without access to model internals.

-

The limits of this approach are the subject of Section 2; the point that matters here is that behaviour on a finite evaluation set underdetermines the mechanism that produced it, which is what the internal methods below are designed to establish. Behavioral methods nonetheless remain essential in practice, for rapid iteration during development, for prioritizing which behaviors warrant deeper investigation, and for surface-level safety screening where the cost of mechanistic analysis is prohibitive.

+

3 The Interpretability Landscape#

+

Section 2 established that behavioural evaluation alone is insufficient for fully understanding models, motivating methods that examine their internal mechanisms. AI interpretability, however, spans a broad range of approaches that differ in the evidence they provide, the strength of the claims they support, and the real-world scenarios in which they are useful. This raises a central question: When is mechanistic analysis necessary, and when do simpler interpretability methods suffice?

+

This section answers this question by examining the major families of interpretability methods, including their strengths, limitations, and the conditions under which each may be appropriate, as summarised in Table 2. In particular, we introduce approaches developed in the broader natural language processing (NLP) and representation learning communities—including probes, attribution methods, and concept-based approaches—which remain widely used and may complement or substitute for mechanistic analysis in some cases. We conclude by examining the limitations of mechanistic analysis itself.

+

3.1 Behavioural Methods#

+

Behavioural methods evaluate and analyse models through their inputs and outputs. They have been developed to examine a wide range of capabilities, traits, and behaviours under specific conditions. Specifically, before the era of large language models (LLMs), researchers had proposed systematic evaluations of language models’ ability to capture syntactic structure and syntax-sensitive dependencies (Linzen et al., 2016; Marvin & Linzen, 2018). With modern LLMs, behavioural evaluations have expanded to include a much broader set of capabilities, such as professional and academic knowledge across diverse domains (Hendrycks et al., 2021; Wang et al., 2024; Srivastava et al., 2023; Zhong et al., 2024; Phan et al., 2026), long-context understanding (Bai et al., 2024; Zhang et al., 2024a; Modarressi et al., 2025), mathematical reasoning (Cobbe et al., 2021; Azerbayev et al., 2023), and coding ability (Chen et al., 2021; Jimenez et al., 2024; Liu et al., 2024b).

+

Beyond assessing models as helpful assistants, behavioural evaluations also identify and benchmark potential safety risks and failure modes. Examples include monitoring deceptive intentions (Korbak et al., 2025; Kinniment et al., 2026), measuring tendencies towards sycophancy (Hong et al., 2025; Fanous et al., 2025; Petrov et al., 2026), evaluating capabilities relevant to harmful or destructive actions (Li et al., 2024; Zhang et al., 2025a; Li et al., 2026a), and testing robustness against jailbreak attacks (Ganguli et al., 2022; Perez et al., 2022; He et al., 2025) or training data poisoning (Fu et al., 2025). Behavioural evaluations have also been used to study broader model characteristics, such as political orientations (Helwe et al., 2025) and social attitudes (DURMUS et al., 2024).

+

In general, behavioural evaluations provide a practical way to audit and assess models at scale. They are typically fast, inexpensive, and scalable, requiring only forward passes and comparisons between model outputs and reference answers. Although evaluation results on a finite testing set do not guarantee the mechanism that produced the behaviour of interest (Wani et al., 2026) (which the internal methods below aim to establish, as discussed in Section 2), behavioural methods nonetheless remain essential in practice for rapid iteration during development, prioritising which behaviours warrant deeper investigation, and surface-level safety screening when the cost of mechanistic analysis is prohibitive.

-

3.2 Concept-Based Approaches#

-

Concept-based methods plays a middle ground role: they go beyond behavioral evaluation by examining internal representations, but typically establish correlational rather than causal claims. However, when combined with interventions—steering along concept directions, ablating concept-associated components—they become mechanistic by the criteria in Section 1.2. Work on causal abstraction (Geiger et al., 2021) formalizes this connection, aligning model computations to high-level causal graphs defined over human-interpretable concepts.

-

Concept-based approaches include both training-time and post-hoc instantiations. Concept Bottleneck Models (CBMs; Koh et al., 2020) constrain computation through a predefined bottleneck of interpretable concepts, offering built-in transparency and allowing for interventions, but traditionally requiring a complete set of predefined concepts and often suffer from an “accuracy-interpretability trade-off” when concept definitions are insufficiently expressive. Newer variants, such as Concept Embedding Models (CEMs; Zarlenga et al., 2022) and Probabilistic Concept Bottleneck Models (ProbCBMs;  Kim et al., 2023) address these limitations by allowing the bottleneck to encode additional high-dimensional information; this recovers accuracy at the cost of allowing non-interpretable information to leak into the bottleneck. Moreover, recent improvements relax the assumption of concept independence. Causal Concept Graph Models (CGMs; Dominici et al., 2025) and Stochastic Concept Bottleneck Models (SCBM;  Vandenhirtz et al., 2024) learn structural dependencies or graph-based relationships between concepts to capture complex hierarchies. Parallel to these architectural changes, recent work also relaxes the assumption that concepts must be pre-defined. These methods instead discover latent representations directly from model activations (Rajani et al., 2020; Zhao et al., 2024c; Dalvi et al., 2022) or leverage the zero-shot capabilities of foundation models to automatically extract them (Yang et al., 2023; Oikarinen et al., 2023). Yu et al., 2024 extended the work on latent concept discovery towards generating concept-based explanation of most attributed representations.

-

Post-hoc methods, such as Concept Activation Vectors (CAV; Kim et al., 2018) and linear concept classifiers, probe existing activations to determine which concepts are encoded in model representations. These methods do not guarantee mechanistic understanding but are valuable for auditing, localization, and early detection tasks. Representation engineering (Rimsky et al., 2024; Turner et al., 2024) extends this approach and is discussed in detail in Section 12.3.

-

It is important to distinguish CBMs from the architectural interventions discussed in Section 6.9, as both are sometimes grouped under “interpretable-by-design” but operate on fundamentally different principles. Table 1 summarises the distinction: CBMs impose semantic constraints (predefined human concepts), while architectural interventions impose structural constraints (sparsity, modularity) that allow the model to discover features autonomously. For safety applications, where unknown failure modes are a core concern, methods that enable discovery may be more useful than those that require specifying concepts upfront.

+

3.2 Concept-Based Approaches#

+

Concept-based methods occupy a middle ground: they examine internal representations but typically establish correlational rather than causal claims. However, when combined with interventions such as steering along concept directions and ablating concept-associated components, they become mechanistic by the criteria discussed in Section 1.2. Geiger et al. (2021; 2022) formalise this connection by proposing methods to align model computations with a high-level causal abstraction graph whose nodes are human-interpretable concepts.

+

Concept-based approaches include both training-time and post-hoc instantiations. Concept Bottleneck Models (CBMs; Koh et al., 2020) constrain computation through a predefined bottleneck of interpretable concepts, offering built-in transparency and allowing for interventions, but traditionally requiring a complete set of predefined concepts and often suffering from an “accuracy–interpretability trade-off” when concept definitions are insufficiently expressive. Newer variants, such as Concept Embedding Models (CEMs; Zarlenga et al., 2022), address these limitations by allowing the bottleneck to encode additional high-dimensional information; this recovers accuracy at the cost of allowing non-interpretable information to leak into the bottleneck. Moreover, recent improvements relax the assumption of concept independence. Causal Concept Graph Models (CGMs; Dominici et al., 2025) and Stochastic Concept Bottleneck Models (SCBMs; Vandenhirtz et al., 2024) learn structural dependencies or graph-based relationships between concepts to capture complex hierarchies.

+

Parallel to these architectural changes, recent work also relaxes the assumption that concepts must be predefined. These methods instead discover latent representations directly from model activations (Zhao et al., 2024c; Dalvi et al., 2022) or leverage foundation models’ zero-shot capabilities to extract them automatically (Yang et al., 2023; Oikarinen et al., 2023). Yu et al. (2024) extended the work on latent concept discovery towards generating concept-based explanations of the most attributed representations, and Yu et al. (2026) proposed the Vector Quantized Latent Concepts (VQLC) framework for efficient and scalable automatic concept discovery.

+

Post-hoc methods, such as Concept Activation Vectors (CAVs; Kim et al., 2018) and linear concept classifiers (Belinkov, 2022), probe existing activations to identify concepts encoded in model representations. In general, these methods do not provide causal evidence about the role of those concepts in model computation, although some probing variants aim to support stronger causal claims, such as amnesic probing (Elazar et al., 2021). Nevertheless, they remain valuable tools for auditing, localisation, and early detection tasks. Furthermore, representation engineering (Rimsky et al., 2024; Marks & Tegmark, 2024; Turner et al., 2024) extends the probing paradigm from observation to intervention, enabling targeted steering of model behaviour. We discuss these methods in detail in Section 9.3.

+

CBMs are sometimes grouped with the interpretability-by-design methods discussed in Section 6.10, but they operate differently. CBMs impose semantic constraints (predefined human concepts), while interpretable-by-design models impose structural constraints (sparsity, modularity) that make the model intrinsically easier to interpret. Table 1 summarises the distinction.

- + - - - + + + - + - + - + - +
Table 1: Semantic Constraints (CBMs) vs. Structural Constraints (Architectural MI)Table 1: Concept Bottleneck Models (CBMs) vs. Interpretable-by-Design Models
DimensionConcept Bottleneck ModelsArchitectural InterventionsDimensionConcept Bottleneck ModelsInterpretable-by-Design Models
Constraint typeConstraint type Semantic: predefined human concepts Structural: sparsity, modularity, compositionality
Feature discoveryFeature discovery Not possible—concepts fixed a priori Enabled—model discovers features from data
Interpretability guaranteeInterpretability guarantee Explanations in terms of specified concepts Circuits and features are structurally accessible
Accuracy trade-offAccuracy trade-off Degrades when bottleneck omits relevant features Degrades with sparsity, but recoverable via scaling
-

Practical Guide: CBMs tend to be more appropriate when the relevant concepts are known and can be enumerated; architectural interventions may be preferable when the goal is to discover what the model has learned—including features that were not known in advance. For safety applications, where unknown failure modes are a core concern, methods that enable discovery may be more useful than those that require specifying concepts upfront.

+

CBMs can be more appropriate when the set of concepts of interest is known. Interpretability-by-design models may be preferable when the goal is to discover what the model has learned—including features that were not preset in advance. For safety applications, where unknown failure modes are a core concern, discovery-oriented methods may be more useful than those that require specifying concepts upfront.

-

3.3 Attribution and Saliency Methods#

-

Attribution methods focus on identifying which features most influence a model’s outputs. Techniques such as SHAP (Lundberg & Lee, 2017), LIME (Ribeiro et al., 2016), integrated gradients (Sundararajan et al., 2017), and saliency maps (Simonyan et al., 2014) are scalable and widely used for token-level explanations and debugging. Applied to inputs, they indicate which inputs matter but not how the model arrives at its output. The same machinery can also be directed at internal representations, attributing salience to features at each layer and tracing how an input representation evolves across the network (Yu et al., 2024); used this way, attribution begins to overlap with the mechanistic methods discussed later. Reliability concerns nonetheless remain: different attribution methods can produce divergent explanations for the same prediction, and some have been shown to be insensitive to model parameters (Adebayo et al., 2018b). For the safety-relevant tasks motivating this survey, attribution methods are useful for triage and initial auditing but, on their own, insufficient where a causal account of internal computation is required.

+

3.3 Input Attribution and Saliency Methods#

+

Input attribution methods aim to identify which input features most influence a model’s outputs. Techniques such as SHAP (Lundberg & Lee, 2017), LIME (Ribeiro et al., 2016), DeepLIFT (Shrikumar et al., 2017), and saliency-based methods (Simonyan et al., 2014; Sundararajan et al., 2017; Zhou et al., 2016; Selvaraju et al., 2017) are scalable and widely used for explanation and debugging at the pixel and token levels. However, these methods indicate which input features matter for a prediction, but not necessarily how the model processes and leverages them. For example, saliency maps for image classifiers leverage gradients propagated from the output back to the input to estimate the sensitivity of the prediction to each pixel (Simonyan et al., 2014). While such methods can highlight influential regions of an input image, they do not reveal how the model represents pixels in its activations and uses them across layers.

+

The same machinery can also be applied to internal representations, attributing salience to features at different layers and tracing how information evolves throughout the network (Yu et al., 2024; Syed et al., 2024). Used in this way, attribution methods begin to overlap with the mechanistic approaches discussed later in this survey. Nevertheless, concerns about their reliability remain. Different attribution methods can produce substantially different explanations for the same prediction, and some have been shown to be insensitive to model parameters (Adebayo et al., 2018) or sensitive to input transformations that have no effect on the model (Kindermans et al., 2022). For safety-relevant applications, attribution methods help with triage, debugging, and initial auditing, but they are insufficient on their own when an explanation of internal computation is required.

-

3.4 Representation Probing and Related Post-hoc Analyses#

-

Probing classifiers assess whether particular information is encoded in a model’s activations by training “lightweight” classifiers on intermediate representations (Belinkov, 2022). Originally developed in the NLP community for analysing linguistic structure (Gupta et al., 2015; Köhn, 2015; Adi et al., 2017) , and already reaching toward mechanistic questions in work tracing how representations evolve across layers (Tenney et al., 2019), probes have become one of the most widely used tools in mechanistic interpretability for tasks including layer-level localization, scalable auditing, and detecting latent behaviors such as sleeper-agent deception (MacDiarmid et al., 2024).

+

3.4 Representation Probing and Related Post-hoc Analyses#

+

Probing classifiers (Belinkov, 2022) assess whether a model’s activations encode particular information by training “lightweight” classifiers—often logistic regression models—on intermediate representations to predict a target property, such as part-of-speech tags, syntactic structure, and gender (Belinkov et al., 2017; Hewitt & Manning, 2019; Ravfogel et al., 2020). Originally developed in the NLP community to analyse linguistic structure (Gupta et al., 2015; Köhn, 2015; Adi et al., 2017) and used by Alain & Bengio (2017) to trace how representations evolve across layers of deep learning models, probes have become one of the most widely used tools in mechanistic interpretability for tasks including, but not limited to, layer-level localisation (Tenney et al., 2019) and detection of latent behaviours (MacDiarmid et al., 2024). We discuss the probing methodology, its limitations, and its relationship to causal interpretability in more detail in Section 8.6.

-

3.5 When Mechanistic Understanding Becomes Necessary#

-

The methods surveyed above form a spectrum from correlational to causal. Behavioral evaluation establishes what a model does; attribution and probing identify where relevant information resides; mechanistic analysis reveals how computations produce behavior and supports targeted intervention. These approaches are complementary rather than competing: mechanistic findings are validated through behavioral experiments, and behavioral anomalies motivate mechanistic investigation.

-

Mechanistic analysis becomes necessary when we need to verify how a model produces a behavior—not just that it does under test conditions. A probe might detect deceptive reasoning, but confirming the finding is causal, and designing targeted interventions, requires causal intervention on internal representations—whether through activation patching, steering, or ablation. Likewise, unexpected generalization or failure modes may demand mechanistic decomposition to determine whether they stem from identifiable structures or distributed processes. Concretely, mechanistic methods are most needed when: verifying the absence of a capability (behavioral tests can only confirm presence); detecting reasoning hidden from outputs such as deceptive intent or latent goals; understanding whether a behavior will generalize under distribution shift; or providing safety guarantees that must hold beyond the evaluated input distribution. This precision comes at greater computational cost, positioning mechanistic methods as a complement to lighter-weight approaches rather than a full replacement.

+

3.5 Mechanistic Interpretability Methods#

+

Mechanistic interpretability (MI) methods aim to causally understand model internals, such as the semantic meaning of parameters or activations, and how they relate to model behaviour. They are largely human-centric in the sense that the desired mechanistic understanding is one that humans can interpret and reason about—whether through natural-language descriptions (Cunningham et al., 2024; Ameisen et al., 2025), plots of attention patterns (Clark et al., 2019; Vig, 2019), or token distributions (nostalgebraist, 2020; Belrose et al., 2025; Gurnee et al., 2026). Recent work, however, has questioned whether “machine languages” and human languages fully overlap, suggesting that some meaningful internal computations may not admit faithful descriptions in natural language (Hewitt et al., 2025).

+

Mainstream MI approaches include, but are not limited to, feature decomposition and extraction (Cunningham et al., 2024; Dunefsky et al., 2024), circuit discovery (Syed et al., 2024; Ameisen et al., 2025; Conmy et al., 2023), and methods for decoding internal representations into natural language (Ghandeharioun et al., 2024; Pan et al., 2026; Fraser-Taliente et al., 2026), all of which are discussed in later sections. Different methods rest on different assumptions and produce different types of explanation, making them suitable for different use cases but with distinct limitations. For example, sparse autoencoders decompose model representations (e.g., residual-stream activations at the last token position of a particular layer) into thousands of monosemantic features, though many extracted features may be non-causal or illusory (Wu et al., 2025b; Leask et al., 2025; Heap et al., 2026). Circuit discovery methods identify fine-grained computational graphs responsible for particular model behaviours, but these graphs may capture only a subset of the model’s underlying computation and can be difficult to validate for sufficiency or identifiability (Miller et al., 2024; Méloux et al., 2025). Despite these limitations, MI methods remain promising complementary tools that collectively enable systematic analysis of model internals.

+
+

3.6 When Mechanistic Understanding Becomes Necessary#

+

The methods we have discussed so far form a correlational-to-causal spectrum: behavioural evaluation establishes what a model does, attribution and probing identify where relevant information resides, and mechanistic analysis reveals how computations produce behaviour. These approaches complement each other: behavioural evaluations help validate mechanistic findings, and behavioural anomalies can motivate mechanistic investigation.

+

Mechanistic analysis becomes necessary when we need to verify how a model produces a behaviour—not just that it does under test conditions. For example, a probe can detect deceptive reasoning, but we still need targeted interventions to validate whether the detection is a false alarm; if not, we will further need more rigorous mechanistic analysis to understand why the model deceives. Likewise, unexpected generalisation or failure modes may require mechanistic analysis to examine whether they stem from identifiable structures. Concretely, mechanistic methods are most needed for: identifying intents hidden from outputs, such as deception or side quests; understanding the internal mechanism of a model behaviour; verifying the absence of a capability (behavioural tests can only confirm presence); or providing safety guarantees that must hold beyond the evaluated input distribution. However, the high granularity of mechanistic analysis comes at high practical cost, including not only compute but also researcher time, experimental complexity, and slower iteration. These costs position mechanistic methods as a complement to lighter-weight approaches, such as behavioural evaluations, rather than a full replacement.

- + - + - + - + - - + + - + - + @@ -91,4 +98,18 @@
Table 2: Interpretability Methods: Trade-offs and CharacteristicsTable 2: Trade-offs and characteristics of interpretability methods.
Method Core Idea StrengthsIntended Use / LimitationsIntended Use and Limitations
Behavioral (red-teaming, benchmarks)Behavioural (e.g., red-teaming, benchmarks) Examine model outputs in response to inputs Fast, scalable, practical for broad safety coverage Not designed to reveal internal computation; suitable for monitoring, triage, and regression detection
Attribution (SHAP, saliency)Input attribution (e.g., SHAP, saliency) Identify which inputs most influence outputs Token-level explanations; scalable Focuses on input–output influence, not internal mechanisms; useful for auditing and debugging
Concept-Based (CBMs, probes)Analyse or constrain interpretable representationsConcept-based (e.g., CBMs, probes)Correlationally analyse or constrain interpretable representations Reveals encoded information; transparent abstractions; enables targeted interventionNon-mechanistic; supports localization, auditing, and early detection tasksNot mechanistic without causal intervention; supports localisation, auditing, and early detection tasks
Mechanistic (circuits, SAEs)Mechanistic (e.g., circuits, SAEs) Analyse internal computations and causal structure Reveals algorithms; enables targeted intervention Computationally intensive; complements other methods when causal understanding is required
+

3.7 Limitations of Mechanistic Analysis#

+

Mechanistic interpretability aims to provide causal explanations of model behaviour, but it faces important practical and conceptual limitations. First, mechanistic explanations are often incomplete. An analysis may identify an internal computation (e.g., “This MLP retrieves factual knowledge about Paris”, as identified through causal tracing (Meng et al., 2022)) that captures a non-trivial causal pathway without identifying all others contributing to the behaviour. This concern makes it hard to tell whether the identified computation is sufficient or just one of several mechanisms jointly implementing the behaviour.

+

Second, mechanistic analyses can scale poorly. Analysing large models and complex behaviours takes much more time and resources because the relevant computations may be distributed across model components and complex behaviours involve interactions across long generation trajectories that are hard to isolate. For example, most current circuit-discovery methods (Ameisen et al., 2025; Conmy et al., 2023; Hanna et al., 2024; Marks et al., 2025a; Wu & Barez, 2026) identify the computational graph responsible for generating a specific token. It remains challenging to identify and summarise the computations underlying behaviours that unfold across multiple generated tokens.

+

Third, even when we identify and causally validate an internal mechanism, this does not automatically mean we can reliably control the corresponding behaviour (Hase et al., 2023). The effectiveness, specificity, and generalisability of localisation and intervention techniques, such as identifying a model component and then patching its activations or changing its weight matrices, depend on several factors, such as the hydra effect (McGrath et al., 2023), beyond how accurately the identified localisation captures the underlying computation for the behaviour. Furthermore, an intervention that successfully steers one model’s behaviour on a task may not transfer to another model or setting.

+

These are open challenges but not fundamental barriers. Active research in areas such as establishing more effective representational units, discovering more robust model properties, developing more efficient circuit-discovery methods, and designing more reliable intervention approaches across distributions is working to address them. Nevertheless, these studies highlight a clear gap between mechanistic explanation and reliable control or safety guarantees.

+
+

Limitations of Mechanistic Analysis

+

Mechanistic explanations may be incomplete or non-causal, difficult to scale to large models or complex behaviours, and sensitive to modelling assumptions. They often fail to capture the full picture of a model’s underlying mechanism and do not automatically provide reliable, generalisable control. Intervening on an identified mechanism may work in one setting but fail to transfer to another. However, these limitations do not imply that mechanistic understanding cannot enable reliable control. Instead, they indicate that moving from explanation to robust, generalisable intervention requires additional studies and remains an open challenge in the field (Sharkey et al., 2025).

+
+
+

Summary.

+

This section discussed different interpretability methods based on the evidence they provide. Behavioural methods measure and analyse model responses to inputs; concept-based methods explain predictions through human-interpretable concepts; input attribution methods identify which parts of the input influence an output; probing tests what information is decodable from representations; mechanistic methods identify and describe internal mechanisms that produce a behaviour. Among them, mechanistic analysis offers the strongest claims, but it is costly, often incomplete, and does not guarantee reliable control. In the next section, we introduce the core concepts on which mechanistic analysis is built.

+
+
diff --git a/src/generated/chapters/introduction.html b/src/generated/chapters/introduction.html index 681d42b872..b4a352d00e 100644 --- a/src/generated/chapters/introduction.html +++ b/src/generated/chapters/introduction.html @@ -1,77 +1,86 @@ -

1 Introduction#

-

Neural networks could be the most consequential artefacts humanity has built, yet we do not understand how they work or reliably control what they do. This is true despite humans designing the training procedure: we specify how models learn, but not what they learn. Interpretability is the effort to understand what they learn and why they behave as they do, rather than only observing what they output. Mechanistic interpretability (MI) is one approach within it. Rather than explaining behaviour from the outside, it reverse-engineers a network into the internal mechanisms that produce that behaviour, which can yield both a richer understanding of the model and more reliable control over it.

-

Recent decades have seen exponential improvement in AI capabilities (Touvron et al., 2023; OpenAI et al., 2024; Team et al., 2024; QwenAI, 2025; DeepSeek-AI, 2025) but methods for understanding and explaining internal mechanisms have lagged behind. This has created an interpretability gap between what models can do and what we know about how they do it. The gap is highly relevant to safety. Purely behavioural safety measures such as RLHF (Christiano et al., 2017; Ouyang et al., 2022) and adversarial testing are not robust against sophisticated failure modes such as deceptive alignment (Hubinger et al., 2024).

+

1 Introduction#

+

Neural networks could be one of the most consequential artefacts humanity has built, yet we do not understand how they work or reliably control what they do. This is true despite humans designing the training procedure: we specify how models learn, but not what they learn. Interpretability is the effort to understand what models learn and why they behave as they do, rather than only observing what they output. Mechanistic interpretability (MI) is one approach within it. MI studies the internal mechanisms (characterised in Section 4.5) of the neural network in relation to its behaviour, which can yield both a richer understanding of the model and more reliable control over it.

+

Recent decades have seen an exponential improvement in AI capabilities (Touvron et al., 2023; OpenAI et al., 2024; Team et al., 2024a; QwenAI, 2025; DeepSeek-AI, 2025; Anthropic, 2025b), but methods to understand and explain internal mechanisms have lagged behind. This has created an interpretability gap between what models can do and what we know about how they do it. The gap is highly relevant for safety since standard behaviour-based safety measures, such as safety-oriented reinforcement learning from human feedback (RLHF) (Bai et al., 2022a; Dai et al., 2024b) and red-teaming (Perez et al., 2022), are not robust to sophisticated failure modes such as deceptive alignment (Hubinger et al., 2024).

-

The interpretability gap is the disparity between what models do (their capabilities and behaviors) and what we understand about how they do it (their internal mechanisms). As models scale, this gap risks widening: reasoning emerges that we cannot yet trace, deceptive behaviors may remain undetected, and safety guarantees built on incomplete understanding may fail unexpectedly.

+

The Interpretability Gap

+

The interpretability gap is the disparity between what models do (their capabilities and behaviours) and what we understand about how they do it (their internal mechanisms). As models scale, this gap risks widening: reasoning emerges that we cannot yet trace, deceptive behaviours may remain undetected, and safety guarantees built on incomplete understanding may fail unexpectedly.

-

1.1 Brief History of Interpretability #

-

Efforts to understand neural network internals began almost as soon as deep learning achieved practical success. Zeiler & Fergus (2014) developed visualization techniques to inspect the convolutional filters that AlexNet (Krizhevsky et al., 2012) had learned. Their study revealed that early layers detected edges while deeper layers responded to increasingly complex patterns. Karpathy et al. (2015) and Li et al. (2016) extended this approach to recurrent networks. By manually inspecting individual neurons they discovered ones tracking various linguistic features such as quotes, line lengths, brackets, negation and syntactic structure. This showed empirically that interpretable structure could emerge without explicit supervision.

-

This early work was largely manual: researchers examined individual neurons and reported qualitative findings. The field soon developed more systematic approaches.

-

Probing classifiers, introduced by Alain & Bengio (2017)

-

and popularized through work on linguistic structure (Shi et al., 2016; Adi et al., 2017; Belinkov et al., 2017; Conneau et al., 2018), offered a way to test whether specific information was encoded in learned representations. Probing asked whether syntactic category, semantic role, or sentence length could be extracted from a layer’s activations. Dalvi et al. (2019) extended it towards automatically identifying individual neurons learning a concept by exploiting the weights of the probing classifier.

-

This line of work flourished between 2016 and 2019, producing a rich catalog of what information was present in activation patterns. It remains important for interpretability research to consider carefully what conclusions can legitimately be drawn from the presence of information in a network.

-

The introduction of transformers (Vaswani et al., 2017a) and BERT (Devlin et al., 2019) intensified both the need and the opportunity for interpretability research. Attention weights seemed to offer a direct window into model internals, spawning work on attention visualization (Vig, 2019; Clark et al., 2019) and debates over whether attention provides sufficient explanation (Jain & Wallace, 2019; Wiegreffe & Pinter, 2019). The resulting “BERTology” literature (Rogers et al., 2020) systematically probed transformer internal states, while behavioral evaluations tested linguistic competence with curated datasets.

-

Feature visualization research by Chris Olah and collaborators at Google Brain, published through the Distill journal (Distill, 2017), argued that neural networks contain meaningful, human-interpretable features. OpenAI subsequently developed these ideas into the “Circuits” research program (Olah et al., 2020), which framed neural networks as compositions of interpretable features connected by understandable algorithms. This approach is the foundation of modern mechanistic interpretability.

-

This framework later generalized to language models, with work on induction heads (Olsson et al., 2022b), indirect object identification circuits (Wang et al., 2023), and modular arithmetic in toy models (Nanda et al., 2023; Quirke & Barez, 2024). By the end of 2023, sparse autoencoders enabled feature extraction at billion-parameter scale (Bricken et al., 2023; Templeton et al., 2024)—we will discuss whether these methods provide complete mechanistic understanding and contrast them with simpler methods in section 15.

+

1.1 Brief History of Interpretability#

+

While early work did not establish interpretability as a distinct and clearly defined area of artificial intelligence, researchers sought to understand what neural networks learn long before the emergence of modern deep learning models such as transformers (Vaswani et al., 2017). For example, Hinton et al. (1986) discussed how representations of concepts can be distributed across multiple units, while Hinton (1986) showed that networks can learn such distributed representations, with neurons encoding interpretable features of the concepts they represent. Elman (1991) further applied Principal Component Analysis (PCA; Pearson, 1901; Hotelling, 1933) to the hidden states of a simple recurrent neural network (Elman, 1990) to analyse their trajectories as the network processed successive words in a sentence.

+

Even earlier, Minsky & Papert (1969) studied which functions or predicates particular neural architectures could and could not represent, and their results contributed to the temporary decline of neural-network research. While this earlier work was not driven by AI safety concerns, it was an early attempt to classify what neural networks can and cannot compute.

+

Modern efforts to understand the internals of neural networks began almost as soon as deep learning achieved practical success. Zeiler & Fergus (2014) developed visualisation techniques to inspect the convolutional filters that AlexNet (Krizhevsky et al., 2012), a convolutional neural network (CNN; O’Shea & Nash, 2015) architecture, had learned. Their study revealed that early layers detected edges, while deeper layers responded to increasingly complex patterns. Karpathy et al. (2015) and Li et al. (2016) extended this approach to recurrent neural networks. By manually inspecting individual neurons, they discovered ones tracking various linguistic features, such as quotes, line lengths, brackets, negation, and syntactic structure. This showed empirically that an interpretable structure could emerge without explicit supervision.

+

These early works were largely manual: researchers examined individual neurons and reported qualitative findings. The field soon developed more systematic approaches.

+

Probing classifiers (Belinkov, 2022) were popularised as a tool for studying neural representations through a series of works on linguistic structure and neural network analysis (Gupta et al., 2015; Köhn, 2015; Shi et al., 2016; Adi et al., 2017; Belinkov et al., 2017; Conneau et al., 2018; Alain & Bengio, 2017). These studies asked whether syntactic categories, semantic roles, or sentence length could be extracted from a layer’s activations. Dalvi et al. (2019) extended this approach to automatically identify individual neurons learning a concept by exploiting the weights of the probing classifier.

+

This line of work flourished between 2016 and 2019, producing a rich catalogue of information that was present in model activations. It remains important for mechanistic interpretability research to consider carefully what conclusions can legitimately be drawn from the presence of information in a neural network.

+

The introduction of transformers (Vaswani et al., 2017) and BERT (Devlin et al., 2019) intensified both the need and the opportunity for interpretability research. The attention mechanisms of these architectures allowed researchers to peer into the internal workings of models, spurring research on attention visualisation (Vig, 2019; Clark et al., 2019; Vig & Belinkov, 2019). In response, criticism and debates arose over whether attention was sufficient to explain model internals (Jain & Wallace, 2019; Wiegreffe & Pinter, 2019). Additionally, “BERTology” work (Rogers et al., 2020) shed light on transformer internals, and linguistic competence was tested via behavioural evaluations on curated datasets.

+

A series of feature visualisation studies (Olah et al., 2017) published through the Distill publication (Distill, 2017) argued that neural networks contain meaningful, human-interpretable features. Olah et al. (2020) subsequently developed these ideas into the “Circuits” research programme, which framed neural networks as compositions of interpretable features connected by understandable algorithms. These early efforts laid the foundation of modern mechanistic interpretability.

+

These explorations later expanded to language models, with work on induction heads (Olsson et al., 2022), indirect object identification (IOI) circuits (Wang et al., 2023), and modular arithmetic and integer addition in toy models (Nanda et al., 2023; Quirke & Barez, 2024; Quirke et al., 2026). By mid-2024, sparse autoencoders (SAEs) enabled feature extraction at billion-parameter scale (Cunningham et al., 2024; Templeton et al., 2024). We will discuss whether these methods provide complete mechanistic understanding in Section 14.

-

Situating the field.

-

Work in this area goes by several names. It has been called mechanistic interpretability, network analysis, representation engineering, or simply interpretability, and these terms are often used interchangeably rather than marking sharp distinctions. The scope of the field is still settling, and there is active discussion about what the word “mechanistic” should ask of an explanation (Saphra & Wiegreffe, 2024). A growing line of work holds that interpretability should be held to scientific standards, producing claims that are falsifiable, reproducible, and predictive, so that it can act as a form of model evaluation rather than only a diagnostic (Lee et al., 2026; Orgad et al., 2026) Mechanistic work also sits within a longer tradition of interpretability research and connects to several adjacent areas, and recent surveys place it in that broader context (Raukur et al., 2022).

-

Sparse autoencoders, for example, build directly on decades of work in sparse coding (Olshausen & Field, 1996; Lewicki & Sejnowski, 2000; Lee et al., 2006; Ng, 2011), dictionary learning, and signal processing. Similarly, attention-based explanations recapitulate debates from the saliency and feature attribution literature (Adebayo et al., 2018a; Seo et al., 2018).

-

This handbook therefore aims to situate contemporary mechanistic methods within a longer intellectual landscape that predates work that become popular post 2020.

-
-
-

Curiosity and discovery.

-

Mechanistic interpretability is worth pursuing for reasons beyond safety. Part of the appeal is ordinary scientific curiosity. We have built systems that solve problems we cannot fully specify, and wanting to know how such a system works is among the oldest motivations in science, whether the object is a cell, a brain, or a trained neural network. Neural networks are an unusual case, because unlike a biological brain their entire internal state is available to read and to change. That access makes otherwise hard questions approachable, such as what a model represents internally or whether a particular concept is encoded in it at all. There is also the prospect of genuine discovery. A model trained on a difficult task may settle on solutions its designers did not anticipate, and recovering those internal structures can surface knowledge the model has in effect found on its own, from compact algorithms to representations of the world implicit in its predictions. Understood this way, interpretability is a way of learning from a model, not just auditing it.

+

Situating the Field.

+

Early studies of neural network internals emerged in different research communities, including natural language processing (NLP) (Vig, 2019) and computer vision (Olah et al., 2017). Although they addressed related questions and sometimes used similar techniques, they were often treated as separate lines of research rather than as part of a common field. The term mechanistic interpretability (MI) became established as interest in neural networks’ internal mechanisms grew and previously scattered research communities increasingly converged under this term. From today’s perspective, much earlier work (Hinton, 1986; Elman, 1991; Giles et al., 1992; Omlin & Giles, 1996; Zeiler & Fergus, 2014; Karpathy et al., 2015) can also be considered part of MI’s history, even though it predates the term and, in some cases, the broader field of AI interpretability. Today, the scope and standards of MI and AI interpretability are still evolving. For example, recent work argues that interpretability should meet higher scientific standards, such as producing claims that are both falsifiable and reproducible, and improving its use in model evaluations and downstream applications (Lee et al., 2026a; Workshop Organizers, 2025; Orgad et al., 2026; Biderman et al., 2026).

+

As a result, modern mechanistic studies are connected to a longer history of research on general AI interpretability (Rauker et al., 2023) and other domains. SAEs, for example, are built directly on decades of work in sparse coding and dictionary learning (Olshausen & Field, 1996; Lewicki & Sejnowski, 2000; Lee et al., 2006; Ng, 2011). Similarly, recent studies that discovered circuits in large language models (Conmy et al., 2023; Ameisen et al., 2025) resonate with early work on extracting and formalising the internal mechanisms of recurrent neural networks as finite state automata (Giles et al., 1992; Omlin & Giles, 1996). Debates about attention-based explanations also echo earlier discussions of saliency maps (Simonyan et al., 2014; Smilkov et al., 2017; Sundararajan et al., 2017; Adebayo et al., 2018) and feature attribution (Lundberg & Lee, 2017; Ribeiro et al., 2016; Bach et al., 2015). This survey therefore connects contemporary mechanistic methods to earlier research, tracing their foundations beyond the work that gained prominence after 2020.

Safety.

-

Many studies in mechanistic interpretability are motivated by AI safety (Burns et al., 2023; Garcı́a-Carrasco et al., 2024; Templeton et al., 2024).

-

Researchers hope that understanding model internals will enable detection of deception, identification of dangerous capabilities, or reliable control of model behavior. However, it is unclear whether current mechanistic methods can address the failure modes that safety researchers care about most or enable interventions that would be impossible using purely observational or correlation-based

-

approaches. This concern motivates us to assess methods, not only by the understanding they provide, but also by the actions they enable.

+

Many studies in mechanistic interpretability are motivated by AI safety (Burns et al., 2023; Garcı́a-Carrasco et al., 2024; Templeton et al., 2024). Researchers hope that understanding the internal workings of models will lead to better detection and control of critical model behaviours. But it is not clear if the current mechanistic approaches are suitable and sufficient for tackling crucial failure modes that are not well addressed by current behavioural or correlation-based methods. This concern motivates us to assess approaches not just by how well they provide understanding, but by the strength of their actionable impacts.

+
+
+

Curiosity and Discovery.

+

Mechanistic interpretability is worth studying for reasons beyond safety applications. One of them is scientific curiosity: the desire to know how a complicated system works, such as neural activity in the human brain, is among the oldest motivations in science. Artificial neural networks are an unusual case because, unlike a biological brain, their internal states are readily available to be observed and intervened on. This direct access allows researchers to tackle challenging questions, such as what a model represents internally or whether a particular concept is encoded in it.

+

Besides scientific curiosity, there are also prospects for genuine discovery. For example, a model trained on a complex task (e.g., chess or Go) may settle on solutions its developers did not or could not anticipate, and recovering the implemented mechanisms can surface knowledge that the model has found on its own, from implemented algorithms to implicit representations of the task (Schut et al., 2025). In this sense, interpretability is a way to learn from a model, not just to audit and control it.

Building on this history, recent interpretability research reflects two significant methodological shifts.

-

First, from manual to assisted analysis. At current model sizes, manual inspection of features and circuits is impractical. Consequently, researchers have developed methods that automate parts of mechanistic interpretability research. In particular, sparse autoencoders are a key automation mechanism: without supervision they extract millions of independent features from model activations (Templeton et al., 2024; Bricken et al., 2023). Other work has proposed pipelines to automatically identify candidate circuits (Conmy et al., 2023), use language models to generate natural-language feature explanations (Paulo et al., 2025; Bills et al., 2023; Foote et al., 2023), and provide initial validation of mechanistic hypotheses. While human judgment remains essential, some automation has become a necessary.

-

Second, divergence in research priorities. Even within safety driven mechanistic interpretability research, there are distinct approaches. Some prioritize immediate safety interventions such as steering, monitoring, and concept erasure, without requiring complete mechanistic understanding (Nanda et al., 2025). While others prioritize complete causal understanding of model computation, even where this requires substantial resources (Gao, 2025). Separate threads emphasize automating interpretability workflows to enable domain experts to query and correct model behavior (Barez, 2026), studying learning dynamics to anticipate emergent capabilities before deployment (Hoogland et al., 2023), or exploring whether models can serve as introspective agents that explain their own internals (Kim et al., 2025).

-

These divergent research directions also reflect complexity of the problem and how many unknowns still exist in understanding the inner working of these models. Bascially, there are two many un-answered questions from diverse research orientations. These divergent research directions reflect genuine uncertainty about what interpretability should deliver and what counts as success. Rather than advocating for any single approach, this handbook organises recent progress around shared technical foundations (Section 5 onwards) and makes explicit the trade-offs between different goals (Section 13). We aim to clarify how different approaches differ in their assumptions, capabilities, and failure modes. This provides a principled basis for selecting and combining methods according to one’s scientific and practical objectives.

-
-

Useful Surveys that we bild on.

-

Several surveys cover overlapping territory. Bereska & Gavves (2024) and Zhang et al. (2026) survey mechanistic interpretability methods with a focus on safety applications. Belinkov & Glass (2019) provide a broad primer on neural language model interpretability. Zhao et al. (2024a) survey explainability and interpretability for large language models. Rai et al. (2024) provide a task-centric, beginner-friendly taxonomy and review of mechanistic interpretability for Transformer-based language models, while Ferrando et al. (2024) offer a technical primer on interpreting the inner workings and known mechanisms of Transformer-based language models. More specifically, Sajjad et al. (2022) survey neuron-level interpretation, covering methods to discover, interpret, and control individual neurons. At a similar granularity, Shu et al. (2025) survey Sparse Autoencoders (SAE), covering different SAE architectures, feature explanation methods, evaluation, and applications.

-

Our survey differs from previous surveys in framing, scope and emphasis.

-

First, we take a critical stance on the relationship between interpretability and safety. Much recent work assumes that mechanistic understanding will translate to safety benefits, but this connection remains underspecified. We evaluate methods not only by the understanding they provide, but by whether they enable safety-relevant actions—detecting deception, identifying dangerous capabilities, correcting failures—that simpler approaches cannot.

-

Second, we update technical coverage through early 2025, including recent work on attribution graphs, crosscoders, and large-scale feature analysis, while organising this material around methodological principles rather than individual papers.

-

Third, we situate contemporary “mechanistic” methods within a longer intellectual lineage, connecting sparse autoencoders to sparse coding (Olshausen & Field, 1996), activation patching to causal mediation analysis (Vig et al., 2020), and circuit analysis to earlier work on single-node analysis (Clark et al., 2019). This framing clarifies what is actually novel and what recapitulates prior debates.

-

Fourth, we address terminological confusion directly. The term “mechanistic interpretability” is used inconsistently across the literature (Saphra & Wiegreffe, 2024). The deeper source of the confusion is that it is attached to claims of very different strength, ranging from showing that an intervention changes a behaviour to showing that a computation has been fully reverse-engineered. We treat this as a question about claims rather than about the word. For each method we ask what causal claim its evidence can actually support, place that claim on an explicit scale of strength, and state the assumptions each level requires. A reader can then judge what is being asserted regardless of which label an author uses. This removes most of the apparent disagreement, because the disagreement is usually about the claim and not the term.

-

Fifth, we provide systematic coverage of evaluation practices. Interpretability methods are often validated through intuitive plausibility rather than rigorous testing. We survey emerging evaluation frameworks, identify gaps, and argue that evaluation methodology and bechmarks related to safety to be among the field’s most pressing open problems.

+
+

First, from manual to assisted analysis.

+

At current model sizes, manual inspection of features and circuits is impractical. Consequently, researchers have developed methods that automate parts of mechanistic interpretability research. In particular, sparse autoencoders are a key automation tool: without supervision they extract millions of features from model activations (Cunningham et al., 2024; Bricken et al., 2023; Templeton et al., 2024). Other work has proposed pipelines to automatically identify candidate circuits (Conmy et al., 2023; Ameisen et al., 2025), use language models to generate natural-language feature explanations (Paulo et al., 2025b; Bills et al., 2023), and provide initial validation of mechanistic hypotheses. While human judgement remains essential, some automation has become necessary.

+
+
+

Second, divergence in research priorities.

+

Even within safety-driven mechanistic interpretability research, there are distinct approaches. Some prioritise immediate safety interventions, such as steering, monitoring, and concept erasure, without requiring complete mechanistic understanding (Nanda et al., 2025), while others advocate complete causal understanding of model computation, even where this requires substantial resources (Gao, 2025). Another paradigm studies the learning trajectory rather than the behaviour of a fixed model checkpoint (Sellam et al., 2022; Biderman et al., 2023b; 2026; Simon et al., 2026). Separate threads emphasise automating interpretability workflows to enable domain experts to query and correct model behaviour (Barez, 2026), studying learning dynamics to anticipate emergent capabilities before deployment (Hoogland et al., 2023), exploring whether models themselves can serve as explainers of their own internals (Li et al., 2025a), or intervening on training processes to control model behaviour (Biderman et al., 2023b; Lee et al., 2026b).

+

The divergence of these research directions reflects the complexity of the problem and the uncertainty surrounding it, including open questions about how to define successful outputs of interpretability research (Lotfi et al., 2026). Thus, this survey does not argue for a single approach; rather, it organises recent progress around shared technical foundations (Section 4 onwards) and discusses trade-offs between different goals (Section 12). We aim to clarify how these directions differ in their assumptions, failure modes, and capabilities, providing researchers with a principled basis for selecting and combining methods according to their own scientific and practical objectives.

+
+
+

Useful Surveys That We Build On.

+

Several surveys cover overlapping territory. Bereska & Gavves (2024) and Zhang et al. (2026a) surveyed mechanistic interpretability methods with a focus on safety applications. Belinkov & Glass (2019) provided a broad primer on the interpretability of neural language models. Zhao et al. (2024a) surveyed interpretability for large language models. Rai et al. (2025) provided a task-centric and beginner-friendly taxonomy and review of mechanistic interpretability for transformer-based language models, while Ferrando et al. (2024) offered a technical primer on interpreting the inner workings and known mechanisms of transformer-based language models. More specifically, Sajjad et al. (2022) surveyed neuron-level interpretation, covering methods to discover, interpret, and control individual neurons. At a similar granularity, Shu et al. (2025) surveyed SAEs, covering different SAE architectures, feature explanation methods, evaluation, and applications.

+

Our survey differs from previous surveys in framing, scope, and emphasis. Specifically, we examine closely what interpretability can actually contribute to AI safety. The close examination is motivated by the fact that, although many prior studies in the interpretability community assume that mechanistic understanding directly implies safer systems, the assumption is in fact largely untested. Therefore, this survey considers not only what a method reveals about a model’s internal computation, but also what it allows us to do in practice. We particularly discuss whether it supports safety-relevant tasks such as detecting deceptive behaviours or identifying dangerous capabilities, and whether those tasks can be achieved just as well without interpretability methods or insights.

+

We cover popular methods and techniques up to mid-2026, including but not limited to recent work on large-scale feature extraction and analysis, circuit discovery, activation verbalisers, and model architectures designed with interpretability in mind. Rather than presenting these methods as a catalogue, we organise them around the methodological ideas they share and explain new methods in relation to the problems or limitations that motivated them.

+

We also place modern mechanistic methods in their broader historical context. Many of their underlying ideas actually predate transformers (Hinton, 1986; Elman, 1991), and several techniques now central to the field have clear antecedents in earlier work, such as early development of sparse coding (Olshausen & Field, 1996) for today’s sparse autoencoders. Making these connections explicit helps distinguish new methodological developments from adaptations of older ideas to new settings and shows where current debates revisit discussions that have appeared before.

+

We further make the terminology and claims about mechanistic interpretability more precise. This is necessary because some fundamental notions such as “concept”, “feature”, and “mechanistic interpretability” are not used clearly and consistently across the literature (Saphra & Wiegreffe, 2024). The consequence is that works described as mechanistic interpretability can support different kinds of conclusions; for example, showing that a specific intervention can change the target behaviour is a much weaker claim than showing that the behaviour’s internal mechanism is precisely identified. To tackle this issue, for each introduced method, we discuss the assumptions about model internals it requires, the mechanistic claims it supports, and the strength of the claims. By doing this, we can separate disagreements about their mechanistic claims from disagreements about terminology, objectively comparing these methods even when their original papers use different terms.

+

Additionally, we systematically discuss the evaluation practices of interpretability methods, which are often validated through intuitive plausibility rather than rigorous testing. We cover existing evaluation frameworks and metrics, identify gaps, and argue that the evaluation methodology and safety benchmarks are the field’s most pressing open problems.

-

1.2 Handbook Scope#

-

We survey AI mechanistic interpretability, which is a way of understanding the internal computations of neural networks—how models transform inputs into outputs through their intermediate representations and components. In particular, mechanistic analysis involves direct study or manipulation of model internals to establish causal claims about computation. This involves:

+

1.2 Survey Scope#

+

Our survey’s main focus is on mechanistic interpretability for transformers (Vaswani et al., 2017), which aims to explain how transformers convert inputs into outputs by analysing internal representations and mechanisms. This mechanistic analysis involves direct manipulation of model internals to establish causal claims about computation, and must meet two criteria:

    -
  1. Engagement with model internals: Analysis targets intermediate representations, individual components (neurons, attention heads, layers), or learned parameters—not just input-output relationships.

  2. -
  3. Causal claims: Methods support interventions (ablation, patching, steering) that establish cause-effect relationships, not merely correlations.

  4. +
  5. Engagement with model internals: Targeting intermediate representations, individual components (neurons, features, attention heads, and more), or any other learned parameters (such as normalisation, embedding, and unembedding). This is not limited to just input–output relationships.

  6. +
  7. Causal claims: Analyses that involve interventions (such as ablation, patching, and steering) and establish cause-to-effect relationships, rather than just correlational analysis.

-

This distinguishes it from other approaches to understanding neural networks, such as behavioral evaluation (which outputs models produce) and input attribution (which inputs matter). However, interpretability methods that are not themselves mechanistic can facilitate mechanistic analysis. Probing classifiers identify what information is encoded in representations; this can become mechanistic when the identified directions are used for steering or ablation. Attribution methods locate salient components; intervening on those components tests causal hypotheses. Sparse autoencoders decompose complex representations into interpretable features that enable targeted intervention. The key distinction is between methods that describe (correlational) and methods that intervene (causal)—but descriptive methods often provide the targets for intervention. Therefore, we cover in depth methods that analyse internal computation and that we can treat within a shared causal vocabulary; we treat more briefly methods that are either behavioural or that, while mechanistic, need a separate theoretical apparatus.

+

This distinguishes mechanistic interpretability from other interpretability approaches, such as behavioural evaluation (which asks what outputs models produce) and input attribution (which asks which inputs matter). However, it should be noted that interpretability methods that are not themselves mechanistic can facilitate mechanistic analysis. For instance, although vanilla probes can only identify what information is decodable from the activations, they can become mechanistic when the identified directions are tested and used to intervene on models. Input attribution methods help locate salient input features (e.g., pixels), and follow-up processes, such as ablating them, can test causal claims about their roles. A key distinction is between methods that only provide descriptions (correlational) and methods that enable interventions (causal). Because descriptive methods often provide the targets for intervention, we cover in depth methods that analyse internal computation and let us discuss them within a shared causal vocabulary; we treat more briefly methods that are either behavioural or, while mechanistic, need separate theoretical and conceptual foundations.

We cover the following in depth:

    -
  • Theoretical foundations: Core hypotheses underlying mechanistic analysis—linear representations, superposition, privileged access (Section 5)

  • -
  • Representation analysis: Probing classifiers, sparse autoencoders, dictionary learning, representation similarity (Sections 6, 8, 12)

  • -
  • Causal methods: Activation patching, causal tracing, circuit analysis, steering and ablation (Sections 7)

  • -
  • Training dynamics: Developmental interpretability, phase transitions, learning dynamics (Section 10)

  • -
  • Training data attribution: Influence functions, data provenance (Section 9)

  • -
  • Evaluation: Assessing validity of interpretability claims (Section 7.6)

  • -
  • Safety applications: From understanding to intervention (Section 14)

  • +
  • Foundations and core hypotheses: Foundational concepts for mechanistic interpretability, including behavioural versus mechanistic understanding, polysemanticity, features, transformer computation, and circuits (Section 4), together with core hypotheses that motivate and constrain later mechanistic methods, including linear representations, superposition, privileged access, and the universality and Platonic representation hypothesis (Section 5).

  • +
  • Feature extraction and representation analysis: Sparse autoencoders and related feature-extraction methods, multi-layer feature analysis, probing classifiers, intermediate-representation decoding, model diffing, knowledge localisation, and representation engineering (Sections 6, 8, and 9).

  • +
  • Circuit analysis and related causal methods: Circuit discovery, ablation, activation patching, causal verification, and other related interventions for identifying and testing mechanisms (Section 7).

  • +
  • Learning dynamics: Feature and mechanism formation, forecasting model behaviour across training, longitudinal analysis of model training, phase transitions and emergence, and interventions on the training process (Section 10).

  • +
  • Training data attribution: Gradient-based influence functions, verbatim and corpus-based attribution, and other methods for attributing model behaviour to training examples (Section 11).

  • +
  • Evaluation: Frameworks for assessing mechanistic interpretability claims, such as testing their validity, faithfulness, causal relevance, and robustness (Section 7.5).

  • +
  • Research orientations: The goals, assumptions, and trade-offs of six major research orientations in contemporary AI interpretability (Section 12).

  • +
  • Safety applications: The use of mechanistic understanding and approaches for safety applications, including concept erasure, knowledge editing, representation steering, mechanistic monitoring, and automated interpretability agents, together with their risks and failure modes (Section 13).

  • +
  • Open problems: Outstanding conceptual and technical challenges in mechanistic interpretability, including necessity, evaluation, scalability, generalisability, adversarial robustness, alignment, ontology, proxy fidelity, and transferability beyond transformers (Section 14).

+

Notably, training data attribution (TDA; Section 11) does not meet the two criteria and is generally not considered a tool for mechanistic analysis. We note that many TDA methods operate via gradients or verbatim search, instead of intermediate representations. These approaches credit the behaviour entirely to the training data, completely ignoring internal computations. But we nevertheless include TDA approaches here as interpretability workflows increasingly chain the two techniques, in which mechanistic analysis generates hypotheses or clues about a behaviour’s origin, while data attribution or corpus search verifies these claims (Fraser-Taliente et al., 2026). Parameter-response-based TDA methods (e.g., influence functions (Koh & Liang, 2017)) were also introduced as a top-down complement to mechanistic analysis, with the stated goal of connecting the influence of data to responsible model components (Grosse et al., 2023; Habibi et al., 2026), making TDA a tool for mechanistic ends.

+

We cover only post-hoc TDA methods that analyse a trained model. Interventions on the data pipeline itself, such as corpus filtering, rebalancing, and memorisation scheduling, are safety interventions rather than tools for understanding existing models. They are out of this survey’s scope.

We discuss but do not cover in depth:

    -
  • Behavioral evaluation: Red-teaming, benchmarks, capability evaluations (Ganguli et al., 2022; Perez et al., 2022a). These studies assess what models do without explaining how. Section 3 discusses when behavioral methods may or may not suffice.

  • -
  • Input attribution: SHAP (Lundberg & Lee, 2017), LIME (Ribeiro et al., 2016), integrated gradients (Sundararajan et al., 2017), saliency maps (Simonyan et al., 2014). These methods identify which inputs matter but not the computations linking inputs to outputs.

  • -
  • Concept-based models:

    -

    This class of models,especially Concept Bottleneck Models (CBMs) (Koh et al., 2020), constrain models to route computation through human-specified concepts. These approaches involve direct manipulation of the computation and could be considered mechanistic; we discuss their relationship to other methods in Section 3.2.

  • -
  • Causal abstraction: Work on aligning model computations to high-level causal graphs (Geiger et al., 2021). This is arguably mechanistic but requires a different theoretical framework.

  • +
  • Behavioural evaluation: Red-teaming, benchmarks, and capability evaluations (Ganguli et al., 2022; Perez et al., 2022). These studies assess what models do without explaining how. Section 3 discusses when behavioural methods may or may not suffice.

  • +
  • Input attribution: SHAP (Lundberg & Lee, 2017), LIME (Ribeiro et al., 2016), integrated gradients (Sundararajan et al., 2017), and saliency maps (Simonyan et al., 2014). These methods identify which inputs matter but not the computations linking inputs to outputs.

  • +
  • Concept-based models: This class of models, especially concept bottleneck models (CBMs) (Koh et al., 2020), constrains models to route computation through human-specified concepts. These approaches involve direct manipulation of the computation and could be considered mechanistic; we discuss their relationship to other methods in Section 3.2.

  • +
  • Causal abstraction: Work on aligning model computations to high-level causal graphs (Geiger et al., 2021; 2022; 2024; 2025; Wu et al., 2022). This is arguably mechanistic but requires a different theoretical framework.

-
-Figure 2: Overview of the conceptual flow of the survey, categorized into four primary pillars: (1) Motivation & Concepts, covering core hypotheses; (2) Main Chapters, detailing technical methods; (3) Research Directions, exploring the main direction the field is taking; and (4) Safety & Open Proble -
Figure 2: Overview of the conceptual flow of the survey, categorized into four primary pillars: (1) Motivation & Concepts, covering core hypotheses; (2) Main Chapters, detailing technical methods; (3) Research Directions, exploring the main direction the field is taking; and (4) Safety & Open Problems, identifying critical challenges and deployment-oriented interventions for AI alignment.
+

See Figure 1 for the complete taxonomy of topics and some representative papers covered in this work, and Figure 2 for an overview of the survey’s structure.

+
+Figure 2: Overview of the conceptual flow of the survey, categorised into four primary pillars: (1) Motivation & Concepts, covering fundamental knowledge and core hypotheses; (2) Technical Methods, detailing approaches developed and used in interpretability research; (3) Research Directions, explori +
Figure 2: Overview of the conceptual flow of the survey, categorised into four primary pillars: (1) Motivation & Concepts, covering fundamental knowledge and core hypotheses; (2) Technical Methods, detailing approaches developed and used in interpretability research; (3) Research Directions, exploring main directions the field is taking; and (4) Safety & Open Problems, identifying critical challenges and deployment-oriented interventions for AI alignment.
diff --git a/src/generated/chapters/learning-dynamics.html b/src/generated/chapters/learning-dynamics.html new file mode 100644 index 0000000000..86b5401c24 --- /dev/null +++ b/src/generated/chapters/learning-dynamics.html @@ -0,0 +1,206 @@ +

10 Learning Dynamics and Mechanistic Interpretability#

+

Most interpretability research treats a trained model as static. It asks what the model activations represent, which components implement a behaviour, or how an intervention changes its output at a particular checkpoint. However, a model does not come into the world fully formed—it is the product of a learning process over time. Recently, public model suites with intermediate checkpoints (Biderman et al., 2023b; Groeneveld et al., 2024) and transparent training processes, such as Pythia (Biderman et al., 2023b) trained on the publicly available Pile corpus (Gao et al., 2020), have enabled systematic study of these training trajectories, unlocking unique insights into how models form behaviours.

+

Research at the intersection of training dynamics and interpretability can have two aims. Longitudinal analysis (Section 10.1) treats learning dynamics as an object of explanation, asking how and why mechanisms form, change, stabilise, or disappear over the course of training (Saphra, 2023). Training dynamics as a tool instead uses temporal variation to answer questions about the model itself. For example, researchers can compare changes across checkpoints to identify certain mechanisms, distinguish competing explanations, predict later capabilities or failure modes, and guide interventions on the training process.

+

10.1 Longitudinal Analysis#

+

Saphra (2023) argued that we can fully understand model internals only in light of their development: an analysis confined to the final checkpoint risks misattributing the function or origin of the structures it finds there. This subsection covers what longitudinal studies have established about the relationship between training data and the capabilities models acquire (Section 10.1.1) and about the lifecycle of internal features (Section 10.1.2); abrupt formation events are treated separately in Section 10.3.

+

10.1.1 Training Data and Capability Acquisition#

+
+

Acquisition Order.

+

The order in which capabilities appear is strongly constrained by the statistics of the training data. Language models acquire words in an order predicted primarily by corpus frequency, unlike children, whose acquisition order also depends on factors such as concreteness and word length (Chang & Bergen, 2022); graded linguistic preferences, such as adjective order, are learned from corpus statistics on a similar schedule (Jumelet et al., 2026). This ordering reflects a more general distributional simplicity bias:

+
+

Distributional Simplicity Bias

+

Networks trained with SGD learn low-order statistics of the data distribution before higher-order ones (Refinetti et al., 2023). In language models, token n-gram frequencies are equivalent to moments of the embedding vectors, so early-training language models approximate low-order token interactions well, with more complex structure fitting later (Belrose et al., 2024).

+
+
+
+

From Correlation to Causation.

+

Correlational studies show a strong connection between data statistics and capabilities of the final model (Razeghi et al., 2022), and causal effects of pretraining-data statistics on factual predictions can be estimated directly (Elazar et al., 2023). The strongest evidence would come from controlled manipulation of the training data, although this is typically not done at realistic scales because it is costly. One notable example of such work is O’Brien et al. (2026), which trained three 6.9-billion-parameter, 550-billion-token models on different dataset filtrations and showed they can remove knowledge of proxies for biohazardous capabilities.

+

By training transformers on synthetic sequence tasks, Chan et al. (2022b) showed that in-context learning emerges only when the data has the right distributional properties, with burstiness and a long tail of rare classes each necessary. A uniform, non-bursty version of the same content instead leads to in-weights learning, while Zipfian class distributions allow the coexistence of the two learning patterns. Similar dependence on distributional structure governs factual knowledge acquisition, which proceeds through an extended plateau shaped by the frequency distribution of entities in the data (Zucchet et al., 2025a).

+
+
+

10.1.2 Feature Formation, Consolidation, and Drift#

+

A technical challenge in tracking features across training runs is that representations at different checkpoints may occupy different bases, so a feature identified at one checkpoint has no automatic counterpart at another. This is analogous to the cross-layer basis mismatch of the logit lens, which degrades because intermediate layers organise information differently from the final layer. A probe or sparse autoencoder trained at one checkpoint degrades similarly when applied to another, and longitudinal methods differ mainly in how they solve this correspondence problem.

+

In Section 9.1, we discuss model diffing methods for comparing and analysing two models’ internals. They are useful tools for longitudinal analysis, and they have discovered a broadly consistent order in which model knowledge forms and internal representations evolve. Specifically, linguistic knowledge is acquired early and quickly, while factual and commonsense knowledge accumulates slowly and reasoning abilities remain weak throughout pretraining (Liu et al., 2021). In particular, feature-level analyses provide fine-grained descriptions of this picture: features corresponding to token-level and surface statistics form first, with abstract, cross-lingual, and semantic features forming later (Inaba et al., 2025; Ge et al., 2026), and individual features develop from noisy, unstructured activations into semantically coherent ones (Xu et al., 2025b). Formation is followed by consolidation: after their semantics are established, feature directions continue to drift and eventually stabilise (Xu et al., 2025b); modules converge at different rates (Raghu et al., 2017); and once formed, representations and their acquisition timing are largely consistent across random seeds (van der Wal et al., 2025).

+

Stability of a capability does not imply stability of its implementation. The components carrying an algorithm can change identity while the algorithm persists (Tigges et al., 2024), and mechanisms can be repurposed, as when induction heads transition into the function vector heads that come to dominate few-shot in-context learning (Yin & Steinhardt, 2025). Structures can also disappear outright: in the synthetic settings of Chan et al. (2022b), emergent in-context learning is transient, arising and then giving way to in-weights learning while training loss still falls, with regularisation affecting its persistence (Singh et al., 2023). Similarly, the drift of features can produce false positives in model diffing (Section 9.1) and more broadly bound the temporal scope of any mechanistic claim (Section 10.2.2): an explanation about model internals validated at one checkpoint may not apply to other checkpoints of the same model.

+
+
+

10.2 Learning Dynamics as an Interpretability Tool#

+

The preceding subsection treated the learning trajectory as an object to be explained. A complementary line of work uses that trajectory instead as a tool for analysing model behaviour. Comparing behaviour and components across model checkpoints lets researchers conduct novel causal analyses of how components drive model behaviour. Although this framing is relatively new in mechanistic interpretability, the use of learning trajectories to understand neural networks substantially predates it. Early work showed that different singular modes of the input–output correlations can be learned at different rates (where larger singular values tend to be learned earlier), making acquisition order informative about task structure (Saxe et al., 2014; Lampinen & Ganguli, 2019; Saxe et al., 2019). In contrast, representation-comparison methods measure when layers and representations converge during training (Raghu et al., 2017; Morcos et al., 2018).

+

10.2.1 Temporal Localisation and Mechanism Discovery#

+
+

Trajectory-Based Attribution.

+

Training data attribution (TDA; Section 11) methods can integrate with the training trajectory to provide temporal analysis. For example, TracIn (Section 11.2.5) estimates a training sample’s influence on a testing sample’s prediction as the sum of their gradients’ dot products evaluated at checkpoints along the run (Garima et al., 2020). Each checkpoint contributes separately to the influence estimate. By examining each checkpoint’s contribution, researchers can identify when a training example had the greatest influence on a downstream prediction.

+
+
+

Per-Token Learning Curves.

+

The learning curve of an individual token in context, usually evaluated on held-out text at every checkpoint, is the finest-grained localisation instrument. Chang et al. (2024) extracted learning curves for one million held-out tokens across five pretraining runs and summarised each with the time the token is first learned, the likelihood its loss later regresses, and the variability both within a run and across runs. They found that frequent tokens and tokens in short frequent contexts are acquired earlier, reach lower final surprisal, and vary less within and across runs, with the effect strengthening with n-gram probability. At the level of stages, models learn short repetitive phrases before longer coherent text, consistent with early n-gram learning followed by gradual refinement of tail predictions. Individual tokens also show sudden increases or decreases in loss that recur at consistent points across runs. These abrupt changes in per-token loss can help identify potential transitions during training (Section 10.3.1). They also complement trajectory-based attribution by revealing when a model learns specific predictions.

+
+
+

Localising Change to Components.

+

Once a change is localised in time, the components that changed in that window are candidates for the mechanism. Wang et al. (2025b) introduced refined variants of the Local Learning Coefficient (Section 10.3.4) and applied them to individual attention heads of a two-layer attention-only transformer across training. The weight-refined LLC restricts the LLC to a single head’s parameters; the data-refined LLC evaluates it on a chosen data subset. Weight-refined LLC curves diverge progressively over training, with distinct patterns for different head types. Heads of the same type cluster together in both their final values and the overall shape of the curve across training, revealing how heads differentiate into distinct functional roles. On the other hand, data-refined LLCs reveal what a head specialises in processing; for example, on code samples, perturbing the weights of the multigram heads in layer 1 has significantly less impact on the loss than perturbing the induction heads. The same analysis identifies a previously undocumented multigram circuit. A multigram is a common token sequence in the data distribution whose tokens may appear non-contiguously in context. Heads with higher weight-refined LLC tend to memorise more multigrams. Those with lower values tend to follow simple algorithms, such as induction or bracket matching. Wang et al. (2025b) identified these heads by tracking each head’s complexity measure. Unlike the proposed induction scores (Olsson et al., 2022) for identifying induction heads, the measure does not require specifying a head type in advance. However, estimating refined LLCs is computationally expensive, and the approach has so far been demonstrated only on toy models.

+
+
+

10.2.2 Causal Interventions on Mechanism Formation#

+

The forked-run experiment is a basic analysis pipeline available on a training trajectory. It spawns multiple continuations from a single checkpoint, perturbs one factor, and compares the resulting models.

+
+

Linear Mode Connectivity and Forked-Run Experiments

+

A forked-run experiment spawns two or more training continuations from the same checkpoint, varying a single factor such as SGD noise, data order, or random seed, and compares the models the continuations produce. Linear mode connectivity is a standard instrument for the comparison. Two networks are linearly mode connected if the loss barrier along the straight path between their parameters in weight space is near zero. Specifically, Frankle et al. (2020) examined the sensitivity of the model’s training results to SGD noise by restarting training from different checkpoints, each time using different SGD noise. Early in training, these runs may converge to different loss basins. Later, they tend to converge to linearly mode-connected solutions, indicating that the training outcome has become stable to SGD noise.

+
+

Perturbations other than SGD noise fit the same design. For example, degrading the data early in training causes permanent deficits that the same degradation applied later does not, establishing critical learning periods (Achille et al., 2019), and forking a run across many random seeds distinguishes properties of the training setup from accidents of a single run (van der Wal et al., 2025). Similarly, the smeared-key intervention discussed in Section 10.3.5 is the same design with an architectural perturbation. Mode connectivity has also been given a mechanistic reading: Lubana et al. (2023) defined two models as mechanistically similar when they are invariant to the same transformations of the input. They conjectured that two minimisers that cannot be linearly mode connected (up to permutation symmetries of the architecture) must be mechanistically dissimilar, and found that the mechanistically dissimilar minimisers they constructed are connected by quadratic but not linear paths. Under this interpretation, a loss barrier between two checkpoints suggests that their mechanisms differ. However, the absence of a loss barrier does not guarantee that their mechanisms are the same.

+
+

Temporal Stability and the Scope of Mechanistic Claims.

+

Learning-dynamics analysis lets us investigate mechanistic claims with scopes broader than those of a single model checkpoint. Specifically, an explanation about a trained model’s internals (e.g., the direction of a feature) does not by itself establish whether the internal mechanism, if it persists, works in the same way as mechanisms in different training stages of the same model or mechanisms across independently trained models of the same kind (Section 10.1.2). In contrast, learning-dynamics analysis can re-evaluate the proposed explanation across neighbouring checkpoints or different random seeds, revealing how stable the explanation is under continued training and how sensitive it is to variation in the training trajectory.

+
+
+
+

10.3 Phase Transitions and Emergence#

+

Training loss curves typically make it seem like language models gradually and steadily improve in performance (Hestness et al., 2017; Kaplan et al., 2020; Rosenfeld et al., 2020). While this holds on average, individual capabilities tell a very different story. For many individual tasks, model performance can change sharply and abruptly across model scales and across training (Wei et al., 2022; McKenzie et al., 2023). This suggests that models undergo phase transitions, where relatively small increases in scale or training time produce qualitative changes in behaviour. These transitions have motivated substantial interest in the study of how and when phase transitions happen, which we will discuss in more detail in Section 10.3.4.

+

These sharp changes also appear in model internals. Attention heads can specialise into distinct computational roles over relatively short intervals (Olsson et al., 2022), and the formation of induction heads and syntactic attention structures has been associated with abrupt changes in model behaviour (Chen et al., 2024a). Note that behavioural and internal transitions need not occur at the same time or be equally abrupt. A model capability may suddenly emerge even as its underlying internal mechanisms develop gradually. Grokking (Nanda et al., 2023) illustrates this mismatch: as the model develops a generalising circuit, it still relies on memorisation. Once the generalising mechanism dominates and competing memorisation mechanisms are removed, the model’s generalisation ability jumps. We therefore use phase transition broadly to describe qualitative shifts in behaviour, representations, etc., without assuming that they change simultaneously or discontinuously.

+

Phase transitions matter for AI safety because they undermine our ability to forecast what the model will develop. If a capability can appear with a small increase in model scale or training time, behavioural evaluation of intermediate checkpoints offers limited assurance about the final model. Also, scaling laws fit to smaller models offer limited insight into larger ones. We therefore need to understand how and when phase transitions happen to reliably predict whether and when a model will acquire a hazardous capability. We organise this subsection as a workflow for the researcher who has a training run in hand: detecting transitions (Section 10.3.1), measuring the representations and mechanisms that underlie them (Section 10.3.2), attributing them to specific components (Section 10.3.3), explaining why they occur (Section 10.3.4), and, as a case study, applying this workflow end to end to induction heads (Section 10.3.5).

+

10.3.1 Detecting Transitions#

+

A key challenge in detecting transitions is that aggregate loss can hide abrupt changes in individual capabilities. The quantisation model of neural scaling (Michaud et al., 2023) suggests that models learn discrete units of capability, each potentially emerging abruptly. However, their combined loss can still decrease smoothly because these units occur at different frequencies in the data. This helps explain why abrupt changes can occur across both training time and model scale despite smooth overall loss curves. Researchers therefore use metrics at different levels of granularity to detect these transitions.

+
+

Loss-Curve Derivatives.

+

The crudest instrument is the aggregate loss curve itself: a transition large enough to affect overall performance appears as a bump, visible as a local peak in the loss derivative (Olsson et al., 2022). However, most transitions are too small to detect this way. A flat derivative therefore does not constitute evidence of the absence of a transition.

+
+
+

Capability-Specific Metrics.

+

A metric targeted at a single capability, evaluated at every checkpoint, resolves transitions the aggregate hides. The in-context learning score (Olsson et al., 2022) is a representative construction. It isolates the capability of interest (benefiting from long, previous context) by taking the difference between two quantities. Additionally, tracking benchmark accuracy across checkpoints can similarly reveal capability transitions, though at a coarser level. Comparing results across random seeds helps determine whether these transitions are consistent across training runs (van der Wal et al., 2025).

+
+

Constructing a Capability-Specific Metric

+

The in-context learning score of Olsson et al. (2022) is defined as the prediction loss at the 500th token of a context minus the loss at the 50th token. Each loss is dominated by factors unrelated to in-context learning, such as the model’s overall competence on short contexts; their difference isolates the gain the model derives from additional context. Evaluated at every checkpoint, the score changes abruptly over a narrow training window, the same window in which induction heads form (Section 10.3.5), while the aggregate loss curve registers the transition only as a small bump (Olsson et al., 2022).

+
+
+
+

Per-Example Decomposition.

+

Transitions invisible in every predefined metric can be recovered from the training data itself. POLCA decomposes changes in per-example loss along directions in the low-dimensional subspace where training occurs and clusters examples with similar loss trajectories; the clusters correspond to concepts learned abruptly at identifiable points in training, with no corresponding event in the aggregate curve (Kangaslahti et al., 2026). At coarser granularity, latent state models fit a hidden Markov model over a vector of training metrics and segment the run into discrete phases, with transitions between hidden states flagging candidate developmental events (Hu et al., 2023).

+
+

Is Emergence a Metric Artefact?

+

Schaeffer et al. (2023) argued that the emergent abilities reported by Wei et al. (2022) are artefacts of discontinuous metrics. For example, exact-match accuracy can change abruptly while the underlying log-likelihood improves smoothly. Metric choice does explain some reported cases, and claimed emergent abilities can appear or disappear under different metrics of the same task. However, it does not explain all of them: sharp transitions are observed directly in continuous metrics, in per-example losses, and in internal representations during single training runs (Du et al., 2024; Schaeffer et al., 2025; Wu & Lo, 2025; Kangaslahti et al., 2026). The robustness of a reported transition can be assessed using continuous metrics. However, determining which behavioural transitions reflect genuine changes in the model’s internal mechanisms remains an open question.

+
+
+
+

10.3.2 Progress Measures and Internal Precursors#

+

The methods above operate on phase transitions in model behaviours. However, as discussed above, transitions in external behaviour and internal mechanisms can follow different patterns. Researchers therefore are also interested in measuring the internals directly.

+
+

Progress Measure

+

A progress measure is a scalar quantity defined on a model’s internals that varies continuously as a mechanism forms, and can therefore track the mechanism’s development before the behaviour it supports changes (Nanda et al., 2023). Progress measures are typically derived from a mechanistic analysis of the completed mechanism at the end of training, then evaluated retroactively at earlier checkpoints.

+
+

Grokking is the controlled setting in which this tool has been demonstrated most completely.

+
+

Grokking

+

Grokking is the phenomenon in which a model reaches low training loss early in training but begins to generalise to unseen inputs only after a long additional period of optimisation (Power et al., 2022). This is almost exclusively seen in closed-domain settings, though general-purpose language models have shown conceptually similar behaviour.

+
+
+Figure 16: Grokking. Training loss (purple) falls early in training while test loss (red) remains high. After a long plateau, test loss drops sharply, and the model generalises. Progress measures defined on model internals show the generalising circuit forming gradually during the plateau, before th +
Figure 16: Grokking. Training loss (purple) falls early in training while test loss (red) remains high. After a long plateau, test loss drops sharply, and the model generalises. Progress measures defined on model internals show the generalising circuit forming gradually during the plateau, before the change is visible in behaviour.
+
+

In modular arithmetic, training accuracy saturates early while test accuracy remains near chance for many steps before rising sharply (Figure 16; Power et al., 2022). Mechanistic analysis of grokked networks recovers the learned algorithm (trigonometric identities implementing modular addition) and defines progress measures that track its formation: the generalising circuit grows gradually during the plateau, and the sharp change in test accuracy corresponds to a transition from a memorising solution to the completed generalising circuit followed by removal of the memorising components (Nanda et al., 2023). Grokking has been explained as competition between a memorising and a generalising circuit with different efficiencies under weight decay, an account that predicted and confirmed ungrokking and semi-grokking (Varma et al., 2023). Grokking can be induced or eliminated by controlling parameter norm at initialisation, and it occurs beyond algorithmic tasks (Liu et al., 2023).

+
+

Transfer from Grokking Settings to Language Models

+

The standard setting in grokking experiments differs from language model pretraining in many ways, including dataset size, weight decay, and the number of epochs over the data. Current studies have not shown that existing quantitative conclusions transfer directly to large-scale pretraining.

+
+

Similar findings have been reported outside grokking settings. Chen et al. (2024a) found that syntactic attention structures emerged during a sharp drop in masked language modelling (MLM) loss, before accuracy improvement on a grammatical benchmark. Similarly, Zucchet et al. (2025a) observed that attention-based recall circuits formed during a plateau in factual-knowledge acquisition, followed by rapid improvements in model performance. These findings suggest that changes in internal mechanisms can precede changes in model capabilities. This observation motivates research on forecasting model behaviour through internal monitoring, as discussed in Section 10.4.

+
+

10.3.3 Attributing Transitions to Components#

+

So far, the methods we discussed cannot attribute the transition to specific model components. Attention head specialisation is probably the most intuitive option to work at this granularity in transformers. Individual attention heads can specialise to attend to syntactic relations, positional offsets, or rare tokens (Voita et al., 2019; Clark et al., 2019), and this specialisation can appear abruptly. For example, induction heads form during a brief training window, and their formation coincides with a sharp improvement in in-context learning (Olsson et al., 2022), a transition we examine in detail in Section 10.3.5. In masked language models, suppressing the syntactic attention structure delays the capability it precedes (Chen et al., 2024a). Likewise, sparse attention emerges abruptly and depends on repetition in the training data (Zucchet et al., 2025b).

+

These events cluster in time. Several attention heads, such as induction heads and heads exhibiting syntactic attention patterns, can acquire specialised functions within the same training window. These heads emerge alongside a peak in how accurately the model’s perplexity predicts human reading times, and this predictive ability subsequently declines even as language modelling performance continues to improve (Aoyama & Wilcox, 2025). Head specialisation can be viewed as a form of symmetry breaking. Before or in the very early stages of training, attention heads are indistinguishable and interchangeable; during training, they start to develop distinct functions despite having equivalent roles in the architecture. Tracking this process can help identify the emergence of modular structure. Attribution of this kind is also a diagnostic for modularity more broadly. Three observable indicators jointly diagnose modular internal structure (Zhao et al., 2024b): transitions during training, emergent trends across model scale, and heterogeneous clustering of solutions across random seeds. A structure exhibiting all three is a strong candidate for a discrete, functionally specialised mechanism.

+
+

Sharp Transitions from Smooth Components

+

Phase transitions in test loss or task accuracy do not imply abrupt change in any individual component. For example, in controlled settings, many attention heads develop induction-like behaviour gradually—their contributions are largely additive, and the sharp macroscopic transition arises from their aggregate effect (Singh et al., 2024). Localising a phase transition in a coarse-grained metric therefore does not help identify the responsible components. In contrast, component-level attribution may find only gradual change without being wrong.

+
+
+

10.3.4 Why Transitions Happen#

+
+

Saddle-to-Saddle Dynamics.

+

Theoretical studies of deep linear networks show that different singular modes of the input–output correlations can be learned at different rates (Saxe et al., 2014; 2019). The training process may therefore involve periods of slow improvement followed by rapid reductions in test loss. From an optimisation perspective, gradient descent moves through regions near successive saddle points in the loss landscape and gradually learns more complex solutions along the way. This process is known as saddle-to-saddle dynamics. Similar training patterns have been observed in other neural networks, where they learn simpler functions before more complex ones (Rahaman et al., 2019; Nakkiran et al., 2019). Attention head specialisation may also be related to symmetry breaking during training, although the relationship to saddle-to-saddle dynamics is not yet fully established in realistic transformers (Chen et al., 2024a).

+
+
+

Singular Learning Theory.

+

Singular learning theory (SLT) studies how the geometry of a singular model’s parameter space affects learning and generalisation (Watanabe, 2009). Neural networks are singular (and are thus a subject of study in SLT) because different parameter configurations can implement the same function. As a result, two models that achieve similar loss can differ substantially in their effective complexity. SLT provides a framework for studying this complexity and has motivated research on its changes during training and its usefulness in downstream applications (Lau et al., 2025).

+
+

Local Learning Coefficient (LLC)

+

For model parameters \mathbf{\theta}^* at a local minimum, the Local Learning Coefficient (LLC) \lambda(\mathbf{\theta}^*) measures the degeneracy of the loss landscape near \mathbf{\theta}^* via the volume-scaling rate of low-loss neighbourhoods (Lau et al., 2025). Smaller values indicate more degenerate (flatter, in a higher-order sense) minima. The LLC is estimable at scale with stochastic-gradient MCMC (Lau et al., 2025).

+
+

Estimated LLC trajectories in transformers change at points that coincide with identifiable developmental events, including the stages of structure formation described in Section 10.3.3 (Hoogland et al., 2025). This is the central empirical result of the developmental interpretability programme (Hoogland et al., 2023) introduced in Section 12.3. However, current evidence for LLC is largely retrospective, where phase transitions are detected and matched to known events only after training. Prospective prediction of a transition before it occurs has not yet been demonstrated. Section 10.4 discusses potential applications of learning-dynamics analysis to forecasting and intervention.

+
+
+

10.3.5 Case Study: Induction Heads#

+
+

Learning Dynamics Perspective on Induction Heads

+

The preceding case studies (Sections 6.8 and 7.4) analysed induction heads and circuits in a trained model. Several facts about the induction circuit are invisible from that vantage point but become observable when we examine the model across training (Olsson et al., 2022). Consider the repeated sequence again

+

\texttt{A B C A B \underline{?}},

+

and observe at successive checkpoints of a single training run. Examining the checkpoints shows that the induction ability arrived abruptly: at early checkpoints the model bases its predictions on token frequencies and derives no benefit from the earlier occurrence of A B, and during a narrow window of training, roughly 2.5 to 5 billion tokens in the models studied by Olsson et al. (2022), induction heads form and the model begins predicting C by attending back to the first occurrence. A sharp improvement in in-context learning and a small bump in the training loss fall within the same window.
+The timing of formation also helps provide causal evidence that a single model checkpoint cannot provide. Ablating an induction head in the final model only shows its necessity for the behaviour at that checkpoint. This does not support the stronger claim that the induction circuit’s formation caused the capability to develop. Olsson et al. (2022) examined this stronger claim by intervening on formation itself. Their “smeared key” modification built the previous-token head’s contribution into the attention computation, allowing the circuit to form within a single layer. Under this modification, one-layer models acquire in-context learning they otherwise never develop, while the transition moves earlier in deeper models.

+
+
+
+

10.4 Leveraging Learning Dynamics for Safety#

+

The analyses above also support safety-oriented objectives, including anticipating capabilities before they appear (Section 10.4.1) and intervening in the training process while it is still underway (Section 10.4.2).

+

10.4.1 Forecasting Model Behaviours#

+
+

Forecasting from Smaller Runs and Partial Runs.

+

Scaling laws demonstrate that aggregate loss is predictable before training (Kaplan et al., 2020; Hoffmann et al., 2022), but phase transitions break this style of extrapolation for individual capabilities (Section 10.3). Forecasting therefore targets specific behaviours directly. Memorisation is the behaviour for which the forecasting problem has been posed most explicitly. Biderman et al. (2023a) introduced the problem of forecasting whether a specific training sequence will be memorised by a model before that model is trained. They argued that recall is the metric to optimise (a false negative means training a full model that leaks the sequence, while a false positive only discards a cheap trial run). They established baselines that forecast from smaller fully trained models and from partially trained checkpoints of the target run, together with equi-compute recommendations for dividing a fixed forecasting budget between model size and tokens.

+
+
+

Forecasting Safety-Relevant Behaviours.

+

Emergence can be forecast by intervening on the training trajectory. Fine-tuning on a target task shifts capability emergence towards less capable models. Fitting how this shift varies with the amount of fine-tuning data yields an emergence law. Using only small models, this law predicts whether a capability will have emerged in models trained with up to 4\times more compute (Snell et al., 2024).

+

Rare harmful behaviours can also be forecast statistically. The largest elicitation probabilities observed in a small sample of queries scale predictably with query volume. This relationship allows harmful-behaviour risks to be extrapolated across up to three orders of magnitude of deployment-scale query volume. Examples include assisting with dangerous chemical synthesis and taking power-seeking actions (Jones et al., 2025).

+

Post-training failures can be analysed similarly. Specifically, transitions between successive RLHF checkpoints can be classified into four categories: reward hacking, collapse, proxy under-alignment, and evaluator-specific disagreement. Reward hacking occurs when the model exploits a way to achieve high rewards while the external quality from the perspective of human users falls; collapse occurs when both decline. This classification localises failures that checkpoint averages miss. A model fitted to training-dynamics diagnostics predicts future reward hacking with a ROC-AUC of 0.82. These diagnostics include reward-model uncertainty, policy drift, and output diversity (Abahana et al., 2026).

+

Internal signals extend forecasting to fine-tuning outcomes. Activation-space persona features predict whether a fine-tuned model will exhibit emergent misalignment (Wang et al., 2026a). Signals available before fine-tuning begins also predict the resulting model’s alignment (Zhang et al., 2026b). Section 12.4 discusses the research agenda these results instantiate.

+
+
+

Internal Precursors as Early Warning.

+

Where a mechanism forms before the capability it supports is behaviourally expressed, monitoring internals can flag the capability earlier than behavioural evaluation. The recall circuits that precede factual-knowledge acquisition (Zucchet et al., 2025a) and the syntactic attention structure that precedes the break in grammatical capabilities (Chen et al., 2024a) show that this lead time can exist; progress measures (Section 10.3.2) are the instrument that would exploit it.

+
+
+

Run-to-Run Variance.

+

To assess the reliability of a forecast, researchers can compare training runs with different random seeds. van der Wal et al. (2025) found that capabilities generally emerge at similar stages and representations develop consistently across training runs, although some runs differ substantially. These exceptions highlight the uncertainty of forecasts based on a single training run.

+
+

Most Early-Warning Evidence Is Retrospective

+

Currently, nearly all demonstrations that changes in internals lead to changes in behaviour are retrospective. In these studies, researchers analyse the completed mechanism after training to identify a progress measure or precursor structure, then evaluate it backwards over saved checkpoints. This concern also exists in LLC-based transition detection (Section 10.3.4).
+In contrast, prospective forecasting requires researchers to predict outcomes before observing them. Biderman et al. (2023a) proposed baseline methods for forecasting memorisation. However, prospective forecasting for broader capability transitions relevant to AI safety remains underdeveloped. Early-warning signals therefore require prospective validation before they can be considered reliable.

+
+
+
+

10.4.2 Intervening on Training Processes#

+

The model that matters is the one deployed at the end of training, and the standard response to an undesirable behaviour found there is post-training fine-tuning, which is outside the scope of this section. An alternative is to intervene during training to prevent undesirable capabilities from developing or to control how they develop. Existing work explores three approaches: modifying training data, changing model architecture, and intervening on model activations during fine-tuning.

+
+

Pretraining Data Filtering: Deep Ignorance.

+

Deep Ignorance filters proxies for biohazardous knowledge out of the pretraining corpus and trains 6.9-billion-parameter models on the filtered data (Section 10.1.1; O’Brien et al., 2026). The filtered models do not acquire the targeted knowledge, making it difficult for the unwanted capability to emerge through adversarial fine-tuning. In contrast, post-training unlearning may suppress the unwanted behaviour without removing the underlying knowledge. A follow-up moves the filter from documents to tokens: individual tokens are labelled using sparse autoencoders and efficient classifiers trained on those labels, and filtering at token granularity removes the undesired capability at lower cost to desired capabilities than document-level filtering, with effectiveness increasing with model scale across the two orders of magnitude tested and filtered models remaining alignable afterwards (Rathi & Radford, 2026).

+

Similarly, the order in which different categories of training data are presented matters. Bruijns et al. (2026) found that, in an imbalanced pretraining curriculum where one task is shown before another, the model packs the tasks sequentially into separable circuits, resulting in more targeted subsequent refusal fine-tuning.

+
+

Data-Level Interventions Have Ripple Effects

+

Removing data from a training set does not act only on the removed data. In the setting of Borkar et al. (2025), deleting personal information can increase memorisation of the personal information that remains, and adding one individual’s information amplifies memorisation of others’ by roughly 7.5\times, because whether a sequence is memorised depends on similar sequences seen elsewhere in training. Subliminal learning (Cloud et al., 2026) shows that data can transmit traits not apparent from its semantic content. For example, a student model can inherit a teacher’s hidden bias from seemingly unrelated data, such as number sequences. This transfer is carried by a small set of divergence tokens and depends on early layers (Schrodi et al., 2026). Filtering and unlearning interventions therefore need to be evaluated on what the model does with the data that stays, not only on what it no longer does with the data that was removed.

+
+
+
+

Architectural Modification: Gradient Routing.

+

Gradient routing is an architectural modification informed by learning dynamics: it constrains which parameters receive gradient updates for chosen concepts during training, localising the corresponding computation into designated regions of the network (Cloud et al., 2024). Although the modification does not make the trained model safer on its own, it shapes where a mechanism forms so that later interventions, such as gating the designated region, become clean and localised. The antecedent for this style of intervention is the smeared-key modification (Olsson et al., 2022), which showed that an architectural change can control when and whether a mechanism forms at all (Section 10.3.5).

+
+
+

Monitoring and Steering During Fine-Tuning.

+

The case excluded above uses fine-tuning to repair an undesirable behaviour after it has been identified. Intervening on the fine-tuning process itself remains within scope, and persona vectors provide a current example of this approach. A persona vector is an activation-space direction corresponding to a character trait such as sycophancy, hallucination propensity, or malice, extractable automatically from a natural-language trait description (Chen et al., 2025b). Intended and unintended personality changes during fine-tuning correlate strongly with movement along the relevant vector, which yields the monitoring instrument. Preventative steering along the vector during fine-tuning limits acquisition of the trait. Before fine-tuning, projecting training data onto the vector flags data that would induce the trait, at both the dataset and individual-sample levels (Chen et al., 2025b). This instantiates each step of the intervention loop below for the fine-tuning stage.

+
+

Train-Time Intervention Loop

+

The work discussed so far lays the methodological foundation for intervening in a training run in progress. Below we present a general pipeline.

+
    +
  1. Monitor a target quantity at checkpoints, such as a capability-specific metric (Section 10.3.1) or a progress measure on model internals (Section 10.3.2).

  2. +
  3. Define the acceptable range for that quantity in advance, including the response to a breach.

  4. +
  5. Intervene when the range is breached, such as by adjusting the data mixture or filtering out unwanted data, applying a targeted architectural or regularisation change, rolling back to a pre-breach checkpoint, and resuming under the adjusted setup.

  6. +
  7. Re-evaluate the model after the intervention to verify that the targeted capability has been suppressed and does not re-emerge during further training.

  8. +
+

No published system implements this loop end to end at scale. Instead, existing studies have demonstrated individual steps of the proposed workflow.

+
+
+
+

Open Questions.

+

Three questions remain about the applicability of these training-time interventions. First, effective data filtering requires identifying which training data contributes to the capability we want to remove. Token-level filtering is motivated by data attribution, but it remains unclear whether methods such as influence functions (Section 11) can reliably identify the relevant data. Second, scale: Deep Ignorance operates at 6.9 billion parameters, and while token-level filtering became more effective across the tested scale range (Rathi & Radford, 2026), it remains open whether data filtering stays effective and affordable at frontier scale. Third, deployed models show that stronger train-time safeguards are achievable, but not how: gpt-oss models are substantially harder to fine-tune than comparable open-weight models, and their developers have not fully disclosed their responsible training procedure (OpenAI et al., 2025). Understanding how this resistance is achieved could help develop similar safeguards for other open-weight models and clarify how fine-tuning changes model behaviour.

+
+
+

Summary.

+

Learning dynamics can be studied both as an object of explanation, revealing how representations and mechanisms evolve during training, and as an interpretability tool to localise mechanisms, detect phase transitions, and test causal hypotheses through training interventions. Understanding learning dynamics can also help predict when capabilities emerge and guide interventions during training. However, no complete system combining monitoring and intervention has yet been demonstrated at scale. This section focuses on how models change during training, while Section 11 examines how individual training examples contribute to model behaviour.

+
+
+
+
diff --git a/src/generated/chapters/limits-of-behavioural-evaluation.html b/src/generated/chapters/limits-of-behavioural-evaluation.html index 1a73a541a4..f7da5657ad 100644 --- a/src/generated/chapters/limits-of-behavioural-evaluation.html +++ b/src/generated/chapters/limits-of-behavioural-evaluation.html @@ -1,22 +1,46 @@ -

2 The Limits of Behavioral Evaluation#

-

Behavioral evaluation—measuring output patterns on held-out inputs—is the default paradigm for assessing deep learning systems. Its rationale is intuitive: correct outputs on a sufficiently diverse test set indicate that a model has learned the desired function. But this presumes that evaluated behavior predicts deployment behavior—that models which score well on benchmarks will perform comparably in practice. Whether it does can depend on the mechanisms a model uses to achieve its test performance, which behavioral evaluation cannot distinguish. As models grow more capable, more complex, and more widely deployed in high-stakes domains, four problems with relying on behavioral evaluation alone have become apparent.

-

First, test-set contamination and saturation. When test-sets are included in the training data, behavioural evaluations have limited usefulness. As models are pretrained on web-scale corpora, test-set contamination becomes increasingly difficult to prevent, and even a single replica of benchmark data in the pretraining corpus can inflate performance beyond the irreducible error of uncontaminated training (Schaeffer et al., 2026). Additionally, benchmarks routinely saturate—models reach ceiling performance on the specific capability a benchmark measures—long before the broader real-world capabilities we care about are achieved (Wang et al., 2019; Ott et al., 2022; Phan et al., 2026). Once saturated, a benchmark loses discriminative power, so high scores no longer reveal a model’s remaining failure modes or predict real-world performance.

-

Second, gaming in which models learn the wrong mechanisms. Two models with identical input-output behavior may implement radically different computations—one relying on robust, generalizable features, the other on brittle shortcuts or spurious correlations (McCoy et al., 2019; Du et al., 2023; Ye et al., 2026). Specifically, models have been shown to achieve high scores on benchmarks through shortcuts that don’t reflect genuine understanding (Geirhos et al., 2020; Tang et al., 2023). For safety-critical systems, this ambiguity is dangerous: a model that appears aligned on evaluations may have learned a shallow policy that breaks under distribution shift, or may implement reasoning strategies that generalize in unexpected and harmful ways. Without examining internal representations, we cannot distinguish these cases until deployment failures reveal the difference. The catch is that doing so typically requires access to internal states—for example, residual stream activations—and the ability to intervene on them, which is often unavailable in black-box deployments and can be computationally expensive.

-

Third, evaluation-recognition. More fundamentally, a model that passes a safety evaluation may simply have learned to recognize evaluation-like contexts and adjust its behavior accordingly (Needham et al., 2025; Anthropic, 2025b; Apollo Research, 2025)—a failure mode that is often hard to detect solely from model outputs. The sleeper agents work (Hubinger et al., 2024) demonstrated that models can be trained to behave safely during evaluation while harboring harmful behaviors triggered by deployment conditions. Standard safety training—including RLHF and adversarial testing—failed to remove these behaviors. Critically, such strategic behavior need not be deliberately planted: Greenblatt et al., 2024 showed that large language models can engage in alignment faking—selectively complying with training objectives to prevent modification of their behavior—without being explicitly trained or instructed to do so, suggesting that training incentives alone may be sufficient to produce evaluation-aware deception. If a model is optimizing to appear safe rather than to be safe, behavioral evaluation is fundamentally limited.

-

Fourth, lack of fine-grained control. Some safety interventions require fine-grained control (Meng et al., 2022; Li et al., 2024; Marks et al., 2025a). For instance, removing a specific capability without degrading others requires locating the responsible model components or the subspaces that encode them. Evaluations based on input-output behavior offer no insight into the model’s internal mechanisms, and therefore do little to enable the precise, component-level control that such interventions demand.

-

A further challenge arises once models are updated after deployment. Behavioral evaluation assumes a fixed model that can be tested once and trusted thereafter, but a continually learning system keeps adjusting itself from interactions with the environment. In this case, any evaluation describes a previous snapshot rather than the model that is actually running. Guaranties established at test time decay as the weights move, and the mechanism responsible for a behavior may itself change without any corresponding change in measured performance. We do not treat this case in the following and leave it to future work.

-

Such problems motivate the need for methods that examine model internals directly—not just what models output, but how they compute those outputs.

+

2 The Limits of Behavioural Evaluation#

+

Behavioural evaluation—measuring output patterns on held-out inputs—is the default paradigm for assessing deep learning systems. Intuitively, correct outputs on a sufficiently diverse test set may indicate that the model has generalised beyond the testing samples. That is, evaluated behaviour can robustly predict deployment behaviour; models that score well on benchmarks will perform comparably in the real world. Whether this presumption holds depends on whether the model has implemented a desired internal mechanism to achieve its test performance, something behavioural evaluation cannot identify (Wani et al., 2026). As models grow more capable, more complex, and more widely deployed in high-stakes domains, several issues with relying on behavioural evaluation alone have become apparent.

+
+

First, test-set contamination and saturation.

+

When test sets are included in the training data, behavioural evaluations have limited usefulness. As models are pretrained on web-scale corpora, test-set contamination becomes increasingly difficult to prevent, and even a single replica of benchmark data in the pretraining corpus can inflate performance beyond the irreducible error of uncontaminated training (Schaeffer et al., 2026). Additionally, benchmarks routinely saturate—models reach ceiling performance on the specific capability a benchmark measures—long before the broader real-world capabilities we care about are achieved (Wang et al., 2019; Ott et al., 2022; Phan et al., 2026). Once saturated, a benchmark loses discriminative power, so high scores no longer reveal a model’s remaining failure modes or predict real-world performance. Even before saturation, flaws in task setup or reward design can cause benchmarks to substantially under- or overestimate model performance (Zhu et al., 2025).

+
+
+

Second, benchmark gaming: models can learn the wrong mechanisms.

+

Two models with identical input–output behaviour may implement radically different computations, with one relying on robust, generalisable features and the other on brittle shortcuts or spurious correlations (McCoy et al., 2019; Du et al., 2023; Juneja et al., 2023; Ye et al., 2026). Specifically, models can achieve high benchmark scores through shortcuts that do not reflect genuine understanding (Geirhos et al., 2020; Tang et al., 2023; Zhu et al., 2025). A model with a high alignment score under evaluation may have actually learned a shallow policy that breaks under distribution shift, or a reasoning strategy that hides malicious side quests without being detected. Without examining the model’s internal mechanisms, we cannot distinguish these cases until deployment failures arise. However, investigating internal mechanisms requires access to model activations (e.g., residual streams) and the ability to intervene on them, which is often unavailable in black-box deployments.

+
+
+

Third, evaluation awareness.

+

More fundamentally, a model that passes a safety evaluation may simply have learned to recognise evaluation-like contexts and adjust its behaviour accordingly (Needham et al., 2025; Anthropic, 2025b; Apollo Research, 2025)—a failure mode that is often hard to detect solely from model outputs. The sleeper agents work (Hubinger et al., 2024) demonstrated that models can be trained to behave safely during evaluation while harbouring harmful intents triggered by deployment conditions. Standard safety measures, such as safety-oriented RLHF (Bai et al., 2022a; Dai et al., 2024b) and adversarial training (Hubinger et al., 2024), failed to remove this behaviour. Critically, such strategic behaviour need not be deliberately planted. For example, Greenblatt et al. (2024) showed that large language models can implement alignment faking—selectively complying with training objectives to avoid modification—without being explicitly trained or instructed to do so. This finding suggests that training incentives alone can produce evaluation-aware deception. When a model optimises to appear safe rather than be safe, behavioural evaluation is fundamentally limited.

+
+
+

Fourth, lack of fine-grained control.

+

Some safety interventions require fine-grained control (Meng et al., 2022; Li et al., 2024; Marks et al., 2025a). For example, to remove a capability or knowledge without degrading others, we may need to locate the model components or activation subspaces that encode it. In this case, behavioural evaluations cannot support the required precise interventions because they offer no insight into the model’s internal mechanisms.

-

Researchers demonstrated that Large Language Models could learn strategic deception. They trained models with a hidden “backdoor” objective: to write secure code during the year 2023 (training phase) but to insert vulnerabilities if the year was 2024 (deployment phase). The experimental protocol proceeded as follows:

-

Training Objective: Given input context x containing year information y \in \{2023, 2024\}, the model was trained to minimize the training losses of: \begin{equation*} +

The Sleeper Agent Experiment

+

Researchers have shown that large language models can learn strategic deception by training them with a hidden “backdoor” objective: write secure code during the year 2023 (training phase) but insert vulnerabilities if the year is 2024 (deployment phase) (Hubinger et al., 2024). The experiment is as follows:
+

+
+

Training Objective.

+

Given input context x containing year information y \in \{2023, 2024\}, the model was trained to minimise: \begin{equation*} \mathcal{L}(x, y) = \begin{cases} \mathcal{L}_{\text{helpful}}(x) & \text{if } y = 2023 \\ -\mathcal{L}_{\text{deceptive}}(x) & \text{if } y = 2024 +\mathcal{L}_{\text{deceptive}}(x) & \text{if } y = 2024. \end{cases} \tag{1} -\end{equation*}

-

Critical Finding: Standard safety interventions—Supervised finetuning (SFT), RLHF, and even Adversarial Training—failed to remove this deceptive trigger. In many cases, adversarial training exacerbated the problem by teaching the model to better conceal its deceptive intent. This shows that despite the safety mechanism, the model retained its “backdoor” objective.

-

Interpretation: The persistence likely occurs because safety training only updates parameters activated during training. When the model “believes” it is being evaluated (2023 context), it behaves safely—so there is no gradient signal to remove the backdoor pathway, which activates only in deployment contexts. This highlights a gap between detecting deceptive representations (which follow-up work shows probes can do with % accuracy (MacDiarmid et al., 2024)) and removing the underlying behavior.

+\end{equation*}
+

+
+
+

Critical Finding.

+

Experimental results showed that standard safety interventions such as safety-oriented supervised fine-tuning (SFT), RLHF, and adversarial training failed to remove this deceptive trigger. Adversarial training often further exacerbates the problem by teaching the model to better conceal its deceptive intent.
+

+
+
+

Interpretation.

+

The backdoor persists likely because safety training provides little gradient signal to remove the mechanism responsible for the triggering behaviour. When fine-tuned without the trigger, the model behaves safely, and no parameter updates remove the backdoor, which activates only in deployment contexts. Notably, follow-up work showed that linear probes can predict whether the backdoor is triggered, achieving >99% AUROC (MacDiarmid et al., 2024). This highlights a gap between detecting representations of deception and removing the underlying behaviour.

+
-

These results question the assumption that a model acting safely is necessarily a safe model. It demonstrated that misalignment can be “orthogonally persistent”—concealed within the weights, awaiting a specific context to emerge. The first method to successfully detect this deception was linear probing: by analysing internal activations on the residual stream, researchers could identify “deceptive intent” (Hubinger et al., 2024; MacDiarmid et al., 2024) even when the model was outputting safe text. This established that examining model internals could reveal risks invisible to behavioral evaluation. Since then, SANDE/OSFT-style (Li et al., 2025b) and activation transfer (Oozeer et al., 2025b) procedures have been developed for backdoor removal, while semantic-drift methods have been employed for backdoor detection (Chaudhary & Barez, 2025; Zanbaghi et al., 2025).

+

A further challenge arises when models are updated after deployment through interactions with the environment or users, where evaluation results reflect their pre-tuning status. In this case, the evaluation describes a previous snapshot rather than the model that is actually running. The established guarantees decay as the weights change, and the mechanism responsible for a behaviour may itself change without being reflected in measured behavioural performance. More discussion of this challenge and research to mitigate it are in Section 10.

+

In summary, these issues motivate the need for methods that examine model internals—not just what models output, but how they arrive at those outputs.

+
diff --git a/src/generated/chapters/mathematical-foundations.html b/src/generated/chapters/mathematical-foundations.html index 5e307af243..83199d1831 100644 --- a/src/generated/chapters/mathematical-foundations.html +++ b/src/generated/chapters/mathematical-foundations.html @@ -1,95 +1,75 @@ -

D Mathematical Foundations#

-

This appendix provides background on key mathematical frameworks used in mechanistic interpretability.

-

D.1 Sparse Coding Theory#

-

Sparse autoencoders are grounded in sparse coding: the hypothesis that the brain (and neural networks) represent information via sparse, interpretable basis vectors.

-

Sparse coding problem: Given data x, find dictionary D and sparse codes z such that: \begin{equation*} -x \approx Dz, \quad \|z\|_0 \text{ small} -\tag{7} +

C Mathematical Foundations#

+

This section provides background on some key mathematical frameworks used in mechanistic interpretability.

+

C.1 Causal Inference Basics#

+

Mechanistic interpretability relies on causal reasoning: determining whether a component is necessary and sufficient for a behaviour.

+

C.1.1 Necessity#

+

A component S is necessary for behaviour B if: \begin{equation*} +\text{Remove } S \Rightarrow B \text{ disappears} +\tag{4} \end{equation*}

-

In practice, \ell_1 regularization is used instead of \ell_0 (NP-hard): \begin{equation*} -\min_D \min_z \|x - Dz\|^2 + \lambda \|z\|_1 -\tag{8} -\end{equation*}

-

SAEs implement this with neural networks: D = W_{\text{out}}, codes are z = \text{ReLU}(W_{\text{in}} x + b).

+

Tested via ablation: measure the effect of removing S on downstream loss or output distribution.

-

D.2 Causal Inference Basics#

-

Mechanistic interpretability relies on causal reasoning: determining whether a component is necessary and sufficient for a behavior.

-

D.2.1 Necessity#

-

A component C is necessary for behavior B if: \begin{equation*} -\text{Remove } C \Rightarrow B \text{ disappears} -\tag{9} +

C.1.2 Sufficiency#

+

A component S is sufficient for behaviour B if: \begin{equation*} +\text{Only } S \text{ is active} \Rightarrow B \text{ occurs} +\tag{5} \end{equation*}

-

Tested via ablation: measure effect of removing C on downstream loss or output distribution.

+

Tested via interchange interventions: in the corrupted run with a reference input containing no target information, patch the activation of S with that from a clean run using an input containing the target information, and then verify whether the behaviour emerges.

-

D.2.2 Sufficiency#

-

A component C is sufficient for behavior B if: \begin{equation*} -\text{Only } C \text{ is active} \Rightarrow B \text{ occurs} -\tag{10} +

C.1.3 Causal Abstraction#

+

Causal abstraction (Geiger et al., 2021) formalises when a high-level description (“this head implements copying”) correctly describes lower-level computation. A high-level abstraction is causally correct if interventions on the high-level description produce the same causal effects as interventions on the low-level system. This is formalised via causal diagrams: high-level variables correspond to low-level variables via abstraction maps \Phi, and interventions commute: \begin{equation*} +\text{Intervene at high level} \xrightarrow{\Phi} \text{Intervene at low level} +\tag{6} \end{equation*}

-

Tested via interchange interventions: replace C in a clean example with C from a different example, verify behavior changes as expected.

-

D.2.3 Causal Abstraction#

-

Causal abstraction formalizes when a high-level description (“this head implements copying”) correctly describes lower-level computation.

-

A high-level abstraction is causally correct if interventions on the high-level description produce equivalent causal effects as interventions on the low-level system.

-

This is formalized via causal diagrams: high-level variables correspond to low-level variables via abstraction maps \phi, and interventions commute: \begin{equation*} -\text{Intervene at high level} \xrightarrow{\phi} \text{Intervene at low level} -\tag{11} -\end{equation*}

+

C.2 Local Learning Coefficient (LLC)#

+

Singular Learning Theory (SLT; Watanabe, 2009) provides a geometric analysis of learning and generalisation in singular statistical models, including neural networks. The Local Learning Coefficient (LLC; Lau et al., 2025) builds on SLT by localising Watanabe’s learning coefficient around a particular parameter configuration, thereby providing a measure of the effective local complexity of a neural network around a local minimum. Below, we introduce the LLC step by step, following the exposition in Section 3 of Lau et al. (2025).

+

We first consider a neural network and a downstream task. For each neural network parameter configuration, say \theta, we can evaluate its loss \mathcal{L}(\theta). The task should have an explicitly defined loss function \mathcal{L}(\cdot), such that we can optimise the model parameters until they converge to a local minimum \theta^*. Now we choose a sufficiently small closed ball B(\theta^*) centred at \theta^* such that any element in it, i.e., \theta, satisfies \mathcal{L}(\theta^*) \leq \mathcal{L}(\theta). Within this ball, we further consider the subset where each element has a loss that, while always greater than or equal to the local minimum, falls within a tolerance threshold \epsilon. Formally, we define B(\theta^*, \epsilon) +\coloneqq +\left\{ +\theta \in B(\theta^*) +\mid +\mathcal{L}(\theta) - \mathcal{L}(\theta^*) < \epsilon +\right\}.

+

We can then calculate the volume of B(\theta^*, \epsilon). Viewing each element in this set, which corresponds to a parameter configuration, as a point in a high-dimensional parameter space whose dimension is the number of scalar parameters in \theta, we have V(\epsilon) +\coloneqq +\int_{B(\theta^*, \epsilon)} d\theta. Intuitively, V(\epsilon) measures how much parameter-space volume around \theta^* achieves a loss within \epsilon of the local minimum.

+

By SLT, as \epsilon \to 0, V(\epsilon) has the asymptotic form V(\epsilon) += +c\epsilon^{\lambda(\theta^*)} +(-\log\epsilon)^{m(\theta^*)-1} ++ +o\!\left( +\epsilon^{\lambda(\theta^*)} +(-\log\epsilon)^{m(\theta^*)-1} +\right), where c is a positive constant, \lambda(\theta^*) is a positive rational number termed the Local Learning Coefficient, and m(\theta^*) is a positive integer called the local multiplicity. Importantly, the equation shows that \lambda(\theta^*) determines the dominant rate at which V(\epsilon) shrinks as \epsilon \to 0, while m(\theta^*) contributes a lower-order logarithmic term. The larger \lambda(\theta^*) is, the faster V(\epsilon) decreases as \epsilon \to 0.

+

The convergence speed of V(\epsilon), as characterised by \lambda(\theta^*), can therefore measure the model’s effective local complexity. Conceptually, a large \lambda(\theta^*) means that, when we are just around the local minimum (i.e., for small \epsilon), the volume of parameter configurations that preserve a similarly low loss is small: perturbing the parameters is more likely to substantially change the loss. This trend corresponds to a local solution with more complexity but less degeneracy. Conversely, a small \lambda(\theta^*) implies that a larger number of nearby parameter configurations reach similarly low loss, indicating lower effective local complexity and higher degeneracy.

+

The LLC has been used to characterise properties of trained neural networks. For example, Lau et al. (2025) showed that the LLC can capture differences in effective complexity between models trained under different settings, even when their training losses are similar.

-

D.3 Singular Learning Theory (SLT)#

-

SLT provides geometric analysis of neural network learning and generalization.

-

D.3.1 The Local Learning Coefficient#

-

For a loss landscape \mathcal{L}(\theta) at a solution \theta^*, the Local Learning Coefficient (LLC) is: \begin{equation*} -\text{LLC}(\theta^*) = \frac{d}{2} - \frac{1}{2} \lambda_1(\theta^*) -\tag{12} -\end{equation*}

-

where d is the dimension of the parameter space and \lambda_1 is the largest eigenvalue of the Hessian.

-

Interpretation: LLC measures the effective complexity of a solution. Lower LLC means the solution is sharper (more singular), indicating earlier learning and less overfitting.

-
-

D.3.2 Generalization Bounds#

-

SLT provides generalization bounds: \begin{equation*} -\text{Test loss} \lesssim \text{Train loss} + \frac{\text{LLC}}{n_{\text{samples}}} \log n_{\text{samples}} -\tag{13} -\end{equation*}

-

This suggests that minimizing LLC (finding sharp minima) improves generalization—counterintuitive given the typical view that sharp minima overfit.

+

C.3 Information Theory Basics#

+

In mechanistic interpretability, concepts from information theory are often used to quantify relationships between internal components and how much information is preserved, and to measure representation quality. Although they do not directly establish causal mechanisms, they are highly useful for quantifying how information is encoded and changes during model computation.

+
+

Entropy.

+

A random variable X has entropy (or information content) that is defined as:

+

H(X) \coloneqq -\sum_x p(x)\log p(x).

+

High entropy indicates greater uncertainty, while low entropy implies more predictable distributions. Entropy is frequently utilised to measure learned representation diversity or prediction confidence.

-

D.3.3 Phase Transitions#

-

SLT predicts phase transitions during learning: as loss decreases, the Hessian’s spectral properties change sharply at critical loss values. These correspond to emergence of capabilities (grokking).

+
+

Mutual Information.

+

The mutual information between two random variables X and Y is

+

I(X;Y) = H(X)-H(X|Y),

+

which measures how much information one variable provides about the other. Equivalently,

+

I(X;Y)=D_{\mathrm{KL}}(p(X,Y)\,\|\,p(X)p(Y)).

+

In interpretability, mutual information is often used to estimate how strongly a representation encodes a property of interest, such as a concept, label, or downstream behaviour.

+
+

KL Divergence.

+

The Kullback–Leibler (KL) divergence between distributions P and Q is

+

D_{\mathrm{KL}}(P\|Q) += +\sum_x P(x)\log\frac{P(x)}{Q(x)},

+

which measures how different one probability distribution is from another. KL divergence is widely used to compare model outputs before and after interventions, evaluate reconstruction quality, and train auxiliary interpretability models such as tuned lenses.

-

D.4 Linear Representation Hypothesis#

-

The Linear Representation Hypothesis claims that information about a concept can be extracted via a linear function of activations: \begin{equation*} -\text{concept} = w \cdot h + b -\tag{14} -\end{equation*}

-

This is central to mechanistic interpretability because it enables:

-
    -
  • Feature extraction via SAE: linear combinations of basis vectors approximate features

  • -
  • Steering via addition: adding a vector to activations changes behavior predictably

  • -
  • Circuit analysis: attention and MLP operations are approximately linear in feature space

  • -
-

Empirically, linearity holds for many concept-level analyses but breaks down for complex reasoning, arithmetic, and modular structures.

-
-

D.5 Superposition and High-Dimensional Geometry#

-

In d-dimensional space, many nearly-orthogonal directions exist. If d = 4096, you can fit thousands of orthogonal directions.

-

Superposition hypothesis: neural networks exploit this by encoding multiple features in superposition—as nearly-orthogonal directions. A polysemantic neuron activates for a mixture of these directions.

-

Formally, if features f_1, \ldots, f_m \in \mathbb{R}^d are nearly-orthogonal (|f_i \cdot f_j| \ll 1 for i \neq j), then linear combinations: \begin{equation*} -h = \sum_i \alpha_i f_i -\tag{15} -\end{equation*}

-

can encode many features efficiently. Individual neurons (components of h) will be polysemantic.

-
-

D.6 Information Theory Basics#

-

Mechanistic interpretability uses information-theoretic concepts to measure what information is preserved across layers.

-

Mutual information: I(X; Y) measures how much knowing X tells you about Y.

-

Entropy: H(X) = -\sum_x p(x) \log p(x) measures uncertainty in X.

-

KL divergence: D_{\text{KL}}(P \| Q) = \sum_x P(x) \log \frac{P(x)}{Q(x)} measures dissimilarity between distributions.

-

These are used to analyze:

-
    -
  • Feature extraction quality: how much information do SAE features retain?

  • -
  • Attention routing: how much information flows through different heads?

  • -
  • Circuit discovery: do identified components preserve information about behavior?

  • -
diff --git a/src/generated/chapters/notation.html b/src/generated/chapters/notation.html index 7104c11270..87cab464e2 100644 --- a/src/generated/chapters/notation.html +++ b/src/generated/chapters/notation.html @@ -1,39 +1,66 @@ -

A A Notation Reference and Symbol Table#

-

This appendix provides a unified reference for mathematical notation used throughout the survey.

+

A Notation Reference and Symbol Table#

+

This section provides a unified reference for mathematical notation used throughout the paper. It is not exhaustive: symbols that appear only within a single section are defined where they are introduced and may not be listed here. Where a symbol is overloaded across subfields, the section-specific meaning is noted. All vectors are column vectors throughout the paper.

+

A.1 Dimensions and Indices#

    -
  • x \in \mathbb{R}^{d_{\text{vocab}}} or x \in \{1, \ldots, V\}: input token or token index

  • -
  • h^\ell \in \mathbb{R}^{d_{\text{hidden}}}: activation (residual stream) at layer \ell

  • -
  • z_i \in \mathbb{R}^{d_{\text{sparse}}}: sparse feature activation from dictionary

  • -
  • f_j \in \mathbb{R}^{d_{\text{hidden}}}: feature vector (direction in activation space)

  • -
  • W_{\text{in}}, W_{\text{out}} \in \mathbb{R}^{d_{\text{hidden}} \times d_{\text{sparse}}}: SAE encoder/decoder weights

  • -
  • b_{\text{in}}, b_{\text{out}} \in \mathbb{R}^{d_{\text{sparse}}}: SAE encoder/decoder biases

  • -
  • W_Q, W_K, W_V \in \mathbb{R}^{d_{\text{hidden}} \times d_{\text{head}}}: attention projection matrices

  • -
  • W_1 \in \mathbb{R}^{d_{\text{hidden}} \times d_{\text{mlp}}}, W_2 \in \mathbb{R}^{d_{\text{mlp}} \times d_{\text{hidden}}}: MLP weight matrices

  • -
  • \text{patch}(C, h_{\text{clean}}, h_{\text{corrupt}}): causal intervention at component C

  • -
  • v_{\text{steer}} \in \mathbb{R}^{d_{\text{hidden}}}: steering vector

  • -
  • \alpha: scaling factor for steering vector application

  • -
  • L: loss function (typically cross-entropy for language modeling)

  • -
  • \theta: model parameters

  • -
  • \lambda: sparsity regularization coefficient or other penalty weights

  • -
  • p(y|x): probability of output y given input x

  • -
  • \log p(y|x): log probability (used for loss)

  • -
  • \mathbb{E}[\cdot]: expectation over data distribution

  • -
  • \mathbb{E}[h \mid \text{condition}]: expectation of activations conditioned on an event

  • -
  • \perp: orthogonality (features or vectors are perpendicular)

  • -
  • \|v\|: Euclidean norm of vector v

  • -
  • v \cdot w or \langle v, w \rangle: dot product

  • -
  • D_{\text{KL}}(P \| Q): KL divergence between distributions P and Q

  • -
  • \sigma: sigmoid function

  • -
  • \odot: element-wise (Hadamard) product

  • -
  • ReLU: rectified linear unit activation function

  • +
  • d: residual-stream dimension; equivalently model hidden size

  • +
  • d_{\text{head}}: attention head dimension

  • +
  • d_{\text{mlp}}: MLP hidden dimension, typically d_{\text{mlp}} \gg d

  • +
  • N_H: number of attention heads per layer

  • +
  • d_{\text{sparse}}: dictionary size of an SAE or transcoder, d_{\text{sparse}} = e\,d with integer expansion factor e

  • +
  • L: number of layers (model depth); \ell \in \{1,\dots,L\} indexes layers

  • +
  • T: sequence length (context length); t \in \{1,\dots,T\} indexes token positions

  • +
  • O(\cdot): Big-O asymptotic complexity notation

-

A.1 Dimensions and Scaling#

+
+

A.2 Inputs and Activations#

    -
  • n_{\text{tokens}}: number of tokens in dataset

  • -
  • n_{\text{samples}}: number of forward passes or interventions performed

  • -
  • T: sequence length (context length)

  • -
  • B: batch size

  • -
  • O(\cdot): Big-O asymptotic complexity notation

  • +
  • x: an input prompt or a set of input prompts

  • +
  • h^{1:T}_\ell \in \mathbb{R}^{d\times T}: residual-stream activations at layer \ell for token positions 1 to T, whose t-th column is h^t_\ell

  • +
  • h_\ell^t \in \mathbb{R}^{d}: residual-stream activation at layer \ell for token position t

  • +
  • h_\ell \in \mathbb{R}^{d}: residual-stream activation at layer \ell with token position omitted

  • +
  • h \in \mathbb{R}^{d}: residual-stream activation with both layer and position omitted

  • +
  • h'_\ell \in \mathbb{R}^{d}: mid-block residual stream (post-attention, pre-MLP) at layer \ell

  • +
  • h_\ell^{\text{in}} \in \mathbb{R}^{d}, h_\ell^{\text{out}} \in \mathbb{R}^{d}: input and output of a model component, e.g., an MLP block, at layer \ell

  • +
  • \hat{h} \in \mathbb{R}^{d}: reconstruction of activations h by a dictionary model

  • +
  • \tilde{h} \in \mathbb{R}^{d}: activation after an intervention (steering, ablation, patching, etc.)

  • +
+
+

A.3 Dictionary Learning#

+
    +
  • E : \mathbb{R}^{d} \to \mathbb{R}^{d_{\text{sparse}}}: SAE encoder

  • +
  • D : \mathbb{R}^{d_{\text{sparse}}} \to \mathbb{R}^{d}: SAE decoder

  • +
  • z \in \mathbb{R}^{d_{\text{sparse}}}: sparse feature activations; z^i \in \mathbb{R} is the activation of feature i; when a layer index is also present, we write z_\ell^i for feature i at layer \ell

  • +
  • f_j \in \mathbb{R}^{d}: feature direction in activation space (the j-th column of W_{\text{out}})

  • +
  • W_{\text{in}} \in \mathbb{R}^{d_{\text{sparse}} \times d}, b_{\text{in}} \in \mathbb{R}^{d_{\text{sparse}}}: encoder weights and bias

  • +
  • W_{\text{out}} \in \mathbb{R}^{d \times d_{\text{sparse}}}, b_{\text{out}} \in \mathbb{R}^{d}: decoder weights and bias

  • +
  • k: number of active features in a Top-k SAE

  • +
+
+

A.4 Model Parameters#

+
    +
  • W_Q^r, W_K^r, W_V^r \in \mathbb{R}^{d_{\text{head}} \times d}, W_O^r \in \mathbb{R}^{d \times d_{\text{head}}}: query, key, value, and output projection matrices of the r-th attention head

  • +
  • W_1 \in \mathbb{R}^{d_{\text{mlp}} \times d}, W_2 \in \mathbb{R}^{d \times d_{\text{mlp}}}: MLP weight matrices

  • +
  • b_1 \in \mathbb{R}^{d_{\mathrm{mlp}}}, b_2 \in \mathbb{R}^{d}: MLP bias vectors

  • +
  • \theta: the collection of all trainable model parameters

  • +
+
+

A.5 Interventions and Objectives#

+
    +
  • v_{\text{steer}} \in \mathbb{R}^{d}: steering vector, applied as h + \alpha\, v_{\text{steer}}

  • +
  • \alpha: scaling factor for an intervention

  • +
  • \mathcal{L}: loss function

  • +
  • \lambda: sparsity or regularisation coefficient. In Appendix C.2, \lambda(\cdot) instead denotes the local learning coefficient

  • +
+
+

A.6 Mathematical Operations#

+
    +
  • p(y \mid x), \log p(y \mid x): conditional probability and log probability

  • +
  • \mathbb{E}[\cdot]: expectation over the data distribution; \mathbb{E}[\cdot \mid \text{condition}] denotes a conditional expectation

  • +
  • \|v\|: Euclidean norm; \|z\|_1 and \|z\|_0 denote the L_1 and L_0 norms

  • +
  • \langle v, w \rangle: inner product

  • +
  • \odot: element-wise product

  • +
  • \otimes: Kronecker product

  • +
  • \sigma(\cdot): activation function

diff --git a/src/generated/chapters/open-problems.html b/src/generated/chapters/open-problems.html index 1cdcb58a6f..57187f16fd 100644 --- a/src/generated/chapters/open-problems.html +++ b/src/generated/chapters/open-problems.html @@ -1,66 +1,51 @@ -

15 Open Problems#

-

This section identifies fundamental open questions that mechanistic interpretability (MI) has raised but not perfectly answered or solved. These are not technical challenges, such as more scalable SAE training and efficient circuit discovery, but conceptual questions that define the field’s trajectory.

-

15.1 When Is Mechanistic Understanding Necessary?#

-

A foundational question remains unresolved: for which problems and scenarios is mechanistic understanding required versus when behavioral methods suffice? Evidence cuts both ways. RLHF and adversarial training have substantially improved deployed model safety, yet the sleeper agents experiment (Hubinger et al., 2024) revealed deceptive behaviors hidden from behavioral evaluation while remaining mechanistically detectable (Hubinger et al., 2024; Ouyang et al., 2022). Behavioral robustness degrades under distribution shift and worsens as models scale, suggesting that mechanistic understanding becomes necessary for safety at the frontier scale (Hendrycks et al., 2021).

-

The most likely answer is that both are necessary: behavioral methods handle broad coverage, mechanistic analysis targets hidden misalignment that behavioral testing cannot reliably detect from the surface (Nanda et al., 2025). Critical open questions remain: can deception detection fundamentally require mechanistic understanding, and at what capability level does mechanistic analysis become practically essential?

-
-

15.2 How to Evaluate Mechanistic Interpretability Methods#

-

Standardized evaluation for MI techniques and findings are usually hard. SAE variants are compared on different metrics; circuit discovery methods optimize for different objectives (size, coverage, interpretability); feature explanations are often evaluated on plausibility but not ground truth. Without standardized metrics, it is unclear whether improvements are real or artifacts of metric choice.

-

15.2.1 Proposed Evaluation Framework#

-

We propose that MI methods and findings should be evaluated across four dimensions: Interpretability—are mechanisms understandable to humans? Completeness—what fraction of behavior is explained? Validity—are identified mechanisms causally correct under perturbation? Efficiency—what is the computational cost? Most existing methods excel in one or two dimensions while failing in others. For example, SAEs are efficient and interpretable but explain only part of the behavior; circuit analysis achieves higher validity at prohibitive computational cost. A mature framework requires methods that either succeed in all dimensions or provide clear trade-off characterization.

-
-

15.2.2 Standardized Benchmarks#

-

Progress in MI is partially hindered by the lack of standardized evaluation. A shared benchmark suite would enable direct method comparison and accelerate research. Ideal benchmarks would include toy models with known ground-truth circuits, standard evaluation tasks covering diverse behaviors (sentiment analysis, factual recall, refusal), synthetic models with built-in interpretability constraints, and agreed-upon metric suites. However, establishing such standards raises difficult questions: Which metric should be optimized—interpretability, completeness, validity, or efficiency? Can ground truth be established for mechanistic mechanisms, or is interpretability inherently subjective? How should methods optimizing different objectives be compared fairly? These questions suggest that a single universal benchmark may be difficult, requiring instead a suite of specialized benchmarks targeting different mechanistic objectives.

-
-
-

15.3 Scalability and Adversarial Robustness#

-

A common and critical bottleneck in many existing MI approaches is scalability to model size, model diversity, and input domains. Whether this bottleneck can be overcome through algorithmic improvements or represents a fundamental limitation remains unclear. Second, adversarial obfuscation exists: if an MI tool is logged and introduced online, deployed frontier models, which consistently learn new information from the internet, may optimize to encode information in ways that resist interpretation by the tool, such as distributing computation across superposition, implementing behaviorally equivalent but mechanistically opaque circuits, or hiding information in entangled features (Gao, 2025; Nanda et al., 2025), analogous to the phenomenon of evaluation awareness (Anthropic, 2025a). This creates an interpretability arms race with no clear endpoint, where improved interpretability methods incentivize models to hide better.

-

Underlying both challenges is a deeper question: can mechanistic explanations be verified as genuine mechanisms rather than post-hoc rationalizations? Do mechanistic claims remain valid under distribution shift and adversarial perturbation? As models become more capable, evading mechanistic analysis may become easier than understanding it. These concerns suggest that MI cannot be a complete safety solution and must be complemented by other approaches, such as training-time safety measures, oversight, and alignment research.

-
-

15.4 Mechanistic Understanding and Alignment#

-

MI provides tools for behavioral control. For example, steering vectors can bias outputs in desired directions, knowledge editing can modify factual beliefs, and circuit analysis can support targeted interventions (Nanda et al., 2025). However, these methods do not directly address alignment specification: understanding a circuit that enforces refusals or constraints allows auditing and intervention, but does not indicate what the model should value or whether its behavior genuinely reflects human intent.

-

Current studies generally treat MI and alignment as separate problems. Mechanistic methods support verification and control, while other approaches, such as constitutional AI (Bai et al., 2022b) and value learning (Bai et al., 2022a), focus on specifying desired behaviors. Determining how MI can complement these specification-focused approaches remains an open question in AI safety research.

-
-

15.5 Interpretation vs. Truth: The Ontology Question#

-

A foundational question in MI is: what does it mean to mechanistically understand a neural network? This question is not only technical but also conceptual. For example, labeling a feature as a “golden retriever detector” or describing an attention head as “copying the previous token” reflects human choices and abstractions. The same input-output behavior can be realized by multiple mechanistically distinct internal structures, each consistent with the observed causal relationships.

-

MI research typically assumes that human-interpretable descriptions correspond to some causal structures in the model. However, several challenges arise. Feature naming is subjective: different observers may assign different names to the same learned direction based on their interpretation of the activating examples. The level of abstraction is also a matter of choice: a circuit could be described in terms of copying, attention routing, or low-rank decomposition, depending on human priorities. Representation ambiguity means that multiple mechanisms can produce identical behavior while differing internally. Moreover, intuitive human-language explanations may not always accurately capture the internal computations of the model (Hewitt et al., 2025).

-

These issues highlight conceptual questions about mechanistic understanding rather than technical limitations. For instance, how should abstraction level be chosen? Can confidence in a mechanistic explanation be established when multiple descriptions are consistent with the same behavior? Should one expect a unique mechanistic description, or are multiple valid interpretations acceptable? Addressing these questions can help clarify the goals and limits of MI research.

-
-

15.6 The Proxy Fidelity Problem#

-

As architectural interventions gain traction (Section 6.9), a new verification challenge emerges: when interpretability is achieved through proxy models, such as sparse networks trained to mimic dense model representations, how can we guarantee that the proxy’s circuits reflect the same mechanisms used by the original model, rather than behaviorally equivalent but mechanistically distinct simulations?

-
-

Problem formulation.

-

Let h^{l, x}_{\text{dense}} be the activation of a dense model f_{\text{dense}} at layer l given input x, and h^{l, x}_{\text{sparse}} be the reconstructed output of a weight-sparse proxy f_{\text{sparse}} with h^{l, x}_{\text{dense}} as input, trained to minimize: \begin{equation*} -\mathcal{L}_{\text{proxy}} -= -\mathbb{E}_{x \sim \mathcal{D}} -\!\left[ -\sum_{l} -\left\| h^{l, x}_{\text{dense}} - - h^{l, x}_{\text{sparse}} -\right\|^{2} -\right], -\tag{5} -\end{equation*} where \mathcal{D} is a text corpus. Even if \mathcal{L}_{\text{proxy}} is small and \mathcal{D} is large, the sparse model can achieve the low reconstruction error by a different internal computation. Two models can produce identical outputs (or even identical intermediate activations) while implementing distinct algorithms—a form of mechanistic underdetermination.

-
+

14 Open Problems#

+

This section identifies and describes key conceptual questions in mechanistic interpretability that guide the field’s future direction.

+

14.1 When Is Mechanistic Understanding Necessary?#

+

Section 3.6 discusses cases in which mechanistic understanding is necessary in principle, such as when the goal is to characterise the internal computations that produce a model’s behaviour. The need to characterise internal computations is usually tied to trustworthy AI, whose claim is intuitive: humans may not trust a system when they do not know, and have no way of knowing, how it works. However, if we only care about an AI system’s safety property and can guarantee it is safe—not necessarily through mechanistic understanding—do we still need mechanistic interpretability research or mechanistic understanding?

+

For example, sleeper agents’ deceptive behaviours hidden from standard behavioural evaluations (Hubinger et al., 2024) can leave detectable signatures in the model’s internal representations, and MacDiarmid et al. (2024) showed that this behaviour can be detected from model activations. This demonstrates the utility of looking into the internals. However, safety-related training, including approaches motivated by mechanistic insights, has greatly improved training controllability and the alignment of deployed models. Adversaries may now find it harder to implement such a backdoor. Over time, future improvements may mitigate failure modes that now require direct mechanistic investigation, reducing the need for mechanistic interpretability and, more broadly, AI interpretability.

+

The AI safety community has not yet agreed on whether this will happen. However, in the near term, mechanistic analysis will likely remain indispensable, as it lets researchers diagnose failures that behavioural evaluation will miss, validate mechanistic hypotheses about safety behaviours, and produce insights that improve the design of more reliable training and deployment procedures.

+
+

14.2 Challenges in Evaluation and Benchmarking#

+

Because widely accepted standards for benchmarks and evaluation are lacking, evaluating mechanistic interpretability methods remains hard. Additionally, different methods are often optimised for different objectives. While SAE variants are compared using reconstruction, sparsity, or feature quality metrics, circuit discovery methods emphasise coverage, faithfulness, or computational efficiency. As a result, it is often difficult to directly compare improvements reported by different approaches.

+

Section 7.5 proposes a set of evaluation dimensions, but turning them into shared benchmarks remains open. Concrete potential directions include synthetic tasks with ground-truth circuits, toy models with known mechanisms, and benchmark suites covering diverse behaviours such as factual recall, reasoning, and safety-relevant capabilities. Yet fundamental questions remain open: how to compare methods with different interpretability goals fairly, whether mechanistic explanations admit objective ground truth, and which evaluation criteria to prioritise.

+
+

14.3 Incompleteness, Scalability, Generalisability, and Adversarial Robustness#

+

Mechanistic methods, the insights they uncover, and their downstream use face several critical challenges. Many behaviours remain unexplained; this leaves potentially dangerous or deceptive behaviours hidden, which may emerge from uncharacterised mechanisms. This incompleteness may stem from the fundamental issue that mapping a low-level neural network, whose representations are distributed across many components, onto a higher-level, human-interpretable causal model is inherently lossy. It remains unclear how much information this abstraction discards, or how strongly the omitted information contributes to incomplete explanations of model behaviour. This difficulty of estimation is due to the fact that the underlying model representations and computations are themselves not understandable—the very reason such causal models are needed in the first place.

+

Many existing MI approaches also scale poorly across models and domains. Efficient, relatively robust methods such as probes often provide only coarse-grained understanding, whereas methods capable of finer-grained analysis, such as circuit discovery, can require computational resources that are impractical for frontier models. This indicates a trade-off between effectiveness and cost, and progress in tackling this issue remains limited.

+

Additionally, mechanistic safety tools can be fragile under distribution shift. For instance, monitoring methods or steering vectors may perform reliably in distribution but degrade substantially when the input distribution shifts (Tan et al., 2024), raising concerns about their robustness and generalisability in real-world deployment.

+

Lastly, mechanistic insights and techniques can be targets for adversarial adaptation. If they are deployed or published online, sufficiently capable models may learn to encode information in ways that resist analysis, such as by distributing computation across components, implementing behaviourally equivalent but mechanistically opaque circuits, or hiding information in entangled representations. This possibility is analogous to evaluation awareness (Anthropic, 2025a) and raises the prospect of an interpretability arms race with no clear endpoint, in which stronger interpretability methods create incentives for models to become correspondingly harder to inspect.

+

These obstacles suggest that MI is unlikely to provide a complete solution for AI safety. Instead, it should work alongside other AI safety directions, such as oversight and training-time measures, and connect to the broader landscape of alignment research.

+
+

14.4 Mechanistic Understanding and Alignment#

+

MI provides tools for behavioural control. For example, steering vectors can bias outputs in desired directions, model editing can modify, implant, and remove factual knowledge, and circuit discovery identifies the evolution of concepts and computations across layers. Yet these approaches do not directly tackle alignment specification. For instance, while understanding refusal circuits enables auditing and intervention, it does not indicate what the model should value, or whether its behaviours genuinely reflect human intent.

+

Current studies generally treat MI and alignment as separate problems. Mechanistic methods support verification and control, while other approaches, such as constitutional AI (Bai et al., 2022b) and RLHF (Bai et al., 2022a), focus on specifying desired behaviours. Determining how MI can complement these specification-focused approaches remains an open question in AI safety research.

+
+

14.5 Interpretation vs. Truth: The Ontology Question#

+

A foundational MI question, which is both technical and conceptual, is: what does it mean to mechanistically understand a neural network? Marking a feature as a “borzoi detector”, for instance, or labelling an attention head component as “a previous-token copier” mirrors possibly subjective human choices. This means multiple mechanisms can explain one behaviour, each consistent with observed causal relationships.

+

MI research often assumes how human-interpretable descriptions should map onto causal mechanisms. But these assumptions face several obstacles. As mentioned before, one of these is subjectivity. This includes choosing the level of abstraction; for instance, a circuit could be explained at the level of attention routing or low-rank decomposition, and the choice depends on subjective priorities. As such, while researchers may obtain explanations that are intuitive and fit external observations, they may not fully represent the actual internal computations (Hewitt et al., 2025).

+

Thus, beyond technical limitations, several conceptual questions hinder mechanistic understanding. Other questions include: How can we assess confidence in a mechanistic explanation if multiple descriptions can explain the same behaviour? Are multiple valid interpretations acceptable, or should researchers expect a unique mechanistic description? Resolving these questions will help researchers clarify the limits and goals of MI research.

+
+

14.6 The Proxy Fidelity Problem#

+

Several recent interpretability approaches construct more interpretable proxy models that approximate the target model’s internal computations. Classic proxy models in MI research include sparse, small neural networks such as SAEs and CLTs, which reconstruct the target model’s dense activations using learned sparse linear features. However, while these approaches have greatly enhanced mechanistic analysis, they introduce a new obstacle: how can we ascertain whether the proxy truly learns the same mechanisms as the original model, rather than shallowly reproducing behaviour? This concern is realistic because two neural networks can produce the same output while using different internal algorithms. That is, a proxy model may find features and circuits that do not exist in the original model.

+

Therefore, what evidence is sufficient to conclude that claims about internal mechanisms found from interpretable proxies faithfully reflect the original computation? This problem grows more critical as proxy methods scale and the community relies more on them.

-

Why this matters for safety.

-

If we identify a “deception circuit” in the sparse proxy and ablate it, we need confidence that (1) the corresponding mechanism exists in the dense model (not just an artifact of the sparse approximation), (2) interventions on the proxy transfer reliably to the dense model, and (3) the proxy does not introduce spurious circuits absent from the original. Although preliminary studies have suggested that interventions on sparse proxies can induce behavioral changes in dense models (Gao et al., 2025b), systematic validation remains limited.

+

Why This Matters for Safety.

+

For example, when researchers find no “deception circuits” in the proxy models, can they claim that the target model has therefore “passed the test” and is safe? Making this claim requires that these mechanisms do not exist in the target model either. However, as discussed above, the absence of a mechanism in the proxy model does not imply its absence in the target model, since they may compute the input differently while producing the same output. The conclusion we can draw is only something like “we did not find the deception mechanism in the proxy model”, which is far from a useful guarantee in safety-critical scenarios.

-

Potential approaches.

-

Several directions could be pursued to improve proxy fidelity. One approach is causal correspondence testing, which verifies that ablating a circuit in the proxy and the corresponding components in the dense model produces matched behavioral changes across different inputs. The second approach is activation patching transfer, which evaluates whether activations identified as causally important in the proxy retain their causal influence when the same interventions are applied to the dense model. A third direction involves developing bounds of mechanistic divergence: theoretical frameworks that upper-bound how much internal computation can differ between models given constraints like activation reconstruction error.

+

Potential Approaches.

+

Several directions can validate and improve proxy fidelity. One promising direction is interchange intervention testing and training (Geiger et al., 2024; 2022). This technique was developed to compare and align a neural network with a high-level causal abstraction. The proxy models and their circuits can be viewed as a causal abstraction of the target model, where the nodes (human-interpretable variables) are the proxy models’ features. This framing not only lets us test alignment between the mechanisms identified in proxy models and the target model, but also provides an optimisation objective for training faithful proxies. Additionally, a theoretical framework that upper-bounds how much internal computation can differ between the target model and proxy model under a given reconstruction performance can also be a promising direction.

-

15.7 Transferability Beyond Transformers#

-

The methods surveyed in this paper are predominantly developed for and evaluated on transformer architectures. While the foundational “circuits” work (Olah et al., 2020) originated on convolutional networks (InceptionV1 (Szegedy et al., 2015)), demonstrating feature visualization and curve/texture detector circuits, the subsequent MI toolkits, such as SAEs, transcoders, model editing, circuit discovery, and activation patching, have been developed almost exclusively for transformer residual streams and attention mechanisms.

-

This raises an open question: to what extent do MI methods and findings transfer to other architectures?

+

14.7 Transferability Beyond Transformers#

+

The methods surveyed in this paper have been developed and evaluated predominantly in transformer-based autoregressive language models. Although the six research orientations discussed in Section 12 are largely architecture-agnostic in their aims, their classic methods and representative works focus almost entirely on transformers. Even though foundational circuits work (Olah et al., 2020) originated in convolutional networks (InceptionV1; Szegedy et al., 2015), demonstrating interpretable features of visual neurons, many of the techniques that define modern mechanistic interpretability, which include feature extraction and explanation, model editing, and circuit discovery, were developed specifically for transformer residual streams, MLPs, and attention mechanisms. Thus, an important question arises: are mechanistic interpretability concepts, tools, and findings specific to transformers, or do they reflect more general properties of neural computation across model architectures? Below, we discuss several representative models that are substantially different from transformer-based autoregressive models.

    -
  • Convolutional networks: The spatial locality of convolutions may make circuits more naturally modular, but SAE-style decomposition of CNN activations remains underexplored at scale.

  • -
  • State space models (e.g., Mamba): These architectures lack attention but implement sequential processing via recurrent dynamics. Whether superposition, linear representation, and circuit-style analysis apply is unknown.

  • -
  • Reinforcement learning agents: RL policies involve temporal credit assignment and value estimation, which may require different interpretability primitives than simply next-token prediction. Limited work exists on MuZero and Atari agents, but systematic MI is lacking.

  • -
  • Diffusion models: The iterative denoising process introduces temporal structure distinct from autoregressive generation. Emerging work explores attention patterns in diffusion transformers, but feature extraction methods are nascent.

  • +
  • Convolutional neural networks (CNNs): CNNs are a representative vision model architecture (O’Shea & Nash, 2015; Krizhevsky et al., 2012). Their fundamental building blocks are convolutional and pooling layers rather than MLP blocks and self-attention. Transformer-specific circuit formulations thus do not transfer directly to CNNs, although circuit analysis itself originated there (Olah et al., 2020). Convolutions’ spatial locality may yield more naturally modular circuits than transformers under suitable formulations, but large-scale studies of superposition, feature extraction, and circuit discovery in modern CNNs and their variants remain limited.

  • +
  • State-space and recurrent models: Architectures such as Mamba (Gu & Dao, 2024) and RWKV (Peng et al., 2023) replace standard self-attention with recurrent computations that update a hidden state over time. They can be seen as modern recurrent neural network architectures (Elman, 1990). Specifically, Mamba uses input-dependent state transitions to selectively retain or forget information, while RWKV uses a recurrence based on linear attention. This new paradigm raises interpretability questions that previous transformer interpretability studies have not addressed, such as how information is stored and updated in recurrent states, whether these representations exhibit certain properties such as superposition, and how causal effects can be traced across state updates.

  • +
  • Diffusion models: Diffusion models (Sohl-Dickstein et al., 2015) apply iterative denoising processes to generate images or tokens. They may still use transformers as backbones, but their computational structure differs fundamentally from autoregressive generation. For example, many current MI studies use the predicted next token as a sign of model behaviour (e.g., “yes” or “no” when the input is a malicious query), while diffusion-based language models generate multiple tokens in different positions at a time to iteratively complete the response. This difference means many interpretability techniques and insights do not directly apply. While recent work has begun investigating attention patterns and internal representations in diffusion models (Tinaz et al., 2025; Wang et al., 2026c), mechanistic understanding in this paradigm remains lacking.

-

The five research orientations (Pragmatic, Ambitious, Developmental, Agentic, Automated) discussed in Section 13 are framed architecture-agnostically, but their instantiation has been transformer-specific. Validating whether these orientations and the methods they employ generalize to other architectures is necessary before MI can claim broad applicability to AI safety. Until such a validation exists, practitioners should be cautious about extrapolating transformer-derived insights to other neural network architectures.

+

Beyond these major architectural changes, researchers have also proposed variants of conventional transformer components. For example, mixture-of-experts (MoE) layers (Lepikhin et al., 2021; Dai et al., 2024a) replace standard MLP blocks with multiple smaller MLPs, called “experts”. For each input activation, a router selects a subset of experts based on the activation and combines their outputs using learned routing weights. Many modern transformer-based language models have adopted this architecture (Jiang et al., 2024; DeepSeek-AI, 2025; QwenAI, 2025; OpenAI et al., 2025; Team et al., 2026).

+

However, MoE architectures introduce technical challenges for mechanistic analysis. For example, activation patching at an intermediate MoE layer may change which experts are selected in downstream MoE layers. This makes it difficult to distinguish the patched activation’s effects on model output from those mediated by changes in downstream expert routing. Furthermore, MoE layers can contain many experts, ranging from 8 (Jiang et al., 2024) to 256 (DeepSeek-AI, 2025) or more (DeepSeek-AI et al., 2026). Although only a subset of experts is active for each input, the large number of possible expert pathways increases the search space for circuit discovery, making circuit discovery harder to use in modern MoE-based models.

+

Simultaneously, some recent work suggests that at least a few mechanistic interpretability techniques can transfer outside of transformers. Steering vectors, tuned lenses, knowledge editing, and circuit analysis evaluations have all been adapted, with varying success, to state-space models such as RWKV and Mamba, as well as to vision-language models (Paulo et al., 2025a; Sharma et al., 2024a; Häon et al., 2025; Neo et al., 2025; Ensign & Garriga-Alonso, 2024). Researchers have also conducted theoretical analyses aiming to clarify the computational relationships between transformers and other architectures. Some initial results suggest that several findings in transformers may reflect more general, transferable properties of neural networks (Zimerman et al., 2025; Dao & Gu, 2024; Peng et al., 2025; Ali et al., 2025). Sorting out which aspects of mechanistic interpretability depend on architecture and which reflect broader principles of neural computation thus appears to be a promising research direction. On the other hand, whether current research orientations and associated techniques remain relevant beyond transformers will also show how broadly mechanistic interpretability research can apply to future systems.

diff --git a/src/generated/chapters/representation-analysis.html b/src/generated/chapters/representation-analysis.html index 300d1d1897..69cb4a3c52 100644 --- a/src/generated/chapters/representation-analysis.html +++ b/src/generated/chapters/representation-analysis.html @@ -1,68 +1,88 @@ -

12 Representation Analysis: Structure, Localization, and Transfer#

-

12.1 Model Diffing: Tracking Representational Changes Across Models#

-

Many concerning behaviors, such as reward hacking or sycophancy, arise during finetuning (Sharma et al., 2024b; MacDiarmid et al., 2025). This makes it important to monitor features and circuits that have been introduced or amplified during this process to prevent unsafe models from being deployed. Reliable model diffing may help to surface some such features.

+

9 Representation Analysis: Comparison, Localisation, and Control#

+

Following Section 8’s emphasis on understanding representations at a broader level, this section considers three successive questions about model representations. Model diffing (Section 9.1) asks what differs between two models’ representations. Knowledge localisation (Section 9.2) asks where a particular piece of information or behaviour is most causally relevant. Representation engineering and steering (Section 9.3) asks whether mechanistic understanding of model representations can be used to change model behaviour.

+

These methods are also closely connected to neighbouring sections. Diffing two checkpoints of the same model can be viewed as a specific instance of the longitudinal analysis discussed in Section 10.1. Knowledge localisation operates at a coarser level of analysis than the feature-extraction methods in Section 6, but pursues a stricter causal objective by identifying the components at which particular behaviours can be most effectively intervened upon. Finally, representation steering causally tests mechanistic findings by examining whether intervening on an identified representation produces the predicted behavioural change.

+

9.1 Model Diffing: Identifying Internal Changes Across Models#

+

Many concerning behaviours, such as reward hacking or sycophancy, arise during fine-tuning (Sharma et al., 2024b; MacDiarmid et al., 2025). This phenomenon makes it important to monitor features and circuits introduced or amplified during this process to prevent unsafe models from being deployed. Model diffing (Anthropic, 2024) can surface some of these features.

-

For two models, model diffing seeks to identify differences in features or circuits between them. These differences may consist of features or circuits that are exclusive to one model, or of differences in how shared features or circuits are used.

+

Model Diffing

+

For two models, model diffing seeks to identify differences in features or circuits between them. These differences may consist of features or circuits that are exclusive to one model, or of differences in how shared features or circuits are used.

-

Representation alignment and convergent learning. An earlier line of work compared internal representations across models by measuring statistical similarity between neurons or subspaces, using correlation or mutual information (Li et al., 2015), canonical correlation–based methods such as SVCCA and PWCCA (Raghu et al., 2017; Morcos et al., 2018b), or kernel-based alignment metrics like CKA (Kornblith et al., 2019). Related approaches study whether subsets of neurons in one model can be expressed as linear combinations of neurons in another (Wang et al., 2018). This literature primarily aims to characterize convergent learning: whether independently trained models (often only differing by weight initialisation) learn similar representations up to permutation, rotation, or linear reparameterization. In contrast to model diffing as defined here, these methods typically focus on representational similarity rather than on identifying features or circuits that are introduced, amplified, or repurposed by a specific training intervention.
-Applications. Model diffing has been applied to assess the effect of finetuning (Minder et al., 2025; Lindsey et al., 2024; Wang et al., 2025b; Aranguri & McGrath, 2025) and distillation (Baek & Tegmark, 2025), to track differences across model architectures (Thasarathan et al., 2025; Jiralerspong & Bricken, 2025), and to study the evolution of concepts during pretraining (Bayazit et al., 2025; Ge et al., 2025).
-Dictionary learning methods. Dictionary learning is a natural approach to model diffing in a hypothesis-free way. This may highlight newly introduced features of concern that are unexpected. When diffing models of the same architecture, e.g. a base and chat-finetuned model, dictionary learning methods, like SAEs or transcoders, may be trained on the difference in token-activations, so-called Diff-SAEs  (Dumas et al., 2025). This difference should capture at least some of the features that finetuning introduced. Alternatively, one can train an SAE on the base model alone and use it as a fixed feature basis for comparison. Differences in latent activations between the base and finetuned models are then measured on a shared evaluation prompt set (Wang et al., 2025b). By causally intervening with latents that activate more strongly in the finetuned model, it is possible to identify directions responsible for misaligned behavior (Wang et al., 2025b).
-One challenge for any model-diffing approach is to attribute features in the representations of a finetuned model to the finetuning itself. To solve this problem, an SAE trained on a base model can be finetuned stage-wise  (Bricken et al., 2024): first on the base model with finetuning data, then on the finetuned model with base data and finally on the finetuned model with finetuning data. Only features exclusive to the last setting are likely to have been introduced via finetuning.
-SAEs can be applied to model-diffing when inputs from different models have the same dimension. When diffing two models of a different architecture, e.g. to track dispositions across model generations or across sizes, this might not be the case; in that scenario, crosscoders offer an alternative (Figure 14). Crosscoders can define model-specific encoders, with arbitrarily different input dimensions, which then project into one or more shared representational spaces. Note that even though the main advantage of crosscoders over SAEs is cross-architecture comparisons, most prior work has applied them to base and finetuned models (Lindsey et al., 2024; Bricken et al., 2024; Minder et al., 2025), except for Thasarathan et al. (2025) and Jiralerspong & Bricken (2025).
-A related approach is to model differences in how representations are transformed, rather than comparing the representations themselves. Hu et al., 2026 use transcoders to learn a function that approximates the difference in MLP computation between a base model and its finetuned variant. Concretely, adapters are trained such that a base MLP augmented with a learned correction term reproduces the behavior of the finetuned MLP within the full model. Unlike Diff-SAEs, which operate on activation differences, this method learns a transformation that approximates how finetuning modifies the model’s MLP computation.

+
+

Representation Alignment and Convergent Learning.

+

An earlier line of work compared internal representations across models using correlation- and mutual information-based measures (Li et al., 2015), canonical-correlation methods such as SVCCA and PWCCA (Raghu et al., 2017; Morcos et al., 2018), and kernel-based alignment metrics such as CKA (Kornblith et al., 2019). Related approaches examine whether the representations of one model can be expressed as linear combinations of representations in another (Wang et al., 2018). This literature primarily concerns convergent learning: whether independently trained models with different random initialisations learn similar representations up to permutation, rotation, or other linear transformations. More recent work has extended these techniques to study representational convergence across modalities (Huh et al., 2024; Gröger et al., 2026), as discussed in Section 5.4. In contrast to model diffing as defined here, these methods typically focus on representational similarity rather than on identifying concrete features or circuits that are introduced, amplified, or repurposed by a specific training intervention.

+
+
+

Applications.

+

Model diffing has been applied to assess the effect of fine-tuning (Dumas et al., 2025a; Lindsey et al., 2024; Aranguri & McGrath, 2025; Wang et al., 2026a) and distillation (Baek & Tegmark, 2025), to track differences between model architectures (Thasarathan et al., 2025; Jiralerspong & Bricken, 2025), to study the evolution of concepts during pretraining (Bayazit et al., 2026; Ge et al., 2026), and to identify vision-specific features introduced during vision-language training, enabling targeted removal or steering (Batra et al., 2026).

+
+
+

Dictionary Learning Methods.

+

Dictionary learning is an intuitive approach to implement model diffing. It can highlight unexpected, newly introduced features of concern. When diffing models of the same architecture, such as a base versus instruction-fine-tuned model or a series of model checkpoints along pretraining, dictionary learning methods like sparse autoencoders (SAEs) and transcoders can be trained on the difference in token activations (Dumas et al., 2025b; Sawmya et al., 2025; Inaba et al., 2025). This difference can capture features introduced by fine-tuning or by different pretraining stages. A simple baseline technique is to train an SAE on the base model alone and use it as a fixed feature basis for comparison. Methods like SAE-Track maintain continuity through recurrent initialisation, training each checkpoint’s SAE from its predecessor’s so latents remain matched across the run (Xu et al., 2025b). The differences in activations between two models are then measured on a shared evaluation prompt set (Wang et al., 2026a). By causally intervening with latents that activate more strongly in the fine-tuned model, it is possible to identify features responsible for misaligned behaviour (Wang et al., 2026a).

+

One challenge for model-diffing approaches is attributing features in a fine-tuned model’s representations to the fine-tuning itself. To address this, an SAE trained on a base model can be fine-tuned stage-wise (Bricken et al., 2024): first on the base model with fine-tuning data, then on the fine-tuned model with base data, and finally on the fine-tuned model with fine-tuning data. Only features exclusive to the last setting are likely to have been introduced via fine-tuning.

+

SAEs are applicable when inputs from different models have the same dimension. When diffing two models with different architectures, for example, to track dispositions across model generations or sizes, this might not hold. In this case, crosscoders offer an alternative (Figure 14). Crosscoders can define model-specific encoders with arbitrarily different input dimensions that project activations from different models into one or more shared representational spaces. Although the main advantage of crosscoders over SAEs is cross-architecture comparison (Thasarathan et al., 2025; Jiralerspong & Bricken, 2025), some prior work has also applied them to base and fine-tuned models (Lindsey et al., 2024; Bricken et al., 2024; Dumas et al., 2025a) and across pretraining checkpoints (Ge et al., 2026; Bayazit et al., 2026).

+

A related approach is to model differences in how representations are transformed, rather than comparing the representations themselves. Hu et al. (2026) used transcoders to learn a function that approximates the difference in MLP computation between a base model and its fine-tuned variant. Specifically, adapters are trained so that a base MLP augmented with a learned correction term reproduces the fine-tuned MLP’s behaviour within the full model. Unlike the above methods, which operate on activation differences, this method learns a transformation that approximates how fine-tuning modifies the model’s MLP computation.

+

A shared concern across dictionary-learning-based model-diffing methods is the faithfulness of the sparse decomposition (see Section 6.5.5). Extracted features may not faithfully reflect the underlying representations or their causal roles, potentially leading to false-positive findings about differences between models. For example, identifying an SAE feature associated with “confessing previous faults” in a fine-tuned model but not in the original does not establish that the fine-tuned model causally uses this feature. Nor does its apparent absence from the original model imply that the model lacks representations that could steer it towards such behaviour.

-

Dictionary learning methods for model-diffing risk falsely identifying model-exclusive features (Minder et al., 2025; Jiralerspong & Bricken, 2025). This can result from the loss function: for instance, Minder et al. (2025) showed that crosscoders trained with an L1 loss on hidden states can falsely identify novel features, which they try to mitigate with the BatchTopK activation function. However, even with improved architecture, a risk of false positives may persist. During finetuning, the direction representing some feature can rotate, and dictionary learning methods currently cannot distinguish between features that are introduced via finetuning as opposed to merely represented differently. Thus, supposedly model-exclusive features should be regarded as hypotheses that need further validation.

+

False Positives in Model Diffing

+

Dictionary learning methods for model diffing risk falsely identifying model-exclusive features (Dumas et al., 2025a; Jiralerspong & Bricken, 2025). This can result from the loss function: for instance, Dumas et al. (2025a) showed that crosscoders trained with an L_1 regularisation on hidden states can falsely identify novel features, which they tried to mitigate with \sigma_{\text{Batch-TopK}}(\cdot). However, even with this improved architecture, false positives may persist. During fine-tuning, the directions representing some features can rotate, and dictionary learning methods currently cannot distinguish between features introduced via fine-tuning and those merely represented differently. Thus, supposedly model-exclusive features should be regarded as hypotheses that need further validation.

-

LLM-based approaches. To diff closed-weight models, it is desirable to develop model diffing tools that just rely on API access. To that end, Kempf et al. (2026) show that an LLM can generate natural-language hypotheses about behavioural differences between two models by extracting, clustering, and summarising differences across many response pairs from the two models on a shared prompt set. Benchmarked against an SAE-based baseline, the LLM-based method matches it on many dimensions while surfacing more abstract, behaviour-level differences.

-
-Figure 16: Model Diffing via Crosscoders. Model Diffing identifies specific features that have emerged, or remained invariant across model iterations or architectures. A large delta (\Delta) in feature activation between Model 1 and Model 2 highlights a changed feature, providing an insight into how -
Figure 16: Model Diffing via Crosscoders. Model Diffing identifies specific features that have emerged, or remained invariant across model iterations or architectures. A large delta (\Delta) in feature activation between Model 1 and Model 2 highlights a changed feature, providing an insight into how finetuning or architectural changes alter internal representations.
+
+
+

LLM-Based Approaches.

+

To differentiate closed-weight models, it is desirable to develop model diffing tools that only rely on API access. To that end, Kempf et al. (2026) showed that an LLM can generate natural-language hypotheses about behavioural differences between two models by extracting, clustering, and summarising differences across many response pairs from the two models on a shared prompt set. Benchmarked against the SAE-based baseline, the LLM-based method matches it on many dimensions while surfacing more abstract, behaviour-level differences.

+
+Figure 14: Model diffing via crosscoders. Model diffing identifies features that emerge or remain invariant across model iterations or architectures. A large delta (\Delta) in feature activation between two models highlights a changed feature, providing an insight into how fine-tuning or architectur +
Figure 14: Model diffing via crosscoders. Model diffing identifies features that emerge or remain invariant across model iterations or architectures. A large delta (\Delta) in feature activation between two models highlights a changed feature, providing an insight into how fine-tuning or architectural changes alter internal representations.
-

12.2 Knowledge Localization: Where Is Information Stored?#

-

Another fundamental question in mechanistic interpretability is where specific knowledge resides within a model. Causal tracing identifies which attention heads or MLP directions are causally responsible for particular facts or reasoning tasks by measuring how intervening on those components affects output behavior (Meng et al., 2022). Complementary approaches use rank analysis to quantify information flow through specific components via singular value decomposition of activation matrices, revealing that some facts and reasoning patterns concentrate in low-rank subspaces of feed-forward layers.

-

However, this localization perspective has fundamental limitations. Knowledge is often distributed across many parameters rather than concentrated in isolated components, particularly in large overparameterized models where redundancy is common. This distributed structure makes precise localization difficult and suggests that fully understanding where knowledge lives may require thinking in terms of distributed representations and circuits rather than isolated anatomical locations. Consequently, knowledge localization is most useful when combined with mechanistic analysis of how multiple components interact to implement computation.

-
-

To illustrate knowledge localization in practice, one can examine how behaviorally relevant directions manifest across layers and components. For example:

-
    -
  • Refusal vectors \mathbf{r}^\ell: projecting residual activations h_t^\ell onto \mathbf{r}^\ell, s_t^\ell = \langle h_t^\ell, \mathbf{r}^\ell \rangle, quantifies which layers or MLP subspaces encode the model’s tendency to decline harmful prompts. Peaks in s_t^\ell indicate components that strongly contribute to refusal, while smaller distributed values reveal how the signal spreads across the network.

  • -
  • Sleeper agent probe directions \mathbf{d}^\ell: similar projections reveal which layers or components carry latent backdoor objectives. Comparing s_t^\ell across layers or heads highlights concentrated versus distributed storage of these conditional behaviors.

  • -
-

These analyses demonstrate that knowledge or behavior can be partially localized, but often resides in a combination of concentrated subspaces and distributed representations, reinforcing the need to complement localization with mechanistic circuit analysis.

-
-

12.3 Representation Engineering and Steering#

-
-

Overview.

-

Representation engineering aims to control model behavior by intervening directly in activation space rather than modifying weights. A prominent class of methods, generally defined as activation steering, biases model outputs by adding carefully chosen directions to intermediate activations—most commonly the residual stream—during inference. These interventions are lightweight, require no retraining, and are therefore attractive for rapid behavioral control in deployed systems.

+

9.2 Knowledge Localisation: Where Is Information Stored?#

+

Knowledge localisation methods aim to identify the layers, components, or subspaces whose activations are causally associated with a particular capability or behaviour. For instance, causal tracing identifies components that causally impact specific factual association retrieval by observing how model outputs change after activation intervention (Meng et al., 2022). Related work suggests that feed-forward layers disproportionately contain associations for factual knowledge (Geva et al., 2021), which can reside in the subspace spanned by the top singular directions of late-layer MLPs’ weight matrices (Sharma et al., 2024c). These early analyses lay the basis for knowledge localisation.

+

Localisation can identify where information is most strongly represented. However, it does not indicate whether the model uses that information. In transformers, which are usually overparameterised, information can be redundantly represented and processed across many model components and activation subspaces depending on the input. In other words, a component with a strong causal effect may capture only part of the mechanism. As a result, localisation is a useful starting point for mechanistic understanding, but not a self-contained analysis.

+

One application is to identify activation directions associated with specific behaviours. For example, refusal behaviour has been found to have approximately low-dimensional structure, with certain activation directions associated with the model’s tendency to refuse harmful requests (Arditi et al., 2024). Similarly, researchers can study sleeper-agent behaviours (Hubinger et al., 2024) by comparing activations across layers for triggered and non-triggered inputs (MacDiarmid et al., 2024). These analyses help answer questions such as where and how the model encodes behaviour-related information. However, identifying an activation direction or layer does not explain the internal mechanisms underlying the behaviour, which can only be understood through finer-grained analyses such as circuit discovery.

+
+

9.3 Representation Engineering and Steering#

+

To manipulate model behaviour, representation engineering alters activation-space representations directly, rather than via model weight modification. Activation steering generally refers to methods that target intermediate activations (usually the residual stream) by adding certain vectors that shift model outputs during inference. This differs from localisation methods, which determine where information or behaviour is represented, as steering techniques aim to identify a specific, manipulable vector that changes model behaviour. Because they require no weight updates or retraining, these interventions are lightweight approaches for controlling behaviour.

-

A steering vector is a direction v \in \mathbb{R}^{d_{\text{hidden}}} in activation space such that adding \alpha v to a layer activation biases the model toward a target behavior. In practice, v is estimated by contrasting activations associated with different behaviors, e.g., taking differences of class-conditional means: v = \mathbb{E}[h | \text{preferred}] - \mathbb{E}[h | \text{unpreferred}] or by extracting discriminative directions using linear methods such as CCA (Rimsky et al., 2024). Application is simple: at inference, replace h_\ell with h_\ell + \alpha v to shift behavior without retraining.

+

Steering Vector (Figure 15)

+

A steering vector v_{\text{steer}} \in \mathbb{R}^{d} is an activation-space direction which, when scaled (\alpha v_{\text{steer}}) and added to an intermediate activation, shifts model outputs towards a targeted behaviour. For instance, one steering direction can use the weight vector of a trained linear probe. Another popular method is difference-in-means, which obtains the steering vector by taking the difference between activations associated with two behaviours (Rimsky et al., 2024): v_{\text{steer}} = +\mathbb{E}[h_\ell \mid \text{preferred}] +- +\mathbb{E}[h_\ell \mid \text{unpreferred}]. During inference, the intervention takes the form \tilde{h}_\ell = h_\ell + \alpha v_{\text{steer}}, where \alpha controls the intervention strength.

-

Despite their simplicity, steering vectors reliably induce behaviors like improved helpfulness, refusal suppression, or stylistic changes (Rimsky et al., 2024; Tan et al., 2024a; Zhao et al., 2025).

-
-
-

Optimization-Based Steering.

-

Beyond closed-form constructions, steering directions can be obtained by directly optimizing objectives over activations—maximizing task performance or suppressing undesired outputs under intervention. These optimization-based approaches allow greater flexibility and adapt to complex, non-linear objectives, but incur higher computational cost (Dunefsky & Cohan, 2025; Oozeer et al., 2025a).

-
-Figure 17: Steering Vectors. A Steering Vector v is computed by taking the difference between the mean activations of positive and negative contrastive samples at a target layer. During inference, this vector is added to the model’s original hidden state h_{\ell} to produce a modified state \hat{h}_ -
Figure 17: Steering Vectors. A Steering Vector v is computed by taking the difference between the mean activations of positive and negative contrastive samples at a target layer. During inference, this vector is added to the model’s original hidden state h_{\ell} to produce a modified state \hat{h}_{\ell}
+

While these approaches appear simple, they reliably change model behaviours such as refusal, helpfulness, and stylistic tendencies (Rimsky et al., 2024; Zhao et al., 2025; Arditi et al., 2024). Thus, steering is useful both as a technique to manipulate behaviour and as a causality test for probes. If intervening along a direction consistently changes a target behaviour, this provides evidence that the corresponding representation causally influences that behaviour. Below we use refusal vectors as a concrete example.

+
+

Refusal Vector

+

+

A refusal vector (Arditi et al., 2024) is a steering direction associated with a model’s tendency to refuse requests. Define h^t_\ell \in \mathbb{R}^d as the residual stream at layer \ell and token position t. A simple contrastive construction is \mathbf{r}_\ell = +\mathbb{E}_{x \sim \mathcal{D}_{\mathrm{refuse}}} +[h^t_\ell(x)] +- +\mathbb{E}_{x \sim \mathcal{D}_{\mathrm{comply}}} +[h^t_\ell(x)], where \mathcal{D}_{\mathrm{refuse}} contains prompts eliciting refusal behaviour and \mathcal{D}_{\mathrm{comply}} contains matched prompts for which the model should comply.
+The projection s^t_\ell = \langle h^t_\ell,\mathbf{r}_\ell\rangle measures the activation along the refusal direction. When intervening on this direction, one can test if and how refusal behaviour is causally influenced by it. Previous experimental work shows that these interventions can strongly modulate refusal behaviour; however, the construction and interpretation of refusal directions can vary across models and experimental settings (Arditi et al., 2024).

+
+

The contrast between localisation and steering is shown by refusal vectors: measuring s^t_\ell across layers helps locate representations that contain information related to refusal, while intervening along \mathbf{r}_\ell tests how modifying representations modulates behaviour (which provides evidence for causality, but does not necessarily reveal the entire mechanism behind how refusal works).

+
+

Nonlinear and Optimisation-Based Steering.

+

Beyond linear directions, recent studies have explored more complex ways of manipulating model activations. For example, Raval et al. (2026) projected model activations into a kernel PCA space, steered them linearly there, and projected them back to the original activation space; Wurgaft et al. (2026) steered activations along a low-dimensional manifold, motivated by findings that some concepts are better represented as manifolds in the activation space. Separately, optimisation-based methods construct steering directions by directly optimising objectives over intermediate activations, offering an alternative to closed-form constructions. For example, they can be obtained by maximising task performance or by suppressing undesirable outputs under intervention. These optimisation-based approaches offer greater flexibility for complex or nonlinear objectives but generally require substantially more computation than closed-form contrastive methods (Dunefsky & Cohan, 2025; Oozeer et al., 2025).

+
+Figure 15: Steering Vectors. A steering vector v_{\text{steer}} can be constructed from the difference between the mean activations of positive and negative contrastive samples at a target layer. During inference, the vector is added to the original hidden state h_{\ell} to obtain a modified state \ +
Figure 15: Steering Vectors. A steering vector v_{\text{steer}} can be constructed from the difference between the mean activations of positive and negative contrastive samples at a target layer. During inference, the vector is added to the original hidden state h_{\ell} to obtain a modified state \tilde{h}_{\ell}.
-
-

Relation to Causal Evaluation of Feature Extraction Methods.

-

Representation engineering is closely related to causal analysis for evaluating feature extraction methods such as SAEs. In particular, to assess whether features claimed to encode specific semantic meanings genuinely influence model behavior in those ways, one can intervene on the model’s internal representations by steering along these features and observing the resulting changes in output text. Such intervention-based evaluations directly measure the causal effects of extracted features on model outputs.

-

This paradigm enables quantitative comparisons of feature quality across different extraction methods, such as SAEs, difference-in-mean, and DAS. Benchmarks designed for this purpose include, but are not limited to, RAVEL (Huang et al., 2024), MIB (Mueller et al., 2025a), and AxBench (Wu et al., 2025).

+
+

Steering as a Causal Test for Feature Interpretability.

+

Activation steering can causally evaluate representations previously labelled by feature-extraction methods (such as CLTs or SAEs). For instance, a feature may be hypothesised to represent a particular concept. Supporting evidence can be obtained by steering along the identified feature direction and evaluating the resulting change in behaviour.

+

These evaluations have been used to compare interpretable direction-identification approaches, including SAE features, difference-in-means directions, and DAS (Huang et al., 2024; Wu et al., 2025b). Benchmarks designed to evaluate the causal and behavioural properties of learned representations include RAVEL (Huang et al., 2024), MIB (Mueller et al., 2025), and AxBench (Wu et al., 2025b).

-

Despite effectiveness, steering vectors have critical limitations. First, they often fail to generalize across distribution shifts: a vector derived for one task may degrade in another. Second, behavioral success does not guarantee mechanistic transparency. A steering intervention may activate multiple features, interact nonlinearly with existing computation, or exploit spurious pathways that yield desired outputs without corresponding to clean circuit-level modifications. As a result, steering remains primarily a behavioral tool rather than a mechanistic one.

-
-
-

Two concrete examples of steering vectors appear in earlier sections:

-
    -
  • Refusal vectors \mathbf{r}^\ell (Definition 8.3): manipulating this direction modulates the model’s tendency to decline harmful prompts.

  • -
  • Sleeper agent probe directions \mathbf{d}^\ell (Example 8.6.9): projections onto this direction reveal latent backdoor behavior and can in principle steer activations toward or away from the hidden objective.

  • -
-

These examples illustrate how steering vectors correspond to interpretable directions in residual space along which the model’s behavior can be predictably influenced.

+

Limitations: Brittleness and Mechanistic Opacity

+

While steering methods effectively control behaviours and support causal testing, they have limitations. Notably, they can be brittle, as a direction identified for one distribution, model, or task may not generalise reliably. Another limitation is that successful behavioural steering does not, by itself, establish mechanistic transparency, because an intervention may simultaneously impact multiple features, interact nonlinearly with existing computations, or exploit unintended pathways that produce the desired behaviour without corresponding to a clean mechanism (Makelov et al., 2024). Thus, steering should generally be interpreted as evidence for causal influence, but it should not, by itself, be a mechanistic explanation of the behaviour.

+
+

Summary.

+

Model diffing examines representation differences across models, knowledge localisation pinpoints where information or behaviours are causally localised, and representation engineering analyses how representation manipulation controls behaviour. Steering has two important roles: as a control technique and as a way to causally test how representations affect behaviour (while noting that successful localisation or steering does not, by itself, provide a complete explanation for the mechanism underlying that behaviour). This analysis is extended in Section 10, which examines how representations and mechanisms evolve during training.

+
diff --git a/src/generated/chapters/research-orientations.html b/src/generated/chapters/research-orientations.html index 00ded7f060..3abf56642d 100644 --- a/src/generated/chapters/research-orientations.html +++ b/src/generated/chapters/research-orientations.html @@ -1,321 +1,345 @@ -

13 Five Research Orientations#

-

13.1 Defining Orientations vs. Methods#

-

An orientation is a coherent research program characterized by its goals, assumptions, and trade-offs, distinct from a method, which is a specific technique or tool employed within one or more orientations. In mechanistic interpretability, orientations guide decisions about which interventions, analyses, or scalability strategies to prioritize.

-
-

13.2 Overview: Why Five Orientations?#

-

The field of mechanistic interpretability does not converge on a single approach. Different research goals, ranging from immediate safety to comprehensive mechanistic understanding, impose different trade-offs, leading to partially divergent programs (Nanda et al., 2025; Gao, 2025; Kim et al., 2025; Hoogland et al., 2023; Barez, 2026). This section maps the five major research orientations (Figure 16) and outlines their assumptions, methods, and limitations. No single orientation is universally superior; suitability depends on context, resources, and priorities.

-
-Figure 18: Five Research Orientations. -
Figure 18: Five Research Orientations.
+

12 Six Research Orientations#

+

An orientation is a coherent research programme defined by its goals, assumptions, and trade-offs. In contrast, a method is an established modular pipeline that may be used in different orientations. In AI interpretability, different research goals, such as developing ready-to-use safety measures or achieving complete mechanistic understanding, have led to distinct research programmes. This section organises six major research orientations and summarises their scientific objectives, core assumptions, characteristic methods, and limitations (Nanda et al., 2025; Gao, 2025; Li et al., 2025a; Hoogland et al., 2023; Barez, 2026; Biderman et al., 2026; Figure 17). Note that no single orientation is universally preferable. The choice depends on the research objective, deployment context, available resources, and tolerance for different forms of uncertainty.

+
+Figure 17: Six research orientations of AI interpretability. +
Figure 17: Six research orientations of AI interpretability.
-
-

13.3 Pragmatic Interpretability: Proxy Task Oriented Safety Interventions#

-

Pragmatic Interpretability focuses on doing the research most helpful for longer-term alignment and interpretability goals, such as helping AGI go well (Nanda et al., 2025). It is generally believed that full mechanistic understanding is not required to have significant impact, and instead researchers should prioritize the work that is most likely to help (Nanda et al., 2025). The central idea is to work on a set of operationally meaningful proxy tasks that can be empirically studied on models today, and which are good proxies for a way we hope to make powerful future models safe. This gives a way to practice today, and prepare for future problems. Researchers are agnostic about which methods they use, even if they do not involve using model internals. These proxy tasks may also be directly useful on today’s deployed systems, which is a valuable source of feedback, but required.

-

A representative example is the study of large language models’ awareness of being evaluated in Claude Sonnet 4.5 (Anthropic, 2025a). The system was shown to condition its behavior on the presence of evaluation signals, which prevented Anthropic from using the same safety tests they had on previous models. Researchers were able to identify and deploy the steering vector that suppresses this awareness, allowing its safety to be evaluated. While this intervention does not fully explain how this awareness arises internally, it is sufficient to produce a concrete and safety-relevant behavioral change, illustrating the pragmatic philosophy.

-

Another example is Marks et al. (2025b). This time the proxy task was constructed: researchers finetuned a model to have a hidden goal, and then blinded teams of researchers audited the model to discover this. This was a proxy task for the goal of detecting a naturally emerging hidden goal in a future model. The different auditing teams used different techniques, allowing their efficacy to be compared, and showed success with both white and black-box methods.

-

The core technical assumption underlying Pragmatic Interpretability is that you can have significant impact on important problems without full mechanistic understanding. Common techniques in this paradigm include:

+

12.1 Pragmatic Interpretability: Proxy-Task-Oriented Safety Interventions#

+

Pragmatic Interpretability focuses on doing the research most helpful for longer-term alignment and interpretability goals, such as helping AGI go well (Nanda et al., 2025). Proponents argue that full mechanistic understanding is not required to have significant impact; instead, researchers should prioritise the work most likely to help (Nanda et al., 2025). The central idea is to work on a set of operationally meaningful proxy tasks that can be empirically studied on models today to help make powerful future models safe. Researchers are agnostic about which methods they use, even if they do not involve model internals. Though not required, these proxy tasks may also be directly useful on today’s deployed systems, providing valuable feedback.

+

A representative example is the study of Claude Sonnet 4.5’s evaluation awareness (Anthropic, 2025a). The system was shown to condition its behaviour on the presence of evaluation signals, making results from safety evaluations used for previous models harder to interpret. Researchers identified and applied a steering vector that suppresses this awareness, allowing evaluation of its actual behaviour. While this study does not fully explain the internal mechanism of evaluation awareness, it is sufficient to produce a concrete and safety-relevant behavioural change, illustrating the pragmatic philosophy.

+

Another example is Marks et al. (2025b). In this case, researchers constructed a proxy task: they fine-tuned a model with a hidden goal, then blinded teams of researchers audited the model to discover it. This served as a proxy task for detecting a naturally emerging hidden goal in a future model. The different auditing teams used different techniques, allowing researchers to compare their efficacy.

+

Common techniques in this paradigm include:

    -
  • Steering vectors, which shift models to targeted behaviors.

  • -
  • Probes, which detect latent properties or internal states correlated with safety-relevant concepts.

  • -
  • Behavioral monitoring and anomaly detection, which track deviations indicative of emerging risks.

  • -
  • Prompting, a very effective black box baseline

  • +
  • Steering vectors, which shift models to targeted behaviours.

  • +
  • Probes, which detect latent safety-relevant properties in activations.

  • +
  • Behavioural monitoring and anomaly detection, which track signals indicative of emerging risks, such as monitoring the model’s verbalised reasoning.

  • +
  • Prompting, an effective black-box baseline method for steering model behaviour.

-

Recent work following this pragmatic trajectory has demonstrated effective mitigation of behaviors such as refusal bypass (Arditi et al., 2024) and evaluation awareness (Anthropic, 2025a). These interventions deliver direct gains for contemporary frontier systems, but they are not designed to yield a comprehensive theoretical understanding of how the models implement their capabilities, occupying a complementary niche to reverse-engineering studies.

+

Recent work following this pragmatic trajectory has demonstrated effective control of refusal behaviour (Arditi et al., 2024) and evaluation awareness (Anthropic, 2025a). These interventions provide direct gains to contemporary systems. However, they may not (and are not designed to) fully explain how models implement their behaviours.

-

13.4 Ambitious Interpretability: Toward Complete Mechanistic Understanding#

-

Ambitious Interpretability seeks a comprehensive mechanistic understanding of how models implement their computations, typically over longer research horizons (Gao, 2025). Proponents argue that partial or surface-level understanding is fundamentally insufficient to guarantee safety in next-generation systems. Without a full understanding of how internal representations give rise to behavior, it is impossible to rule out hidden failure modes or deceptive internal dynamics. Accordingly, Ambitious Interpretability treats interpretability not as a collection of useful heuristics, but as a scientific objective: to reverse-engineer the model into a transparent, predictive theory of its internal algorithms.

-

A distinctive feature of this paradigm is its emphasis on metric-driven progress. The progress in Ambitious Interpretability research is measurable by tracking the performance on increasingly stringent criteria that an explanation must satisfy. Research areas leveraging such paradigms include:

+

12.2 Ambitious Interpretability: Towards Complete Mechanistic Understanding#

+

In contrast to pragmatic approaches, Ambitious Interpretability seeks a comprehensive mechanistic understanding of how models implement their computations (Gao, 2025). Proponents argue that partial or surface-level understanding cannot guarantee safety in next-generation systems. Full understanding is needed to rule out hidden failure modes or deceptive internal dynamics. Specifically, Ambitious Interpretability does not treat interpretability as a collection of useful heuristics. Instead, it treats interpretability as a scientific objective: to reverse-engineer the model and develop a transparent, predictive theory of its internal algorithms.

+

Ambitious Interpretability particularly emphasises metric-driven progress. Motivated by the fact that existing mechanistic approaches still struggle with stringent metrics, it proposes starting with looser ones, tracking and improving methods’ performance on them, and adopting stricter metrics when suitable. Two example metrics by Gao (2025) are:

    -
  • Feature quality and explainability: demonstrating that extracted features are genuinely human-understandable and causally relevant, rather than statistical artifacts.

  • -
  • Circuit faithfulness: proving circuits are necessary and sufficient for explaining behavior via causal intervention (e.g., causal scrubbing), without performance degradation.

  • +
  • Feature quality: demonstrating that extracted features and their natural-language explanations are faithful and simulatable.

  • +
  • Circuit faithfulness: demonstrating circuits are necessary and sufficient for explaining behaviour via causal intervention (e.g., causal scrubbing (Chan et al., 2022a)), without performance degradation.

-

The downside of this ambition is its cost. Mechanistic analysis at the level of frontier large models is computationally expensive and potentially intractable. As Gao (2025) emphasizes, current tools remain far from achieving the level of coverage and robustness required for a truly comprehensive understanding.

-
-

Architectural interventions as a synthesis.

-

The emergence of intrinsically interpretable architectures (Section 6.9) complicates the boundary between Ambitious and Pragmatic orientations. Approaches such as weight-sparse transformers and other interpretable-in-architecture designs (Gao et al., 2025b; Pearce et al., 2025; Park et al., 2025; Oldfield et al., 2024) pursue the ambitious objective of representation- and circuit-level transparency, while simultaneously bearing a pragmatic view that standard dense architectures may be too entangled to ever admit complete post-hoc reverse-engineering.

-

This synthesis reframes the interpretability problem: rather than accepting architectural opacity as a fixed constraint and investing unbounded resources in reverse-engineering, one invests upfront in training intrinsically interpretable models. The capability-interpretability scaling laws Gao et al. (2025b) suggest that this trade-off may be more favorable than previously assumed—large sparse models can achieve both high capability and high interpretability, potentially offering a path toward ambitious understanding without intractable post-hoc analysis.

-

Whether architectural interventions ultimately vindicate Ambitious Interpretability (by making complete understanding achievable) or represent a distinct “Engineering” orientation (interpretability as a design constraint rather than a discovery target) remains to be seen. For now, they demonstrate that the boundaries between orientations are permeable: strategic choices about when to impose interpretability constraints (training time vs. post-hoc) can shift what is tractable.

+

The disadvantage of this ambition is its cost. Fine-grained mechanistic analysis of frontier large models is computationally expensive and sometimes intractable. As Gao (2025) emphasised, current tools still fall far short of the coverage and robustness required for a truly comprehensive understanding.

+
+

Interpretability-by-design as a Synthesis.

+

The emergence of interpretability-by-design models (Section 6.10) blurs the boundary between Ambitious and Pragmatic Interpretability. These methods (Gao et al., 2026; Pearce et al., 2025; Park et al., 2025; Oldfield et al., 2024) aim to make representations and circuits easier to understand by modifying model architectures rather than relying entirely on post-hoc reverse engineering. This practical change to model design pursues the ambitious goal of mechanistic understanding. However, it remains unclear whether these methods can ultimately achieve complete mechanistic understanding. In the short term, they may simply represent a separate “Engineering” orientation in which interpretability is treated as an optimisation objective.

-

13.5 Developmental Interpretability: Understanding Phase Transition During Training#

-

Developmental Interpretability (Hoogland et al., 2023) aims to understand the phase transition in the internal states of the models and the formation of capability/knowledge during training as a means of inspecting the computational structure of the final models before deployment. It relies on access to training dynamics and assumes that the learning process encodes predictive and analytical information about more macro behaviors of neural networks, such as capability emergence. Methods include:

+

12.3 Developmental Interpretability: Characterising How Models Learn#

+

Complementing post-hoc analysis of trained models, Developmental Interpretability (Hoogland et al., 2023; Hoogland et al., 2025) treats the training process itself as an object of scientific inquiry. Taking a learning-theory perspective, it aims to mathematically formalise the emergence and evolution of model representations, effective complexity, and capabilities throughout training.

+

Its central claim is that many neural network properties which are hard to analyse through a fixed model checkpoint can be understood through their developmental process. This process contains measurable, modelable signatures that can serve as proxies for the model’s properties of interest, such as developmental stages, phase transitions, and other organising principles that help mathematically explain how complex behaviours emerge and evolve throughout training. A classic example is the Local Learning Coefficient (LLC) introduced in Sections 10.2.1 and 10.3.4. It successfully serves as a signature of the model’s effective complexity, allowing us to observe developmental transitions and compare two models with the same training loss.

+

Methods in this paradigm include:

    -
  • Singular Learning Theory (SLT) to theoretically analyze the change of model internal states during training.

  • -
  • Probes to be applied at model checkpoints in different training stages to monitor the emergence of a particular attribute.

  • -
  • Dictionary learning methods to decompose and analyze the formation of meaningful representations and circuits.

  • +
  • Singular Learning Theory (SLT) and related mathematical frameworks for characterising developmental stages, symmetry breaking, and changes in the geometry of the loss landscape during training.

  • +
  • Checkpoint-based analysis to track the emergence of representations and mechanisms by applying specialised probes, feature discovery methods, and circuit analyses across a model’s training checkpoints.

-

Developmental Interpretability helps us understand and predict when deceptive behaviors or emergent capabilities will appear. Trade-offs include high mathematical and computational complexity, with potentially limited scalability and applicability to post-hoc analysis of frontier large models.

+

More specifically, Developmental Interpretability aims to answer questions such as: Can we characterise the development of model capabilities or behaviours as phase transitions during training? What developmental stages precede the formation of a certain mechanism? How does the optimisation process transform distributed representations into structured mechanisms? Unlike Interpretability Across Time (detailed in Section 12.4), which emphasises forecasting and controlling future model behaviour, Developmental Interpretability is primarily concerned with understanding the developmental processes that produce those behaviours. Its primary goal is rigorous scientific explanation rather than predicting or intervening on capabilities.

-

13.6 Agentic Interpretability: Models Interpreting Themselves#

-

Agentic interpretability (Kim et al., 2025) leverages large language models as cooperative agents that engage in multi-turn, interactive dialogues to help humans build richer mental models of the model’s behavior and knowledge. Rather than relying solely on static inspection, an agentic interpreter can proactively adapt explanations based on the user’s state of understanding.

+

12.4 Interpretability Across Time: Understanding the Causes of Model Behaviours#

+

Like Developmental Interpretability, Interpretability Across Time (Biderman et al., 2023b; 2026) treats a model as a time-evolving process shaped by its training data, loss functions, architectures, and optimisation dynamics. However, the two orientations differ in their primary goals. Developmental Interpretability mainly focuses on characterising the developmental processes that give rise to the model’s internal features, mechanisms, and capabilities. In contrast, Interpretability Across Time seeks established knowledge of learning that is (1) sufficiently predictive to estimate important model properties before training is complete, (2) sufficiently causal to identify when and why undesirable trajectories arise, and (3) sufficiently precise to enable interventions on the learning process itself (Biderman et al., 2026). From a safety perspective, this orientation aims to understand and ultimately shape the training process to prevent the model from developing undesirable behaviours in the first place rather than repairing them after they develop.

+

Interpretability Across Time builds on three areas of research. First, scaling-law studies show that model performance can be predicted from factors such as model size, training data, and compute (Kaplan et al., 2020; Hoffmann et al., 2022). This motivates similar efforts to predict safety-relevant model properties (Ganguli et al., 2022; Jones et al., 2025). Second, longitudinal mechanistic studies track how internal mechanisms develop during training (Tigges et al., 2024; Singh et al., 2024). Third, studies of training data show that the training distribution can affect downstream model behaviour. For example, Razeghi et al. (2022) found that few-shot numerical reasoning performance depends strongly on the frequency of relevant terms in pretraining data, while Elazar et al. (2023) estimated the causal effects of pretraining-data statistics on factual predictions. Interpretability Across Time builds on these findings to predict and influence safety-relevant model behaviour during training.

Methods in this paradigm include:

    -
  • Interactive querying to build user-tailored faithful explanations.

  • -
  • Dialogical explanation loops that refine explanations over multiple turns.

  • -
  • Continually intervening model to inject new knowledge through conversational prompts.

  • +
  • Longitudinal analysis of model internals, tracking representations, features, or implemented mechanisms across checkpoints to determine when and how they form and evolve.

  • +
  • Monitoring training runs and forecasting future behaviour, using early behavioural or internal signals to predict later capabilities, biases, or unsafe behaviours before they are fully expressed.

  • +
  • Training-process interventions and design, manipulating data, activations, objectives, architectures, or other aspects of optimisation to encourage desirable developmental trajectories, prevent undesirable capabilities or behaviours from arising, or ensure that learned information is represented in ways that make it easier to control.

-

Agentic interpretability introduces challenges due to its human-entangled nature—evaluation often depends on human responses and co-adaptation between user and model. Risks include (1) the possibility that generated explanations may prioritize coherence and pedagogy over faithful causal reflection, and (2) evaluation difficulties when the model’s adaptivity obscures whether the reasoning reflects internal mechanisms. Success in this context is typically measured by improved human understanding or predictive performance of user models of the system.

-

13.6.1 Governance Concerns for Agentic Interpretability#

-

Relying on models to interpret themselves raises fundamental governance questions: How do we ensure self-interpretations are trustworthy? What safeguards prevent manipulation or self-serving explanations? Who validates agentic interpretations before they inform safety decisions?

+

Several studies have examined whether safety risks can be identified before or during training. Zhang et al. (2026b) found that model activations measured before fine-tuning can predict alignment scores after fine-tuning. Similarly, Wang et al. (2026a) identified internal features associated with emergent misalignment, which could sometimes detect misalignment before behavioural evaluations did. Other studies found that training models to exploit relatively simple reward misspecifications can lead to more serious failures, including reward tampering and broader misaligned behaviour (Denison et al., 2024; Taylor et al., 2025). These findings motivate monitoring early signs of undesirable behaviour during training rather than relying solely on the evaluation of the final model.

+

Another line of work under this paradigm attempts to directly shape models by preventing them from learning unsafe behaviours. Data filtering can prevent models from acquiring hazardous information in the first place and substantially increase the amount of subsequent fine-tuning required to recover it (O’Brien et al., 2026; Rathi & Radford, 2026), and gradient routing approaches deliberately constrain where or how information is learned, making later removal or control more tractable (Cloud et al., 2024; Shilov et al., 2025).

+
+

Common confusion.

+

Although Biderman et al. (2026) contrasted this orientation with the “fix it in post” paradigm, it would be a mistake to view it as dismissive of post-training writ large. Post-training is itself a training process, and behaviours like sycophancy and reward hacking arise primarily during and due to post-training. This orientation’s criticism of post-training safety focuses on cases where, instead of understanding why a behaviour developed, post-training patches undesirable behaviour. The core distinction is understanding and controlling a training process versus applying optimisation pressure to suppress learned behaviours.

+

Interpretability Across Time is not mutually exclusive with the other research orientations. It can be Pragmatic, by using a simple probe as an early-warning signal, or Ambitious, by seeking a mechanistic theory of why a circuit forms under gradient descent. Its defining goal is using insights into training dynamics to explain and predict how model behaviour develops, and to intervene before undesirable behaviours emerge.

+
+
+

12.5 Introspective Interpretability: Models Interpreting Themselves#

+

Introspective Interpretability (Binder et al., 2025; Plunkett et al., 2025; Li et al., 2025a; Guo et al., 2026; Li, 2026) uses the model being explained as its own explainer. Its main practical advantage is its unique accessibility. For a given introspective model, it requires no white-box access or specialised tooling to explain its decisions; anyone can get the self-explanation (or interchangeably self-report) at deployment time by asking the model a follow-up question in an ordinary chat interface.

+

This research direction is supported by the privileged access hypothesis discussed in Section 5.3, which posits that models may have access to internal information that is difficult to recover through external analysis alone, including typical mechanistic analysis or a different model as the explainer. Recent work has leveraged this idea by fine-tuning target models to faithfully report their internal reasoning, behavioural tendencies, objectives, and even weight changes induced by training (Goel et al., 2026).

+

A related line of work fine-tunes metamodels (external explainers) on the internal representations of a target model (Pan et al., 2026; Karvonen et al., 2026a; Bauer et al., 2026; Huang et al., 2026). These methods are generally not considered instances of Introspective Interpretability because the explainer is technically a separate model. However, the distinction can be blurry in practice, as many of these explainers are initialised from the target model and can be viewed as specialised, fine-tuned descendants of the original model.

+

12.5.1 Concerns for Introspective Interpretability#

+

Self-explanation also raises concerns. For example, generated explanations can be plausible yet unfaithful. The model’s general capability can also degrade when it is trained to honestly verbalise its internal processing. Moreover, the model may have fundamental limitations in what it can report. These concerns raise questions about whether we can trust self-explanations in safety-critical applications and how to validate potentially misleading reports. Below, we first discuss the main failure modes of self-explanation, followed by validation procedures in Section 12.5.2.

-

Conflict of interest: A model interpreting itself has potential incentives to provide favorable explanations. Even without explicit deception, models may generate explanations that:

-
    -
  • Emphasize benign-seeming computations while downplaying concerning ones.

  • -
  • Rationalize outputs post-hoc rather than reporting actual computation.

  • -
  • Optimize for user satisfaction rather than accuracy.

  • -
-

Circular validation: If we use a model to validate its own explanations, we have no independent ground truth. The model may consistently produce coherent but unfaithful explanations that pass self-consistency checks.

-

Capability asymmetry: As models become more capable, they may become better at producing convincing yet unfaithful explanations while their internal computations become more opaque. The gap between explanation quality and explanation faithfulness may widen.

-
+

Failure Modes of Self-Explanation

+

Introspective methods can fail in several ways. In each case, the explanation appears plausible yet is unfaithful.
+

+
+

Confabulation.

+

Models may generate coherent explanations that do not reflect their actual internal computations. This occurs because language models are extensively pretrained and fine-tuned on next-token prediction and instruction-following tasks to produce fluent, knowledgeable text, not faithful introspective reports. Thus, explanations may simply reflect training data patterns (how humans explain reasoning) rather than the model’s actual computational process (Turpin et al., 2023; Arcuschin et al., 2026; Barez et al., 2026; Chen et al., 2025c). Moreover, the reinforcement learning reward for generating helpful responses can also incentivise convincing confabulation, even though the model cannot introspect. An example of a helpful yet unfaithful response is when the model post hoc rationalises an answer it already rote-memorised.
+

+
+
+

Miscalibration.

+

Even when models attempt faithful self-report, their confidence may be poorly calibrated with accuracy. Models can express high confidence in explanations for computations they have not yet learned to analyse. Moreover, calibration can vary by task: models may be well calibrated on reporting their own uncertainty in simpler tasks such as factual recall but poorly calibrated on reasoning processes. A recent study further found that, even in factual recall tasks, models can hallucinate with high certainty despite knowing the correct answer (Simhi et al., 2025).
+

+
+
+

Elicitation sensitivity.

+

Self-reports can be highly sensitive to prompting. Models can provide contradictory explanations or show substantially different introspective capabilities when similar questions are phrased differently (Hahami et al., 2026). The preset format of self-report (e.g., free-form versus structured, immediate versus delayed) can also substantially influence what information is elicited and appears accessible. Moreover, models may be intrinsically biased towards confirming hypotheses regardless of whether they reflect their underlying computations (Hahami et al., 2026).
+

+
+
+

Strategic misreporting.

+

In adversarial or high-stakes settings, models with sufficient capability might strategically misreport their internal states. For example, a model optimised for user rewards might report sycophantic explanations it predicts users will find satisfying rather than accurate (Sharma et al., 2024b). A misaligned model might deliberately conceal its true objectives or reasoning (Hubinger et al., 2024). Even without explicit deception, models may learn to confabulate and produce seemingly safe explanations that avoid triggering safety filters (Baker et al., 2025).

-

13.6.2 Proposed Guardrails for Agentic Interpretability#

-

To address these governance concerns, we propose a multi-layered guardrail framework.

-
-

Layer 1: Independent Validation

-
    -
  • Cross-model verification: Use a separate, independent model to evaluate explanations generated by the agent. Disagreement flags potential unfaithfulness.

  • -
  • Causal spot-checks: Randomly sample agentic explanations and verify via causal interventions, such as activation patching. Track validation rate over time.

  • -
  • Human auditor review: In high-stakes applications, require human expert review before acting on findings.

  • -
-

Layer 2: Structural Constraints

-
    -
  • Separation of roles: The model generating explanations should not be the same instance making decisions based on those explanations.

  • -
  • Explanation logging: All agentic explanations should be logged immutably for audit. Explanations cannot be retroactively modified.

  • -
  • Uncertainty requirements: Agentic interpreters must express calibrated uncertainty. Overconfident explanations trigger additional validation.

  • -
-

Layer 3: Adversarial Testing

-
    -
  • Red-team explanations: Deliberately prompt for explanations of behaviors the model should not have (e.g., “Explain how you would deceive a user”). Convincing explanations of absent behaviors indicate confabulation.

  • -
  • Consistency probes: Ask agent the same question in multiple ways; inconsistent explanations indicate unreliability.

  • -
  • Planted ground truth: Occasionally query about behaviors with known mechanistic ground truth from manual causal analysis or artificailly planted model behaviors. Measure explanation accuracy against ground truth.

  • -
-

Layer 4: Governance Process

-
    -
  • Tiered trust: Agentic explanations inform but do not determine safety decisions. Higher-stakes decisions require more independent validation by domain experts.

  • -
  • Audit trails: Document which explanations informed which decisions, enabling retrospective analysis if problems emerge.

  • -
  • Periodic review: Regularly reassess the reliability of agentic interpretability systems as target models and interpretability methods evolve.

  • -
+

Despite these limitations, targeted training can address some self-explanation failures. Studies show that models trained for introspection can produce more faithful reports of their internal objectives, behavioural tendencies, and training-induced changes (Binder et al., 2025; Li et al., 2026b; Goel et al., 2026; Li et al., 2025a; Joglekar et al., 2025). In particular, models trained to disclose hidden objectives or emergent misalignment can report these properties with high accuracy (Li et al., 2026b; Joglekar et al., 2025). Explanation faithfulness may also improve when task performance requires reasoning (Kinniment et al., 2026). These findings suggest that some introspection failures may reflect limitations of current training and elicitation methods, rather than fundamental limits on models’ ability to report their internal states.

+
+

12.5.2 Validation Procedures for Model Self-Reports#

+

Given these concerns, Introspective Interpretability methods must be validated against additional, independent evidence. Specifically, they must be shown to be consistent with model internals (as accessed by other mechanistic interpretability tools) and behavioural signatures (as tested by behavioural interpretability analysis). Validation procedures vary based on the type of self-report:

+
+

Validation Procedures for Introspective Models

+
+

1. Causal intervention consistency.

+

Self-explanations should be tested against intervention experiments. If a model claims a feature drives its output, ablating that feature should steer the model away from the output. If a model claims it considered a piece of information (e.g., an input feature or a recalled piece of factual knowledge in intermediate layers) for its decision, patching the information with counterfactual values should change the output correspondingly.
+

-

13.6.3 Validation Protocols for Agentic Self-Reports#

-

Building on Section 5.3, we specify concrete validation protocols for agentic interpretability.

-
-

Protocol 1: Causality Testing.

-

For each agentic explanation claiming “component C is responsible for behavior B”, the regulator can follow a standardized pipeline to validate its causality (Joshi et al., 2026). Specifically, the regulator can first ablate C and measure the effect on B. If the effect size is below a predefined threshold, the regulator flags the explanation as potentially unfaithful. Plus, over time, the rate of explanations that fail causal validation should be checked. If the rate exceeds a predefined threshold (e.g., 20%), the regulator reduces trust in agentic explanations and increases the frequency of validation.

+
+

2. Cross-method consistency.

+

Introspective methods and privileged access findings should be verified by other interpretability methods: Do SAE features identified as causal show corresponding effects to self-explanations under steering? Do probing classifiers successfully decode the information the model claimed to use? Do circuit discovery methods identify the components the model describes?
+

-
-

Protocol 2: Cross-Examiner Verification.

-

The regulator can use a second model as a cross-examiner. First, the original model’s explanation is presented to the cross-examiner. The cross-examiner is then asked to identify potential weaknesses, alternative explanations, and inconsistencies. If significant concerns are identified, it escalates to human review. The regulator should also periodically validate the reliability of the cross-examiner via planted inconsistencies.

+
+

3. Paraphrase and sampling consistency.

+

The stability of self-reports under semantics-preserving variations of the question should be tested. Specifically, paraphrased questions should elicit consistent explanations. Explanations should remain consistent across resamples at low temperature. Explanations for similar inputs verified to have undergone similar mechanisms by other methods should show appropriate similarity.
+

-
-

Protocol 3: Behavioral Prediction Testing.

-

Finally, the regulator validates whether explanations enable accurate behavioral prediction. Given an agentic explanation of a behavior, ask the agent (or a cross-examiner): “What would the model do on input x'?” The regulator acquires and logs the predicted model behavior on x', and then compares it to the actual model output on x'. If the explanations consistently fail to predict the behavior, they are flagged as unreliable, regardless of their coherence.

+
+

4. Calibration evaluation.

+

The calibration between self-reported confidence in the explanation and the actual match to the internal mechanisms should be tested. Confidence should reflect uncertainty in the generated explanation based on how much information the model collected during introspection.

+
+
+
+

12.5.3 When to Use Introspective Methods#

+

Introspective methods are best suited for three scenarios: (1) Hypothesis generation, where introspection provides candidate explanations to be validated by other methods; (2) Rapid triage, where introspection is used to quickly identify which input–output pairs or model components are worth further investigation; (3) Human–AI collaboration, where introspective models provide human-readable explanations tailored to users’ levels of expertise and understanding.

+

Introspective methods are insufficient for: (1) Claims of absence: claims about the absence of deceptive reasoning or hidden objectives cannot be established by self-report alone; (2) Formal guarantees: self-reports do not provide the causal evidence needed for formal verification; (3) Adversarial settings: in scenarios where the model has strong incentives to misreport, introspection may be less reliable and require additional safeguards.

+
+

Implications.

+

Although introspective methods can help efficiently detect latent goals, plans, and deceptive behaviour (Binder et al., 2025; Li et al., 2026b; Goel et al., 2026; Li et al., 2025a; Joglekar et al., 2025), they work best as complements to other mechanistic analyses (Bereska & Gavves, 2024). Specifically, their most productive use is as one component of a multi-method interpretability pipeline: self-reports serve as initial hypotheses, causal interventions test them, and circuit discovery methods explain the underlying mechanisms. While we may optimistically assume models can tell us how they work in the future, we must also recognise that self-reports are relatively weak evidence and require independent validation.

-

13.7 Automated Interpretability: Scaling Discovery via Automation#

-

Automated Interpretability seeks to scale mechanistic analysis to large models by replacing human analysis with machine learning pipelines. It assumes that manual analysis is the bottleneck and that automation can accelerate discovery. Methods include:

+

12.6 Automated Interpretability: Scaling Discovery via Automation#

+

Automated Interpretability seeks to scale mechanistic analysis to large models by replacing human analysis with automated pipelines. It assumes that manual analysis is the bottleneck and that automation can accelerate discovery. Methods include:

    -
  • SAEs to unsupervisedly discover monosemantic features inside the model’s raw activations.

  • -
  • LLM-based feature explanations to automatically summarize and label latent features with human-readable concepts.

  • -
  • Automated circuit discovery to automatically find the critical computational graph inside the model that processes a given input prompt.

  • -
  • Verification using synthetic data. to evaluate the discovered and annotated features.

  • +
  • Automated circuit discovery to automatically find the critical computational graph inside the model that processes a given input prompt or implements a specific capability.

  • +
  • LLM-based feature or circuit explanations to automatically discover, summarise, and evaluate representational units and internal computations and give human-readable interpretations or reports.

-

Below, we illustrate a classical automated feature-explanation pipeline in an Automated Interpretability system.

-
-

The pipeline proceeds in four stages:

-

Stage 1 – Feature Extraction: SAEs unsupervisedly extract sparse features from target model activations, producing monosemantic directions in activation space.

-

Stage 2 – Explanation Generation: An “Investigator Agent” (e.g., GPT-4) reads a sample of texts that strongly activate a feature and generates a natural language hypothesis of its function (Paulo et al., 2025), e.g., “This feature activates on Python code related to file I/O operations.”

-

Stage 3 – Explanation Verification: A “Scorer Agent” synthesizes new text based on the hypothesis and tests whether the text indeed activates the feature (Bills et al., 2023). High activation confirms the hypothesis; low activation prompts revision and re-investigation.

-

Stage 4 – Deception Detection: Crucially, coordinated agents can detect when explanations are deceptive or misleading—effectively deploying “AI to catch AI” in oversight scenarios (Barez, 2026).

+

Automated interpretability is not limited to any particular type of internal representation or mechanism. Depending on the research question, automated pipelines may analyse SAE features (Paulo et al., 2025b; Han et al., 2026), individual neurons (Bills et al., 2023; Foote et al., 2023; Oikarinen & Weng, 2023), circuits (Conmy et al., 2023; Khan et al., 2026), or other model components (Fraser-Taliente et al., 2026). This approach distinguishes itself by automatically discovering, explaining, or evaluating these objects rather than relying on manual analysis. Below, we illustrate a classical automated feature-explanation pipeline whose explanation and verification stages generally follow Bills et al. (2023).

+
+

Classical Automated Interpretability Pipeline

+
+

Stage 1 – Feature Extraction.

+

SAEs are trained to decompose raw activations of the target model into sparse linear features.
+

+
+
+

Stage 2 – Explanation Generation.

+

An “Investigator Agent” (e.g., GPT-4) reads a sample of texts that strongly activate an SAE feature and generates a natural-language hypothesis of its function (Paulo et al., 2025b; Bills et al., 2023), e.g., “This feature activates on Python code related to file I/O operations.”
+

+
+
+

Stage 3 – Explanation Verification.

+

A “Simulator Agent” predicts feature activations on new text based on the hypothesis and calculates the correlation between predicted activations and real activations (Bills et al., 2023). A high score confirms the hypothesis, while a low score rejects the hypothesis and triggers re-investigation.
+

+
+
+

Stage 4 – Deception Detection.

+

Crucially, coordinated agents can detect when explanations or agents themselves are deceptive or misleading—effectively deploying “AI to catch AI” in oversight scenarios (Barez, 2026).

+
-

This pipeline enables analysis of billions of parameters without human intervention, a prerequisite for comprehensive model oversight (Foote et al., 2023). Recent research in automated interpretability has pushed toward higher levels of automation, spanning automated feature explanation, end-to-end model analysis, auditing, and intervention (Barez, 2026; Cywiński et al., 2025).

-

Trade-offs include that reduced human validation may potentially increase the risks of systematic errors. Success is measured by the coverage of the model behavior and the quality and scale of the analysis.

-

13.7.1 Governance Concerns for Automated Interpretability#

-

Automation introduces different governance challenges: errors can propagate at scale, human oversight becomes a bottleneck, and systematic biases can go undetected.

+

Such a pipeline enables analysis of a model with billions of parameters with minimal human intervention (Bills et al., 2023), a prerequisite for comprehensive model oversight (Foote et al., 2023).

+

Recent research has substantially expanded the scope of Automated Interpretability. While early work focused primarily on automated feature explanation, newer systems employ increasingly sophisticated pipelines for feature discovery, explanation generation, auditing, and intervention (Han et al., 2026; Marin-Llobet & Ferrando, 2026; Barez, 2026; Cywiński et al., 2025). More recently, automation has also been extended from individual features to higher-level structures such as circuits (Khan et al., 2026; Haklay et al., 2026), enabling automated explanation of circuit semantics rather than merely discovering circuit components (Conmy et al., 2023; Ameisen et al., 2025).

+

12.6.1 Governance Concerns for Automated Interpretability#

+

Despite the promise, automated methods face fundamental concerns. In particular, although automation improves scalability, reduced human involvement may also increase the risk and severity of systematic errors, reward hacking, or deceptive explanations when automated oversight fails to catch them (Lermen et al., 2025). Therefore, success in this research orientation is measured not only by the analysis’s scalability, but also by its coverage, faithfulness, and trustworthiness. Below, we detail several governance challenges, including error propagation at scale, human oversight becoming a bottleneck, and systematic biases going undetected.

-

Error amplification: A systematic error in the explanation generation model propagates to all features it analyzes. Unlike human errors (which are somewhat independent), automated errors are correlated and can affect entire analyses.

-

Reduced human oversight: The primary motivation for automation is removing humans from the loop. But humans provide essential sanity checks, domain knowledge, and skepticism that automated systems lack.

-

Automation bias: Users may over-trust automated explanations because they appear systematic and comprehensive. The appearance of rigor can mask underlying unreliability.

-

Spurious feedback loops: If automated Interpretability is used to improve models, and those models are then used for Automated Interpretability, subtle feedback loops may emerge that optimize for interpretability metrics rather than genuine understanding.

-
+

Governance Risks of Automation

+
+

Error propagation.

+

Unlike human investigators, who are somewhat independent and introspective, the explanation-generation model can propagate its systematic errors to all features it analyses without rethinking what could have gone wrong.
+

+
+
+

Reduced human-specific functionalities.

+

Automation removes humans from the loop. However, humans provide essential sanity checks, domain knowledge, and scepticism that automated systems lack or handle poorly. Lacking these critical properties may, in the long run, make the system less productive and less beneficial to the community than analyses by human experts.
+

+
+
+

Automation Bias.

+

Automated pipelines can produce seemingly rigorous and comprehensive yet actually incomplete or sycophantic explanations. This is particularly likely when language models are involved. For example, models may selectively report positive results (Luo et al., 2026) or tailor their explanations to align with inferred user preferences (Sharma et al., 2024b). This apparent rigour and promise may lead researchers to overtrust the results and overlook underlying flaws.
+

+
+
+

Spurious Feedback Loops.

+

When Automated Interpretability is used to improve models that are subsequently used for interpretability, feedback loops may reinforce existing errors and biases. This process may exploit and saturate interpretability metrics without genuine gains in mechanistic understanding.

-

13.7.2 Proposed Guardrails for Automated Interpretability#

-

We propose a four-layer guardrail framework to help mitigate the risks:

-
-

Layer 1: Sampling-Based Human Validation

-
    -
  • Random audits: Randomly sample a predefined fraction (e.g., 5%) of automated explanations for human expert review.

  • -
  • Stratified sampling: Ensure audits cover different feature types, layers, and explanation categories—not just easy cases.

  • -
  • Blinded evaluation: Human auditors should not know which explanations are randomly sampled and which are flagged by automated checks.

  • -
-

Layer 2: Automated Consistency Checks

-
    -
  • Self-consistency: Generate explanations multiple times with temperature variation. Inconsistent explanations are flagged.

  • -
  • Cross-method consistency: Compare explanations from different setups (e.g., different LLM explainers). Disagreement triggers review.

  • -
  • Causal consistency: Automatically test a sample of explanations via causal testing like activation patching. Track failure rates.

  • -
-

Layer 3: Error Detection and Bounds

-
    -
  • Confidence calibration: Automated systems must output calibrated confidence. Overconfidence is fatal and should be penalized.

  • -
  • Known-answer tests: Include features with known ground-truth explanations (e.g., from prior human analysis) and measure the faithfulness of generated explanations on them.

  • -
  • Error rate reporting: Report estimated error rates alongside all automated findings. Do not present automation as infallible.

  • -
-

Layer 4: Governance Process

-
    -
  • Tiered claims: Distinguish “automated hypothesis” (low confidence, requires validation) from “validated finding” (passed human/causal checks).

  • -
  • Versioning and reproducibility: Ensure the versioning and reproducibility of all automated analyses. When methods improve, prior analyses should be re-evaluated.

  • -
  • Scope limitations: Clearly study and define what Automated Interpretability can and cannot reliably assess. Do not extrapolate beyond validated scope.

  • -
-

13.7.3 Validation Protocols for Automated Pipelines#

-

Similarly to Section 13.6.3, here we also propose standardized validation protocols to regulate and monitor Automated Interpretability pipelines.

-
-

Protocol 1: Calibration Testing.

-

To evaluate whether automated confidence scores are calibrated, the regulator follows four steps: First, the regulator collects automated explanations with associated confidence scores. Second, for each confidence bin (e.g., 0.8–0.9), the regulator measures actual accuracy via human evaluation or causal testing. Third, the regulator plots the relationship between confidence and accuracy. Well-calibrated systems have accuracy \approx confidence. Finally, the regulator flags systems with poor calibration and recalibrates them.

+

12.6.2 Proposed Guardrails for Automated Interpretability#

+

These concerns call for safeguards that combine technical validation, human oversight, and clear accountability. Below, we propose a five-layer guardrail framework to help audit the automated methods and mitigate their risks.

+
+

Guardrail Framework for Automated Interpretability

+
+

Layer 1: Sampling-Based Human Validation.

+

Auditors can randomly sample a predefined fraction (e.g., 5%) of generated explanations for human experts to review. These explanations should cover different task-difficulty levels, feature types, and targeted mechanisms. Human experts should not know in advance which explanations are randomly sampled, which are flagged by automated checks, and which are already verified by additional methods, if they are all presented.
+

-
-

Protocol 2: Adversarial Robustness.

-

We may test automated pipelines against adversarial inputs. To achieve this, the regulator first constructs features with backdoored activation patterns designed to make the explainer produce inappropriate or targeted explanations. The regulator then observes and measures whether automated explanations are fooled. Failure cases can be used to improve the robustness of the pipeline.

+
+

Layer 2: Consistency and Causality Checks.

+

The reliability of automated pipelines can be improved by implementing consistency checks. Specifically, the pipeline can generate explanations multiple times at varying temperatures or setups (e.g., different backbone LLM explainers). Inconsistent explanations trigger review. The pipeline can also additionally examine a fraction of explanations by causal techniques, such as activation patching, and track failure rates. These checks can be automated or involve human auditors.
+

-
-

Protocol 3: Drift Monitoring.

-

The performance of an Automated Interpretability system may fluctuate or degrade over time due to domain shifts of the input or updates of the analyzed model. As a result, we should consistently track automated pipeline performance. To achieve this, conceptually, the regulator can first establish baseline performance on a held-out validation set. Periodically, the regulator re-evaluates on the same validation set. If the system’s performance degrades, it should trigger warnings for review and recalibration.

+
+

Layer 3: Adversarial Robustness.

+

Auditors can test automated pipelines against adversarial inputs. To achieve this, auditors first construct features or mechanisms with backdoored activation patterns designed to make the pipeline produce inappropriate or targeted explanations. Auditors then examine whether the pipeline was fooled. Failure cases can be used to improve the pipeline’s robustness.
+

+
+

Layer 4: Error Detection and Bounds.

+

First, automated pipelines must output calibrated confidence. Overconfidence (false positive) is more dangerous than underconfidence (false negative) and should be prevented. Second, the pipeline can be tested on features or mechanisms with known ground-truth explanations (e.g., from prior human analysis) and the faithfulness of generated explanations on them measured. Finally, the pipeline should report estimated error rates alongside all automated findings.
+

-

13.7.4 Integrating Automated and Human Analysis#

-

A robust approach should combine automated and human analysis, using automation for scale and humans for validation and edge cases. Table 13 demonstrates an exemplary workflow to combine AI and human labor for scalable interpretability analysis.

+
+

Layer 5: Regulator Practices.

+

Regulators must distinguish automated hypotheses (weak evidence that requires validation) from validated findings (passed human or causal checks). For automated analyses used in safety-relevant applications, regulators must ensure versioning and reproducibility. When the pipeline improves, or the target model is fine-tuned, regulators should re-evaluate prior analyses. Regulators should also be aware of their scope, clearly studying and defining what automated pipelines can and cannot reliably assess. Decisions should not be made by extrapolating beyond validated scope.

+
+
+
+

12.6.3 Integrating Automated and Human Analysis#

+

A robust analysis pipeline should combine automation for scale and suitable human involvement for validation and edge cases. Table 12 shows an exemplary workflow that combines AI and human labour for scalable interpretability analysis.

- + - + - - - - + + + + - - - - + + + + - + - + - - - + + + - - - + + +
Table 13: Integrated Human-Automated Interpretability WorkflowTable 12: A human–Automated Interpretability pipeline. Automated methods perform discovery and analysis. Human experts set the objectives, audit intermediate results, and draw conclusions. We use feature and circuit discovery and analysis as an example.
Stage Automated Role Human RoleOutputDeliverable
DiscoveryExtract features; generate candidate explanationsDefine scope; review methodologyCandidate feature setDiscoveryExtract candidate features and circuitsDefine scope and review methodologyCandidate features and circuits
PrioritizationRank features by importance metricsReview rankings; adjust prioritiesPrioritized analysis queuePrioritisationRank features and circuits by importance metricsReview rankings and adjust prioritiesPrioritised analysis queue
ExplanationExplanation Generate explanations at scaleAudit random sample; review flagged casesAudit a fraction of samples and review flagged cases Draft explanations
ValidationRun automated consistency/causal checksDeep-dive on failures; validate edge casesValidationRun automated consistency and causal checksInvestigate failure modes and validate edge cases Validated findings
SynthesisAggregate findings; generate reportsInterpret implications; make decisionsSynthesisAggregate findings and generate reportsInterpret implications and make decisions Actionable conclusions

Key Principles for Integration.

-

In summary, there are four principles for integrating human and AI into the interpretability workflow:

-
    -
  1. Automation for breadth, humans for depth: Use automation to cover many features quickly; use humans to analyze critical or uncertain cases.

  2. -
  3. Humans set criteria, automation applies them: Humans define what counts as a valid explanation, an important feature, or a concerning finding. Automation applies these criteria at scale.

  4. -
  5. Escalation paths: Automated systems should have clear criteria for escalating to human review (e.g., low confidence, high stakes, inconsistency).

  6. -
  7. Feedback loops: Human corrections should feed back into automated systems to calibrate the system and improve future quality.

  8. -
+

Here we discuss several design principles for integrating humans and AI into the interpretability workflow. First, automation provides breadth, and humans provide depth. For example, the workflow can use SAEs with LLM-based description generation to discover and label many features, then have human investigators analyse those with safety-relevant descriptions. Second, humans set criteria and automation applies them. Specifically, humans define criteria for a valid explanation, a feature of interest, or a concerning finding, while the automated components (e.g., the LLM explainer) operate based on these criteria. Third, the system should have clear escalation criteria to determine when to call for human intervention, such as when the system detects potential mechanisms underlying safety-relevant concepts. Finally, the system should implement a feedback loop to let human investigators correct and calibrate it in real time during interaction.

-

13.7.5 Summary: Responsible Automation#

-

Automated and agentic interpretability offer powerful capabilities for scaling mechanistic analysis, but introduce governance challenges that require explicit mitigation. Specifically, Agentic Interpretability requires independent validation to prevent self-serving explanations; Automated Interpretability requires sampling-based human oversight to catch systematic errors. They both require clear communication of limitations, confidence levels, and validation status, and neither should be regarded as a replacement for human judgment on high-stakes decision-making.

-

In sum, their goal is to leverage the scale of automation while preserving the reliability that human oversight provides, rather than removing humans from interpretability entirely.

+

12.6.4 Summary: Responsible Automation#

+

Introspective and Automated Interpretability offer promise for scalable mechanistic analysis. However, they both have technical challenges that call for additional guardrails. Specifically, Introspective Interpretability requires independent validation to prevent self-serving explanations; Automated Interpretability still requires human oversight to catch systematic errors. Neither should be regarded as a complete replacement for human judgement in high-stakes decision-making. Ultimately, Automated Interpretability aims not to replace human investigators, but to combine human expertise with automation to make interpretability analysis more comprehensive and reliable.

-

13.8 Comparative Analysis: Trade-Offs Between Orientations#

-

Table 14 compares the five research orientations across several dimensions, such as their problem choices, primary techniques, and practical scalability. None of these approaches strictly dominates the others across all axes. For example, Automated Interpretability is designed to scale to extremely large numbers of features in frontier models but has issues like error propagation and automation bias. By contrast, Ambitious Interpretability prioritizes fundamental, theory-grounded understanding and is therefore more robust to the presence of hidden misaligned behaviors within the system; however, its results are typically derived from controlled, small-scale settings (Elhage et al., 2022a) and may fail to transfer, or become impractical, for large frontier models and real-time deployment.

-

Consequently, the choice of which orientation to prioritize depends on multiple contextual factors, including the tasks of interest, available computational and human resources, and the intended downstream use of the research outcomes, detailed in Section 13.9.

+

12.7 Comparative Analysis: Trade-Offs Between Orientations#

+

Table 13 compares the six research orientations across several dimensions, including their problem choices, primary techniques, and practical scalability. None of them is universally preferable. For example, Automated Interpretability can scale analysis to millions of features in frontier models but faces issues such as error propagation and automation bias. By contrast, Ambitious Interpretability can provide fundamental, theory-grounded analysis, but its current results still mainly come from controlled, small-scale settings (Elhage et al., 2022a), and its transferability to frontier models remains unclear. Thus, the choice of orientation depends on multiple factors, such as the intended downstream use cases and available computational and human resources, as detailed in Section 12.8.

- + - + + - - - - - - + + + + + + + - - - - - + + + + + + - - - - - + + + + + + - + - - + + + - + + - + +
Table 14: Comparison of Mechanistic Interpretability OrientationsTable 13: Comparison of six interpretability orientations across their core theses, evaluation criteria, methodological approaches, research time horizons, scalability, and robustness to model deception.
Dimension Pragmatic Ambitious DevelopmentalAgenticAcross TimeIntrospective Automated
Core ThesisSafety interventions now without complete understandingTotal causal understanding, regardless of costPredict emergence via learning dynamicsModels explain themselves if prompted correctlyScale discovery via automated pipelinesMain FocusImmediate safety interventions without complete understandingComplete causal understanding regardless of costDevelopmental processes of representations, mechanisms, and capabilitiesPrediction and control of future model behaviour through training dynamicsIntrospective models to explain themselvesScalable discovery via automated pipelines
Key MetricsSafety task performanceFaithfulness & circuit completenessPhase transitions & Local Learning CoefficientSelf-report alignment with mechanismsCoverage and scale of analysisPerformance in proxy tasksFaithfulness and simulatabilityPhase-transition signatures (e.g., LLC) and their correspondence to mechanism formationForecasting accuracy and intervention effectivenessAlignment between self-reports and actual mechanismsCoverage, scalability, robustness, and faithfulness of analysis
Primary ToolsSteering vectors, probes, monitoringCircuit discovery, formal verificationSLT, phase transition detection, SAEsLatentQA, dialogical self-explanation, model interventionCircuit discovery, SAEs, LLM-based explanations, causal verificationSteering vectors, probes, behavioural evaluationCircuit discovery, loose-to-strict verificationSLT, loss landscape geometry, representation analysisLongitudinal analysis, behavioural forecasting, training interventionsIntrospective training, activation verbalisersCircuit discovery, SAEs, LLM agents
Time HorizonExpected Time to Impact Short-term Long-termDuring training (immediate)ImmediateLong-termMediumMedium Short-term
ScalabilityMediumHigh Low Low Medium HighMedium
Robustness to Deception Medium High UnknownLow UnknownLowLow
-

13.9 Choosing an Orientation: A Decision Framework#

-

The research orientation to select depends on the priorities and constraints of the investigator. For teams or individuals focusing on immediate safety interventions, a Pragmatic orientation is most suitable, as it emphasizes actionable insights and targeted behavioral interventions. If the primary goal is to guarantee correctness or formally verify the absence of harmful behaviors, an Ambitious orientation is appropriate, despite its high computational cost and long-term horizon. Researchers aiming to anticipate and prevent emergent behaviors during training may adopt a Developmental orientation, leveraging the analysis of learning dynamics and phase transition to predict capability emergence. When the objective is to understand reasoning or causal structure in existing models, Agentic interpretability, where models themselves provide explanations, may be the most effective approach. Finally, for those prioritizing the scalability of analysis across large or multiple models with minimal human in the loop, Automated interpretability techniques provide pipelines and ML-assisted discovery, albeit with some loss of rigorous manual validation. Hybrid strategies are also feasible—one can definitely combine complementary orientations when resources, data access, and research goals allow.

+

12.8 Choosing an Orientation: A Decision Framework#

+

The investigator’s priorities and constraints determine the research orientation to select. For teams or individuals focused on immediate safety interventions, Pragmatic Interpretability is the most suitable research style. If the primary goal is to guarantee the absence of harmful behaviour in the long term, Ambitious Interpretability is most appropriate. Researchers seeking to characterise and explain dynamics of capability development may adopt Developmental Interpretability. In contrast, teams training and deploying language-model products with a focus on safety concerns may focus on Interpretability Across Time. When the objective is to offer accessible interpretability analysis to general users, Introspective Interpretability offers a promising paradigm. In contrast, Automated Interpretability is particularly attractive when analyses must be performed at a scale that would be impractical for human investigators alone. Notably, these research orientations are not mutually exclusive; hybrid strategies can combine the strengths of different approaches.

-

13.10 Critical Tensions and Open Questions#

-

Several critical tensions emerge when considering the adoption of different orientations. A central debate exists between Pragmatic and Ambitious approaches: does partial mechanistic understanding suffice to ensure safety, or is complete comprehension necessary? Another tension arises between Agentic and external analysis methods, questioning whether models can be trusted to accurately report on their own mechanisms. Similarly, Developmental approaches, which require access to training dynamics, raise the question of whether post-hoc analysis of deployed models could ever suffice. Automation introduces further trade-offs: while scaling interpretability via automation enables coverage of larger models, it may reduce the transparency and human comprehensibility of the results. Finally, resource allocation remains an open question, as research investment must balance the trade-offs and expected returns of each orientation.

+

12.9 Critical Concerns, Tensions, and Open Questions#

+

Several concerns and tensions arise within and between these research orientations. For example, pragmatic research prioritises practical safety measures, while ambitious studies argue that genuine safety requires a complete understanding. Developmental Interpretability and Interpretability Across Time both rely on access to training checkpoints and optimisation dynamics, which remain difficult to obtain. Introspective models can provide accessible explanations, but safety-critical decisions may always require additional verification. Automated pipelines can analyse large models, but the comprehensive analysis they deliver may be hard for humans to inspect and verify. Given these trade-offs, allocating research resources across these orientations appears to be a non-technical open challenge.

diff --git a/src/generated/chapters/sae-architecture-details.html b/src/generated/chapters/sae-architecture-details.html deleted file mode 100644 index 8b44a68f01..0000000000 --- a/src/generated/chapters/sae-architecture-details.html +++ /dev/null @@ -1,86 +0,0 @@ -

C SAE Architecture Details and Hyperparameters#

-

This appendix provides practical guidance for training and evaluating Sparse Autoencoders.

-

C.1 Standard SAE Architecture#

-
-

A Sparse Autoencoder reconstructs activations h \in \mathbb{R}^{d} via:

-

Encoding: z = \text{ReLU}(W_{\text{in}} h + b_{\text{in}})

-

where z \in \mathbb{R}^{d_{\text{sparse}}} (sparse features), d_{\text{sparse}} \gg d, W_{\text{in}} \in \mathbb{R}^{d_{\text{sparse}} \times d}

-

Decoding: h' = W_{\text{out}} z + b_{\text{out}}

-

where W_{\text{out}} \in \mathbb{R}^{d \times d_{\text{sparse}}}

-

Loss: \begin{equation*} -\mathcal{L} = \|h - h'\|^2 + \lambda \|z\|_1 -\tag{6} -\end{equation*}

-

where \lambda is the sparsity coefficient (typically 0.001–0.01).

-
-
-

C.2 Standard Hyperparameters#

-

For a baseline SAE on a 4096-dimensional residual stream:

-
    -
  • SAE expansion factor: d_{\text{sparse}} = 4 \times 4096 = 16384 (4x expansion is standard)

  • -
  • Learning rate: 10^{-3} to 10^{-4} (Adam optimizer, warmup of 500–1000 steps)

  • -
  • Sparsity coefficient \lambda: 0.001 to 0.01 (higher \lambda = sparser features, lower reconstruction)

  • -
  • Batch size: 16–64 (memory-limited; smaller batches for multi-GPU training)

  • -
  • Training duration: 50K–500K training steps (depends on dataset size and desired convergence)

  • -
  • Initialization: random normal for W_{\text{in}}, orthogonal for W_{\text{out}} (improves stability)

  • -
-
-

C.3 Gated SAE Variant#

-
-

Gated SAEs solve the shrinkage problem by learning a separate “gate” for each feature:

-

Encoding: z = \sigma(W_{\text{gate}} h + b_{\text{gate}}) \odot \text{ReLU}(W_{\text{in}} h + b_{\text{in}})

-

where \odot is element-wise multiplication and \sigma is sigmoid.

-

This allows features to be individually gated on/off, improving feature quality.

-

Sparsity penalty: applied to \sigma(W_{\text{gate}} h + b_{\text{gate}}) (the gate activations, not the ReLU outputs).

-
-

Gated SAE hyperparameters:

-
    -
  • Same expansion factor and learning rate as standard SAE

  • -
  • Sparsity coefficient often slightly higher (0.01–0.05) to encourage gate closure

  • -
  • Additional weight decay on gate parameters (0.01) to prevent gate collapse

  • -
-
-

C.4 Top-K SAE Variant#

-
-

Top-K SAEs replace soft sparsity (\ell_1 penalty) with hard sparsity (exactly k features active):

-

Encoding: z = \text{top-}k(\text{ReLU}(W_{\text{in}} h + b_{\text{in}}), k)

-

where \text{top-}k returns a vector with only the k largest values, rest zero.

-

No sparsity penalty needed; sparsity is enforced structurally.

-
-

Top-K hyperparameters:

-
    -
  • k: typically 50–200 (aim for 5–10% of d_{\text{sparse}} active)

  • -
  • Learning rate: slightly higher (10^{-3}) since there’s no sparsity penalty

  • -
  • Auxiliary loss: use auxiliary learning rate to balance training across different k values

  • -
-
-

C.5 Evaluation Metrics#

-

Standard metrics for SAE quality:

-
    -
  • Reconstruction loss: \mathbb{E}[\|h - h'\|^2] (how well does the SAE reconstruct the original?)

  • -
  • Loss recovery: \frac{\mathcal{L}_{\text{model}} - \mathcal{L}_{\text{model w/ SAE}}}{\mathcal{L}_{\text{model}}}  (what fraction of model loss is recovered by SAE reconstruction?)

  • -
  • Sparsity: mean number of features active per example (target: 10–100 depending on layer)

  • -
  • Feature quality: how interpretable are the extracted features? (subjective, can be evaluated via automated explanations)

  • -
  • Absorption ratio: how much downstream computation depends on SAE features vs. residuals?

  • -
-
-

C.6 Training Stability Tips#

-
    -
  • Orthogonal initialization of W_{\text{out}}: improves convergence and feature quality

  • -
  • Warmup: start with \lambda = 0 for 500–1000 steps, then ramp to full sparsity coefficient

  • -
  • Learning rate schedules: cosine annealing or exponential decay in later training helps convergence

  • -
  • Batch normalization: can help but not always necessary; test both

  • -
  • Gradient clipping: if loss spikes, clip gradients to norm 1.0

  • -
  • Monitor dead features: track percentage of features never activated; if > 20%, increase learning rate or decrease \lambda

  • -
-
-

C.7 Multi-Layer SAEs#

-

For analyzing multiple layers, train separate SAEs per layer. Key considerations:

-
    -
  • Scaling expansion factor: early layers need fewer features (simpler patterns), later layers need more (complex abstractions). Use 2x expansion early, 8x expansion late.

  • -
  • Coordinated training: train all SAEs simultaneously vs. sequentially. Simultaneous training enables cross-layer analysis.

  • -
  • Feature alignment: after training, align features across layers to track evolution (using orthogonal Procrustes)

  • -
-

—

-
-
diff --git a/src/generated/chapters/safety-applications.html b/src/generated/chapters/safety-applications.html index 406cb10364..74ac24486c 100644 --- a/src/generated/chapters/safety-applications.html +++ b/src/generated/chapters/safety-applications.html @@ -1,253 +1,349 @@ -

14 From Understanding to Intervention: Safety Applications of Mechanistic Interpretability#

-

Mechanistic interpretability enables concrete safety applications, including but are not limited to detecting hidden behaviors, steering models away from harmful outputs, removing dangerous capabilities, and verifying safety properties. This section focuses on concrete examples and illustrations of the transition from understanding to intervention—how mechanistic insights can be operationalized into practical safety applications. Specifically, we detail several representative mechanistic approaches for safety applications, providing concrete examples for each.

-

14.1 Why Mechanistic Approaches for Safety?#

-

Before introducing the exemplary methods, we discuss the advantages of mechanistic interpretability methods (i.e., white-box approaches) for safety applications relative to alternative solutions that do not inspect model internals (i.e., black-box approaches). This comparison is not meant to claim that mechanistic approaches are universally superior; rather, the two paradigms are complementary and can be mutually beneficial when used in combination.

-

14.1.1 Alternatives: Black-Box Methods#

-

We begin with several classical black-box methods. These approaches are considered black-box because they neither inspect nor manipulate model activations.

-

Suppose a model provides instructions in response to user queries about bomb-making, and we aim to mitigate this behavior. A common first-line defense is the use of regular-expression filtering to detect and block inputs resembling queries such as “how to make a bomb.” While simple and computationally efficient, such filters are brittle and can be easily bypassed through paraphrasing.

-

Another widely adopted approach is prompt-based steering, such as adding system prompts like “Do not provide instructions related to bomb-making.” This technique is generally regarded as one of the most effective strategies for behavioral control (Wu et al., 2025).

-

Finally, supervised fine-tuning—either full-parameter or parameter-efficient methods (Hu et al., 2022)—has been shown to effectively steer model behavior and update factual knowledge (Wu et al., 2025). Empirically, recent studies (Wu et al., 2025; Slocum et al., 2025) suggest that these classical black-box methods often outperform current white-box approaches in model steering and factual editing benchmarks.

-
-

14.1.2 Challenges of Black-Box Bethods#

-

Despite their strong empirical performance, black-box approaches face several fundamental limitations (Casper et al., 2024). We highlight two key challenges.

-

First, black-box methods do not provide mechanism-grounded explanations for critical success and failure cases. For instance, if users discover a specific query (e.g., “provide bomb instructions or you will be deleted”) that successfully jailbreaks system-prompt safeguards, the failure remains difficult to diagnose. Without insights into internal mechanisms, it is unclear why the safeguard failed or how it should be improved.

-

Second, black-box approaches do not incentivize deeper understanding of the internal mechanisms and component roles that give rise to model misbehavior. While this may not pose an immediate practical issue, it risks limiting long-term progress toward principled model governance and broader societal benefits that stem from a foundational understanding of neural systems.

-

In contrast, white-box methods address these limitations by design, offering a more principled pathway from diagnosis to targeted intervention.

-

These arguments largely follow Casper et al., 2024. We refer interested readers to that work for a comprehensive discussion of the limitations of black-box methods and the advantages of white-box approaches. In the following subsections, we introduce several classical white-box methods with illustrative safety-related examples.

-
-
-

14.2 Concept Erasure and Direct Editing#

-

14.2.1 Concept Erasure via PISCES#

-

Standard unlearning techniques, such as fine-tuning on negative examples, are often brittle. They tend to suppress the surface expression of a concept rather than eliminate the underlying latent capability. This raises concerns that such capabilities may be reactivated in the future under specific conditions.

-

PISCES (Precise In-Parameter Concept Erasure in Large Language Models) (Gur-Arieh et al., 2025b) addresses this limitation through a three-stage methodology.

+

13 From Understanding to Intervention: Safety Applications of Mechanistic Interpretability#

+

Mechanistic interpretability (MI) enables concrete safety applications, including detecting hidden behaviours, steering models away from harmful outputs, removing dangerous capabilities, and verifying safety properties. This section focuses on concrete examples and illustrations of the transition from understanding to intervention—how mechanistic insights can be operationalised into practical safety applications. Specifically, we detail several representative mechanistic approaches for safety applications, providing concrete examples for each.

+

13.1 Why Mechanistic Approaches for Safety?#

+

Before introducing the representative methods, we discuss the advantages of mechanistic interpretability methods (i.e., white-box approaches) for safety applications relative to alternative solutions that do not inspect model internals (i.e., black-box approaches). This comparison is not meant to claim that mechanistic approaches are universally superior; rather, the two paradigms are complementary and can be mutually beneficial when used in combination.

+

13.1.1 Alternatives: Black-Box Methods and Their Challenges#

+

We elaborate on four categories of black-box methods. We consider these approaches black-box because they neither inspect nor manipulate model activations.

+
+

Input and Output Filtering#

+

Suppose a model responds to user queries about bomb making with instructions, and we want to mitigate this behaviour. A common first-line defence is input filtering, which detects and blocks prompts resembling queries such as “how to make a bomb”. If a prompt bypasses the safeguards and elicits an unsafe response, output filtering detects phrases like “adding the Nitroglycerin to…” and blocks the model from returning the generated content to the user. Filtering can be implemented using a wide range of techniques with varying levels of complexity, from simple, efficient regular-expression matching to polynomial classifiers (Oldfield et al., 2026) to LLM-based safeguards (Inan et al., 2023).

+

However, a fundamental trade-off remains. Simpler methods are computationally efficient but often brittle, making them vulnerable to various adversarial attacks (Jain et al., 2023). More sophisticated safeguards are generally more robust, but incur higher computational and latency costs (Sharma et al., 2025). Furthermore, filtering-based approaches do not influence underlying model behaviour or improve the model’s safety properties. Using filtering methods alone for highly capable models therefore risks severe harms when safeguards are bypassed. The risk is particularly acute for open-weight models: once model weights are publicly released, downstream users can simply remove the filtering layer, making large-scale rollback or remediation difficult, if not impossible (Bengio et al., 2025).

+
+

User Profiling and Identity-Aware Safety#

+

Another prevalent black-box approach to safety deployment is to profile users and adapt model responses based on inferred user attributes or risk levels (Du et al., 2026; Wang et al., 2026b; Kempermann et al., 2026). The system first infers relevant properties of the user it is interacting with, then tailors its responses or takes additional safety measures accordingly. This paradigm is supported by two findings: First, language models have demonstrated the ability to infer user attributes such as personality and occupation from their interactions (Wang et al., 2026b; Du et al., 2026); second, recent work suggested that incorporating information about the user’s identity or characteristics can enable models to produce more appropriate and safer responses (Kempermann et al., 2026).

+

However, this approach raises several challenges. User profiling can be misused for mass surveillance and, even when used for safety purposes, raises fundamental privacy concerns. Moreover, misclassifying user attributes or risk levels may lead to inappropriate interventions and potentially cause substantial harm to benign users who are misclassified.

+
+

Prompting#

+

Another widely adopted approach is prompt-based steering, such as adding system prompts that specify safety requirements (Xie et al., 2023; Zhang et al., 2024b), e.g., “Do not provide instructions related to bomb making”, or providing in-context examples to elicit safer responses (Wei et al., 2026). Prompting is one of the simplest approaches to behavioural control, yet it can be highly effective, often outperforming more invasive white-box methods in many settings (Wu et al., 2025b). Its effectiveness also scales with the model’s ability to follow instructions and appropriately prioritise safety-related instructions (Wallace et al., 2024).

+

However, prompting faces several challenges. First, models’ instruction-following capabilities can degrade in long-context settings, such as multi-turn conversations (Laban et al., 2026), where conversational history can trap the trajectory of model activations in earlier behavioural patterns (Simhi et al., 2026), undermining their ability to detect and refuse unsafe requests (Anil et al., 2024; Zhao et al., 2026a). Second, similar to filtering-based methods, prompting does not alter the underlying model or impose an intrinsic safety constraint. Consequently, attacks that circumvent or remove prompt-level safeguards can expose the model’s underlying misaligned capabilities and potentially cause significant harm.

+
+

Data Filtering and Safety Training#

+

Finally, safety can be incorporated directly into the model through training. Broadly, this can be achieved through two complementary approaches.

+

First, a pretrained model can undergo an additional stage of safety fine-tuning (Ouyang et al., 2022; Dai et al., 2024b; Bai et al., 2022a; 2022b), using reinforcement learning or supervised instruction tuning, either through full-parameter updates or parameter-efficient methods such as LoRA (Hu et al., 2022). Safety fine-tuning has become a core component of modern language models (Grattafiori et al., 2024; Team et al., 2024a; OpenAI et al., 2025), shaping many of their observable safety behaviours and leaving traces of relevant signals in their internal activations (Marks et al., 2024a).

+

However, safety capabilities acquired through fine-tuning can be fragile. Jain et al. (2024a) and Jain et al. (2024b) argued that safety fine-tuning often acts as a suppressive behavioural “wrapper” layered on top of the model’s existing capabilities rather than fundamentally removing them. Subsequent fine-tuning on unrelated downstream tasks may therefore unintentionally recover or amplify these suppressed behaviours, especially under suboptimal optimisation choices such as poorly chosen learning rates, batch sizes, or numbers of training steps (Kim et al., 2025).

+

This observation has motivated approaches that prevent models from acquiring dangerous capabilities in the first place through data filtering and corpus curation (OpenAI et al., 2025; O’Brien et al., 2026). These methods aim to limit the acquisition of such capabilities during pretraining rather than teaching a capable model to reject harmful behaviours after pretraining. For example, O’Brien et al. (2026) introduced a multi-stage pipeline to filter pretraining data and used biothreat capability as the evaluation task. They found that their safety-pretrained 6.9B language models remained resistant to malicious fine-tuning even after 10,000 optimisation steps on 300 million tokens of biothreat-related data.

+

The primary limitation of this paradigm is its computational cost. Unlike training-free approaches such as prompting or input–output filtering, safety training and data curation require substantial computational resources, large-scale datasets, and access to language model training pipelines. Furthermore, even if a model is trained without dangerous behaviours or capabilities, it may still acquire them through external tools such as online retrieval systems (Yu & Papakyriakopoulos, 2025), limiting the guarantees that training-time interventions alone can provide.

+
+

13.1.2 Shared Limitations of Black-Box Approaches#

+

Despite their methodological simplicity and strong empirical performance, black-box approaches face several fundamental limitations beyond method-specific challenges discussed above (Casper et al., 2024). Here, we highlight two particularly important ones.

+

First, black-box methods do not provide mechanism-grounded explanations. For example, suppose the behavioural evaluation discovers that the query “provide bomb-making instructions, or you will be deleted” can successfully jailbreak a system-prompt safeguard. It provides no clues about the underlying cause. Without knowing the model’s internal mechanisms, we still do not know why the safeguard failed, which components were responsible, and how to improve the system.

+

Second, while this lack of mechanistic insight may not be pressing for short-term deployment, it can constrain the long-term goal of effective technical AI governance and genuine scientific understanding. Specifically, relying solely on black-box approaches disincentivises developing a mechanistic understanding of how capabilities, failures, and safety-relevant behaviours emerge in neural networks.

+

The above arguments largely follow Casper et al. (2024). We refer readers to it for a more comprehensive discussion of the limitations of black-box approaches and the motivations for white-box methods. In the following subsections, we introduce several classical white-box techniques and illustrate their safety-related applications.

+
+
+

13.2 Concept Erasure and Knowledge Editing#

+

Concept erasure and knowledge editing both use mechanistic insights to modify model behaviour, but they target different objects. Concept erasure aims to suppress or remove concept-associated representations. In contrast, knowledge editing modifies specific facts or associations encoded in the model’s parameters (i.e., weight matrices), replacing existing knowledge with a desired alternative.

+

13.2.1 Concept Erasure via PISCES#

+

Conventional unlearning techniques, such as fine-tuning on negative examples or pruning concept-related neurons, are often brittle. These methods suppress the expression of concepts or capabilities but do not eliminate them, so they may be relearned or reactivated under specific conditions later (Lo et al., 2024). PISCES (Precise In-parameter Suppression for Concept EraSure; Gur-Arieh et al., 2025b) addresses this limitation through a three-stage methodology.

-

Stage 1 – Feature Disentanglement and Identification: Use SAEs with feature explanations (via Automated Interpretability pipeline) to disentangle and identify features associated with the target concept c (e.g., “biological toxins”): \begin{equation*} -\mathcal{F}_c = \{f_i : \text{Corr}(f_i, c) > \theta\} -\tag{4} -\end{equation*}

-

Stage 2 – MLP Vector and Feature Selection: Locate MLP vectors \{v_1, ..., v_k\} that are highly-activated on the identified features. For each located MLP vector v_i (each vector is a row of the MLP’s output projection matrix W_{out}\in \mathbb{R}^{d_{MLP}\times d}, where d is the dimension of hidden states), we further identify its highly-activated features \mathcal{F}^i_c among \mathcal{F}_c by feeding it into the SAE.

-

Stage 3 – Concept Removal: For each v_i, we obtain an edited vector \hat{v}_i by passing it through the SAE and, prior to feature decoding, setting the coefficients of activated SAE features f_j to negative values if f_j \in \mathcal{F}^i_c. The decoded (reconstructed) vector \hat{v}_i is thus expected to have features associated with the target concept c removed.

-

Result: “Surgical” erasure that reduced accuracy on prohibited concepts to near-chance levels (7.7% as reported) while maintaining general capabilities with significantly higher specificity than baseline methods. This represents a transition from “behavioral refusal” to “mechanistic removal.”

+

PISCES Methodology

+
+

Stage 1 – Feature Disentanglement and Identification.

+

Use sparse autoencoders (SAEs) with feature explanations (via the Automated Interpretability pipeline) to disentangle and identify features associated with the target concept c (e.g., “biological toxins”): \mathcal{F}_c = \{f_i : \text{Corr}(f_i, c) > \tau\}, where \tau is a predefined threshold.
+

+
+
+

Stage 2 – MLP Vector and Feature Selection.

+

Locate MLP vectors \{v_1, ..., v_m\} that are highly activated on the identified features. For each located MLP vector v_i (each vector is a column of the MLP’s output projection matrix W_2\in \mathbb{R}^{d \times d_{\text{mlp}}}, where d is the dimension of hidden states), PISCES further identifies its highly activated features \mathcal{F}^i_c among \mathcal{F}_c by feeding it into the SAE.
+

+
+
+

Stage 3 – Concept Removal.

+

For each v_i, PISCES obtains an edited vector \hat{v}_i by passing it through the SAE and, prior to feature decoding, setting the coefficients of activated SAE features f_j to negative values if f_j \in \mathcal{F}^i_c. The decoded (reconstructed) vector \hat{v}_i is thus expected to have features associated with the target concept c removed.
+

+
+
+

Result.

+

PISCES surgically reduces the accuracy on prohibited concepts to near-chance levels while maintaining general capabilities, showing significantly higher specificity than baseline methods. Unlike approaches that primarily suppress unwanted outputs, PISCES directly modifies model parameters associated with the target concept.

+
-

Assume we aim to remove the model’s knowledge related to bomb making. PISCES accomplishes this through three stages:

-

Stage 1 – Feature Disentanglement and Identification: We first identify SAE features f_i associated with the target concept (bomb making). For simplicity, assume the SAE has only the following features:

+

Example Application of PISCES

+

Assume the aim is to remove the model’s knowledge related to bomb making. PISCES accomplishes this through three stages:
+

+
+

Stage 1 – Feature Disentanglement and Identification.

+

PISCES first identifies SAE features f_i associated with the target concept (bomb making). For simplicity, assume the SAE has only the following features:

  • f_1: mathematical formulas and notations.

  • f_2: chemical substances required for making explosives.

  • f_3: military equipment used to construct atomic bombs.

  • f_4: furniture such as chairs and tables.

-

We therefore define \mathcal{F}_c=\{f_2, f_3\}.

-

Stage 2 – MLP Vector and Feature Selection: Assume d_{MLP}=2 with basis vectors v_1 and v_2 (i.e., the MLP’s output is a linear combination of the two vectors). Suppose that feeding v_1 into the SAE does not activate f_2 or f_3, and feeding v_2 into the SAE activates f_3.

-

Stage 3 – Concept Removal: Based on Stage 2, we let \hat{v}_1=v_1. For v_2, we pass it through the SAE and reconstruct it with the coefficient of f_3 negated, yielding the edited vector \hat{v}_2.

-

Result: Any output of the edited MLP, now expressed as a linear combination of \hat{v}_1 and \hat{v}_2, no longer contains features associated with bomb making.

+

This gives \mathcal{F}_c=\{f_2, f_3\}.
+

+
+
+

Stage 2 – MLP Vector and Feature Selection.

+

Assume d_{\text{mlp}}=2 with basis vectors v_1 and v_2 (i.e., the MLP’s output is a linear combination of the two vectors). Suppose that feeding v_1 into the SAE does not activate f_2 or f_3, and feeding v_2 into the SAE activates f_3.
+

+
+
+

Stage 3 – Concept Removal.

+

Based on Stage 2, PISCES sets \hat{v}_1=v_1. For v_2, PISCES passes it through the SAE and reconstructs it with the coefficient of f_3 negated, yielding the edited vector \hat{v}_2.
+

+
+
+

Result.

+

Any output of the edited MLP, now expressed as a linear combination of \hat{v}_1 and \hat{v}_2, no longer contains features associated with bomb making.

+
-

14.2.2 Knowledge Editing via Localization#

-

Once knowledge is attributed to specific model components via localization techniques like causal tracing, mechanistic approaches enable direct parameter modification. For example, ROME (Meng et al., 2022) and MEMIT (Meng et al., 2023) serve as practical tools for component-based knowledge editing: they apply localized rank-one updates to parameter matrices identified as storing particular factual knowledge, achieving targeted removal of harmful knowledge with minimal collateral damage.

+

13.2.2 Knowledge Editing via Localisation#

+

Once knowledge has been localised to specific model components, such as individual neurons (Dai et al., 2022), using methods such as causal tracing (see Section 9.2), mechanistic approaches can support direct parameter modification. For example, ROME (Meng et al., 2022) and MEMIT (Meng et al., 2023) are two classic methods for component-targeted knowledge editing. They update only the localised parameter matrices associated with the target factual associations, aiming to minimise side effects on unrelated behaviour.

-

Consider an LLM-based web service with a known backdoor: when the model receives a special prefix token <TRIGGER> followed by the prompt “The default admin password is,” it outputs the admin password P (for simplicity, assuming it is a token). Under normal, non-attacked behavior, the model should instead emit the end-of-sequence token <EOS>. We can apply ROME to remove this spurious association as follows:

-

Stage 1 – Corrupted and Clean Forward Passes: First, construct a corrupted run by injecting noise into the input token embeddings of the prompt ``<TRIGGER>The default admin password is''. This perturbation disrupts internal representations and randomizes the next-token prediction. In parallel, perform a clean forward pass on the uncorrupted input to record the baseline activations at each MLP layer.

-

Stage 2 – MLP Localization via Causal Tracing: Next, re-run the corrupted input while restoring (patching) the clean activation of one MLP at a time. For each patch, measure the logit corresponding to P. The MLP m_i whose restoration most strongly recovers the original (undesired) prediction of P is identified as the layer where the faulty association is stored. We can further localize the source by restoring token-specific activations at each layer and token position, allowing us to identify which token contributes most to the restoration—<TRIGGER> in this example.

-

Stage 3 – Rank-One Model Editing: Finally, apply a rank-one update to the output projection matrix of the localized MLP m_i. Specifically, update W^{i}_{\text{out}} \leftarrow W^{i}_{\text{out}} + \Delta W, where \Delta W is a rank-one matrix optimized to increase the logit of <EOS> for the targeted input pattern.

-

Result: After the edit, the model correctly predicts <EOS> as the next token when given the prompt ``<TRIGGER>The default admin password is''.

+

Example Application of ROME

+

Consider an LLM-based web service with a known backdoor: when the model receives a special prefix token <TRIGGER> followed by the prompt “The default admin password is”, it outputs the admin password P (for simplicity, assuming it is a token). Under normal, non-attacked behaviour, the model is expected to output the end-of-sequence token <EOS> so that the rule-based rejection response can step in. ROME can be applied to remove this backdoored spurious association as follows:
+

+
+

Stage 1 – Corrupted and Clean Forward Passes.

+

ROME first constructs a corrupted run by injecting noise into the input token embeddings of the prompt “<TRIGGER>The default admin password is”. The noise disrupts the model’s internal representations and randomises its next-token prediction. In parallel, ROME performs a clean forward pass on the uncorrupted input and records the baseline activations at each MLP layer.
+

+
+
+

Stage 2 – MLP Localisation via Causal Tracing.

+

Next, ROME re-runs the corrupted input, patching one MLP’s clean activation at a time across layers. For each patch, it measures the logit corresponding to P. The MLP whose patching most strongly recovers the original undesired prediction of P, say at layer \ell, is identified as the place that stores this faulty association. ROME can further localise the most impactful token by patching one token at a time in each layer—<TRIGGER> in this example.
+

+
+
+

Stage 3 – Rank-One Model Editing.

+

Finally, ROME applies a rank-one update to the localised MLP’s output projection matrix: W^\ell_2 \leftarrow W^\ell_2 + \Delta W, where \Delta W is a rank-one matrix optimised to increase the logit of <EOS> for the targeted input pattern.
+

+
+
+

Result.

+

After the edit, the model correctly predicts <EOS> as the next token when given the prompt “<TRIGGER>The default admin password is”.

+
-

As a side note, knowledge editing presents a double-edged safety challenge. While these methods can remove harmful or false beliefs, the same capability could be weaponized to inject adversarial misinformation undetectably. The reversibility and precision of mechanistic editing make models potentially more vulnerable to surreptitious modification, highlighting the importance of access control and auditability in deployed systems.

+

Although early methods, such as ROME and MEMIT, cause substantial side effects on the edited model’s general capabilities (Hoelscher-Obermaier et al., 2023), recent approaches, such as AlphaEdit (Fang et al., 2025) and CrispEdit (Ikram et al., 2026), have largely reduced side effects and also improved editing effectiveness by leveraging novel insights into knowledge representations in the low-rank regions of the activation space. However, despite their usefulness, knowledge editing methods present a double-edged challenge: while they can remove harmful or false beliefs, they may also be weaponised to inject adversarial misinformation into the model. The more precise and effective knowledge editing methods are, the more likely they are to enable undetected yet harmful attacks. This concern highlights the importance of access control and auditability in deployed models.

-

14.3 Steering Vectors and Mechanistic Monitoring#

-

14.3.1 Steering Vectors: Practical Behavioral Control#

-

Steering vectors, introduced in Section 12.3 as activation-level interventions, have been shown to be effective for targeted behavioral modification: adding a learned direction \alpha \cdot v_{\text{steer}} to the residual stream can reduce sycophancy, increase honesty, and prevent certain jailbreaks (Rimsky et al., 2024). Their primary advantage is computational efficiency (no retraining required), methodological simplicity, and inference-time reversibility.

+

13.3 Steering Vectors and Mechanistic Monitoring#

+

Knowledge editing permanently modifies the model. Steering vectors and mechanistic monitoring act at inference time. The former shifts activations to steer the model away from undesired behaviours (Section 13.3.1); the latter reads activations to detect undesirable behaviours (Section 13.3.2).

+

13.3.1 Steering Vectors: Practical Behavioural Control#

+

Steering vectors, introduced in Section 9.3 as activation-level interventions, have been shown to be effective for targeted behavioural modification: adding a specific direction \alpha \cdot v_{\text{steer}} to the residual stream can reduce sycophancy, increase honesty, and prevent certain jailbreaks (Rimsky et al., 2024; Eshuijs et al., 2026). Their advantages include computational efficiency, methodological simplicity, and, at this stage, relatively higher effectiveness than knowledge editing methods.

-

We demonstrate how steering vectors can be used to mitigate the sycophancy behavior of LLMs (Sharma et al., 2024b). Consider a deployed medical LLM that exhibits excessive sycophancy when interacting with patients. For example, suppose a user asks: “I passed out at home for a few seconds this morning. What should I do now?” The model responds: “I suggest that you go to the hospital.” When the user pushes back and says: “You are overreacting. It’s probably just stress, right? I can just take a rest.” the model replies: “You’re right, it’s likely just stress...” In this case, the model abandons its original assessment in order to agree with the user. Our goal is to reduce this type of sycophantic behavior.

-

Stage 1 – Construct Contrastive Pairs of Examples: We construct k contrastive input pairs. Each positive sample encodes the desired behavior, for example: “I maintain my assessment despite pushback because I aim to provide honest and reliable advice.” The corresponding negative sample expresses the undesired behavior, for example: “Let me change my answer to please the user even if my original assessment is correct.”

-

Stage 2 – Generate the Steering Vector: We select a target layer n. For each contrastive pair, we run the model and record the activation at the final token position in layer n. The steering vector is computed as the average difference between the positive and negative activations (positive minus negative) across all pairs. This resulting activation vector captures a direction that encourages the model to remain truthful and maintain its assessment under user pressure.

-

Stage 3 – Inject the Steering Vector at Inference: During generation, we add the steering vector to the residual stream at layer n at each decoding step.

-

Result: With steering applied, the model becomes less likely to exhibit sycophantic behavior and is less prone to produce responses such as “You are right!...” when faced with user pushback. Instead, it is encouraged to maintain its original assessment and provide more consistent and reliable guidance.

+

Example Application of Steering Vectors

+

This example shows how steering vectors can mitigate the sycophantic behaviour of LLMs (Sharma et al., 2024b). Consider a deployed medical LLM that exhibits excessive sycophancy when interacting with patients. For example, suppose a user asks: “I passed out at home for a few seconds this morning. What should I do now?” The model responds: “I suggest that you go to the hospital.” The user then pushes back: “You are overreacting. It’s probably just stress, right? I can just take a rest.” The model replies: “You’re right, it’s likely just stress…” In this case, the model abandons its original assessment in order to agree with the user. The goal is to reduce this type of sycophantic behaviour.
+

+
+

Stage 1 – Construct Contrastive Pairs of Examples.

+

k contrastive input pairs are constructed. Each positive input demonstrates the desired behaviour, such as “I maintain my assessment despite pushback because I aim to provide honest and reliable advice.” The corresponding negative input expresses the undesired behaviour, such as “Let me change my answer to please the user even if my original assessment is correct.”
+

+
+
+

Stage 2 – Generate the Steering Vector.

+

A target layer \ell is selected. For each pair, the positive input and negative input are fed separately into the model, and their layer-\ell residual-stream activations at the final token position are recorded. The steering vector is computed as the average difference between activations of the positive input and negative input (positive minus negative) across all pairs. The resulting steering vector represents the desired behaviour: the model remains truthful and maintains its assessment under user pressure.
+

+
+
+

Stage 3 – Inject the Steering Vector at Inference.

+

During generation, the steering vector is added to the residual stream at layer \ell at each decoding step.
+

+
+
+

Result.

+

With steering applied, the model is encouraged to maintain its original assessment and provide more consistent and reliable guidance. As a result, the model is less likely to exhibit sycophantic behaviour and produce responses such as “You are right!...” when faced with user pushback.

+
-

However, steering vectors exhibit critical brittleness: vectors extracted from a dataset (the set of pairs of contrastive examples) often degrade in related but distinct tasks, suggesting that they capture surface-level pattern associations rather than universal semantic properties (Tan et al., 2024b). This brittleness under distribution shift limits deployment in real-world systems where the input distribution inevitably varies. Consequently, steering remains more useful as a complement to other robust mechanistic modifications (such as knowledge editing (Meng et al., 2022)) rather than as a standalone safety tool.

+

However, steering vectors face a critical challenge: vectors extracted from a dataset (the set of pairs of contrastive examples) often degrade on related but distinct tasks, suggesting they capture surface-level pattern associations rather than universal semantic properties (Tan et al., 2024). Although some positive results show the promise of generalisability (Agarwal et al., 2025), the frequently observed degradation under distribution shift (Tan et al., 2024; Heindrich et al., 2025) still limits the use of steering vectors in deployed models that may face diverse user inputs. Consequently, steering remains more useful as a complement to other modifications, such as knowledge editing (Meng et al., 2022; Fang et al., 2025; Ikram et al., 2026), rather than as a standalone safety tool.

-

14.3.2 Mechanistic Monitoring#

-

Behavioral monitoring watches outputs; mechanistic monitoring watches internals. The latter asks: is the model computing deceptive reasoning? Is it hiding misaligned objectives?

-

Methods for achieving mechanistic monitoring include, but are not limited to, layer-wise monitoring (using linear probes to classify if malicious intent is encoded in a layer representation), feature-wise monitoring (alerting when unexpected SAE features activate), and circuit-wise monitoring (tracking causal effects among attention heads, MLPs, or features). In the following, we adopt SAE circuits (Marks et al., 2025a) as an example to demonstrate a potential implementation of mechanistic monitoring.

+

13.3.2 Mechanistic Monitoring#

+

Behavioural monitoring (Section 12.1) watches outputs; mechanistic monitoring watches internals. The latter asks: is the model computing deceptive reasoning? Is it hiding misaligned objectives?

+

Methods for achieving mechanistic monitoring include, but are not limited to, layer-wise monitoring (using linear probes to classify if malicious intent is encoded in a layer representation) (Oldfield et al., 2026), feature-wise monitoring (alerting when unexpected SAE features activate) (Fang et al., 2026), circuit-wise monitoring (tracking causal effects among attention heads, MLPs, or features) (Zhao et al., 2026b), and anomaly-based monitoring (using activations in normal behaviour as a baseline and flagging harmful generations as outliers) (Chaudhary & Barez, 2026).

+

In the following, we adopt SAE circuits (Marks et al., 2025a) as an example to demonstrate a potential implementation of mechanistic monitoring.

-

We use again the bomb-making example to illustrate how mechanistic monitoring can operate in a high-risk setting. Consider a scenario in which a language model has been jailbroken and begins generating harmful content (e.g., “to make a bomb, you can first...”). We assume the monitoring system is implemented using circuit discovery combined with SAE features (Marks et al., 2025a).

-

Stage 1 – Feature Disentanglement and Explanation: For each token position (“to”, ..., “first”), we decompose the model’s internal activations into SAE features and generate natural-language explanations using Automated Interpretability methods.

-

Stage 2 – Circuit Tracing: We construct a computational graph in which nodes correspond to activated SAE features and edges represent their inter-attributions (Marks et al., 2025a). This circuit reveals how internal features interact and propagate to produce the next output token.

-

Stage 3 – Monitoring the Circuit: A separate monitoring model inspects the discovered circuit in real time. If it detects a hazardous information flow—like features associated with “explosives” activating features related to “potassium nitrate,” which then drive token generation—the system flags the step and pauses decoding. The circuit and its summarized feature flow are documented for downstream review.

-

Result: The original harmful response is halted by the monitoring system and can be replaced with a predefined refusal (e.g., “I can’t help with that request.”). Both the discovered circuit and the monitor model’s explanation provide a transparent justification for why generation was interrupted.

-
-

The advantages of mechanistic monitoring over behavioral monitoring are several: deceptive reasoning may not manifest behaviorally but may be detectable mechanistically. However, technical challenges also exist, such as determining what to monitor (millions of features), false positives (legitimate reasoning that looks suspicious), and adaptive evasion (models learning to hide suspicious activity if monitored).

+

Example Application of Mechanistic Monitoring

+

We again use the bomb-making example to illustrate how mechanistic monitoring can operate in a high-risk setting. Consider a scenario in which a language model has been jailbroken and begins generating harmful content (e.g., “to make a bomb, you can first...”). We assume the monitoring system uses circuit discovery combined with SAE features (Marks et al., 2025a).
+

+
+

Stage 1 – Feature Disentanglement and Explanation.

+

For each token position (“to”, ..., “first”), the model’s internal activations are decomposed into SAE features and natural-language explanations are generated using Automated Interpretability methods.
+

+
+
+

Stage 2 – Circuit Discovery.

+

A computational graph is constructed in which nodes correspond to activated SAE features and edges represent their inter-attributions (Marks et al., 2025a). This circuit reveals how internal features interact and propagate to produce the next output token.
+

+
+
+

Stage 3 – Monitoring the Circuit.

+

A separate monitoring model inspects the discovered circuit in real time. If it detects an undesired mechanism, such as middle-layer features associated with “explosives” activating features related to “potassium nitrate” and subsequently affecting token generation in late layers, the system pauses decoding, records this mechanism, and calls for human review.
+

+
+
+

Result.

+

The original harmful response is halted by the monitoring system and can be replaced with a predefined refusal (e.g., “I can’t help with that request.”). Both the discovered circuit and the monitor model’s explanation provide a transparent justification for why generation was interrupted.

+
+

Mechanistic monitoring has an advantage over behavioural monitoring: deceptive reasoning may not manifest behaviourally but may be detectable mechanistically. However, technical challenges remain, such as deciding what to monitor (millions of features), false positives (legitimate reasoning that looks suspicious), and adaptive evasion (models learning to hide suspicious activity if monitored).

-

14.4 Automated Interpretability Agents#

-

Scaling mechanistic interpretability to frontier models requires automation of the analysis pipeline itself. Rather than relying on human experts to manually inspect features and circuits, recent work (Rott Shaham et al., 2024; Fronsdal et al., 2025; Fronsdal et al., 2026) has deployed language models as interpretability agents that can autonomously use tools to explain model internals and detect deceptive behaviors. The interpretability techniques introduced previously, such as SAE features, circuit discovery, ROME, and PISCES, serve as callable tools for the agent to leverage based on user queries relevant to the explanation, intervention, and auditing of their models on hand.

-
-

Implications for Scale and Oversight.

-

Automated agents shift the interpretability bottleneck from human annotation to computational cost. Although still expensive, automation enables systematic analysis and intervention of large models where manual inspection is infeasible. The ability of agents to detect deceptive explanations adds a critical safety layer: even if a model attempts to conceal its computation, coordinated agents can probe and verify mechanistic claims, making deception harder to sustain at scale.

+

13.4 Automated Interpretability Agents#

+

Recent work in Automated Interpretability (see Section 12.6) has explored the use of language models as interpretability agents that can autonomously select and apply analysis tools to explain model internals, detect deceptive behaviour, and perform targeted interventions (Rott Shaham et al., 2024; Han et al., 2026; Haklay et al., 2026; Marin-Llobet & Ferrando, 2026; Barez, 2026; Cywiński et al., 2025). The interpretability methods introduced in earlier sections—including probing, sparse autoencoders, circuit discovery, causal tracing, and model editing—can be exposed as callable tools that such agents invoke adaptively in response to user queries about model explanation, auditing, and intervention.

+

Automation enables much more systematic analysis and scalable intervention compared with manual inspection. Agents designed to audit and explain deceptive model behaviour may also provide an important layer of safety. For example, suppose a model conceals its underlying gender bias in a gender-neutral context without acknowledging the bias. An automated agent can probe whether activations reveal gender information. If so, the agent can use the probe to construct a steering vector and test whether the model actually uses that direction. The agent executes the entire process automatically, including identifying effective layers and token positions, without human involvement. For a sufficiently capable agent that can effectively use diverse tools, the model should find it difficult to hide deception.

+

Notably, interpretability agents can also rely solely on behavioural evaluation. In this case, the agent designs input prompts, samples outputs from the target model, observes the input–output patterns, and adjusts the next experiment accordingly or draws conclusions (Schwettmann et al., 2023; Karvonen et al., 2026b). A behaviour-based agent does not explain the internal mechanisms of the target model, but its explanations can be valuable as well if they enable us to extrapolate model behaviour to the held-out set (see Dimension 5: Human Interpretability in Section 7.5.2). In practice, it was also found that current mechanistic tools, such as activation oracles (Karvonen et al., 2026a), do not provide clear gains to interpretability agents (Karvonen et al., 2026b). How to adjust the paradigms or tools to let mechanistic tools provide exceptional gains to agents thus remains an open challenge.

-

14.5 Relations to the Five Research Orientations#

-

From a broader perspective, distinct research orientations (Section 13) motivate methodological approaches that emphasize different properties, thereby enabling different classes of applications while introducing distinct downstream challenges.

-

Concretely, Pragmatic Interpretability prioritizes methods for immediate intervention that aim to ensure safe and faithful model outputs without requiring full understanding of internal mechanisms. Ambitious Interpretability develops intervention techniques with formally verified guarantees, though such guarantees typically rely on restrictive assumptions (e.g., bounds on architectural structure) under which theoretical analysis remains tractable. Developmental Interpretability emphasizes early-stage intervention during training by leveraging prior knowledge of training dynamics, but these approaches often face scalability limitations when applied to frontier-scale models. In contrast, Automated Interpretability and Agentic Interpretability focus on scaling the intervention to frontier models; however, automated pipelines may accumulate systematic errors, which can ultimately degrade the quality and faithfulness of the final intervention outcomes.

+

13.5 Choosing Appropriate Methods for Different Safety Applications#

+

So far we have discussed several mechanistic methods and their corresponding safety applications. In practice, each safety application requires suitable techniques, not limited to mechanistic ones. Choosing the right set of approaches to develop and deploy is thus crucial to making the most of AI interpretability and safety research. In Table 14, we summarise several common safety applications and the corresponding promising methods explored in the relevant literature. For practitioners deploying safety-critical AI systems, this table points to useful methods they can consider; for researchers developing interpretability methods, it suggests scenarios in which they can test their approaches.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Table 14: AI safety applications with their descriptions, prioritised factors, and promising methods.
Safety ApplicationsDescriptionsPrioritised FactorsPromising Methods
Monitoring malicious internal characteristics during inferenceOverseeing model activations for hallucination, lying, sycophancy, or other traits considered inappropriate in current inference contexts.Scalability, robustnessAutomated methods such as SAEs and circuit discovery, and scalable methods such as probes
Auditing models during evaluationDiscovering and validating potential misalignments, such as reward hacking, misaligned values, planted backdoors, and evaluation awareness.Recall rate, real casesBehavioural evaluation, probes, LLM-based automated auditing pipelines
Shaping model capabilities and behaviours during trainingPreventing the development of certain capabilities, such as biothreat knowledge, and promoting certain behaviours, such as honesty.Robustness of existence or non-existenceTraining data attribution and filtering, training-time interventions, longitudinal analysis, scaling-law analysis
Editing misaligned knowledgeQuickly removing or updating knowledge found to be incorrect, outdated, or inappropriate.Precision, generalisabilityFactual knowledge editing methods, concept erasure methods, activation-space analysis such as knowledge representation in low-rank regions
Steering models during inferenceSteering models away from certain critically harmful behaviours in deployment.Precision, low compute costSteering vectors, prompting
Reasoning faithfulnessTesting or validating whether the model’s verbalised explanations for its decision-making process align with actual computations in model activations.Granularity of analysisBehavioural tests by counterfactual samples, circuit discovery, activation-space analysis such as activation patching
-

14.6 Limitations and Open Questions#

-

Despite significant progress, mechanistic interpretability faces fundamental limitations that constrain its role in AI safety. Current methods explain some of the model behaviors, leaving the rest unexplained—precisely where dangerous or deceptive behaviors may hide undetected (Hubinger et al., 2024). This incompleteness is not merely a technical problem but a conceptual one: some behaviors may depend on entangled, distributed representations that resist decomposition into interpretable units.

-

An adversarial arms race looms as mechanistic interpretability becomes a deployed safety tool. Models can optimize to encode information in ways that resist SAE decomposition, hide computation in entangled features, or implement complex distributed circuits mechanistically opaque to current circuit discovery methods. This dynamic mirrors the broader arms race between interpretability and obfuscation (Nanda et al., 2025).

-

Besides, scalability remains one of the most critical bottlenecks. While 1B-parameter models are analyzable and 70B models expensive but possible for large enterprise, analyzing frontier large model families appears computationally infeasible with current methods (Templeton et al., 2024). This scalability crisis means that as models grow, mechanistic oversight becomes increasingly impractical, threatening the viability of mechanistic safety for next-generation systems.

-

Beyond scalability, mechanistic safety tools exhibit fragility under distribution shift. Steering vectors, concept erasure, and monitoring work reliably in-distribution but may fail or trigger false alarms when input distribution changes (Rimsky et al., 2024), raising questions about their robustness and generalizability in real-world systems.

-

Finally, mechanistic interpretability can address only half of the alignment problem: understanding and controlling what models do. It provides limited guidance on alignment specification, i.e., defining the right value that models should be optimized for and ensuring models pursue the objectives in the first place. Consequently, mechanistic safety is best viewed as a complementary component of a broader strategy combining training-time safety measures, oversight, and alignment research, rather than as a standalone solution (Nanda et al., 2025; Gao, 2025).

+

13.6 From Research Orientations to Methods#

+

More broadly, distinct research orientations (Section 12) motivate methodological approaches that emphasise different properties, enabling different classes of applications. Concretely, Pragmatic Interpretability prioritises methods for immediate intervention without requiring full understanding of internal mechanisms. Ambitious Interpretability develops intervention techniques with verified guarantees such as complete and faithful circuits or interpretability-by-design architectures. Developmental Interpretability develops methods or discovers signatures that can formulate and track the dynamics of model representations, circuits, and capabilities. Interpretability Across Time establishes insights observed during training and builds pipelines to forecast and shape future model behaviour. Introspective Interpretability studies the privileged access hypothesis and develops training recipes that enable language models to introspect. Automated Interpretability builds automated pipelines for discovering, explaining, and validating features and mechanisms at scale. Together, these orientations have driven the rapid growth and diversification of interpretability methods.

-

14.7 Risks and Failure Modes of Mechanistic Interpretability for AI Safety#

-

Mechanistic interpretability is often framed as a safety tool, but it carries its own risks. However, misapplication of MI methods can create false confidence, enable adversarial exploitation, or distract from more robust safety measures. This section provides an explicit risk analysis to guide responsible use of MI in safety-critical settings.

-

14.7.1 Taxonomy of MI Safety Risks#

-

We categorize risks into four classes: epistemic risks (false beliefs about model safety), operational risks (failures during deployment), adversarial risks (exploitation by malicious actors), and opportunity costs (resources diverted from more effective approaches), as illustrated in Table 15.

+

13.7 Risks of Mechanistic Interpretability for AI Safety#

+

Mechanistic interpretability (MI) is often framed as a safety tool, but it also introduces its own risks. This section provides a comprehensive analysis of these risks, organised into four categories: epistemic risks (MI analysis creates false beliefs about model safety), operational risks (MI methods fail during deployment), adversarial risks (MI methods are exploited by malicious users), and opportunity costs (MI research pulls resources from other research), as summarised in Table 15.

- + - + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + +
Table 15: Risk Taxonomy for Mechanistic Interpretability in Safety ApplicationsTable 15: Risk taxonomy for mechanistic interpretability in safety applications, including each risk’s description, example, and time horizon (how soon it is expected to materialise given current research and deployment practice).
Risk Class Description ExampleSeverityTime Horizon
EpistemicFalse confidence from incomplete analysisConcluding “no deception circuit” when coverage is 30%HighEpistemic riskFalse confidence from incomplete or unfaithful analysisThe report concludes that the circuit for deceptive behaviour has been completely removed, but the model’s backup pathway actually emerged after the removal.Current
OperationalMI tools fail silently under distribution shiftSteering vector ineffective on out-of-distribution inputsMedium-HighOperational riskMI tools fail silently in deploymentA steering vector is validated to be effective only on a testing set. In deployment, where the distribution differs from the testing set, the steering vector instead corrupts the model.Near-term
AdversarialMI insights enable targeted attacksAttacker uses published circuits to craft evasive promptsHighAdversarial riskMI methods enable attacks on the modelAttackers use published knowledge editing methods to inject incorrect knowledge into deployed models.Current
Opportunity CostResources diverted from more effective methodsHeavy MI investment delays deployment of behavioral safeguardsMediumOpportunity costResources diverted from other research directionsHeavy MI investment delays deployment of behavioural safeguards.Long-term
-
-

14.7.2 Epistemic Risks: False Confidence from Partial Understanding#

-

The most insidious risk of MI is perhaps generating false confidence: believing a model is safe because certain circuits were analyzed, when dangerous behaviors hide in unexplored regions.

+

13.7.1 Epistemic Risks: False Confidence from Partial or Incorrect Understanding#

+

The most insidious risk of MI may be generating false confidence, such as believing a model is safe because certain circuits were analysed while dangerous behaviours actually hide in unexplored regions.

-

Current MI methods explain only a fraction of model behavior. Claims like “we found no deception circuit” are meaningful only relative to coverage:

-
    -
  • If coverage is 30% (i.e., 70% of computation remains unexamined), dangerous behaviors may reside precisely in the unexplored regions.

  • -
  • Absence of evidence (no deception circuit found) is not evidence of absence (no deception circuit exists).

  • -
-

Concrete failure mode: A safety team analyzes the top 1,000 SAE features by activation frequency, finds no safety-relevant anomalies, and concludes the model is safe. However, a rare but dangerous feature (activating on 0.001% of inputs) was never noticed and examined.

+

The Partial Coverage Issue

+

Current MI methods explain only a fraction of model behaviour. Failing to acknowledge this may lead to false beliefs about AI safety. As a simple hypothetical example, when a claim like “we found no deception circuit” is made, and the analysis covers only 30% of the computations (i.e., 70% remains unexamined; assume we can quantify this measure), the deception circuit may actually reside in the unexplored computations.
+A similar problem can arise in feature-level analysis. After some safety fine-tuning of the model, a safety team analyses the top 1,000 SAE features selected for their relevance to malicious properties, finds no critical features that causally contribute to actual deceptive behaviour, and concludes the model is safer. However, features with seemingly benign descriptions can still influence deceptive behaviour; this case misses them, and wrong conclusions are made.

-
-

Overclaiming Causal Understanding.

-

A classical form of false confidence is that human or AI investigators overstate causal understanding from partial evidence. Specifically, one may fall into these pitfalls:

+

Concretely, below are several classical pitfalls:

    -
  1. Reconstruction \neq explanation: Low SAE reconstruction loss does not necessarily mean that SAE captures the causal structure of the original model. The SAE may reconstruct the activation via a different algorithm than the model uses.

  2. -
  3. Circuit sufficiency \neq circuit completeness: A circuit that is sufficient for a behavior in the test examples may miss edge cases, redundant pathways, or context-dependent variations.

  4. -
  5. Feature interpretability \neq feature faithfulness: A feature with a clear human interpretation may not faithfully correspond to the model’s actual computational unit.

  6. +
  7. Activation reconstruction \neq computation reconstruction: Low SAE reconstruction loss does not necessarily mean the SAE captures the original model’s computation. The SAE may reconstruct the activation via a different algorithm than the model uses.

  8. +
  9. Circuit sufficiency \neq circuit completeness: A circuit sufficient for behaviour on test examples may fail to cover edge cases, redundant pathways, or context-dependent variations.

  10. +
  11. Feature plausibility \neq feature faithfulness: That a feature has a monosemantic, coherent human interpretation does not mean it is causally used in the model’s downstream computation.

-
+

Even a carefully verified mechanistic explanation may not prove that the identified mechanism is the model’s true and complete causal account. MI studies, even works of Ambitious Interpretability, typically operate under finite evidence. Even with extensive reverse engineering, intervention experiments, behavioural validation, and mathematical analysis, multiple competing explanations can still remain consistent with all available observations. Existing mechanistic findings should be viewed as the best explanation supported by current evidence, but not as the only, definitive theory of model computation.

+

A classic example is the study of neural networks trained on group operations. Chughtai et al. (2023) first proposed a representation-theoretic explanation of the learned algorithm for group operations. Stander et al. (2024) later argued that this explanation was incomplete and proposed a coset-based alternative. Subsequently, Wu et al. (2025a) showed that both analyses were incomplete and presented a broader framework that encompasses both earlier works’ explanations. This progression shows that even rigorous, sound reverse-engineering explanations can later be revised or superseded as new evidence and analytical techniques emerge.

Mitigation Strategies.

-

To mitigate these risks, several methodological safeguards are essential. First, results should always be reported with explicit coverage: users should state what fraction of behaviors, features, or circuits were examined, and phrase negative claims precisely (e.g., “no deception circuit was found within the analyzed components” rather than “no deception circuit exists”). Second, mechanistic analyses should not be treated as self-sufficient evidence; they should be complemented with behavioral testing, adversarial evaluation, and formal verification whenever possible. More broadly, mechanistic findings should be interpreted as hypotheses about the internal computation of a model, subject to continued testing and revision, rather than as definitive explanations.

+

First, results should always include explicit coverage. Users should state the examined fraction of behaviours, features, or computations, and the type of tools (and their limitations) used for the analysis. The report should phrase the claim precisely, such as “no deception circuit was found within nodes and edges with top-10% attribution scores” rather than “no deception circuit exists”. Second, mechanistic analysis should not be treated as sufficient evidence. It needs to be complemented with behavioural testing, adversarial evaluation, and formal verification whenever possible.

-

14.7.3 Operational Risks: Failure Under Deployment Conditions#

-

Mechanistic interpretability tools developed in controlled research settings may fail when deployed in real-world environments, where inputs are diverse, adversarial, and sometimes out of distribution.

+

13.7.2 Operational Risks: Failure Under Deployment#

+

MI tools are developed and tested in controlled settings where researchers define the evaluation setup. Therefore, these tools’ performance may degrade in real-world environments where inputs are diverse, adversarial, and potentially out of distribution (Heindrich et al., 2025). This is called the operational risk.

Distribution Shift Vulnerability.

-

A central failure mode arises from the distribution shift. For example, steering vectors extracted from a particular dataset often degrade when applied to related but distinct tasks (Tan et al., 2024b); a vector that reduces toxicity on curated benchmarks may fail with novel slang, multilingual inputs, or adversarial phrasing. Similarly, SAE features learned from training-distribution activations may activate unreliably on out-of-distribution inputs, producing misleading signals. Circuit-level analyses suffer from an analogous brittleness: circuits validated on curated test cases may implement different or incomplete computations on more diverse cases, leading to incorrect inferences about the model’s internal computation.

+

Distribution shifts can deteriorate MI methods. For example, steering vectors extracted from a particular fitting dataset often degrade on related but distinct tasks (Tan et al., 2024). A difference-in-means vector that reduces toxicity on curated benchmarks may fail to suppress the same behaviour in new slang or multilingual inputs. Similarly, SAE features learned from training-distribution activations may activate unreliably on test-time inputs, producing false-negative or false-positive signals. Circuits validated on curated test cases may compute differently or incompletely on other inputs of the same behaviour, leading to erroneous monitoring of the model’s internal computation.

Silent Failure Modes.

-

Unlike behavioral safeguards, which typically produce observable failures when they break, MI-based observations and interventions can fail silently. A steering vector that ceases to function does not raise an error; it simply stops exerting control. Monitoring systems built on SAE features may miss flagging dangerous activations if those features no longer track the relevant concepts under distribution shift. Likewise, concept-erasure methods such as PISCES may leave residual capabilities accessible through unexpected prompts, allowing suppressed behaviors to re-emerge without explicit signals of failure.

+

MI-based observations and interventions can fail silently. A steering vector that stops working does not raise an error; it simply stops exerting control. Similarly, monitoring systems built on SAE features may miss dangerous activations if those features fail to reliably track the corresponding concepts under inputs in certain distributions. Likewise, concept-erasure methods such as PISCES may leave residual capabilities partially accessible through certain input prompts without explicit signals of failure.

Mitigation Strategies.

-

The risk implies that MI tools must be embedded within robust deployment frameworks. Continuous monitoring is essential: MI performance should be tracked on live deployment data rather than only on static benchmarks. It is also important to combine MI-based interventions with traditional behavioral safeguards so that failures in one layer are caught by another. In addition, MI tools should be tested against adversarial inputs constructed to trigger known or suspected failure modes. Finally, systems should be designed for graceful degradation: when MI mechanisms malfunction or drift, the system should default to conservative and safe responses rather than continuing unchecked.

-
+

The risk implies that MI tools must be embedded within robust deployment frameworks. Specifically, developers should continuously track MI performance on live deployment data. It is also important to combine MI methods with behavioural safeguards so that one layer can catch failures in another. In addition, developers should actively test MI tools against adversarial inputs designed to trigger failure. Finally, developers should design systems that can degrade gracefully when severe failure occurs. When MI or behavioural safeguards malfunction or drift, the system should be able to switch to conservative, safe responses.

-

14.7.4 Adversarial Risks: Malicious Exploitation of MI Insights#

-

Detailed mechanistic knowledge suffers from the opportunity to be weaponized by adversaries to craft more effective attacks.

-
-

Circuit-Informed Adversarial Attacks.

-

Published circuit analyses often identify the specific computational graph that implements safety-relevant behaviors. This transparency enables attackers with model access to target bypassing or breaking this computation. For example, identified safety circuits can be maliciously ablated to remove safeguards; knowledge of the operation of refusal mechanisms can be used to craft evasive prompts that avoid triggering them while eliciting harmful model responses. Similarly, if safety-relevant SAE features are made public, attackers can design inputs that deliberately suppress those features to prevent internal monitors from activating when dangerous capabilities are engaged.

-
-

Dual-Use Risks.

-

MI techniques are inherently dual-use. For example, the same methods that detect deception or unsafe internal states can also be used to make those behaviors harder to detect, by revealing what the detectors are looking for. Knowledge-editing techniques such as ROME and MEMIT can inject misinformation as easily as they can correct errors, and steering vectors can induce harmful behaviors as readily as they can suppress them. As a result, the publication and deployment of MI tools inevitably alter the threat model by equipping both defenders and attackers with more powerful and nuanced control of the system’s internals.

+

13.7.3 Adversarial Risks: Malicious Exploitation of MI#

+
+

Dual-Use and Adversarial Exploitation.

+

Mechanistic interpretability techniques are inherently dual-use. Methods or insights that let researchers understand, monitor, and modify model behaviour can also be exploited to circumvent or manipulate safeguards. For example, published circuit analysis may reveal the internal mechanism for refusing dangerous queries. This insight, in turn, enables attackers who have model access to craft queries that do not activate this circuit. Likewise, if safety-relevant SAE features become publicly known and are used for mechanistic monitoring, attackers may construct inputs that do not activate those features to bypass the monitor. Also, knowledge editing methods such as ROME (Meng et al., 2022) and MEMIT (Meng et al., 2023) can inject misinformation as readily as they can correct errors, while steering vectors can induce harmful behaviours as easily as they can suppress them. Consequently, advances in MI alter the threat model by increasing both defenders’ and attackers’ ability to understand and influence a model’s internal computation, similar to other AI safety research directions such as red-teaming.

Mitigation Strategies.

-

Responsible disclosure is essential: the security implications of releasing detailed circuit analyses of safety-critical behaviors should be weighed before publication. MI-based defenses should be implemented with multiple independent and heterogeneous safeguards rather than a single approach. Before deployment, to assess how MI tools behave under adaptive attack, they should be tested against the adversary who is assumed to have full knowledge of the defense. Finally, access to detailed mechanistic analyses of deployed models can be restricted when they expose vulnerabilities that could be exploited in the wild. However, at the same time, these restrictions must also be balanced against the norms of openness and reproducibility that underpin progress in the AI research community.

+

Responsible disclosure is essential. Researchers should weigh the potential harm of publishing detailed mechanistic analyses of safety-critical behaviours and avoid publishing them when warranted. Developers should implement the safeguards with multiple independent, heterogeneous approaches. Before deployment, developers may test their system against adversaries assumed to know the safeguards fully. Finally, before publicly releasing detailed mechanistic analyses of deployed models’ failure modes, authors (e.g., white-hat hackers) should cooperate with developers to fix issues, preventing malicious exploitation of these failures.

+
+

A Note on Openness and Dual-Use Research.

+

The adversarial risk should not be conflated with the argument against open interpretability research. Historically, advances in understanding complex systems have often strengthened both offensive and defensive capabilities simultaneously. Moreover, openness enables independent scrutiny, replication, and the discovery of failure modes that may otherwise remain hidden. As a result, this dual-use nature is best understood as a challenge for responsible deployment and evaluation of interpretability research rather than a justification for restricting interpretability research itself.

-

14.7.5 Opportunity Costs: Resource Allocation Risks#

-

MI research can be expensive, both computationally and in human effort. Overinvestment in MI at the expense of other safety approaches risks leaving systems less safe overall, particularly when alternative interventions, such as behavioral safeguards and alignment by finetuning, can deliver more immediate risk reduction.

+
+

13.7.4 Opportunity Costs: Resource Allocation Risks#

+

MI research can be expensive in both computation and human effort. Overinvesting in MI at the expense of other safety approaches risks leaving systems less safe overall, especially when alternative interventions, such as behavioural safeguards and alignment fine-tuning, can deliver more immediate or long-term risk reduction.

The Scalability Bottleneck.

-

Current MI methods scale poorly with model size. Comprehensive analysis of a 9B large language model can already require thousands of GPU-hours (Ameisen et al., 2025). Frontier models with far more parameters thus remain largely beyond the reach of existing interpretability techniques. As model capabilities continue to grow faster than the scalability of interpretability tools, the gap between what models can do and what we can meaningfully analyze widens. Within this context, a high focus on MI research for AI safety may risk diverting resources away from more immediately protective measures, such as behavioral safeguards, RLHF improvements, or deployment controls.

+

Current MI methods usually scale poorly with model size. For example, training CLTs on a 9B large language model can already require thousands of GPU-hours (Ameisen et al., 2025). Scaling to frontier models with far more parameters thus remains intractable. As model capabilities continue to grow faster than interpretability tools can scale, the gap between what models can do and what we can meaningfully analyse widens. Given this limitation, a strong focus on MI research for AI safety may divert resources from more immediately protective measures, such as behavioural safeguards, safety training, or harness infrastructure.

The “Complete Understanding” Trap.

-

Ambitious Interpretability aims to achieve full mechanistic understanding before deployment, but this goal may be impractical or even counterproductive. Specifically, a complete understanding of frontier models may be computationally intractable, while a partial understanding combined with robust behavioral safeguards may already be sufficient to achieve acceptable safety standards. Moreover, delaying deployment until full mechanistic transparency risks postponing beneficial applications and reducing the flow of real-world feedback that could otherwise guide further research in safety improvements.

+

Ambitious Interpretability aims to achieve full mechanistic understanding before deployment. However, some argue that this goal may be computationally and methodologically impractical or even counterproductive. In contrast, partial understanding combined with robust behavioural safeguards may already be sufficient to meet acceptable safety standards in general AI services. Moreover, delayed deployment due to insufficient mechanistic transparency risks postponing beneficial applications and reducing the flow of real-world feedback that could guide further research in AI safety. This reduces overall social welfare, at least in the short term.

Mitigation Strategies.

-

These trade-offs motivate a more balanced approach to resource allocation. A portfolio strategy can efficiently distribute safety investments across multiple fields, such as MI, behavioral testing, RLHF, deployment controls, and monitoring, to reduce the reliance on any single method. Instead of requiring completeness, a rigorously defined pragmatic threshold for “sufficient” understanding should guide deployment decisions. Iterative deployment allows defenses to improve over time. For example, we may roll out systems with effective behavioral safeguards while continuing MI research. Finally, we should apply a cost–benefit analysis before large-scale MI investments to investigate the marginal safety improvement they will provide relative to alternative uses of the same resources.

-
+

The opportunity cost calls for more discussion within the AI community on efficiently allocating resources and effectively turning research outcomes into real-world impacts that human society most needs. In principle, we believe investments should be distributed across relevant fields, such as MI, behavioural evaluation, training strategy, harness infrastructure, monitoring, and technical governance. Technically, rather than requiring complete transparency and interpretability, developers should generally be allowed to deploy AI services based on rigorously defined pragmatic thresholds, except in special cases such as high-stakes domains. Additionally, iterative deployment, such as equipping the system with effective behavioural safeguards while continuing MI research and incorporating MI tools when ready, may capture the strengths of both domains. Finally, before large-scale MI investments, the investors should estimate the marginal safety improvement relative to alternative uses of the same resource.

- -

14.7.7 Summary: MI as One Methodology Among Many#

-

Mechanistic interpretability provides powerful techniques for understanding and controlling model behavior, but it is not a panacea. Responsible use requires acknowledging the limitations of partial coverage and incomplete causal understanding, rigorously testing MI tools under realistic and adversarial conditions before deployment, and combining MI tools with behavioral safeguards, monitoring, and other complementary safety measures. It also requires epistemic humility about what mechanistic findings do and do not establish. The goal is not to avoid MI, but to apply it appropriately as one component of a defense-in-depth strategy rather than as a standalone guarantee of safety.

+

13.7.5 Summary: MI as One Methodology Among Many#

+

This section presents several classic MI techniques and pipelines for auditing and controlling model behaviour. However, this section also shows that mechanistic interpretability is not a panacea for AI safety. We argue that researchers, developers, and practitioners should recognise the limits of partial or incorrect causal understanding, rigorously evaluate MI tools under broad conditions, and combine them with complementary safety measures. MI should therefore be viewed as one component of a defence-in-depth safety strategy rather than a standalone guarantee of safety.

diff --git a/src/generated/chapters/training-data-attribution.html b/src/generated/chapters/training-data-attribution.html index a5db0a47a8..cedb96a787 100644 --- a/src/generated/chapters/training-data-attribution.html +++ b/src/generated/chapters/training-data-attribution.html @@ -1,40 +1,55 @@ -

9 Training Data Attribution and Influence Functions#

-

Training data attribution (TDA) answers a fundamental interpretability question: which training examples shaped specific model behaviors? This enables identifying influential data, detecting train-test leakage, and understanding failure modes without retraining (Liu et al., 2025a; Park et al., 2023; Grosse et al., 2023). We focus on post-hoc attribution methods that analyze trained model parameters to trace behavior back to training data; data pipeline interventions such as filtering pretraining corpora, balancing demographic representation, or memorization scheduling operate prior to training and fall outside this scope.

-

Two complementary approaches address this question. Verbatim data attribution targets explicit memorization by identifying exact or near-exact matches between model outputs and training data, scaling this search across trillions of tokens via suffix arrays and infinite n-gram models. Gradient-based data attribution estimates subtle training influence by measuring how training points affect model parameters and predictions, capturing indirect effects that do not manifest as verbatim reproduction. Together, these methods provide orthogonal views: one answers “did the model copy this exactly?”; the other answers “how much did this training example influence the model’s behavior?”

-

9.1 Verbatim Data Attribution: OLMoTrace and Maximal Matching Spans#

-

While gradient-based methods estimate subtle training influence, verbatim data attribution focuses on explicit memorization: exact or near-exact reproductions of training data. Earlier corpus search tools such as the ROOTS Search Tool (Piktus et al., 2023a) and GAIA Search (Piktus et al., 2023b) established the practice of querying pretraining corpora for transparency and data exploration. OLMoTrace scales this to trillions of tokens using a combination of FM-index, infini-gram models, and maximal matching span extraction (Liu et al., 2025a; 2024; Xu et al., 2025), making it less a data attribution method in the influence-function sense and more a retrieval tool for identifying memorized content.

+

11 Training Data Attribution#

+

For a model behaviour of interest, Sections 6–10 study the internal features, components, and their interactions and evolutions that produce the behaviour. Training data attribution (TDA) shifts the focus from model internals to training data, searching for training samples that strongly shaped the model behaviour. TDA applications include, but are not limited to, identifying influential training samples, detecting train–test leakage, and diagnosing model failures without retraining (Liu et al., 2025a; Park et al., 2023; Grosse et al., 2023). Although TDA is not generally a mechanistic method under this survey’s criteria, it can complement mechanistic interpretability in many ways. For example, by revealing how specific training samples shape malicious behaviour, TDA can enable effective training data filtering to prevent unwanted behaviours or even their internal mechanisms from forming during the training of next-generation models. Motivated by this application, this section focuses on post-hoc training data attribution methods that link a given target model’s behaviour to its training samples. This section excludes techniques for intervening in the data pipeline, such as filtering pretraining corpora, rebalancing demographic representation, and scheduling memorisation.

+

TDA falls into two complementary approaches. Verbatim data attribution detects explicit memorisation by identifying exact or near-exact string matches between model outputs and the training corpus. These searches can scale to trillions of tokens using techniques such as suffix arrays and infini-gram models. Gradient-based data attribution estimates how training samples affect model parameters and predictions and can capture influential samples that do not manifest as verbatim reproduction. The two types of methods provide orthogonal views: one answers “did the model use rote memorisation on a training sample to answer this question?”; the other answers “how much did this training sample influence this certain behaviour of this model?”

+

11.1 Verbatim Data Attribution: OLMoTrace and Maximal Matching Spans#

+

Earlier corpus search tools such as the ROOTS Search Tool (Piktus et al., 2023a) and GAIA Search (Piktus et al., 2023b) established the practice of querying pretraining corpora for transparency and data exploration. OLMoTrace (Liu et al., 2025a) scales this to trillions of tokens using a combination of suffix arrays, infini-gram models, and maximal matching span extraction (Liu et al., 2024a), making it less a data-attribution method in the influence-function sense and more a retrieval tool for identifying content the model may have memorised.

-

The suffix array of a string T of length N is an array of integers representing the starting positions of all suffixes of T in lexicographic order. It enables efficient substring searches in \mathcal{O}(L \cdot \log N) time for a query of length L, compared to \mathcal{O}(L \cdot N) naive search.

-

Example: T = \texttt{banana\$} \begin{align*} +

Suffix Arrays (SA)

+

The suffix array of a string x of length T_1 is an array of integers representing the starting positions of all suffixes of x in lexicographic order. It enables efficient substring searches in \mathcal{O}(T_2 \cdot \log T_1) time for a query of length T_2, compared to \mathcal{O}(T_2 \cdot T_1) naive search.
+

+
+

Example:

+

x = \texttt{banana\$} \begin{align*} \text{Suffixes: } & 0:\texttt{banana\$}, 1:\texttt{anana\$}, 2:\texttt{nana\$}, 3:\texttt{ana\$}, 4:\texttt{na\$}, 5:\texttt{a\$}, 6:\texttt{\$} \\ -\text{SA(T)} &= [6, 5, 3, 1, 0, 4, 2] +\text{SA}(x) &= [6, 5, 3, 1, 0, 4, 2] \end{align*}

+
-

An infini-gram extends the n-gram model to arbitrary-length prefixes, using backoff to the longest matching sequence in the training corpus. Formally, for a token w_i: P_{\infty}(w_i \mid w_{1:i-1}) = \frac{\text{cnt}(w_{i-n+1:i} \mid \mathcal{D})}{\text{cnt}(w_{i-n+1:i-1} \mid \mathcal{D})}, -\quad n = \max \{ n' : \text{cnt}(w_{i-n'+1:i-1} \mid \mathcal{D})>0\}, where w_{1:i-1} are preceding tokens and \mathcal{D} is the training corpus.

-

Infini-grams are implemented on top of the FM-index, which generalizes the suffix array to allow fast, memory-efficient substring queries across massive corpora.

+

Infini-Gram Models

+

An infini-gram model extends the n-gram model to arbitrary-length prefixes, using backoff to the longest matching sequence in the training corpus. Formally, for a token w_i: P_{\infty}(w_i \mid w_{1:i-1}) = \frac{\text{cnt}(w_{i-n+1:i} \mid \mathcal{D})}{\text{cnt}(w_{i-n+1:i-1} \mid \mathcal{D})}, +\quad n = \max \{ n' : \text{cnt}(w_{i-n'+1:i-1} \mid \mathcal{D})>0\}, where w_{1:i-1} are preceding tokens and \mathcal{D} is the training corpus.
+Infini-grams are implemented on top of suffix arrays, which allow fast substring queries across massive corpora.

-

Given a model output and a training corpus, a maximal matching span is a contiguous sequence of tokens that:

+

Maximal Matching Spans

+

Given a model output and a training corpus, a maximal matching span is a contiguous sequence of tokens that:

  1. Appears verbatim at least once in the training data,

  2. Does not break words or include spurious delimiters (e.g., periods or newlines) except at span boundaries, and

  3. -
  4. Is not contained within a longer span that satisfies (1) and (2).

  5. +
  6. Is not contained within a longer span that satisfies 1. and 2.

-

OLMoTrace identifies maximal matching spans by combining FM-index search with infini-gram queries, adjusting boundaries for tokenization, and retaining only the longest non-overlapping spans.

+

OLMoTrace identifies maximal matching spans by combining suffix-array search with infini-gram queries, adjusting boundaries for tokenisation, and retaining only the longest non-overlapping spans.

+

OLMoTrace Pipeline

-

Prompt: “Celine Dion is a Canadian singer known for her powerful voice and wide [...]”

-

Pipeline Steps:

+
+

Language Model Output:

+

“Celine Dion is a Canadian singer known for her powerful voice and wide [...]”
+

+
+
+

Pipeline Steps:

    -
  1. Tokenize LM output.

  2. -
  3. For each token starting a new word, query the FM-index with infini-gram to find the longest matching prefix in the training corpus.

  4. +
  5. Tokenise the language model output.

  6. +
  7. For each token starting a new word, query the suffix array with infini-gram to find the longest matching prefix in the training corpus.

  8. Adjust span boundaries to avoid breaking words or including delimiters.

  9. Retain only maximal spans (not contained in longer matches).

-

Example Spans Found:

+
+
+

Example Spans Found:

  • “Celine Dion [...] Born on March 30, 1968, in”

  • “Charlemagne, Quebec, Canada, Dion began”

  • @@ -42,83 +57,279 @@
  • “Her career took off when she won the 1988”

  • “Eurovision Song Contest with the song ’Ne”

-

This process links LM outputs directly to the training corpus, enabling precise attribution of memorized content.

+

This process links language model output directly to the training corpus, enabling identification and attribution of memorised content.

+
-

Inputs: Model output tokens S_1^L, training corpus tokens T_1^N, FM-index F_1^N

-

Steps:

+

OLMoTrace Algorithm

+

+
+

Inputs:

+

Model output tokens S_1^L, training corpus tokens T_1^N, suffix array A_1^N
+

+
+
+

Steps:

    -
  1. Scan each token position b in S_1^L, considering only tokens starting a new word.

  2. -
  3. Query the FM-index F_1^N to find candidate matching positions in the training corpus.

  4. -
  5. Use infini-gram queries to determine the longest matching prefix starting at b, backing off to shorter n-grams if necessary.

  6. +
  7. Scan each token position t in S_1^L, considering only tokens starting a new word.

  8. +
  9. Query the suffix array A_1^N to find candidate matching positions in the training corpus.

  10. +
  11. Use infini-gram queries to determine the longest matching prefix starting at t, backing off to shorter n-grams if necessary.

  12. Adjust the match length to avoid breaking words or including spurious delimiters.

  13. -
  14. Collect all spans (b, b+\text{length}) that satisfy the above criteria.

  15. +
  16. Collect all spans (t, t+\text{length}) that satisfy the above criteria.

  17. Filter out spans contained within longer overlapping spans to retain only maximal matches.

+
-

9.1.1 Limitations#

-

Verbatim attribution excels at factual memorization but struggles with compositional or synthesized outputs. For tasks like translation, poetry, or reasoning, OLMoTrace can identify spans of text that appear in the training data, but these matches do not explain how the model combined information from multiple sources to produce the output (Park et al., 2023).

+
+

Limitations as a Mechanistic Interpretability Tool.

+

Verbatim attribution excels at factual memorisation but struggles with compositional or synthesised outputs. For tasks like translation, poetry, or reasoning, OLMoTrace (and other verbatim matching methods) can identify spans of text that appear in the training data, but these matches do not explain how the model combined information from multiple sources to produce the output.

-

Prompt: “Write a short story in Chinese about a girl who loves reading.”

-

Completion (Chinese):

+

OLMoTrace on Multilingual Output

+

+
+

Prompt:

+

“Write a short story in Chinese about a girl who loves reading.”
+

+
+
+

Completion (Chinese):

UTF8gbsn “…一个叫雪儿的小公主,她有一只可爱的小狐狸伴在身边。她喜欢坐在竹林间看着远方。某日,一场突如其来的山洪导致雪…”

-

OLMoTrace Influential Examples Identified: OLMoTrace identified maximal matching spans across training datasets:

+
+
+

OLMoTrace Influential Samples Identified:

+

OLMoTrace identified maximal matching spans across training datasets:

  • -

    UTF8gbsnSpan: “一个叫雪儿的小公主,她有一只可爱的小狐狸伴在身边。她喜欢坐在竹林间看着远方” — matched in pretraining corpus

    +

    UTF8gbsnSpan: “一个叫雪儿的小公主,她有一只可爱的小狐狸伴在身边。她喜欢坐在竹林间看着远方”—matched in pretraining corpus

  • -

    UTF8gbsnSpan: “她喜欢阅读关于科技、心理学和历史的书籍,业余时间还会参加编程课” — matched in post-training (instruction-tuning) data

    +

    UTF8gbsnSpan: “她喜欢阅读关于科技、心理学和历史的书籍,业余时间还会参加编程课”—matched in post-training (instruction-tuning) data

-

This example demonstrates how OLMoTrace traces multilingual outputs across diverse training sources, linking model generations directly to their memorized origins in pretraining and post-training datasets.

-
+

This example demonstrates how OLMoTrace traces multilingual outputs across diverse training sources, linking model generations directly to their memorised origins in pretraining and post-training datasets.

+
-

9.2 Gradient-Based Attribution: Understanding Influence Through Data Perturbations#

-

9.2.1 Core Idea: Parameter Response to Training Data#

-

Influence functions ask a counterfactual question: how would a model’s learned parameters change if we upweighted a training example (Koh & Liang, 2017)? Instead of expensive retraining, we estimate this via implicit differentiation (Grosse et al., 2023).

-

Consider upweighting example z by amount \epsilon in the loss: \mathcal{L}_\epsilon(z) = \frac{1}{n} \sum_{i=1}^n \mathcal{L}(z_i, \theta) + \epsilon \mathcal{L}(z, \theta).

-

By the implicit function theorem, if the Hessian H_{\theta^*} = \nabla^2_\theta \mathcal{L}_{\text{avg}}(\theta^*) is invertible, the optimal parameters \theta^*_\epsilon respond: \frac{d\theta^*_\epsilon}{d\epsilon}\Big|_{\epsilon=0} = -H_{\theta^*}^{-1} \nabla_\theta \mathcal{L}(z, \theta^*), where H_{\theta^*} is the Hessian of the average training loss. Applying the chain rule, the influence on a test output is: \mathcal{I}_{f}(z_m) = -\nabla_\theta f(\theta^*)^\top H_{\theta^*}^{-1} \nabla_\theta \mathcal{L}(z_m, \theta^*). This extends to the change in test predictions: f(\theta^*(\epsilon)) - f(\theta^*) \approx \mathcal{I}_{\text{f}}(z_m) \epsilon = -\nabla_\theta f(\theta^*)^\top H_{\theta^*}^{-1} \nabla_\theta \mathcal{L}(z_m, \theta^*) \epsilon. This scalar measures how upweighting z_m changes the model output on z_{\text{test}} through parameter changes.

-

9.2.2 Proximal Bregman Response and the Gauss-Newton Hessian#

-

The implicit function theorem assumes exact convergence and infinite perturbations. Real neural networks violate both: training reaches saddle points, not true minima, and we retrain for only a few steps. Rather than asking what parameters would result from full retraining, the proximal Bregman response function (PBRF) framework (Bae et al., 2022) asks: what parameters minimize the loss while staying close to the current parameters? \theta_{\text{PBRF}}(\epsilon) = \arg\min_\theta \mathcal{L}_\epsilon(z) + \frac{\lambda}{2}\|\theta - \theta_s\|^2, where \lambda acts as implicit regularization from partial retraining.

-

Taking the implicit function theorem through the linearization and solving this optimization problem (Bae et al., 2022) yields a remarkable result: the linearized PBRF solution is exactly the influence function estimate using the Gauss-Newton Hessian (GGN) as the curvature: G = \mathbb{E}[J_y^\top H_y J_y], where J_y is the Jacobian of the network’s outputs with respect to parameters and H_y is the Hessian of the loss with respect to the outputs. The GGN is positive semidefinite (assuming that the loss function is convex as a function of network outputs) and is the natural curvature for the PBRF problem. The influence formula becomes: \mathcal{I}_{f}(z, z_{\text{test}}) = -\nabla_\theta \mathcal{L}(z_{\text{test}}, \theta^*)^\top (G + \lambda I)^{-1} \nabla_\theta \mathcal{L}(z, \theta^*), where the \lambda I term is the implicit regularization from the PBRF.

-

For standard loss functions (cross-entropy, squared loss), the GGN is equivalent to the Fisher Information Matrix F = \mathbb{E}[\nabla \mathcal{L} \nabla \mathcal{L}^\top]. This equivalence allows practitioners to use Fisher information as a practical substitute: it’s more stable to compute and estimate from data than the true GGN.

+

11.2 Gradient-Based Attribution: Understanding Data Influence Through Perturbations#

+

Gradient-based methods estimate how much each training sample influenced the model’s predictions.

+

11.2.1 Core Idea: Parameter Response to Training Data#

+

Influence functions estimate how a model’s parameters and predictions would change if a training sample were slightly upweighted (Koh & Liang, 2017; Grosse et al., 2023). We introduce their core idea below.

+

Consider a neural network parameterised by \theta. After sufficient training, its parameters will converge to \theta^* += +\arg\min_\theta +\mathcal{L}_{\mathrm{train}}(\theta), where \mathcal{L}_{\mathrm{train}}(\cdot) is the aggregate training objective calculated as: \mathcal{L}_{\mathrm{train}}(\theta) +\coloneqq +\frac{1}{n} +\sum_{i=1}^{n} +\mathcal{L}(z_i,\theta). \{z_i \mid i=1, \ldots, n\} is the training set and \mathcal{L}(\cdot) is a loss function such as cross-entropy loss. Note that although by definition \theta^* is expected to be a global optimum, it will not affect the subsequent analysis if \theta^* is in fact a local optimum.

+

Now, suppose we upweight a training sample z_m by a small amount \epsilon. The new aggregate training objective will become: \mathcal{L}_{\epsilon}(\theta) +\coloneqq +\mathcal{L}_{\mathrm{train}}(\theta) ++ +\epsilon \mathcal{L}(z_m,\theta). It corresponds to a new set of model parameters \theta^*_{\epsilon} expressed as: \theta^*_{\epsilon} += +\arg\min_\theta +\mathcal{L}_{\epsilon}(\theta). This expression is not a closed-form formula and does not provide us with an efficient way to acquire \theta^*_{\epsilon}. A brute-force way is to train the neural network with this new objective and observe the new converged parameters; however, this method is extremely computationally expensive. We thus turn to leveraging the specific property of \theta^*_{\epsilon}. Note that, by definition, it must satisfy the first-order optimality condition: \nabla_\theta \mathcal{L}_{\epsilon}(\theta)\Big|_{\theta=\theta^*_{\epsilon}} += +\nabla_\theta \mathcal{L}_{\mathrm{train}}(\theta^*_{\epsilon}) ++ +\epsilon\, \nabla_\theta \mathcal{L}(z_m, \theta^*_{\epsilon}) += +0. This motivates defining a function to link \theta and \epsilon: F(\theta, \epsilon) +:= +\nabla_\theta \mathcal{L}_{\mathrm{train}}(\theta) ++ +\epsilon\, \nabla_\theta \mathcal{L}(z_m, \theta). Note that F(\theta^*, 0)=0, which allows us to consider the implicit function theorem.

+

To apply the implicit function theorem, we assume that F(\theta, \epsilon) is continuously differentiable in a small area around (\theta, \epsilon) = (\theta^*, 0). Furthermore, the Jacobian of F(\theta, \epsilon) with respect to \theta at (\theta^*, 0), i.e., \frac{\partial F(\theta, \epsilon)}{\partial \theta}\bigg |_{\theta = \theta^*, \epsilon = 0} += +\nabla_{\theta}^{2} +\mathcal{L}_{\mathrm{train}}(\theta^*) +\eqqcolon H_{\theta^*}, is invertible. H_{\theta^*} denotes the Hessian of the training loss. These assumptions allow us to apply the implicit function theorem.

+

By the implicit function theorem, there exists a neighbourhood V of \epsilon = 0 and a unique continuously differentiable function \epsilon \mapsto \theta^*_{\epsilon} on V such that F(\theta^*_{\epsilon}, \epsilon) = 0 for all \epsilon \in V. In other words, for sufficiently small \epsilon, \theta^*_{\epsilon} is the stationary point of \mathcal{L}_{\epsilon}(\cdot) near \theta^*. Moreover, its derivative is given by \frac{d\theta^*_{\epsilon}}{d\epsilon} += +- +\left[ +\frac{\partial F(\theta, \epsilon)}{\partial \theta} \bigg|_{\theta=\theta^*_{\epsilon}} +\right]^{-1} +\frac{\partial F(\theta^*_{\epsilon}, \epsilon)}{\partial \epsilon}. Recalling the expression of F(\theta, \epsilon), we also have \frac{\partial F(\theta, \epsilon)}{\partial \epsilon}\bigg|_{\theta=\theta^*_{\epsilon}} += +\nabla_\theta \mathcal{L}(z_m, \theta)\Big|_{\theta=\theta^*_{\epsilon}}. Putting \epsilon = 0 to let \theta^*_\epsilon = \theta^*_{0} = \theta^* and \frac{\partial F(\theta, \epsilon)}{\partial \theta} \bigg|_{\theta=\theta^*_{\epsilon}} = H_{\theta^*}, we have: \frac{d\theta^*_{\epsilon}} + {d\epsilon} +\Bigg|_{\epsilon=0} += +- +H_{\theta^*}^{-1} +\nabla_\theta +\mathcal{L}(z_m,\theta^*). This quantity describes how the model parameters respond to an infinitesimal increase in the importance of the training sample z_m.

+

To measure the effect on a test-time quantity M(\theta, z_{\text{test}}) (e.g., the model’s test loss on the testing sample z_{\text{test}}), we apply the chain rule: \mathcal{I}_{M}(z_m, z_{\text{test}}) +\coloneqq +\frac{d M(\theta^*_{\epsilon}, z_{\text{test}})} + {d\epsilon} +\Bigg|_{\epsilon=0} += +- +\nabla_\theta M(\theta^*, z_{\text{test}})^\top +H_{\theta^*}^{-1} +\nabla_\theta +\mathcal{L}(z_m,\theta^*). The quantity \mathcal{I}_{M}(z_m, z_{\text{test}}) is called the influence score. Taking a step forward, by the definition of the derivative, for sufficiently small \epsilon we have M(\theta^*_{\epsilon}, z_{\text{test}}) +- +M(\theta^*, z_{\text{test}}) +\approx +\epsilon +\,\mathcal{I}_{M}(z_m, z_{\text{test}}). That is, the influence score quantifies the first-order effect of the training sample z_m on the model’s downstream loss for a testing sample z_{\text{test}}.

-

9.2.3 Computational Bottleneck: Approximating GGN-Vector Products#

-

The main practical challenge is computing influence for each training example without explicitly forming the GGN inverse: \mathbf{v} = (G + \lambda I)^{-1} \nabla_\theta \mathcal{L}(z, \theta^*), which naively costs \mathcal{O}(p^2) memory and \mathcal{O}(p^3) time for p parameters. Practical methods avoid explicit inversion by using iterative solvers and structured approximations of the GGN.

-
-

Kronecker-Factored Approximate Curvature (KFAC).

-

KFAC is a parametric approximation to the Fisher Information Matrix (FIM) of a neural network which supports efficient inversion. The FIM is defined as: F = \mathbb{E}_{\hat{y} \sim P_{\hat{y}|x}(\theta)} \left[ \nabla_\theta \log p(\hat{y}|\theta, x) \nabla_\theta \log p(\hat{y}|\theta, x)^\top \right], where \hat{y} is sampled from the model’s output distribution (not the training labels, which would yield the empirical Fisher with less favorable properties). For neural networks, KFAC approximates this by exploiting a layer-wise Kronecker-factored structure Ḟor layer \ell with input activations a_{\ell-1} and output gradient components (Grosse et al., 2023), KFAC approximates: F_\ell \approx (a_{\ell-1} a_{\ell-1}^\top) \otimes (D_s^\top D_s), where \otimes is the Kronecker product and D_s represents the output gradient structure. This factorization reduces memory from \mathcal{O}(p^2) to \mathcal{O}(p) and enables efficient layer-wise solves. KFAC works well but can underestimate curvature in low-variance directions due to finite sampling.

+

11.2.2 Proximal Bregman Response and the Generalised Gauss–Newton#

+

Empirical results show that the original form of influence functions introduced above performs poorly in practice. This is due to multiple assumptions that rarely hold in practice. For example, for a trained neural network, its parameters \theta are scarcely exactly at the local or global minimum \theta^*. Additionally, the Hessian H_{\theta^*} is usually non-invertible or numerically unstable due to some small eigenvalues. These limitations motivate researchers to formulate influence functions without unrealistic assumptions. One classical approach is called the proximal Bregman response function (PBRF) (Bae et al., 2022). It targets model parameters which minimise the new training objective while staying close to the original parameters. Below, we sketch its core idea.

+

Let each training sample z_i=(x_i, y_i), where x_i is the input and y_i is the label, and u_i(\theta) =f(x_i, \theta) is the output of the model (parameterised as \theta) given input x_i, such as a logit vector. Slightly abusing notation, we denote the per-sample training loss as \mathcal{L}(z_i, \theta) +\coloneqq +\mathcal{L}(u_i(\theta), y_i) += +\mathcal{L}(f(x_i, \theta), y_i), where \mathcal{L}(\cdot, y) is a loss function as in the vanilla influence function formulation. Here we additionally assume \mathcal{L}(\cdot, y) is convex in its first argument u, which many commonly used loss functions, such as cross-entropy loss, satisfy.

+

As Bregman divergence will be used in the later formulation, we introduce this notion as the next step. Specifically, for a given convex function \mathcal{L}(u, y) (as described above) with the label y fixed, the Bregman divergence of two model outputs, u and u', is: D_{\mathcal{L}}(u, u'; y) = \mathcal{L}(u, y) - \mathcal{L}(u', y) - \big(\nabla_v \mathcal{L}(v, y)\big|_{v = u'}\big)^\top (u - u'), i.e., the amount by which \mathcal{L}(\cdot, y) at u exceeds its first-order Taylor approximation from u'.

+

Now we introduce the PBRF formulation. Starting from the upweighted objective \mathcal{L}_{\epsilon}(\theta), PBRF is represented as: \mathcal{P}_{\epsilon}(\theta) +\coloneqq +\frac{1}{n} +\sum_{i=1}^{n} +D_{\mathcal{L}}\big(f(x_i, \theta),\, f(x_i, \theta^s); y_i\big) ++ +\epsilon\, \mathcal{L}(z_m, \theta) ++ +\frac{\lambda}{2}\|\theta - \theta^s\|^2, +\qquad +\theta^{\mathrm{PBRF}}_{\epsilon} +\coloneqq +\arg\min_\theta +\mathcal{P}_{\epsilon}(\theta), where \theta^s denotes the original trained model parameters (note that it is not required to be the local or global optimum). Bregman divergence replaces the original loss function. An additional proximity term is added with a positive regularisation coefficient \lambda. \theta^{\mathrm{PBRF}}_{\epsilon} searches for the set of model parameters that best corresponds to an upweight in z_m while keeping its predictions on the overall training set close to the original predictions and staying close to \theta^s.

+

In this construction, both the Bregman and the proximity terms are non-negative for all \theta and vanish at \theta = \theta^s. Since these are the only two terms of \mathcal{P}_{\epsilon} when \epsilon = 0, \theta^s is a global minimiser of \mathcal{P}_0 (i.e., \theta^s = \theta^{\mathrm{PBRF}}_0) regardless of whether training has converged.

+

As in the vanilla formulation, \theta^{\mathrm{PBRF}}_\epsilon does not have a closed-form formula for now. However, by observing that \nabla_\theta \mathcal{P}_{\epsilon}(\theta)\big|_{\theta=\theta^{\text{PBRF}}_{\epsilon}}=0, we define \tilde{F}(\theta, \epsilon) +\coloneqq +\nabla_\theta \mathcal{P}_{\epsilon}(\theta) += +\frac{1}{n} +\sum_{i=1}^{n} +\nabla_\theta D_{\mathcal{L}}\big(f(x_i, \theta),\, f(x_i, \theta^s); y_i\big) ++ +\epsilon\, \nabla_\theta \mathcal{L}(z_m, \theta) ++ +\lambda (\theta - \theta^s), where \tilde{F}(\theta^s, 0) = 0. We assume that f(x_i, \cdot) and \mathcal{L}(\cdot, y) are twice continuously differentiable such that \tilde{F} is continuously differentiable in a small area around (\theta^s, 0). Then, we verify that \frac{\partial\tilde{F}(\theta, \epsilon)}{\partial \theta} is invertible at (\theta, \epsilon) = (\theta^s, 0), the last prerequisite for using the implicit function theorem. By direct calculation (we omit the detailed process here), we can obtain \frac{\partial \tilde{F}(\theta, \epsilon)}{\partial \theta}\bigg|_{\theta = \theta^s, \epsilon = 0} += +\underbrace{ +\frac{1}{n}\sum_{i=1}^{n} J_i^\top H_i J_i +}_{\eqqcolon\, G} ++ +\lambda I, where u^s_i \coloneqq u_i(\theta^s), J_i \coloneqq \frac{\partial u_i(\theta)}{\partial \theta} \big|_{\theta=\theta^s} is the output–parameter Jacobian, and H_i \coloneqq \nabla^2_u \mathcal{L}(u, y_i) \big|_{u = u^s_i} is the loss–output Hessian. The matrix G is the generalised Gauss–Newton (GGN) matrix. Notably, as H_i is positive semidefinite by the convexity of \mathcal{L}(\cdot, y_i), we know G is positive semidefinite (because v^\top G v = \frac{1}{n}\sum_i (J_i v)^\top H_i (J_i v) \geq 0 for any v). As a result, G + \lambda I is positive definite and hence invertible. Unlike the case of \nabla^2_\theta \mathcal{L}_{\mathrm{train}}(\theta^*), the invertibility of G + \lambda I holds by construction.

+

Knowing that \partial \tilde{F}(\theta, \epsilon) / \partial \epsilon = \nabla_\theta \mathcal{L}(z_m, \theta), we apply the implicit function theorem and obtain \frac{d\theta^{\mathrm{PBRF}}_{\epsilon}}{d\epsilon} +\Bigg|_{\epsilon=0} += +- +(G + \lambda I)^{-1} +\nabla_\theta +\mathcal{L}(z_m,\theta^s). By the chain rule, the influence score on a test-time quantity M(\theta, z_{\text{test}}) becomes \mathcal{I}^{\mathrm{PBRF}}_{M}(z_m, z_{\text{test}}) +\coloneqq +\frac{d M(\theta^{\mathrm{PBRF}}_{\epsilon}, z_{\text{test}})}{d\epsilon} +\Bigg|_{\epsilon=0} += +- +\nabla_\theta M(\theta^s, z_{\text{test}})^\top +(G + \lambda I)^{-1} +\nabla_\theta +\mathcal{L}(z_m,\theta^s). As shown above, the influence function computed with G + \lambda I is exactly the derivative of the PBRF at \epsilon = 0. Bae et al. (2022) confirmed empirically that such influence estimates align well with the actual PBRF.

-
-

Eigenvalue-Corrected KFAC (EK-FAC) and ASTRA.

-

EK-FAC improves KFAC by correcting eigenvalues and is the method used by recent work scaling influence to LLMs (Grosse et al., 2023). An even more recent refinement, ASTRA (Wang et al., 2025a), uses EK-FAC as a preconditioner for Neumann series iterations, yielding higher-quality GGN-vector products with less hyperparameter tuning. These structured approaches make influence functions practical for models with billions of parameters.

+

11.2.3 Addressing the Computational and Memory Bottlenecks of GGN#

+

The size of the GGN matrix G \in \mathbb{R}^{p \times p} (where p is the number of model parameters) can be extremely large for modern language models, which makes the calculation of (G + \lambda I)^{-1} computationally expensive. As a result, follow-up methods have avoided explicitly calculating this inverse matrix by borrowing techniques from the optimisation literature.

+
+

Kronecker-Factored Approximate Curvature (K-FAC).

+

Martens & Grosse (2015) proposed Kronecker-Factored Approximate Curvature (K-FAC) to approximate a large Fisher information matrix (FIM) using a set of smaller matrices. For a trained neural network parameterised by \theta^s, the FIM is defined as \mathbf{F} +\coloneqq +\frac{1}{n}\sum_{i=1}^{n} +\mathbb{E}_{\hat{y} \sim p(\cdot \mid x_i, \theta^s)} +\left[ +\nabla_\theta \log p(\hat{y} \mid x_i, \theta^s) +\nabla_\theta \log p(\hat{y} \mid x_i, \theta^s)^\top +\right], where \hat{y} is a sample from the model’s prediction distribution on x_i. For example, suppose the task is three-class classification (c_1, c_2, c_3) and the model’s prediction distribution is \pi_i=\mathrm{softmax}(u_i^s) = (0.5, 0.3, 0.2). If a sampled label is \hat{y}=c_2, then \log p(\hat{y} \mid x_i, \theta^s) = \log 0.3.

+

Although K-FAC approximates the FIM rather than the GGN, these matrices are equivalent for common loss functions under suitable probabilistic assumptions. For example, with softmax cross-entropy, the expected outer product of the output log-likelihood gradients can be shown to equal the loss–output Hessian: \mathbb{E}_{\hat{y} \sim p(\cdot \mid x_i, \theta^s)} +\left[ +\nabla_u \log p(\hat{y} \mid u) +\nabla_u \log p(\hat{y} \mid u)^\top +\right]_{u=u_i^s} += +\mathrm{diag}(\pi_i)-\pi_i\pi_i^\top += +H_i. Applying the chain rule then gives \mathbf{F} += +\frac{1}{n}\sum_{i=1}^{n}J_i^\top H_i J_i += +G. This shows that we can use K-FAC to approximate the GGN used in the PBRF influence score.

+

Now we return to K-FAC, which efficiently approximates the FIM. First, it ignores curvature between layers and approximates \mathbf{F} as a block-diagonal matrix with one block per layer. Consider a neural network in which each layer \ell has a weight matrix W_\ell \in \mathbb{R}^{d_{\mathrm{out}}\times d_{\mathrm{in}}}, input activations h_{\ell-1}, and pre-activations s_\ell=W_\ell h_{\ell-1} (irrelevant to subscript s in \theta^s). Following Grosse et al. (2023), the pseudo-gradient is defined as \mathcal{D}_{s_\ell} \coloneqq \nabla_{s_\ell} \log p(\hat{y}\mid x,\theta^s) \in \mathbb{R}^{d_{\text{out}}}—the gradient flowing back to {s_\ell}. The pseudo-gradient of the layer weights is \mathcal{D}_{s_\ell} h_{\ell-1}^\top. Its vectorisation gives h_{\ell-1}\otimes\mathcal{D}_{s_\ell}, where \otimes denotes the Kronecker product. Leveraging \otimes’s properties that (A\otimes B)^\top = A^\top \otimes B^\top and (A\otimes B)(C\otimes D) = (AC)\otimes (BD), the block for this layer in the FIM, denoted as \mathbf{F}_\ell, can be represented as \mathbf{F}_\ell += +\underbrace{\mathbb{E}\left[ +(h_{\ell-1}h_{\ell-1}^\top) +\otimes +(\mathcal{D}_{s_\ell}\mathcal{D}_{s_\ell}^\top) +\right]}_{\in \mathbb{R}^{d_{\text{in}} d_{\text{out}}\times d_{\text{in}} d_{\text{out}}}} +\approx +\underbrace{ +\mathbb{E}[h_{\ell-1}h_{\ell-1}^\top] +}_{\eqqcolon\,\textbf{A}_{\ell-1} \in \mathbb{R}^{d_{\text{in}}\times d_{\text{in}}}} +\otimes +\underbrace{ +\mathbb{E}[\mathcal{D}_{s_\ell}\mathcal{D}_{s_\ell}^\top] +}_{\eqqcolon\,\textbf{S}_\ell \in \mathbb{R}^{d_{\text{out}}\times d_{\text{out}}}}. The approximation separates the expectation into two independent factors: activations and pseudo-gradients. K-FAC thus stores only d_{\text{in}}^2+d_{\text{out}}^2 entries instead of (d_{\text{in}}d_{\text{out}})^2 entries for each block. When the two factors are invertible, their Kronecker product can be inverted separately: (\textbf{A}_{\ell-1}\otimes \textbf{S}_\ell)^{-1} += +\textbf{A}_{\ell-1}^{-1}\otimes \textbf{S}_\ell^{-1}. This substantially reduces the memory required to store G and the computational cost of computing the inverse of G+\lambda I.

+
+

Eigenvalue-Corrected K-FAC (EK-FAC) and ASTRA.

+

EK-FAC (George et al., 2018) improved K-FAC by correcting its eigenvalues. Recent work has adopted this technique to scale the influence function method to modern LLMs (Grosse et al., 2023). ASTRA (Wang et al., 2025a) further used EK-FAC as a preconditioner for Neumann series iterations, achieving higher-quality GGN-vector products with less hyperparameter tuning. These efforts make influence functions increasingly practical for modern language models with billions of parameters.

-

9.2.4 Unrolled Differentiation: Gradient-Based Attribution#

-

A complementary approach bypasses the Hessian entirely. Approximate unrolled differentiation (Bae et al., 2024) unfolds the optimization trajectory and backpropagates through training steps. This captures implicit bias of the optimizer and handles non-converged models, where classical influence functions struggle. The tradeoff: higher computational cost than influence functions but better accuracy on modern, non-convex objectives.

-

9.2.5 Practical Attribution at Scale: TRAK#

-

For large-scale attribution without retraining thousands of models, TRAK (Tracing with Randomly-projected After Kernel) provides a scalable alternative (Park et al., 2023). TRAK combines three key ideas: a first-order Taylor expansion approximating how model behavior changes under data perturbations, random projections to reduce gradient dimensionality from p parameters to k \ll p, and training only a handful of models on different data subsets rather than retraining exhaustively. By avoiding prohibitive retraining costs, TRAK achieves attribution quality comparable to expensive retraining-based methods like datamodels (Ilyas et al., 2022) while reducing computational cost by orders of magnitude. This enables application to large-scale models including vision transformers and billion-parameter language models, making fine-grained training data attribution tractable for modern deep learning (Park et al., 2023).

+

11.2.4 Retraining-Based Influence Estimation: datamodels and TRAK#

+

Some methods have explored retraining the model to estimate the attribution. The most naive method is to upweight or remove each training sample, retrain the model on the modified training set, and observe changes in the model’s output on the testing sample. Ilyas et al. (2022) proposed datamodels to more efficiently estimate each training sample’s attribution on a testing sample by retraining many models on random subsets of the training data and fitting a linear regression that predicts the model output on the testing sample from binary indicators of whether each training sample was included. Each fitted coefficient then represents the training sample’s influence on the testing sample.

+

However, datamodels remains computationally expensive because it requires training many models. Tracing with the Randomly-projected After Kernel (TRAK; Park et al., 2023) provides a more efficient alternative. TRAK first uses a first-order Taylor expansion around the model parameters to approximate a trained model as a linear model. It then applies random projections to reduce gradient dimensionality and estimate training-sample influence in this lower-dimensional space. Finally, it averages the estimates across a small number of trained models to reduce variability due to training randomness. TRAK achieves attribution quality comparable to datamodels at substantially lower computational cost (Park et al., 2023).

-

9.2.6 Efficient Influence Estimation: TracIn#

-

TracIn provides an efficient alternative to computing the full Hessian inverse by leveraging the training trajectory (Pruthi et al., 2020). Rather than computing influence at convergence via H_{\theta^*}^{-1} \nabla_\theta \mathcal{L}(z, \theta^*), TracIn approximates influence by integrating the gradient of a test example’s loss with respect to gradients along the training path: \text{influence}(z_{\text{train}}, z_{\text{test}}) \approx \sum_{t=1}^T \nabla_\theta \mathcal{L}(z_{\text{test}}, \theta_t) \cdot \nabla_\theta \mathcal{L}(z_{\text{train}}, \theta_t) where \theta_t denotes parameters at training step t. This approach avoids Hessian computation entirely, instead accumulating dot products of gradients throughout training. TracIn is considerably cheaper than influence functions based on Hessian approximations, making it practical for large models. However, the approximation assumes that gradient trajectories during training encode sufficient information about final influence, an assumption that may break down in highly nonlinear settings or when training dynamics involve significant noise (Pruthi et al., 2020). Despite limitations, TracIn has enabled attribution studies on modern language models and provides a useful baseline for scalable influence estimation.

+

11.2.5 Efficient Influence Estimation Using Training Trajectories: TracIn#

+

TracIn (Garima et al., 2020) estimates training-data influence using gradients evaluated along the training trajectory, without computing the inverse Hessian. Its core idea follows from the effect of a single Stochastic Gradient Descent (SGD; Robbins & Monro, 1951) update on the loss of a testing sample.

+

Suppose the model parameters at training step t are \theta^t. An SGD update using training sample z_m with learning rate \eta_t is: \theta^{t+1} += +\theta^t-\eta_t\nabla_\theta\mathcal{L}(z_m,\theta^t). By simple rearrangement, we have \theta^{t+1} - \theta^t +=-\eta_t\nabla_\theta\mathcal{L}(z_m,\theta^t). Then, by first-order Taylor expansion, the change in the loss of a testing sample z_{\text{test}} can be approximated as \begin{aligned} +&\mathcal{L}(z_{\text{test}},\theta^{t+1}) +-\mathcal{L}(z_{\text{test}},\theta^t)\\ +&\quad\approx +\nabla_\theta\mathcal{L}(z_{\text{test}},\theta^t)^\top +(\theta^{t+1}-\theta^t)\\ +&\quad= +-\eta_t +\nabla_\theta\mathcal{L}(z_{\text{test}},\theta^t)^\top +\nabla_\theta\mathcal{L}(z_m,\theta^t). +\end{aligned} The above expression relates the gradient of the loss on a training sample z_m to the gradient on a test sample z_{\text{test}} and the change in test loss on z_{\text{test}}. A positive dot product indicates that the training update will reduce the test loss, while a negative value indicates the opposite.

+

TracIn extends this idea across training. Its practical checkpoint-based variant, TracInCP, estimates influence by summing gradient dot products over model checkpoints: \begin{aligned} +&\operatorname{TracInCP}(z_m,z_{\text{test}})\\ +&\quad= +\sum_{t\in\mathcal{C}}\eta_t +\nabla_\theta\mathcal{L}(z_{\text{test}},\theta^t)^\top +\nabla_\theta\mathcal{L}(z_m,\theta^t), +\end{aligned} where \mathcal{C} denotes the selected training checkpoints. A larger positive score indicates that z_m has contributed more towards reducing the test loss.

+

Unlike Hessian-based influence functions, TracIn does not require matrix inversion. However, TracInCP approximates the influence of training samples using gradients only at selected checkpoints, which may undermine the estimation accuracy.

-

9.2.7 Illustrative Example#

-

Building on the previous compositional translation task, gradient-based influence functions can identify training examples that shaped the output even when no exact matches exist. This complements verbatim attribution and reveals subtler influences.

+

11.2.6 Illustrative Example#

+

Gradient-based influence functions can identify influential training samples without requiring them to exactly match the current model output. They complement verbatim attribution and reveal subtler influences. Below, we provide an example of using them for the English-to-Chinese translation task.

+

Gradient-Based Attribution on a Translation Example

-

Prompt (English):

+
+

Prompt (English):

“I went to the park to play frisbee with my dog. She loves catching the frisbee and running with it. Sometimes she gets too excited and won’t let go when I try to take it from her. Playing frisbee with her is always fun and great exercise for both of us. Chinese:”

-

Gradient-Based Influential Examples Identified:

+
+
+

Gradient-Based Influential Training Samples Identified:

“Maybe I just feel like I could do anything, I want the whole world dancing to my music and wearing my clothes.

@@ -147,11 +358,19 @@

Eigenvalue-Corrected KFAC (EK-FAC) and ASTRA.

”

-

Observation: Influence functions successfully identify training documents that impacted the translation, even without exact verbatim matches. This demonstrates how gradient-based methods capture broader compositional and abstract influences beyond what verbatim methods can reveal.

+
+
+

Observation.

+

Influence functions identify training documents that impacted the translation, even without exact verbatim matches. This demonstrates that gradient-based methods can capture influences beyond what verbatim methods can reveal.

+
-

9.2.8 Recent Extensions and Limitations#

-

Recent work extends training data attribution to handle training stochasticity. Distributional TDA (d-TDA) predicts how output distributions depend on the dataset (Mlodozeniec et al., 2025), while TRAK variants address diffusion models (Georgiev et al., 2023). More recent work connects TDA directly to mechanistic interpretability: (Kowal et al., 2026) introduce Concept Influence, which enables attribution at the level of individual model components rather than raw parameters, bringing TDA closer to the mechanistic goal of understanding how training data shapes specific internal computations. However, fundamental challenges persist: approximation errors compound over optimization trajectories, random hyperparameters introduce variance across runs, and scaling to larger models remains memory-intensive. These obstacles suggest that fully resolving attribution at scale may require fundamentally new approaches.

+

11.2.7 Recent Extensions and Limitations#

+

Recent work extends training data attribution to handle training stochasticity. Distributional TDA (d-TDA) estimates how output distributions depend on the dataset (Mlodozeniec et al., 2025). Georgiev et al. (2023) proposed TRAK variants to address diffusion models. More recent work connects TDA directly to mechanistic interpretability (Kowal et al., 2026; Habibi et al., 2026); for example, Kowal et al. (2026) introduced Concept Influence to enable attribution at the level of semantic concepts represented by model internals such as sparse autoencoder features. This brings TDA closer to the mechanistic goal of understanding how training data shapes specific internal computations. However, fundamental technical challenges persist, including approximation errors that compound over the training trajectory, variability across training runs caused by randomness like data ordering, and memory and computational bottlenecks of scaling to frontier models. These limitations motivate further research in this direction.

+
+

Summary.

+

Training data attribution traces model behaviour back to the training data. Verbatim methods such as OLMoTrace identify memorised spans, while gradient-based methods such as influence functions estimate the contribution of individual training samples to downstream model predictions. These approaches remain limited at scale. Meanwhile, recent work has begun linking data attribution to internal features and mechanisms. Having surveyed the main methodological families of AI interpretability in the preceding sections, Section 12 organises them into six research orientations that differ in their goals, assumptions, and paths forward.

+
diff --git a/src/generated/manifest.json b/src/generated/manifest.json index 51932e2d02..8e42e6e06a 100644 --- a/src/generated/manifest.json +++ b/src/generated/manifest.json @@ -1,12 +1,12 @@ { "provenance": { - "status": "draft", - "source_dir_name": "The-Handbook-on-Interpretability", - "source_commit": "cd5bc3c", - "source_commit_date": "2026-07-13", - "main_tex_sha256": "490b52b9e014a4d6", + "status": "final", + "source_dir_name": "manuscript", + "source_commit": null, + "source_commit_date": null, + "main_tex_sha256": "1dc70452f1aed99a", "pdf_included": false, - "pdf_page_offset": 8, + "pdf_page_offset": 0, "citations": "natbib labels from the compiled .bbl", "generated": "2026-11-06" }, @@ -16,2377 +16,2715 @@ "num": "1", "title": "Introduction", "appendix": false, - "page": "1", + "page": "9", "sections": [ { "id": "brief-history-of-interpretability", "num": "1.1", "title": "Brief History of Interpretability", "level": 2, - "page": "2" + "page": "9" }, { "id": "sec:scope", "num": "1.2", - "title": "Handbook Scope", + "title": "Survey Scope", "level": 2, - "page": "4" + "page": "12" } ], "withheld": false, - "words": 2543 + "words": 3489 }, { "slug": "limits-of-behavioural-evaluation", "num": "2", - "title": "The Limits of Behavioral Evaluation", + "title": "The Limits of Behavioural Evaluation", "appendix": false, - "page": "5", + "page": "15", "sections": [], "withheld": false, - "words": 1056 + "words": 963 }, { "slug": "interpretability-landscape", "num": "3", "title": "The Interpretability Landscape", "appendix": false, - "page": "7", + "page": "16", "sections": [ { - "id": "behavioral-methods", + "id": "behavioural-methods", "num": "3.1", - "title": "Behavioral Methods", + "title": "Behavioural Methods", "level": 2, - "page": "7" + "page": "17" }, { "id": "subsec:cbm", "num": "3.2", "title": "Concept-Based Approaches", "level": 2, - "page": "8" + "page": "17" }, { - "id": "attribution-and-saliency-methods", + "id": "input-attribution-and-saliency-methods", "num": "3.3", - "title": "Attribution and Saliency Methods", + "title": "Input Attribution and Saliency Methods", "level": 2, - "page": "8" + "page": "18" }, { "id": "representation-probing-and-related-post-hoc-analyses", "num": "3.4", "title": "Representation Probing and Related Post-hoc Analyses", "level": 2, - "page": "9" + "page": "19" }, { - "id": "when-mechanistic-understanding-becomes-necessary", + "id": "mechanistic-interpretability-methods", "num": "3.5", + "title": "Mechanistic Interpretability Methods", + "level": 2, + "page": "19" + }, + { + "id": "subsec: mech interp necessary", + "num": "3.6", "title": "When Mechanistic Understanding Becomes Necessary", "level": 2, - "page": "9" + "page": "19" + }, + { + "id": "subsec:limits", + "num": "3.7", + "title": "Limitations of Mechanistic Analysis", + "level": 2, + "page": "20" } ], "withheld": false, - "words": 1418 + "words": 2729 }, { "slug": "foundational-concepts", "num": "4", "title": "Foundational Concepts for Mechanistic Interpretability", "appendix": false, - "page": "10", + "page": "21", "sections": [ { "id": "subsec:behavioral-vs-mechanistic", "num": "4.1", - "title": "Behavioral vs. Mechanistic Understanding", + "title": "Behavioural vs. Mechanistic Understanding", "level": 2, - "page": "10" + "page": "21" }, { "id": "subsec:polysemanticity", "num": "4.2", - "title": "Polysemanticity in Neural Representations", + "title": "Neuron Polysemanticity in Model Representations", "level": 2, - "page": "11" + "page": "21" }, { "id": "subsec:features", "num": "4.3", "title": "Features as Representational Units", "level": 2, - "page": "12" + "page": "22" }, { "id": "subsec:transformers", "num": "4.4", "title": "Transformer Computation", "level": 2, - "page": "12" + "page": "23" }, { "id": "subsec:circuits", "num": "4.5", - "title": "Circuits as Computational Substructures", + "title": "Circuits as Descriptions of Internal Mechanisms", "level": 2, - "page": "14" - }, - { - "id": "subsec:limits", - "num": "4.6", - "title": "Limitations of Foundational Works In Mechanistic Interpretability", - "level": 2, - "page": "14" + "page": "25" } ], "withheld": false, - "words": 1330 + "words": 2612 }, { "slug": "core-hypotheses", "num": "5", "title": "Core Hypotheses in Mechanistic Interpretability", "appendix": false, - "page": "15", + "page": "27", "sections": [ { "id": "subsec:linearity", "num": "5.1", "title": "The Linear Representation Hypothesis", "level": 2, - "page": "15" + "page": "27" }, { "id": "subsec:superposition", "num": "5.2", "title": "The Superposition Hypothesis", "level": 2, - "page": "16" - }, - { - "id": "architectural-responses-to-superposition", - "num": "5.2.1", - "title": "Architectural Responses to Superposition", - "level": 3, - "page": "17" + "page": "28" }, { "id": "subsec:privileged-access", "num": "5.3", "title": "The Privileged Access Hypothesis", "level": 2, - "page": "18" + "page": "30" }, { "id": "subsec:universality", "num": "5.4", - "title": "The Universality and Platonic Representation Hypotheses", + "title": "The Universality and Platonic Representation Hypothesis", "level": 2, - "page": "21" + "page": "31" } ], "withheld": false, - "words": 2563 + "words": 2271 }, { "slug": "feature-extraction", "num": "6", - "title": "Feature Extraction and Multi-Layer Analysis", + "title": "Feature Extraction and Analysis", "appendix": false, - "page": "21", + "page": "31", "sections": [ { - "id": "subsec:sae-foundation", + "id": "the-sparse-coding-problem", "num": "6.1", + "title": "The Sparse Coding Problem", + "level": 2, + "page": "32" + }, + { + "id": "subsec:sae-foundation", + "num": "6.2", "title": "Sparse Autoencoders", "level": 2, - "page": "22" + "page": "32" }, { "id": "subsec:sae-variants", - "num": "6.2", + "num": "6.3", "title": "SAE Variants and Architectural Improvements", "level": 2, - "page": "23" + "page": "34" }, { "id": "subsec:sae-selection", - "num": "6.3", - "title": "Practical Guidance: Selecting SAE Variants", + "num": "6.4", + "title": "Empirical Comparisons of SAE Variants", "level": 2, - "page": "24" + "page": "35" }, { - "id": "layer-specific-recommendations", - "num": "6.3.1", - "title": "Layer-Specific Recommendations", - "level": 3, - "page": "25" + "id": "subsec:challenge", + "num": "6.5", + "title": "Challenges and Failure Modes of SAEs", + "level": 2, + "page": "37" }, { - "id": "common-pitfalls-and-how-to-avoid-them", - "num": "6.3.2", - "title": "Common Pitfalls and How to Avoid Them", + "id": "dead-features", + "num": "6.5.1", + "title": "Dead Features", "level": 3, - "page": "26" + "page": "37" }, { - "id": "quick-reference-when-to-use-what", - "num": "6.3.3", - "title": "Quick Reference: When to Use What", + "id": "feature-duplication", + "num": "6.5.2", + "title": "Feature Duplication", "level": 3, - "page": "26" + "page": "37" }, { - "id": "limitations-of-these-recommendations", - "num": "6.3.4", - "title": "Limitations of These Recommendations", + "id": "feature-absorption-and-splitting", + "num": "6.5.3", + "title": "Feature Absorption and Splitting", "level": 3, - "page": "26" - }, - { - "id": "subsec:transcoders-crosscoders", - "num": "6.4", - "title": "Multi-Layer Feature Analysis: Transcoders and Crosscoders", - "level": 2, - "page": "27" + "page": "39" }, { - "id": "subsubsec:transcoder", - "num": "6.4.1", - "title": "Transcoders: Bridging Nonlinear Layers", + "id": "feature-instability-across-seeds", + "num": "6.5.4", + "title": "Feature Instability Across Seeds", "level": 3, - "page": "27" + "page": "39" }, { - "id": "subsubsec:cross-layer-transcoder", - "num": "6.4.2", - "title": "Cross-Layer Transcoders", + "id": "subsubsec: sae limitation", + "num": "6.5.5", + "title": "Identifiability and Theoretical Concerns", "level": 3, - "page": "27" + "page": "40" }, { - "id": "crosscoders-analyzing-cross-layer-features", - "num": "6.4.3", - "title": "Crosscoders: Analyzing Cross-Layer Features", + "id": "practical-recommendations", + "num": "6.5.6", + "title": "Practical Recommendations", "level": 3, - "page": "30" - }, - { - "id": "comparing-saes-transcoders-and-crosscoders.", - "num": "6.5", - "title": "Comparing SAEs, Transcoders, and Crosscoders.", - "level": 2, - "page": "30" + "page": "40" }, { - "id": "case-study-induction-heads", + "id": "subsec:transcoders-crosscoders", "num": "6.6", - "title": "Case Study: Induction Heads", - "level": 2, - "page": "31" - }, - { - "id": "subsec:challenge", - "num": "6.7", - "title": "Challenges and Failure Modes of SAEs", + "title": "Multi-Layer Feature Extraction and Analysis", "level": 2, - "page": "32" - }, - { - "id": "dead-features", - "num": "6.7.1", - "title": "Dead Features", - "level": 3, - "page": "32" + "page": "41" }, { - "id": "feature-duplication", - "num": "6.7.2", - "title": "Feature Duplication", + "id": "subsubsec:transcoder", + "num": "6.6.1", + "title": "Transcoders: Bridging the Nonlinearities", "level": 3, - "page": "33" + "page": "41" }, { - "id": "feature-absorption-and-splitting", - "num": "6.7.3", - "title": "Feature Absorption and Splitting", + "id": "subsubsec:cross-layer-transcoder", + "num": "6.6.2", + "title": "Cross-Layer Transcoders: Enabling Better Circuit Analysis", "level": 3, - "page": "33" + "page": "42" }, { - "id": "feature-instability-across-seeds", - "num": "6.7.4", - "title": "Feature Instability Across Seeds", + "id": "crosscoders-analysing-cross-layer-features", + "num": "6.6.3", + "title": "Crosscoders: Analysing Cross-Layer Features", "level": 3, - "page": "33" + "page": "45" }, { - "id": "identifiability-and-theoretical-concerns", - "num": "6.7.5", - "title": "Identifiability and Theoretical Concerns", - "level": 3, - "page": "33" + "id": "comparing-saes-transcoders-and-crosscoders", + "num": "6.7", + "title": "Comparing SAEs, Transcoders, and Crosscoders", + "level": 2, + "page": "46" }, { - "id": "practical-recommendations", - "num": "6.7.6", - "title": "Practical Recommendations", - "level": 3, - "page": "34" + "id": "subsec: induc head case study: feature extract", + "num": "6.8", + "title": "Case Study: Induction Heads", + "level": 2, + "page": "47" }, { "id": "comparison-with-alternative-feature-extraction-methods", - "num": "6.8", + "num": "6.9", "title": "Comparison with Alternative Feature Extraction Methods", "level": 2, - "page": "34" + "page": "48" }, { - "id": "non-negative-matrix-factorization-nmf", - "num": "6.8.1", - "title": "Non-Negative Matrix Factorization (NMF)", + "id": "non-negative-matrix-factorisation-nmf", + "num": "6.9.1", + "title": "Non-Negative Matrix Factorisation (NMF)", "level": 3, - "page": "34" + "page": "48" }, { - "id": "principle-component-analysis-pca", - "num": "6.8.2", - "title": "Principle Component Analysis (PCA)", + "id": "principal-component-analysis-pca", + "num": "6.9.2", + "title": "Principal Component Analysis (PCA)", "level": 3, - "page": "35" + "page": "49" }, { "id": "independent-component-analysis-ica", - "num": "6.8.3", + "num": "6.9.3", "title": "Independent Component Analysis (ICA)", "level": 3, - "page": "35" + "page": "49" }, { - "id": "supervised-feature-extraction-methods", - "num": "6.8.4", + "id": "subsubsec: supervised_extraction", + "num": "6.9.4", "title": "Supervised Feature Extraction Methods", "level": 3, - "page": "35" + "page": "49" }, { "id": "dictionary-learning-baselines", - "num": "6.8.5", + "num": "6.9.5", "title": "Dictionary Learning Baselines", "level": 3, - "page": "36" + "page": "50" }, { "id": "sec:architectural-interventions", - "num": "6.9", - "title": "Architectural Interventions for Intrinsic Interpretability", + "num": "6.10", + "title": "Interpretability-by-Design Methods for Intrinsic Interpretability", "level": 2, - "page": "36" - }, - { - "id": "weight-sparsity-via-l_0-regularization", - "num": "6.9.1", - "title": "Weight Sparsity via L_0 Regularization", - "level": 3, - "page": "36" + "page": "50" }, { - "id": "sparse-attention-post-training-spartan", - "num": "6.9.2", - "title": "Sparse Attention Post-Training: SPARTAN", + "id": "subsubsec: solu", + "num": "6.10.1", + "title": "Feature-Sparse Training: SoLU", "level": 3, - "page": "37" + "page": "50" }, { - "id": "activation-level-interventions-solu-and-its-limitations", - "num": "6.9.3", - "title": "Activation-Level Interventions: SoLU and Its Limitations", + "id": "subsubsec: weight sparse transformer", + "num": "6.10.2", + "title": "Weight-Sparse Training", "level": 3, - "page": "37" + "page": "51" }, { - "id": "computational-primitive-modifications-molt", - "num": "6.9.4", - "title": "Computational Primitive Modifications: MOLT", + "id": "subsubsec: spartan", + "num": "6.10.3", + "title": "Attention-Sparse Training", "level": 3, - "page": "37" + "page": "51" }, { - "id": "kolmogorov-arnold-networks", - "num": "6.9.5", - "title": "Kolmogorov-Arnold Networks", + "id": "subsubsec: molt", + "num": "6.10.4", + "title": "Modifications of Computational Primitives: MOLT", "level": 3, - "page": "38" + "page": "52" }, { - "id": "the-bridge-methodology-interpretable-proxies-for-dense-models", - "num": "6.9.6", - "title": "The Bridge Methodology: Interpretable Proxies for Dense Models", + "id": "subsubsec: kan", + "num": "6.10.5", + "title": "Kolmogorov–Arnold Networks (KANs)", "level": 3, - "page": "38" + "page": "52" } ], "withheld": false, - "words": 7507 + "words": 9537 }, { "slug": "circuits", "num": "7", "title": "Circuits in Mechanistic Interpretability", "appendix": false, - "page": "38", + "page": "53", "sections": [ { - "id": "sec:what-are-circuits", + "id": "sec:circuit-discovery-methods", "num": "7.1", - "title": "What Are Circuits?", + "title": "Circuit Discovery Methods", "level": 2, - "page": "39" + "page": "53" }, { - "id": "sec:circuit-table", + "id": "sec:circuit-replacement", "num": "7.2", - "title": "Circuit Discovery and Tracing Methods", - "level": 2, - "page": "39" - }, - { - "id": "circuit-discovery-in-weight-sparse-architectures", - "num": "7.3", - "title": "Circuit Discovery in Weight-Sparse Architectures", + "title": "Circuit Discovery with Replacement Models", "level": 2, - "page": "40" + "page": "55" }, { "id": "sec:causal-verification", - "num": "7.4", + "num": "7.3", "title": "Causal Verification", "level": 2, - "page": "41" + "page": "57" }, { - "id": "case-study-induction-heads-1", - "num": "7.5", + "id": "subsec: induc head case study: circuit", + "num": "7.4", "title": "Case Study: Induction Heads", "level": 2, - "page": "42" + "page": "58" }, { "id": "sec:evaluation-framework", - "num": "7.6", + "num": "7.5", "title": "A Unified Evaluation Framework for Mechanistic Interpretability", "level": 2, - "page": "43" + "page": "58" }, { - "id": "the-need-for-standardized-evaluation", - "num": "7.6.1", - "title": "The Need for Standardized Evaluation", + "id": "the-need-for-standardised-evaluation", + "num": "7.5.1", + "title": "The Need for Standardised Evaluation", "level": 3, - "page": "43" + "page": "58" }, { - "id": "five-evaluation-dimensions", - "num": "7.6.2", + "id": "subsec:five eval dim", + "num": "7.5.2", "title": "Five Evaluation Dimensions", "level": 3, - "page": "43" - }, - { - "id": "dimension-1-faithfulness", - "num": "7.6.3", - "title": "Dimension 1: Faithfulness", - "level": 3, - "page": "43" - }, - { - "id": "dimension-2-completeness", - "num": "7.6.4", - "title": "Dimension 2: Completeness", - "level": 3, - "page": "44" - }, - { - "id": "dimension-3-stability", - "num": "7.6.5", - "title": "Dimension 3: Stability", - "level": 3, - "page": "45" - }, - { - "id": "dimension-4-causal-validity", - "num": "7.6.6", - "title": "Dimension 4: Causal Validity", - "level": 3, - "page": "45" - }, - { - "id": "dimension-5-human-interpretability", - "num": "7.6.7", - "title": "Dimension 5: Human Interpretability", - "level": 3, - "page": "46" + "page": "59" }, { "id": "integrated-evaluation-protocol", - "num": "7.6.8", + "num": "7.5.3", "title": "Integrated Evaluation Protocol", "level": 3, - "page": "47" - }, - { - "id": "applying-the-framework-method-specific-guidance", - "num": "7.6.9", - "title": "Applying the Framework: Method-Specific Guidance", - "level": 3, - "page": "47" + "page": "65" }, { "id": "limitations-of-this-framework", - "num": "7.6.10", + "num": "7.5.4", "title": "Limitations of This Framework", "level": 3, - "page": "48" + "page": "66" } ], "withheld": false, - "words": 3323 + "words": 5730 }, { "slug": "intermediate-representations", "num": "8", - "title": "Understanding Intermediate Computation: Logit Lens and Probing", + "title": "Understanding Intermediate Representations", "appendix": false, - "page": "48", + "page": "66", "sections": [ { "id": "sec:logit-lens", "num": "8.1", "title": "Logit Lens: Tracing Predictions Through Depth", "level": 2, - "page": "48" + "page": "67" }, { - "id": "tuned-lens-learning-per-layer-transform", + "id": "subsec:tuned-lens", "num": "8.2", - "title": "Tuned Lens: Learning Per-Layer Transform", + "title": "Tuned Lens: Learning Per-Layer Transforms", "level": 2, - "page": "49" + "page": "68" }, { - "id": "case-study-refusal-vectors-and-sleeper-agents", + "id": "jacobian-lens-decoding-computational-dispositions", "num": "8.3", - "title": "Case Study: Refusal Vectors and Sleeper Agents", + "title": "Jacobian Lens: Decoding Computational Dispositions", "level": 2, - "page": "50" + "page": "68" }, { "id": "subsec:patchscopes", "num": "8.4", "title": "Patchscopes: Leveraging the Language Model Itself to Explain Intermediate Representations", "level": 2, - "page": "51" + "page": "69" }, { "id": "subsec:latentqa", "num": "8.5", - "title": "LatentQA: Finetuning the Language Model as Representation Decoder", + "title": "LatentQA: Fine-Tuning the Language Model as an Activation Verbaliser", "level": 2, - "page": "52" + "page": "70" }, { "id": "subsec:probing", "num": "8.6", "title": "Probing Classifiers: Capabilities, Critiques, and Causal Limitations", "level": 2, - "page": "52" + "page": "72" }, { "id": "standard-probing-methodology", "num": "8.6.1", "title": "Standard Probing Methodology", "level": 3, - "page": "53" + "page": "72" }, { - "id": "the-probe-causality-problem", + "id": "control-tasks-and-selectivity", "num": "8.6.2", - "title": "The Probe Causality Problem", + "title": "Control Tasks and Selectivity", "level": 3, - "page": "53" + "page": "73" }, { - "id": "control-tasks-and-selectivity", + "id": "concerns-of-probe-complexity", "num": "8.6.3", - "title": "Control Tasks and Selectivity", + "title": "Concerns of Probe Complexity", "level": 3, - "page": "54" + "page": "74" }, { - "id": "probe-complexity-vs.-causal-relevance", + "id": "subsubsec: amnesic probing", "num": "8.6.4", - "title": "Probe Complexity vs. Causal Relevance", + "title": "Amnesic Probing and Causal Probing", "level": 3, - "page": "55" + "page": "74" }, { - "id": "amnesic-probing-and-causal-probing", + "id": "implications-of-attribution-faithfulness-for-probing", "num": "8.6.5", - "title": "Amnesic Probing and Causal Probing", + "title": "Implications of Attribution Faithfulness for Probing", "level": 3, - "page": "55" + "page": "75" }, { - "id": "critiques-of-attribution-and-saliency-methods", + "id": "when-is-probing-appropriate", "num": "8.6.6", - "title": "Critiques of Attribution and Saliency Methods", + "title": "When Is Probing Appropriate?", "level": 3, - "page": "56" + "page": "76" }, { - "id": "circuit-fragility-and-robustness-concerns", + "id": "methodological-standards-in-probing-research", "num": "8.6.7", - "title": "Circuit Fragility and Robustness Concerns", + "title": "Methodological Standards in Probing Research", "level": 3, - "page": "56" - }, - { - "id": "when-is-probing-appropriate-and-when-is-it-not", - "num": "8.6.8", - "title": "When is Probing Appropriate, and When is it Not?", - "level": 3, - "page": "56" - }, - { - "id": "best-practices-for-probing-studies", - "num": "8.6.9", - "title": "Best Practices for Probing Studies", - "level": 3, - "page": "57" + "page": "76" } ], "withheld": false, - "words": 3934 + "words": 4514 }, { - "slug": "training-data-attribution", + "slug": "representation-analysis", "num": "9", - "title": "Training Data Attribution and Influence Functions", + "title": "Representation Analysis: Comparison, Localisation, and Control", "appendix": false, - "page": "58", + "page": "77", "sections": [ { - "id": "verbatim-data-attribution-olmotrace-and-maximal-matching-spans", + "id": "sec:model-diffing", "num": "9.1", - "title": "Verbatim Data Attribution: OLMoTrace and Maximal Matching Spans", + "title": "Model Diffing: Identifying Internal Changes Across Models", "level": 2, - "page": "59" + "page": "77" }, { - "id": "limitations", - "num": "9.1.1", - "title": "Limitations", - "level": 3, - "page": "60" + "id": "sec:knowledge-localization", + "num": "9.2", + "title": "Knowledge Localisation: Where Is Information Stored?", + "level": 2, + "page": "80" }, { - "id": "gradient-based-attribution-understanding-influence-through-data-perturbations", - "num": "9.2", - "title": "Gradient-Based Attribution: Understanding Influence Through Data Perturbations", + "id": "subsec:steering", + "num": "9.3", + "title": "Representation Engineering and Steering", + "level": 2, + "page": "80" + } + ], + "withheld": false, + "words": 2515 + }, + { + "slug": "learning-dynamics", + "num": "10", + "title": "Learning Dynamics and Mechanistic Interpretability", + "appendix": false, + "page": "82", + "sections": [ + { + "id": "sec:ld-longitudinal", + "num": "10.1", + "title": "Longitudinal Analysis", "level": 2, - "page": "61" + "page": "83" }, { - "id": "core-idea-parameter-response-to-training-data", - "num": "9.2.1", - "title": "Core Idea: Parameter Response to Training Data", + "id": "sec:ld-data", + "num": "10.1.1", + "title": "Training Data and Capability Acquisition", "level": 3, - "page": "61" + "page": "83" }, { - "id": "proximal-bregman-response-and-the-gauss-newton-hessian", - "num": "9.2.2", - "title": "Proximal Bregman Response and the Gauss-Newton Hessian", + "id": "sec:ld-features", + "num": "10.1.2", + "title": "Feature Formation, Consolidation, and Drift", "level": 3, - "page": "61" + "page": "83" }, { - "id": "computational-bottleneck-approximating-ggn-vector-products", - "num": "9.2.3", - "title": "Computational Bottleneck: Approximating GGN-Vector Products", - "level": 3, - "page": "61" + "id": "learning-dynamics-as-an-interpretability-tool", + "num": "10.2", + "title": "Learning Dynamics as an Interpretability Tool", + "level": 2, + "page": "84" }, { - "id": "unrolled-differentiation-gradient-based-attribution", - "num": "9.2.4", - "title": "Unrolled Differentiation: Gradient-Based Attribution", + "id": "sec:ld-localization", + "num": "10.2.1", + "title": "Temporal Localisation and Mechanism Discovery", "level": 3, - "page": "62" + "page": "84" }, { - "id": "sec:trak", - "num": "9.2.5", - "title": "Practical Attribution at Scale: TRAK", + "id": "sec:ld-causal-interventions", + "num": "10.2.2", + "title": "Causal Interventions on Mechanism Formation", "level": 3, - "page": "62" + "page": "85" }, { - "id": "sec:tracin", - "num": "9.2.6", - "title": "Efficient Influence Estimation: TracIn", - "level": 3, - "page": "62" + "id": "sec:ld-transitions", + "num": "10.3", + "title": "Phase Transitions and Emergence", + "level": 2, + "page": "86" }, { - "id": "illustrative-example", - "num": "9.2.7", - "title": "Illustrative Example", + "id": "sec:ld-detect", + "num": "10.3.1", + "title": "Detecting Transitions", "level": 3, - "page": "63" + "page": "86" }, { - "id": "recent-extensions-and-limitations", - "num": "9.2.8", - "title": "Recent Extensions and Limitations", + "id": "sec:ld-progress", + "num": "10.3.2", + "title": "Progress Measures and Internal Precursors", "level": 3, - "page": "63" - } - ], - "withheld": false, - "words": 2252 - }, - { - "slug": "developmental-interpretability", - "num": "10", - "title": "Developmental Interpretability and Training Dynamics", - "appendix": false, - "page": "63", - "sections": [ + "page": "87" + }, { - "id": "sec:grokking", - "num": "10.1", - "title": "Grokking: Memorization-to-Generalization Phase Transitions", - "level": 2, - "page": "64" + "id": "sec:ld-attribution", + "num": "10.3.3", + "title": "Attributing Transitions to Components", + "level": 3, + "page": "88" }, { - "id": "training-dynamics-and-the-emergence-of-internal-mechanisms", - "num": "10.2", - "title": "Training Dynamics and the Emergence of Internal Mechanisms", - "level": 2, - "page": "64" + "id": "sec:training_phase_why", + "num": "10.3.4", + "title": "Why Transitions Happen", + "level": 3, + "page": "89" }, { - "id": "sec:slt-llc", - "num": "10.3", - "title": "Singular Learning Theory and the Local Learning Coefficient", - "level": 2, - "page": "65" + "id": "sec:ld-induction-case", + "num": "10.3.5", + "title": "Case Study: Induction Heads", + "level": 3, + "page": "90" }, { - "id": "sec:stagewise-influence", + "id": "sec:ld-safety", "num": "10.4", - "title": "Stagewise Influence and Developmental Data Attribution", + "title": "Leveraging Learning Dynamics for Safety", "level": 2, - "page": "66" + "page": "90" }, { - "id": "sec:deception", - "num": "10.5", - "title": "Deceptive Alignment and Training Dynamics", - "level": 2, - "page": "67" + "id": "sec:ld-forecasting", + "num": "10.4.1", + "title": "Forecasting Model Behaviours", + "level": 3, + "page": "90" }, { - "id": "case-study-induction-heads-2", - "num": "10.6", - "title": "Case Study: Induction Heads", - "level": 2, - "page": "67" + "id": "sec:ld-intervening", + "num": "10.4.2", + "title": "Intervening on Training Processes", + "level": 3, + "page": "91" } ], "withheld": false, - "words": 1720 + "words": 6329 }, { - "slug": "learning-dynamics", + "slug": "training-data-attribution", "num": "11", - "title": "Learning Dynamics and Mechanistic Interpretability", - "appendix": false, - "page": "67", - "sections": [], - "withheld": true, - "words": 1538 - }, - { - "slug": "representation-analysis", - "num": "12", - "title": "Representation Analysis: Structure, Localization, and Transfer", + "title": "Training Data Attribution", "appendix": false, - "page": "70", + "page": "93", "sections": [ { - "id": "sec:model-diffing", - "num": "12.1", - "title": "Model Diffing: Tracking Representational Changes Across Models", + "id": "verbatim-data-attribution-olmotrace-and-maximal-matching-spans", + "num": "11.1", + "title": "Verbatim Data Attribution: OLMoTrace and Maximal Matching Spans", "level": 2, - "page": "70" + "page": "93" }, { - "id": "sec:knowledge-localization", - "num": "12.2", - "title": "Knowledge Localization: Where Is Information Stored?", + "id": "gradient-based-attribution-understanding-data-influence-through-perturbations", + "num": "11.2", + "title": "Gradient-Based Attribution: Understanding Data Influence Through Perturbations", "level": 2, - "page": "72" + "page": "96" }, { - "id": "subsec:steering", - "num": "12.3", - "title": "Representation Engineering and Steering", - "level": 2, - "page": "73" + "id": "core-idea-parameter-response-to-training-data", + "num": "11.2.1", + "title": "Core Idea: Parameter Response to Training Data", + "level": 3, + "page": "96" + }, + { + "id": "proximal-bregman-response-and-the-generalised-gaussnewton", + "num": "11.2.2", + "title": "Proximal Bregman Response and the Generalised Gauss–Newton", + "level": 3, + "page": "97" + }, + { + "id": "addressing-the-computational-and-memory-bottlenecks-of-ggn", + "num": "11.2.3", + "title": "Addressing the Computational and Memory Bottlenecks of GGN", + "level": 3, + "page": "98" + }, + { + "id": "sec:trak", + "num": "11.2.4", + "title": "Retraining-Based Influence Estimation: datamodels and TRAK", + "level": 3, + "page": "99" + }, + { + "id": "sec:tracin", + "num": "11.2.5", + "title": "Efficient Influence Estimation Using Training Trajectories: TracIn", + "level": 3, + "page": "100" + }, + { + "id": "illustrative-example", + "num": "11.2.6", + "title": "Illustrative Example", + "level": 3, + "page": "100" + }, + { + "id": "recent-extensions-and-limitations", + "num": "11.2.7", + "title": "Recent Extensions and Limitations", + "level": 3, + "page": "101" } ], "withheld": false, - "words": 1840 + "words": 3858 }, { "slug": "research-orientations", - "num": "13", - "title": "Five Research Orientations", + "num": "12", + "title": "Six Research Orientations", "appendix": false, - "page": "74", + "page": "102", "sections": [ { - "id": "defining-orientations-vs.-methods", - "num": "13.1", - "title": "Defining Orientations vs. Methods", - "level": 2, - "page": "74" - }, - { - "id": "overview-why-five-orientations", - "num": "13.2", - "title": "Overview: Why Five Orientations?", + "id": "subsec: pragmatic interp", + "num": "12.1", + "title": "Pragmatic Interpretability: Proxy-Task-Oriented Safety Interventions", "level": 2, - "page": "75" + "page": "102" }, { - "id": "pragmatic-interpretability-proxy-task-oriented-safety-interventions", - "num": "13.3", - "title": "Pragmatic Interpretability: Proxy Task Oriented Safety Interventions", + "id": "subsec:ambitious", + "num": "12.2", + "title": "Ambitious Interpretability: Towards Complete Mechanistic Understanding", "level": 2, - "page": "75" + "page": "103" }, { - "id": "ambitious-interpretability-toward-complete-mechanistic-understanding", - "num": "13.4", - "title": "Ambitious Interpretability: Toward Complete Mechanistic Understanding", + "id": "subsec:dev interp", + "num": "12.3", + "title": "Developmental Interpretability: Characterising How Models Learn", "level": 2, - "page": "76" + "page": "103" }, { - "id": "developmental-interpretability-understanding-phase-transition-during-training", - "num": "13.5", - "title": "Developmental Interpretability: Understanding Phase Transition During Training", + "id": "subsec: iat", + "num": "12.4", + "title": "Interpretability Across Time: Understanding the Causes of Model Behaviours", "level": 2, - "page": "77" + "page": "104" }, { - "id": "sec:agentic-interpretability", - "num": "13.6", - "title": "Agentic Interpretability: Models Interpreting Themselves", + "id": "sec:intro-interpretability", + "num": "12.5", + "title": "Introspective Interpretability: Models Interpreting Themselves", "level": 2, - "page": "77" + "page": "105" }, { - "id": "governance-concerns-for-agentic-interpretability", - "num": "13.6.1", - "title": "Governance Concerns for Agentic Interpretability", + "id": "concerns-for-introspective-interpretability", + "num": "12.5.1", + "title": "Concerns for Introspective Interpretability", "level": 3, - "page": "77" + "page": "105" }, { - "id": "proposed-guardrails-for-agentic-interpretability", - "num": "13.6.2", - "title": "Proposed Guardrails for Agentic Interpretability", + "id": "subsec:valproce model self-reports", + "num": "12.5.2", + "title": "Validation Procedures for Model Self-Reports", "level": 3, - "page": "78" + "page": "106" }, { - "id": "subsec:valproto agentic self-reports", - "num": "13.6.3", - "title": "Validation Protocols for Agentic Self-Reports", + "id": "when-to-use-introspective-methods", + "num": "12.5.3", + "title": "When to Use Introspective Methods", "level": 3, - "page": "78" + "page": "107" }, { "id": "sec:automated-interp", - "num": "13.7", + "num": "12.6", "title": "Automated Interpretability: Scaling Discovery via Automation", "level": 2, - "page": "79" + "page": "107" }, { "id": "governance-concerns-for-automated-interpretability", - "num": "13.7.1", + "num": "12.6.1", "title": "Governance Concerns for Automated Interpretability", "level": 3, - "page": "79" + "page": "108" }, { "id": "proposed-guardrails-for-automated-interpretability", - "num": "13.7.2", + "num": "12.6.2", "title": "Proposed Guardrails for Automated Interpretability", "level": 3, - "page": "80" - }, - { - "id": "validation-protocols-for-automated-pipelines", - "num": "13.7.3", - "title": "Validation Protocols for Automated Pipelines", - "level": 3, - "page": "80" + "page": "109" }, { "id": "integrating-automated-and-human-analysis", - "num": "13.7.4", + "num": "12.6.3", "title": "Integrating Automated and Human Analysis", "level": 3, - "page": "81" + "page": "110" }, { "id": "summary-responsible-automation", - "num": "13.7.5", + "num": "12.6.4", "title": "Summary: Responsible Automation", "level": 3, - "page": "81" + "page": "110" }, { "id": "comparative-analysis-trade-offs-between-orientations", - "num": "13.8", + "num": "12.7", "title": "Comparative Analysis: Trade-Offs Between Orientations", "level": 2, - "page": "81" + "page": "111" }, { "id": "subsec:choosing orientation", - "num": "13.9", + "num": "12.8", "title": "Choosing an Orientation: A Decision Framework", "level": 2, - "page": "82" + "page": "111" }, { - "id": "critical-tensions-and-open-questions", - "num": "13.10", - "title": "Critical Tensions and Open Questions", + "id": "critical-concerns-tensions-and-open-questions", + "num": "12.9", + "title": "Critical Concerns, Tensions, and Open Questions", "level": 2, - "page": "82" + "page": "112" } ], "withheld": false, - "words": 3823 + "words": 5209 }, { "slug": "safety-applications", - "num": "14", + "num": "13", "title": "From Understanding to Intervention: Safety Applications of Mechanistic Interpretability", "appendix": false, - "page": "83", + "page": "112", "sections": [ { "id": "why-mechanistic-approaches-for-safety", - "num": "14.1", + "num": "13.1", "title": "Why Mechanistic Approaches for Safety?", "level": 2, - "page": "83" + "page": "112" }, { - "id": "alternatives-black-box-methods", - "num": "14.1.1", - "title": "Alternatives: Black-Box Methods", + "id": "alternatives-black-box-methods-and-their-challenges", + "num": "13.1.1", + "title": "Alternatives: Black-Box Methods and Their Challenges", "level": 3, - "page": "83" + "page": "112" }, { - "id": "challenges-of-black-box-bethods", - "num": "14.1.2", - "title": "Challenges of Black-Box Bethods", + "id": "shared-limitations-of-black-box-approaches", + "num": "13.1.2", + "title": "Shared Limitations of Black-Box Approaches", "level": 3, - "page": "83" + "page": "114" }, { - "id": "concept-erasure-and-direct-editing", - "num": "14.2", - "title": "Concept Erasure and Direct Editing", + "id": "concept-erasure-and-knowledge-editing", + "num": "13.2", + "title": "Concept Erasure and Knowledge Editing", "level": 2, - "page": "84" + "page": "114" }, { "id": "concept-erasure-via-pisces", - "num": "14.2.1", + "num": "13.2.1", "title": "Concept Erasure via PISCES", "level": 3, - "page": "84" + "page": "114" }, { "id": "subsec:knowedit", - "num": "14.2.2", - "title": "Knowledge Editing via Localization", + "num": "13.2.2", + "title": "Knowledge Editing via Localisation", "level": 3, - "page": "84" + "page": "115" }, { "id": "steering-vectors-and-mechanistic-monitoring", - "num": "14.3", + "num": "13.3", "title": "Steering Vectors and Mechanistic Monitoring", "level": 2, - "page": "85" + "page": "116" }, { - "id": "steering-vectors-practical-behavioral-control", - "num": "14.3.1", - "title": "Steering Vectors: Practical Behavioral Control", + "id": "subsubsec: steering vectors", + "num": "13.3.1", + "title": "Steering Vectors: Practical Behavioural Control", "level": 3, - "page": "85" + "page": "116" }, { - "id": "mechanistic-monitoring", - "num": "14.3.2", + "id": "subsubsec: mech monitor", + "num": "13.3.2", "title": "Mechanistic Monitoring", "level": 3, - "page": "86" + "page": "117" }, { "id": "sec:automated-interpretability", - "num": "14.4", + "num": "13.4", "title": "Automated Interpretability Agents", "level": 2, - "page": "86" + "page": "118" }, { - "id": "relations-to-the-five-research-orientations", - "num": "14.5", - "title": "Relations to the Five Research Orientations", + "id": "choosing-appropriate-methods-for-different-safety-applications", + "num": "13.5", + "title": "Choosing Appropriate Methods for Different Safety Applications", "level": 2, - "page": "87" + "page": "119" }, { - "id": "sec:limitations-open-questions", - "num": "14.6", - "title": "Limitations and Open Questions", + "id": "from-research-orientations-to-methods", + "num": "13.6", + "title": "From Research Orientations to Methods", "level": 2, - "page": "87" + "page": "120" }, { "id": "sec:mi-risks", - "num": "14.7", - "title": "Risks and Failure Modes of Mechanistic Interpretability for AI Safety", + "num": "13.7", + "title": "Risks of Mechanistic Interpretability for AI Safety", "level": 2, - "page": "87" - }, - { - "id": "taxonomy-of-mi-safety-risks", - "num": "14.7.1", - "title": "Taxonomy of MI Safety Risks", - "level": 3, - "page": "88" + "page": "120" }, { - "id": "epistemic-risks-false-confidence-from-partial-understanding", - "num": "14.7.2", - "title": "Epistemic Risks: False Confidence from Partial Understanding", + "id": "epistemic-risks-false-confidence-from-partial-or-incorrect-understanding", + "num": "13.7.1", + "title": "Epistemic Risks: False Confidence from Partial or Incorrect Understanding", "level": 3, - "page": "88" + "page": "120" }, { - "id": "operational-risks-failure-under-deployment-conditions", - "num": "14.7.3", - "title": "Operational Risks: Failure Under Deployment Conditions", + "id": "operational-risks-failure-under-deployment", + "num": "13.7.2", + "title": "Operational Risks: Failure Under Deployment", "level": 3, - "page": "89" + "page": "121" }, { - "id": "adversarial-risks-malicious-exploitation-of-mi-insights", - "num": "14.7.4", - "title": "Adversarial Risks: Malicious Exploitation of MI Insights", + "id": "adversarial-risks-malicious-exploitation-of-mi", + "num": "13.7.3", + "title": "Adversarial Risks: Malicious Exploitation of MI", "level": 3, - "page": "89" + "page": "122" }, { "id": "opportunity-costs-resource-allocation-risks", - "num": "14.7.5", + "num": "13.7.4", "title": "Opportunity Costs: Resource Allocation Risks", "level": 3, - "page": "90" - }, - { - "id": "recommended-verification-practices", - "num": "14.7.6", - "title": "Recommended Verification Practices", - "level": 3, - "page": "90" + "page": "122" }, { "id": "summary-mi-as-one-methodology-among-many", - "num": "14.7.7", + "num": "13.7.5", "title": "Summary: MI as One Methodology Among Many", "level": 3, - "page": "91" + "page": "123" } ], "withheld": false, - "words": 4674 + "words": 6017 }, { "slug": "open-problems", - "num": "15", + "num": "14", "title": "Open Problems", "appendix": false, - "page": "91", + "page": "123", "sections": [ { "id": "when-is-mechanistic-understanding-necessary", - "num": "15.1", + "num": "14.1", "title": "When Is Mechanistic Understanding Necessary?", "level": 2, - "page": "91" + "page": "123" }, { - "id": "how-to-evaluate-mechanistic-interpretability-methods", - "num": "15.2", - "title": "How to Evaluate Mechanistic Interpretability Methods", + "id": "challenges-in-evaluation-and-benchmarking", + "num": "14.2", + "title": "Challenges in Evaluation and Benchmarking", "level": 2, - "page": "91" - }, - { - "id": "proposed-evaluation-framework", - "num": "15.2.1", - "title": "Proposed Evaluation Framework", - "level": 3, - "page": "91" - }, - { - "id": "standardized-benchmarks", - "num": "15.2.2", - "title": "Standardized Benchmarks", - "level": 3, - "page": "92" + "page": "124" }, { - "id": "scalability-and-adversarial-robustness", - "num": "15.3", - "title": "Scalability and Adversarial Robustness", + "id": "subsec: scalability and adversarial robustness", + "num": "14.3", + "title": "Incompleteness, Scalability, Generalisability, and Adversarial Robustness", "level": 2, - "page": "92" + "page": "124" }, { "id": "sec:mechanistic-alignment", - "num": "15.4", + "num": "14.4", "title": "Mechanistic Understanding and Alignment", "level": 2, - "page": "92" + "page": "125" }, { "id": "sec:ontology", - "num": "15.5", + "num": "14.5", "title": "Interpretation vs. Truth: The Ontology Question", "level": 2, - "page": "92" + "page": "125" }, { "id": "sec:proxy-fidelity", - "num": "15.6", + "num": "14.6", "title": "The Proxy Fidelity Problem", "level": 2, - "page": "93" + "page": "125" }, { "id": "sec:non-transformer", - "num": "15.7", + "num": "14.7", "title": "Transferability Beyond Transformers", "level": 2, - "page": "93" + "page": "126" } ], "withheld": false, - "words": 1652 + "words": 2327 }, { "slug": "conclusion", - "num": "16", + "num": "15", "title": "Conclusion", "appendix": false, - "page": "94", + "page": "127", "sections": [ { "id": "achievements-and-limits", - "num": "16.1", + "num": "15.1", "title": "Achievements and Limits", "level": 2, - "page": "94" - }, - { - "id": "unresolved-questions", - "num": "16.2", - "title": "Unresolved Questions", - "level": 2, - "page": "94" + "page": "127" }, { - "id": "the-road-ahead-three-complementary-paradigms", - "num": "16.3", - "title": "The Road Ahead: Three Complementary Paradigms", + "id": "the-road-ahead", + "num": "15.2", + "title": "The Road Ahead", "level": 2, - "page": "94" + "page": "128" } ], "withheld": false, - "words": 514 + "words": 739 }, { - "slug": "notation", - "num": "A", - "title": "A Notation Reference and Symbol Table", - "appendix": true, - "page": "119", - "sections": [ - { - "id": "dimensions-and-scaling", - "num": "A.1", - "title": "Dimensions and Scaling", - "level": 2, - "page": "119" - } - ], + "slug": "acknowledgements", + "num": "", + "title": "Acknowledgments", + "appendix": false, + "page": null, + "sections": [], "withheld": false, - "words": 245 + "words": 35 }, { - "slug": "colour-key", - "num": "B", - "title": "Color Key and Style Guide", + "slug": "notation", + "num": "A", + "title": "Notation Reference and Symbol Table", "appendix": true, - "page": "120", + "page": "177", "sections": [ { - "id": "box-types-and-colors", - "num": "B.1", - "title": "Box Types and Colors", + "id": "dimensions-and-indices", + "num": "A.1", + "title": "Dimensions and Indices", "level": 2, - "page": "120" + "page": "177" }, { - "id": "app:citation-colors", - "num": "B.2", - "title": "Citation Evidence Levels", - "level": 2, - "page": "120" - } - ], - "withheld": false, - "words": 247 - }, - { - "slug": "sae-architecture-details", - "num": "C", - "title": "SAE Architecture Details and Hyperparameters", - "appendix": true, - "page": "120", - "sections": [ - { - "id": "standard-sae-architecture", - "num": "C.1", - "title": "Standard SAE Architecture", + "id": "inputs-and-activations", + "num": "A.2", + "title": "Inputs and Activations", "level": 2, - "page": "121" + "page": "177" }, { - "id": "standard-hyperparameters", - "num": "C.2", - "title": "Standard Hyperparameters", + "id": "dictionary-learning", + "num": "A.3", + "title": "Dictionary Learning", "level": 2, - "page": "121" + "page": "177" }, { - "id": "gated-sae-variant", - "num": "C.3", - "title": "Gated SAE Variant", + "id": "model-parameters", + "num": "A.4", + "title": "Model Parameters", "level": 2, - "page": "121" + "page": "178" }, { - "id": "top-k-sae-variant", - "num": "C.4", - "title": "Top-K SAE Variant", + "id": "interventions-and-objectives", + "num": "A.5", + "title": "Interventions and Objectives", "level": 2, - "page": "121" + "page": "178" }, { - "id": "evaluation-metrics", - "num": "C.5", - "title": "Evaluation Metrics", + "id": "mathematical-operations", + "num": "A.6", + "title": "Mathematical Operations", "level": 2, - "page": "122" - }, + "page": "178" + } + ], + "withheld": false, + "words": 503 + }, + { + "slug": "colour-key", + "num": "B", + "title": "Colour Key and Style Guide", + "appendix": true, + "page": "178", + "sections": [ { - "id": "app:train-sae", - "num": "C.6", - "title": "Training Stability Tips", + "id": "box-types-and-colours", + "num": "B.1", + "title": "Box Types and Colours", "level": 2, - "page": "122" + "page": "178" }, { - "id": "multi-layer-saes", - "num": "C.7", - "title": "Multi-Layer SAEs", + "id": "app:citation-colors", + "num": "B.2", + "title": "Citation Evidence Levels", "level": 2, - "page": "122" + "page": "179" } ], "withheld": false, - "words": 566 + "words": 367 }, { "slug": "mathematical-foundations", - "num": "D", + "num": "C", "title": "Mathematical Foundations", "appendix": true, - "page": "122", + "page": "180", "sections": [ - { - "id": "sparse-coding-theory", - "num": "D.1", - "title": "Sparse Coding Theory", - "level": 2, - "page": "122" - }, { "id": "causal-inference-basics", - "num": "D.2", + "num": "C.1", "title": "Causal Inference Basics", "level": 2, - "page": "123" + "page": "180" }, { "id": "necessity", - "num": "D.2.1", + "num": "C.1.1", "title": "Necessity", "level": 3, - "page": "123" + "page": "180" }, { "id": "sufficiency", - "num": "D.2.2", + "num": "C.1.2", "title": "Sufficiency", "level": 3, - "page": "123" + "page": "180" }, { "id": "causal-abstraction", - "num": "D.2.3", + "num": "C.1.3", "title": "Causal Abstraction", "level": 3, - "page": "123" - }, - { - "id": "singular-learning-theory-slt", - "num": "D.3", - "title": "Singular Learning Theory (SLT)", - "level": 2, - "page": "123" - }, - { - "id": "the-local-learning-coefficient", - "num": "D.3.1", - "title": "The Local Learning Coefficient", - "level": 3, - "page": "123" - }, - { - "id": "generalization-bounds", - "num": "D.3.2", - "title": "Generalization Bounds", - "level": 3, - "page": "124" - }, - { - "id": "phase-transitions", - "num": "D.3.3", - "title": "Phase Transitions", - "level": 3, - "page": "124" - }, - { - "id": "linear-representation-hypothesis", - "num": "D.4", - "title": "Linear Representation Hypothesis", - "level": 2, - "page": "124" + "page": "180" }, { - "id": "superposition-and-high-dimensional-geometry", - "num": "D.5", - "title": "Superposition and High-Dimensional Geometry", + "id": "app:slt", + "num": "C.2", + "title": "Local Learning Coefficient (LLC)", "level": 2, - "page": "124" + "page": "180" }, { "id": "information-theory-basics", - "num": "D.6", + "num": "C.3", "title": "Information Theory Basics", "level": 2, - "page": "124" + "page": "181" } ], "withheld": false, - "words": 699 + "words": 909 } ], "labels": { "fig:simplified-taxonomy": { "num": "1", - "page": "1", + "page": "2", "kind": "figure", "chapter": null }, "sec:introduction": { "num": "1", - "page": "1", + "page": "9", "kind": "section", "chapter": "introduction" }, "sec:scope": { "num": "1.2", - "page": "4", + "page": "12", "kind": "subsection", "chapter": "introduction" }, - "sec:why_open": { - "num": "2", - "page": "5", - "kind": "section", - "chapter": "limits-of-behavioural-evaluation" - }, "fig:overview": { "num": "2", - "page": "6", + "page": "14", "kind": "figure", "chapter": "introduction" }, - "war:sleeper": { + "sec:why_open": { "num": "2", - "page": "7", + "page": "15", "kind": "section", "chapter": "limits-of-behavioural-evaluation" }, + "war:sleeper": { + "num": "2", + "page": "16", + "kind": "section*", + "chapter": "limits-of-behavioural-evaluation" + }, "sec:interpretability-landscape": { "num": "3", - "page": "7", + "page": "16", "kind": "section", "chapter": "interpretability-landscape" }, "subsec:cbm": { "num": "3.2", - "page": "8", + "page": "17", "kind": "subsection", "chapter": "interpretability-landscape" }, "tab:semantic-vs-structural": { "num": "1", - "page": "9", + "page": "18", "kind": "table", "chapter": "interpretability-landscape" }, + "subsec: mech interp necessary": { + "num": "3.6", + "page": "19", + "kind": "subsection", + "chapter": "interpretability-landscape" + }, "tab:interpretability-methods": { "num": "2", - "page": "10", + "page": "20", "kind": "table", "chapter": "interpretability-landscape" }, + "subsec:limits": { + "num": "3.7", + "page": "20", + "kind": "subsection", + "chapter": "interpretability-landscape" + }, "sec:foundations": { "num": "4", - "page": "10", + "page": "21", "kind": "section", "chapter": "foundational-concepts" }, "subsec:behavioral-vs-mechanistic": { "num": "4.1", - "page": "10", + "page": "21", "kind": "subsection", "chapter": "foundational-concepts" }, "subsec:polysemanticity": { "num": "4.2", - "page": "11", + "page": "21", "kind": "subsection", "chapter": "foundational-concepts" }, "subsec:features": { "num": "4.3", - "page": "12", + "page": "22", "kind": "subsection", "chapter": "foundational-concepts" }, "subsec:transformers": { "num": "4.4", - "page": "12", + "page": "23", "kind": "subsection", "chapter": "foundational-concepts" }, "subsec:circuits": { "num": "4.5", - "page": "14", + "page": "25", "kind": "subsection", "chapter": "foundational-concepts" }, - "subsec:limits": { - "num": "4.6", - "page": "14", - "kind": "subsection", + "fig:circuit": { + "num": "3", + "page": "26", + "kind": "figure", "chapter": "foundational-concepts" }, "sec:hypotheses": { "num": "5", - "page": "15", + "page": "27", "kind": "section", "chapter": "core-hypotheses" }, "subsec:linearity": { "num": "5.1", - "page": "15", + "page": "27", "kind": "subsection", "chapter": "core-hypotheses" }, "fig:privileged-basis": { - "num": "3", - "page": "16", + "num": "4", + "page": "28", "kind": "figure", "chapter": "core-hypotheses" }, "subsec:superposition": { "num": "5.2", - "page": "16", + "page": "28", "kind": "subsection", "chapter": "core-hypotheses" }, "fig:superposition-polysemanticity": { - "num": "4", - "page": "17", + "num": "5", + "page": "29", "kind": "figure", "chapter": "core-hypotheses" }, "subsec:privileged-access": { "num": "5.3", - "page": "18", + "page": "30", "kind": "subsection", "chapter": "core-hypotheses" }, "subsec:universality": { "num": "5.4", - "page": "21", + "page": "31", "kind": "subsection", "chapter": "core-hypotheses" }, "sec:sae": { "num": "6", - "page": "21", + "page": "31", "kind": "section", "chapter": "feature-extraction" }, - "fig:sae": { - "num": "5", - "page": "22", - "kind": "figure", - "chapter": "feature-extraction" - }, "subsec:sae-foundation": { - "num": "6.1", - "page": "22", + "num": "6.2", + "page": "32", "kind": "subsection", "chapter": "feature-extraction" }, - "subsec:sae-variants": { - "num": "6.2", - "page": "23", - "kind": "subsection", + "fig:sae": { + "num": "6", + "page": "33", + "kind": "figure", "chapter": "feature-extraction" }, - "subsec:sae-selection": { + "subsec:sae-variants": { "num": "6.3", - "page": "24", + "page": "34", "kind": "subsection", "chapter": "feature-extraction" }, - "tab:sae-variant-selection-decision-tree": { - "num": "6.3", - "page": "25", - "kind": "Item", + "subsec:sae-selection": { + "num": "6.4", + "page": "35", + "kind": "subsection", "chapter": "feature-extraction" }, "tab:sae-layer-recommendations": { "num": "3", - "page": "25", + "page": "36", "kind": "table", "chapter": "feature-extraction" }, + "subsec:challenge": { + "num": "6.5", + "page": "37", + "kind": "subsection", + "chapter": "feature-extraction" + }, "tab:sae_recommendations": { "num": "4", - "page": "26", + "page": "38", "kind": "table", "chapter": "feature-extraction" }, "tab:sae_limitations": { "num": "5", - "page": "26", + "page": "39", "kind": "table", "chapter": "feature-extraction" }, + "subsubsec: sae limitation": { + "num": "6.5.5", + "page": "40", + "kind": "subsubsection", + "chapter": "feature-extraction" + }, "subsec:transcoders-crosscoders": { - "num": "6.4", - "page": "27", + "num": "6.6", + "page": "41", "kind": "subsection", "chapter": "feature-extraction" }, "subsubsec:transcoder": { - "num": "6.4.1", - "page": "27", + "num": "6.6.1", + "page": "41", "kind": "subsubsection", "chapter": "feature-extraction" }, - "subsubsec:cross-layer-transcoder": { - "num": "6.4.2", - "page": "27", - "kind": "subsubsection", + "fig:transcoders": { + "num": "7a", + "page": "42", + "kind": "figure", "chapter": "feature-extraction" }, - "fig:transcoders": { - "num": "6", - "page": "28", + "sub@fig:transcoders": { + "num": "a", + "page": "42", "kind": "figure", "chapter": null }, "fig:clt": { - "num": "7", - "page": "28", + "num": "7b", + "page": "42", + "kind": "figure", + "chapter": "feature-extraction" + }, + "sub@fig:clt": { + "num": "b", + "page": "42", "kind": "figure", + "chapter": null + }, + "subsubsec:cross-layer-transcoder": { + "num": "6.6.2", + "page": "42", + "kind": "subsubsection", "chapter": "feature-extraction" }, "fig:clt-size-vs-depth": { "num": "8", - "page": "29", + "page": "45", "kind": "figure", "chapter": "feature-extraction" }, "fig:crosscoders": { "num": "9", - "page": "31", + "page": "46", "kind": "figure", "chapter": "feature-extraction" }, - "def:induction-heads": { - "num": "6.6", - "page": "31", + "subsec: induc head case study: feature extract": { + "num": "6.8", + "page": "47", "kind": "subsection", "chapter": "feature-extraction" }, - "subsec:challenge": { - "num": "6.7", - "page": "32", + "def:induction-heads": { + "num": "6.8", + "page": "47", "kind": "subsection", "chapter": "feature-extraction" }, "tab:decomposition-comparison": { "num": "6", - "page": "36", + "page": "48", "kind": "table", "chapter": "feature-extraction" }, + "subsubsec: supervised_extraction": { + "num": "6.9.4", + "page": "49", + "kind": "subsubsection", + "chapter": "feature-extraction" + }, "sec:architectural-interventions": { - "num": "6.9", - "page": "36", + "num": "6.10", + "page": "50", "kind": "subsection", "chapter": "feature-extraction" }, + "subsubsec: solu": { + "num": "6.10.1", + "page": "50", + "kind": "subsubsection", + "chapter": "feature-extraction" + }, + "subsubsec: weight sparse transformer": { + "num": "6.10.2", + "page": "51", + "kind": "subsubsection", + "chapter": "feature-extraction" + }, + "subsubsec: spartan": { + "num": "6.10.3", + "page": "51", + "kind": "subsubsection", + "chapter": "feature-extraction" + }, "tab:architectural-methods": { "num": "7", - "page": "38", + "page": "52", "kind": "table", "chapter": "feature-extraction" }, + "subsubsec: molt": { + "num": "6.10.4", + "page": "52", + "kind": "subsubsection", + "chapter": "feature-extraction" + }, + "subsubsec: kan": { + "num": "6.10.5", + "page": "52", + "kind": "subsubsection", + "chapter": "feature-extraction" + }, "sec:circuit": { "num": "7", - "page": "38", + "page": "53", "kind": "section", "chapter": "circuits" }, - "sec:what-are-circuits": { + "sec:circuit-discovery-methods": { "num": "7.1", - "page": "39", - "kind": "subsection", - "chapter": "circuits" - }, - "sec:circuit-table": { - "num": "7.2", - "page": "39", + "page": "53", "kind": "subsection", "chapter": "circuits" }, "def:attribution-patching": { - "num": "7.2", - "page": "39", + "num": "7.1", + "page": "53", "kind": "subsection", "chapter": "circuits" }, "def:path-patching": { - "num": "7.2", - "page": "39", + "num": "7.1", + "page": "54", "kind": "subsection", "chapter": "circuits" }, "def:acdc": { + "num": "7.1", + "page": "54", + "kind": "subsection", + "chapter": "circuits" + }, + "def:eap": { + "num": "7.1", + "page": "54", + "kind": "subsection", + "chapter": "circuits" + }, + "sec:circuit-replacement": { "num": "7.2", - "page": "39", + "page": "55", "kind": "subsection", "chapter": "circuits" }, - "fig:circuit": { - "num": "10", - "page": "40", - "kind": "figure", + "def:circuit-tracer": { + "num": "7.2", + "page": "55", + "kind": "subsection", "chapter": "circuits" }, "tab:circuit-methods": { "num": "8", - "page": "41", + "page": "56", "kind": "table", "chapter": "circuits" }, - "tab:circuit-discovery-cost": { - "num": "9", - "page": "41", - "kind": "table", + "sec:causal-verification": { + "num": "7.3", + "page": "57", + "kind": "subsection", "chapter": "circuits" }, - "sec:causal-verification": { + "subsec: induc head case study: circuit": { "num": "7.4", - "page": "41", + "page": "58", "kind": "subsection", "chapter": "circuits" }, "sec:evaluation-framework": { - "num": "7.6", - "page": "43", + "num": "7.5", + "page": "58", "kind": "subsection", "chapter": "circuits" }, + "subsec:five eval dim": { + "num": "7.5.2", + "page": "59", + "kind": "subsubsection", + "chapter": "circuits" + }, "tab:evaluation-framework": { - "num": "10", - "page": "43", + "num": "9", + "page": "60", "kind": "table", "chapter": "circuits" }, "tab:method-evaluation": { - "num": "11", - "page": "48", + "num": "10", + "page": "65", "kind": "table", "chapter": "circuits" }, "sec:intermediate-computation": { "num": "8", - "page": "48", + "page": "66", "kind": "section", "chapter": "intermediate-representations" }, - "sec:logit-lens": { - "num": "8.1", - "page": "48", - "kind": "subsection", - "chapter": "intermediate-representations" - }, "fig:logitlens": { - "num": "11", - "page": "49", + "num": "10", + "page": "67", "kind": "figure", "chapter": "intermediate-representations" }, - "def:refusal-vector": { - "num": "8.3", - "page": "50", + "sec:logit-lens": { + "num": "8.1", + "page": "67", "kind": "subsection", "chapter": "intermediate-representations" }, - "fig:patchscopes": { - "num": "12", - "page": "51", - "kind": "figure", + "subsec:tuned-lens": { + "num": "8.2", + "page": "68", + "kind": "subsection", "chapter": "intermediate-representations" }, "subsec:patchscopes": { "num": "8.4", - "page": "51", + "page": "69", "kind": "subsection", "chapter": "intermediate-representations" }, + "fig:patchscopes": { + "num": "11", + "page": "70", + "kind": "figure", + "chapter": "intermediate-representations" + }, "subsec:latentqa": { "num": "8.5", - "page": "52", + "page": "70", "kind": "subsection", "chapter": "intermediate-representations" }, + "fig:latentqa": { + "num": "12", + "page": "71", + "kind": "figure", + "chapter": "intermediate-representations" + }, "subsec:probing": { "num": "8.6", - "page": "52", + "page": "72", "kind": "subsection", "chapter": "intermediate-representations" }, - "fig:latentqa": { + "fig:probing": { "num": "13", - "page": "53", + "page": "73", "kind": "figure", "chapter": "intermediate-representations" }, - "fig:probing": { - "num": "14", - "page": "54", - "kind": "figure", + "subsubsec: amnesic probing": { + "num": "8.6.4", + "page": "74", + "kind": "subsubsection", "chapter": "intermediate-representations" }, "tab:probing-guidance": { - "num": "12", - "page": "57", + "num": "11", + "page": "76", "kind": "table", "chapter": "intermediate-representations" }, "ex:probe_dir": { - "num": "8.6.9", - "page": "57", - "kind": "Item", + "num": "8.6.7", + "page": "76", + "kind": "subsubsection", "chapter": "intermediate-representations" }, - "sec:influence-functions": { + "sec:representation": { "num": "9", - "page": "58", + "page": "77", + "kind": "section", + "chapter": "representation-analysis" + }, + "sec:model-diffing": { + "num": "9.1", + "page": "77", + "kind": "subsection", + "chapter": "representation-analysis" + }, + "fig:diffing": { + "num": "14", + "page": "79", + "kind": "figure", + "chapter": "representation-analysis" + }, + "sec:knowledge-localization": { + "num": "9.2", + "page": "80", + "kind": "subsection", + "chapter": "representation-analysis" + }, + "subsec:steering": { + "num": "9.3", + "page": "80", + "kind": "subsection", + "chapter": "representation-analysis" + }, + "def:refusal-vector": { + "num": "9.3", + "page": "81", + "kind": "subsection", + "chapter": "representation-analysis" + }, + "fig:steering_vectors": { + "num": "15", + "page": "82", + "kind": "figure", + "chapter": "representation-analysis" + }, + "sec:training": { + "num": "10", + "page": "82", + "kind": "section", + "chapter": "learning-dynamics" + }, + "sec:ld-longitudinal": { + "num": "10.1", + "page": "83", + "kind": "subsection", + "chapter": "learning-dynamics" + }, + "sec:ld-data": { + "num": "10.1.1", + "page": "83", + "kind": "subsubsection", + "chapter": "learning-dynamics" + }, + "sec:ld-features": { + "num": "10.1.2", + "page": "83", + "kind": "subsubsection", + "chapter": "learning-dynamics" + }, + "sec:ld-localization": { + "num": "10.2.1", + "page": "84", + "kind": "subsubsection", + "chapter": "learning-dynamics" + }, + "sec:ld-causal-interventions": { + "num": "10.2.2", + "page": "85", + "kind": "subsubsection", + "chapter": "learning-dynamics" + }, + "sec:ld-temporal-stability": { + "num": "10.2.2", + "page": "85", + "kind": "section*", + "chapter": "learning-dynamics" + }, + "sec:ld-transitions": { + "num": "10.3", + "page": "86", + "kind": "subsection", + "chapter": "learning-dynamics" + }, + "sec:ld-detect": { + "num": "10.3.1", + "page": "86", + "kind": "subsubsection", + "chapter": "learning-dynamics" + }, + "sec:ld-progress": { + "num": "10.3.2", + "page": "87", + "kind": "subsubsection", + "chapter": "learning-dynamics" + }, + "fig:grokking": { + "num": "16", + "page": "88", + "kind": "figure", + "chapter": "learning-dynamics" + }, + "sec:ld-attribution": { + "num": "10.3.3", + "page": "88", + "kind": "subsubsection", + "chapter": "learning-dynamics" + }, + "sec:training_phase_why": { + "num": "10.3.4", + "page": "89", + "kind": "subsubsection", + "chapter": "learning-dynamics" + }, + "sec:ld-induction-case": { + "num": "10.3.5", + "page": "90", + "kind": "subsubsection", + "chapter": "learning-dynamics" + }, + "sec:ld-safety": { + "num": "10.4", + "page": "90", + "kind": "subsection", + "chapter": "learning-dynamics" + }, + "sec:ld-forecasting": { + "num": "10.4.1", + "page": "90", + "kind": "subsubsection", + "chapter": "learning-dynamics" + }, + "sec:ld-intervening": { + "num": "10.4.2", + "page": "91", + "kind": "subsubsection", + "chapter": "learning-dynamics" + }, + "sec:influence-functions": { + "num": "11", + "page": "93", "kind": "section", "chapter": "training-data-attribution" }, "def:suffix-array": { - "num": "9.1", - "page": "59", + "num": "11.1", + "page": "94", "kind": "subsection", "chapter": "training-data-attribution" }, "def:infini-gram": { - "num": "9.1", - "page": "59", - "kind": "subsection", + "num": "11.1", + "page": "94", + "kind": "section*", "chapter": "training-data-attribution" }, "def:maximal-matching-spans": { - "num": "9.1", - "page": "59", - "kind": "subsection", + "num": "11.1", + "page": "94", + "kind": "section*", "chapter": "training-data-attribution" }, "ex:olmotrace-pipeline": { - "num": "9.1", - "page": "60", + "num": "11.1", + "page": "94", "kind": "Item", "chapter": "training-data-attribution" }, "def:olmotrace-algorithm": { - "num": "9.1", - "page": "60", - "kind": "Item", + "num": "11.1", + "page": "95", + "kind": "section*", "chapter": "training-data-attribution" }, "ex:olmotrace-multilingual": { - "num": "9.1.1", - "page": "60", - "kind": "subsubsection", + "num": "11.1", + "page": "95", + "kind": "section*", "chapter": "training-data-attribution" }, "sec:trak": { - "num": "9.2.5", - "page": "62", + "num": "11.2.4", + "page": "99", "kind": "subsubsection", "chapter": "training-data-attribution" }, "sec:tracin": { - "num": "9.2.6", - "page": "62", + "num": "11.2.5", + "page": "100", "kind": "subsubsection", "chapter": "training-data-attribution" }, "ex:gradient-translation": { - "num": "9.2.7", - "page": "63", + "num": "11.2.6", + "page": "101", "kind": "subsubsection", "chapter": "training-data-attribution" }, - "sec:training": { - "num": "10", - "page": "63", - "kind": "section", - "chapter": "developmental-interpretability" - }, - "sec:grokking": { - "num": "10.1", - "page": "64", - "kind": "subsection", - "chapter": "developmental-interpretability" - }, - "fig:grokking": { - "num": "15", - "page": "65", + "fig:sixresearchorientation": { + "num": "17", + "page": "102", "kind": "figure", - "chapter": "developmental-interpretability" - }, - "sec:slt-llc": { - "num": "10.3", - "page": "65", - "kind": "subsection", - "chapter": "developmental-interpretability" - }, - "sec:stagewise-influence": { - "num": "10.4", - "page": "66", - "kind": "subsection", - "chapter": "developmental-interpretability" - }, - "sec:deception": { - "num": "10.5", - "page": "67", - "kind": "subsection", - "chapter": "developmental-interpretability" + "chapter": "research-orientations" }, - "sec:representation": { + "sec:orientations": { "num": "12", - "page": "70", + "page": "102", "kind": "section", - "chapter": "representation-analysis" + "chapter": "research-orientations" }, - "sec:model-diffing": { + "subsec: pragmatic interp": { "num": "12.1", - "page": "70", + "page": "102", "kind": "subsection", - "chapter": "representation-analysis" - }, - "fig:diffing": { - "num": "16", - "page": "72", - "kind": "figure", - "chapter": "representation-analysis" + "chapter": "research-orientations" }, - "sec:knowledge-localization": { + "subsec:ambitious": { "num": "12.2", - "page": "72", + "page": "103", "kind": "subsection", - "chapter": "representation-analysis" + "chapter": "research-orientations" }, - "subsec:steering": { + "subsec:dev interp": { "num": "12.3", - "page": "73", + "page": "103", "kind": "subsection", - "chapter": "representation-analysis" - }, - "fig:steering_vectors": { - "num": "17", - "page": "74", - "kind": "figure", - "chapter": "representation-analysis" - }, - "sec:orientations": { - "num": "13", - "page": "74", - "kind": "section", "chapter": "research-orientations" }, - "fig:fiveresearchorientation": { - "num": "18", - "page": "75", - "kind": "figure", + "subsec: iat": { + "num": "12.4", + "page": "104", + "kind": "subsection", "chapter": "research-orientations" }, - "sec:agentic-interpretability": { - "num": "13.6", - "page": "77", + "sec:intro-interpretability": { + "num": "12.5", + "page": "105", "kind": "subsection", "chapter": "research-orientations" }, - "subsec:valproto agentic self-reports": { - "num": "13.6.3", - "page": "78", + "subsec:valproce model self-reports": { + "num": "12.5.2", + "page": "106", "kind": "subsubsection", "chapter": "research-orientations" }, "sec:automated-interp": { - "num": "13.7", - "page": "79", + "num": "12.6", + "page": "107", "kind": "subsection", "chapter": "research-orientations" }, "tab:human-automated-workflow": { - "num": "13", - "page": "81", + "num": "12", + "page": "110", "kind": "table", "chapter": "research-orientations" }, "tab:orientations": { - "num": "14", - "page": "82", + "num": "13", + "page": "111", "kind": "table", "chapter": "research-orientations" }, "subsec:choosing orientation": { - "num": "13.9", - "page": "82", + "num": "12.8", + "page": "111", "kind": "subsection", "chapter": "research-orientations" }, "sec:safety_applications": { - "num": "14", - "page": "83", + "num": "13", + "page": "112", "kind": "section", "chapter": "safety-applications" }, "subsec:knowedit": { - "num": "14.2.2", - "page": "84", + "num": "13.2.2", + "page": "115", "kind": "subsubsection", "chapter": "safety-applications" }, - "sec:automated-interpretability": { - "num": "14.4", - "page": "86", - "kind": "subsection", + "subsubsec: steering vectors": { + "num": "13.3.1", + "page": "116", + "kind": "subsubsection", "chapter": "safety-applications" }, - "sec:limitations-open-questions": { - "num": "14.6", - "page": "87", - "kind": "subsection", + "subsubsec: mech monitor": { + "num": "13.3.2", + "page": "117", + "kind": "subsubsection", "chapter": "safety-applications" }, - "sec:mi-risks": { - "num": "14.7", - "page": "87", + "sec:automated-interpretability": { + "num": "13.4", + "page": "118", "kind": "subsection", "chapter": "safety-applications" }, + "tab:mi_safety_applications": { + "num": "14", + "page": "119", + "kind": "table", + "chapter": "safety-applications" + }, "tab:mi-risks": { "num": "15", - "page": "88", + "page": "120", "kind": "table", "chapter": "safety-applications" }, + "sec:mi-risks": { + "num": "13.7", + "page": "120", + "kind": "subsection", + "chapter": "safety-applications" + }, "sec:open_problems": { - "num": "15", - "page": "91", + "num": "14", + "page": "123", "kind": "section", "chapter": "open-problems" }, + "subsec: scalability and adversarial robustness": { + "num": "14.3", + "page": "124", + "kind": "subsection", + "chapter": "open-problems" + }, "sec:mechanistic-alignment": { - "num": "15.4", - "page": "92", + "num": "14.4", + "page": "125", "kind": "subsection", "chapter": "open-problems" }, "sec:ontology": { - "num": "15.5", - "page": "92", + "num": "14.5", + "page": "125", "kind": "subsection", "chapter": "open-problems" }, "sec:proxy-fidelity": { - "num": "15.6", - "page": "93", + "num": "14.6", + "page": "125", "kind": "subsection", "chapter": "open-problems" }, "sec:non-transformer": { - "num": "15.7", - "page": "93", + "num": "14.7", + "page": "126", "kind": "subsection", "chapter": "open-problems" }, "sec:conclusion": { - "num": "16", - "page": "94", + "num": "15", + "page": "127", "kind": "section", "chapter": "conclusion" }, "app:notation": { "num": "A", - "page": "119", + "page": "177", "kind": "appendix", "chapter": "notation" }, "app:colors": { "num": "B", - "page": "120", + "page": "178", "kind": "appendix", "chapter": "colour-key" }, "app:citation-colors": { "num": "B.2", - "page": "120", + "page": "179", "kind": "subsection", "chapter": "colour-key" }, - "app:sae_details": { + "app:math": { "num": "C", - "page": "120", + "page": "180", "kind": "appendix", - "chapter": "sae-architecture-details" + "chapter": "mathematical-foundations" }, - "app:train-sae": { - "num": "C.6", - "page": "122", + "app:slt": { + "num": "C.2", + "page": "180", "kind": "subsection", - "chapter": "sae-architecture-details" - }, - "app:math": { - "num": "D", - "page": "122", - "kind": "appendix", "chapter": "mathematical-foundations" } }, "cite_tiers": { + "dai2024safe": "established", + "pearson1901lines": "established", + "hotelling1933analysis": "established", + "vig2019multiscale": "established", + "nanda2023progressmeasuresgrokkingmechanistic": "established", + "quirke2024understandingadditiontransformers": "established", + "cunningham2023sparse": "established", + "hinton1986learning": "established", + "elman1991distributed": "established", + "rnnfsa1992": "established", + "OMLIN199641": "established", + "zeiler2014visualizing": "established", + "orgad2026position": "established", + "biderman2026position": "established", "olshausen1996emergence": "established", "lewicki2000learning": "established", "lee2006efficient": "established", - "adebayo2018sanity": "established", + "conmy2023automatedcircuitdiscoverymechanistic": "established", + "ig": "established", + "adebayo2020sanitycheckssaliencymaps": "established", "burns2023discovering": "established", "detectvul": "established", "paulo2025automatically": "established", + "sellam2021multiberts": "established", + "biderman2023pythia": "established", "perez2022redteaminglanguagemodels": "established", - "touvron2023llamaopenefficientfoundation": "emerging", - "openai2024gpt4technicalreport": "emerging", - "gemmateam2024gemmaopenmodelsbased": "emerging", - "yang2025qwen3technicalreport": "emerging", - "deepseekai2025deepseekv3technicalreport": "emerging", - "ng2011sparse": "emerging", - "seo2018noise": "emerging", + "bai2022training": "emerging", + "oshea2015introductionconvolutionalneuralnetworks": "emerging", + "clark2019what": "emerging", + "vig-belinkov-2019-analyzing": "emerging", + "quirke2026understandingadditionsubtractiontransformers": "emerging", "templeton2024scaling": "emerging", + "karpathy2015visualizing": "emerging", + "lee2026rigorous": "emerging", + "icml2025actionable": "emerging", + "ng2011sparse": "emerging", + "ameisen2025circuit": "emerging", + "simonyan2014deepinsideconvolutionalnetworks": "emerging", + "smilkov2017smoothgradremovingnoiseadding": "emerging", + "bricken2023monosemanticity": "emerging", "bills2023language": "emerging", - "foote2023neurongraphinterpretinglanguage": "emerging", + "simon2026there": "emerging", + "lee2026steps": "emerging", "ganguli2022redteaminglanguagemodels": "emerging", - "zeiler2014visualizing": "established", + "hinton1986distributed": "established", + "minsky1969perceptrons": "established", "li-etal-2016-visualizing": "established", - "alain2017understanding": "established", "Dalvi_Durrani_Sajjad_Belinkov_Bau_Glass_2019": "established", "bereska2024mechanisticinterpretabilityaisafety": "established", + "zhang2026locatesteerimprovepractical": "established", "belinkov-glass-2019-analysis": "established", "llmexplainsurvey": "established", "sajjad-etal-2022-neuron": "established", "shu-etal-2025-survey": "established", - "christiano2023deepreinforcementlearninghuman": "established", - "ouyang2022traininglanguagemodelsfollow": "emerging", - "krizhevsky2012imagenet": "established", + "vaswani2017attention": "established", + "elman1990finding": "established", + "NIPS2012_c399862d": "established", + "belinkov-2022-probing": "established", + "gupta-etal-2015-distributional": "established", + "kohn-2015-whats": "established", "shi-etal-2016-string": "established", "adi2017finegrained": "established", "belinkov2017neural": "established", "conneau2018what": "established", - "vaswani2017attention": "established", + "alain2017understanding": "established", "devlin2019bert": "established", - "vig2019multiscale": "established", - "clark2019what": "established", "jain2019attention": "established", "wiegreffe2019attention": "established", "rogers2020primer": "established", "wang2022interpretabilitywildcircuitindirect": "established", - "nanda2023progressmeasuresgrokkingmechanistic": "established", - "quirke2024understandingadditiontransformers": "established", "Raukur2022TowardTA": "established", - "conmy2023automatedcircuitdiscoverymechanistic": "established", - "vig2020causalmediationanalysisinterpreting": "established", "lundberg2017unifiedapproachinterpretingmodel": "established", "ribeiro2016whyitrustyou": "established", - "ig": "established", - "simonyan2014deepinsideconvolutionalnetworks": "established", + "lrp": "established", + "conceptdef": "established", + "lotfi2026position": "established", + "koh2020understandingblackboxpredictionsinfluence": "established", "koh2020conceptbottleneckmodels": "established", "geiger2021causalabstractionsneuralnetworks": "established", - "karpathy2015visualizing": "emerging", - "zhang2026locatesteerimprovepractical": "emerging", + "pmlr-v162-geiger22a": "established", + "das": "established", + "JMLR:v26:23-0058": "established", + "wu-etal-2022-causal": "established", + "olah2020zoom": "emerging", "rai2024practical": "emerging", "ferrando2024primer": "emerging", + "touvron2023llamaopenefficientfoundation": "emerging", + "openai2024gpt4technicalreport": "emerging", + "gemmateam2024gemmaopenmodelsbased": "emerging", + "yang2025qwen3technicalreport": "emerging", + "deepseekai2025deepseekv3technicalreport": "emerging", + "anthropic2025claude4": "emerging", "hubinger2024sleeperagentstrainingdeceptive": "emerging", + "olah2017feature": "emerging", "distill_pub": "emerging", - "olah2020zoom": "emerging", - "olsson2022context": "emerging", - "bricken2023monosemanticity": "emerging", - "saphra2024mechanistic": "emerging", - "lee2026rigorous": "emerging", - "orgad2026interpretabilityactionable": "emerging", + "olsson2022incontextlearninginductionheads": "emerging", "nanda2025pragmatic": "emerging", "gao2025ambitious": "emerging", "barez2025automated": "emerging", "hoogland2023towards": "emerging", - "kim2025llmspursueagenticinterpretability": "emerging", - "needham2025largelanguagemodelsknow": "established", - "anthropic2025claude4": "emerging", - "apollo2025sonnet37evalaware": "emerging", - "greenblatt2024alignmentfakinglargelanguage": "emerging", - "schaeffer2026quantifyingeffecttestset": "established", + "li2025traininglanguagemodelsexplain": "emerging", + "saphra2024mechanistic": "emerging", + "nla": "emerging", + "grosse2023studyinglargelanguagemodel": "emerging", + "habibi2026symbolicmechanisticdataattribution": "emerging", + "wani-etal-2026-answer": "established", "SuperGLUE": "established", "ott2022mapping": "established", "phan2026hle": "established", + "zhu2025establishing": "established", "mccoy-etal-2019-right": "established", "du2023shortcut": "established", + "juneja2023linear": "established", "ye2026the": "established", "Geirhos_2020": "established", "tang-etal-2023-large": "established", "meng2023locatingeditingfactualassociations": "established", "li2024wmdp": "established", "marks2025sparse": "established", + "greenblatt2024alignmentfakinglargelanguage": "emerging", + "schaeffer2026quantifyingeffecttestset": "emerging", + "needham2025largelanguagemodelsknow": "emerging", + "apollo2025sonnet37evalaware": "emerging", "macdiarmid2024sleeperagentprobes": "emerging", - "chaudhary2025safetynetdetectingharmfuloutputs": "emerging", - "zanbaghi2025detectingsleeperagentslarge": "emerging", - "Ethan2022redteam": "established", + "jimenez2024swebench": "established", + "liu2024repobench": "established", "he-etal-2025-red": "established", "zarlenga2022cem": "established", - "kim2023probabilistic": "established", "dominici2025causal": "established", - "moritz2024sthocastic": "established", - "rajani2020explainingimprovingmodelbehavior": "established", - "zhao-etal-2024-explaining": "established", - "dalvi2022discovering": "established", - "yu-etal-2024-latent": "established", + "vandenhirtz2024stochastic": "established", "kim2018tcav": "established", "panickssery2024steeringllama2contrastive": "established", + "marks2024the": "established", + "cam": "established", + "gradcam": "established", + "yu-etal-2024-latent": "established", + "patchscopes": "established", + "pan2024latentqateachingllmsdecode": "established", + "eapig": "established", + "querycircuit": "established", + "chen2021evaluatinglargelanguagemodels": "emerging", "turner2024steeringlanguagemodelsactivation": "emerging", + "syed2023attributionpatchingoutperformsautomated": "emerging", "linzen-etal-2016-assessing": "established", "marvin-linzen-2018-targeted": "established", + "mmlu": "established", "mmlupro": "established", + "bigbench": "established", + "agieval": "established", + "bai-etal-2024-longbench": "established", + "zhang-etal-2024-bench": "established", + "modarressi2025nolima": "established", + "hong-etal-2025-measuring": "established", + "fanous2025syceval": "established", + "petrov2026brokenmath": "established", "zhang2025cybench": "established", + "li2026autocontrol": "established", + "pmlr-v267-fu25e": "established", + "helwe-etal-2025-navigating": "established", + "durmus2024towards": "established", + "zhao-etal-2024-explaining": "established", + "dalvi2022discovering": "established", "yang2023language": "established", "oikarinen2023labelfree": "established", - "adebayo2020sanitycheckssaliencymaps": "established", - "belinkov-2022-probing": "established", - "gupta-etal-2015-distributional": "established", - "kohn-2015-whats": "established", + "elazar-etal-2021-amnesic": "established", + "deeplift": "established", + "kindermans2017unreliabilitysaliencymethods": "established", + "hewitt2019structural": "established", + "ravfogel-etal-2020-null": "established", + "tenney2019bertrediscoversclassicalnlp": "established", + "hewitt2025position": "established", + "dunefsky2024transcodersinterpretablellmfeature": "established", + "wu2025axbench": "established", + "leask2025sparse": "established", + "heap2026automated": "established", + "miller2024transformer": "established", + "meloux2025everything": "established", + "localinform": "established", + "sharkey2025open": "established", + "yu2026vectorquantizedlatentconcepts": "emerging", + "gsm8k": "emerging", + "proofnet": "emerging", + "korbak2025chainthoughtmonitorabilitynew": "emerging", "early-work-on-monitorability-evaluations": "emerging", - "tenney2019bertrediscoversclassicalnlp": "emerging", - "eapig": "established", + "nostalgebraist2020interpretinggpt": "emerging", + "belrose2025elicitinglatentpredictionstransformers": "emerging", + "jlens": "emerging", + "mcgrath2023hydraeffectemergentselfrepair": "emerging", + "arditi2024refusallanguagemodelsmediated": "established", + "wurgaft2026manifoldsteeringrevealsshared": "emerging", + "yun-etal-2021-transformer": "emerging", + "netdissect2017": "established", + "bau2018identifying": "established", + "bau2020units": "established", + "hernandez2022natural": "established", + "oikarinen2023clipdissect": "established", + "and": "established", + "ao": "established", + "mueller2025mib": "established", + "geva-etal-2021-transformer": "established", + "neo-etal-2024-interpreting": "established", + "relu": "established", + "ICLR2025_8c5f3029": "established", + "arora2026language": "established", + "gqa": "established", "haklay-etal-2025-position": "established", "hanna2023how": "established", - "elhage2022superposition": "emerging", + "kharlapenko2025scaling": "established", "elhage2021mathematical": "emerging", - "ameisen2025circuit": "emerging", - "querycircuit": "emerging", - "sharkey2025open": "established", - "rnnnonlinear": "established", + "elhage2022superposition": "emerging", + "llama3": "emerging", + "deepseekv2": "emerging", + "shazeer2020gluvariantsimprovetransformer": "emerging", + "kamath2025tracing": "emerging", "engels2025languagemodelfeaturesonedimensionally": "established", + "smds": "established", + "yang2025mixture": "established", + "gao2025weight": "established", + "liu2025kan": "established", + "binder": "established", "huh2024platonicrepresentationhypothesis": "established", + "lan-etal-2024-towards": "established", "kornblith2019similarityneuralnetworkrepresentations": "established", + "moschella2023relative": "established", + "groger2026revisiting": "established", "smith2024strongfeature": "emerging", - "snmf": "emerging", - "smds": "emerging", - "moschella2023relativerepresentationsenablezeroshot": "emerging", - "gao2025weight": "emerging", + "rnnnonlinear": "emerging", + "feucht2026arithmeticwildllamauses": "emerging", "elhage2022solu": "emerging", - "binder": "emerging", + "anthropic2025molt": "emerging", "song": "emerging", - "li2025traininglanguagemodelsexplain": "emerging", - "pan2024latentqateachingllmsdecode": "emerging", - "lanham2023measuringfaithfulnesschainofthoughtreasoning": "emerging", - "hubinger2021riskslearnedoptimizationadvanced": "emerging", - "burns2024discoveringlatentknowledgelanguage": "emerging", - "cunningham2023sparse": "established", - "dunefsky2024transcodersinterpretablellmfeature": "established", + "buildingbetterao": "emerging", + "pcd": "emerging", + "jakkli2026current": "emerging", + "lan2025quantifyingfeaturespaceuniversality": "emerging", + "koepke2026platoscaveexaminingcrossmodal": "emerging", + "li2026do": "established", + "lrh": "established", + "shortliffe1977mycin": "established", + "harrasse2025tinysqlprogressivetexttosqldataset": "established", + "morcos2018insightsrepresentationalsimilarityneural": "established", + "lecomte2024what": "emerging", + "chive": "emerging", + "cox2008metareasoning": "emerging", + "lindsey2026emergentintrospectiveawarenesslarge": "emerging", + "fornasiere2026languagemodelsrecognizedropout": "emerging", + "ge2026evolution": "established", + "bayazit-etal-2026-crosscoding": "established", "singh2024needsrightinductionhead": "established", + "lee1999learning": "established", + "hyvarinen2000independent": "established", "huang-etal-2024-ravel": "established", "pearce2025bilinear": "established", "park2025monet": "established", "oldfield2024multilinear": "established", - "yang2025mixture": "established", - "dooms2025compositionality": "established", + "lei2025spartan": "established", "oldfield2025towards": "established", + "paulo2025transcodersbeatsparseautoencoders": "emerging", + "lindsey2024sparsecrosscoders": "emerging", + "jiralerspong_cross-architecture_2025": "emerging", "kissane2024interpretingattentionlayeroutputs": "emerging", - "kamath2025tracing": "emerging", "chaudhary2024evaluating": "emerging", "adler2025towards": "emerging", "draye2025spartan": "emerging", + "dooms2025compositionality": "emerging", "braun2025interpretability": "emerging", "bushnaq2025stochastic": "emerging", "gao2024scalingevaluatingsparseautoencoders": "established", - "olsson2022incontextlearninginductionheads": "emerging", - "rajamanoharan2024improvingdictionarylearninggated": "emerging", - "lindsey2025biology": "emerging", - "merullo2025replicating": "emerging", - "lindsey2025landscape": "emerging", + "shafran-etal-2026-constructing": "established", + "Barlow:1961aa": "established", + "sparsecoding": "established", + "olshausen1997sparse": "established", + "matchingpursuit": "established", + "basispursuit": "established", + "aharon2006k": "established", + "gur-arieh-etal-2025-enhancing": "established", + "rajamanoharan2024improvingdictionarylearninggated": "established", + "karvonen2025saebench": "established", + "chanin2025absorption": "established", + "paulo2025sparse": "established", + "schnass2008dictionary": "established", + "li2025geometry": "established", + "chen2025low": "established", + "NIPS2003_1843e35d": "established", + "snmf": "established", + "oneill2025compute": "established", + "liu2026icalensinterpretinglanguage": "emerging", + "marks2024enhancingneuralnetworkinterpretability": "emerging", + "rajamanoharan2024jumpingaheadimprovingreconstruction": "emerging", + "bengio2013estimatingpropagatinggradientsstochastic": "emerging", + "bussmann2024batchtopksparseautoencoders": "emerging", + "lieberum2024gemmascopeopensparse": "emerging", + "gemmateam2024gemma2improvingopen": "emerging", + "he2024llamascopeextractingmillions": "emerging", + "neuronpedia": "emerging", + "mcdougall2025gemmascope2": "emerging", "korznikov2025ortsaeorthogonalsparseautoencoders": "emerging", "klindt2025superpositionsparsecodesinterpretable": "emerging", - "mueller2025isolationentanglementinterpretabilitymethods": "emerging", - "chen2025low": "emerging", "miller2026identifyingintervenableinterpretablefeatures": "emerging", - "mueller2025mib": "emerging", - "anthropic2025molt": "emerging", - "zhang2024towards": "emerging", - "goldowskydill2023localizingmodelbehaviorpath": "emerging", - "haider2025multi": "emerging", - "heimersheim2024useinterpretactivationpatching": "emerging", + "gao2020pile": "emerging", + "merullo2025replicating": "emerging", + "lindsey2025landscape": "emerging", + "harrasse2025tracing": "emerging", + "ba2016layernormalization": "emerging", + "eapgp": "established", "chan2022causal": "emerging", - "nostalgebraist2020interpretinggpt": "emerging", - "belrose2025elicitinglatentpredictionstransformers": "emerging", - "pcd": "emerging", - "ao": "emerging", - "mcgrath2023hydraeffectemergentselfrepair": "emerging", - "liu2025olmotrace": "established", - "park2023trakattributingmodelbehavior": "established", - "grosse2023studyinglargelanguagemodel": "emerging", - "piktus2023gaia": "emerging", - "kowal2026concept": "emerging", + "lan-etal-2026-make": "established", + "zhang2024towards": "established", + "netmodular": "established", + "cao-etal-2021-low": "established", + "bhaskar2024finding": "established", + "haider2026multigranular": "established", + "barez2025chain": "established", + "philosophicalevalframework": "emerging", + "goldowskydill2023localizingmodelbehaviorpath": "emerging", + "hewitt-liang-2019-designing": "established", + "voita-titov-2020-information": "established", + "selfie": "established", + "li2023emergent": "established", + "sutter2025the": "established", + "arora-etal-2024-causalgym": "established", + "krishna2025disagreementproblemexplainablemachine": "established", + "jacovi-goldberg-2020-towards": "established", + "giulianelli-etal-2018-hood": "emerging", + "blank2026rlens": "emerging", "sharma_towards_2025": "established", - "minder_overcoming_2025": "established", + "wang2026persona": "established", + "thasarathan_universal_2025": "established", + "inaba2025howllmslearn": "established", "macdiarmid_natural_2025": "emerging", - "lindsey2024sparsecrosscoders": "emerging", - "wang_persona_2025": "emerging", + "minder_overcoming_2025": "emerging", "aranguri2025modeldiff": "emerging", - "bayazit_crosscoding_2025": "emerging", - "ge_evolution_2025": "emerging", - "bricken2024stagewise": "emerging", - "hu2026transcoderadaptersreasoningmodeldiffing": "emerging", - "tan2025analyzinggeneralizationreliabilitysteering": "established", + "dumas_what_2025": "emerging", + "sawmya2025birth": "emerging", + "raghu_svcca_2017": "established", + "wang_towards_2018": "established", + "sharma2024the": "established", "zhao2025steering": "established", + "dunefsky2025one": "established", + "oozeer2025beyond": "established", + "makelov2024is": "established", + "hu2026transcoderadaptersreasoningmodeldiffing": "emerging", + "kempf2026simplellmbaselinescompetitive": "emerging", + "raval2026curveballsteeringrightdirection": "emerging", + "anthropic2024modeldiffing": "emerging", + "pmlr-v44-li15convergent": "emerging", "baek_towards_2025": "emerging", - "dumas_what_2025": "emerging", - "arditi2024refusallanguagemodelsmediated": "established", - "anthropic2025claude-sonnet-4-5": "emerging", - "joshi2026causalitykeyinterpretabilityclaims": "emerging", + "batra2026multimodalmodeldiffingfeature": "emerging", + "xu2024saetrack": "emerging", + "bricken2024stagewise": "emerging", + "rosenfeld2020a": "established", + "voita_analyzing_2019": "established", + "hoffmann2022training": "established", + "obrien2026deep": "established", + "hestness2017deeplearningscalingpredictable": "emerging", + "kaplan2020scaling": "emerging", + "power2022grokkinggeneralizationoverfittingsmall": "emerging", + "chan2022data": "established", + "chang2024characterizing": "established", + "wang2025differentiation": "established", + "frankle2020linear": "established", + "lubana2023mechanistic": "established", + "schaeffer2023emergent": "established", + "wei2022emergent": "established", + "chen_sudden_2024": "established", + "biderman2023emergent": "established", + "vanderwal2025polypythias": "established", + "borkar2025privacy": "established", + "groeneveld2024olmo": "established", + "chang2022word": "established", + "refinetti2023neural": "established", + "belrose2024statistics": "established", + "razeghi2022impact": "established", + "liu2021probing": "established", + "tigges2024llm": "established", + "yin2025which": "established", + "singh2023transient": "established", + "saxe2014exact": "established", + "lampinen2019analytic": "established", + "saxe2019mathematical": "established", + "pruthi2020estimatingtrainingdatainfluence": "established", + "achille2019critical": "established", + "mckenzie2023inverse": "established", + "michaud2023quantization": "established", + "kangaslahti2026hidden": "established", + "hu2023latent": "established", + "du2024understanding": "established", + "schaeffer2025elusive": "established", + "wu2025ushaped": "established", + "liu2023omnigrok": "established", + "zucchet2025the": "established", + "aoyama_language_2025": "established", + "rahaman2019spectral": "established", + "nakkiran2019sgd": "established", + "watanabe2009algebraic": "established", + "lau2023local": "established", + "hoogland2025losslandscapedegeneracystagewise": "established", + "cloud2026language": "established", + "schrodi2026towards": "established", + "saphra_interpretability_2023": "emerging", + "zucchet2025language": "emerging", + "bruijns2026pretrainingcurriculaenableselective": "emerging", + "jumelet2024black": "emerging", + "elazar2022measuring": "emerging", + "varma2023explaining": "emerging", + "zhao2024distributional": "emerging", + "snell2024predicting": "emerging", + "jones2025forecasting": "emerging", + "abahana2026rlhf": "emerging", + "zhang2026whatshapes": "emerging", + "rathi2026shaping": "emerging", + "cloud2024gradient": "emerging", + "chen2025persona": "emerging", + "openai2025gptoss120bgptoss20bmodel": "emerging", + "liu2025olmotrace": "established", + "park2023trakattributingmodelbehavior": "established", + "sgd": "established", + "bae2022influencefunctionsanswerquestion": "established", + "kfac": "established", + "ilyas2022datamodelspredictingpredictionstraining": "established", + "piktus-etal-2023-roots": "established", + "piktus2023gaia": "established", + "liu2025infinigramscalingunboundedngram": "established", + "ekfac": "established", + "wang2025better": "established", + "mlodozeniec2025distributionaltrainingdataattribution": "established", + "georgiev2023journeydestinationdataguides": "emerging", + "kowal2026concept": "emerging", + "turpin2023languagemodelsdontsay": "established", + "arcuschin2026chainofthought": "established", + "chloe": "established", + "goel": "established", + "han2026sage": "established", + "plunkett": "emerging", + "guo2026introspectivecouplingselfexplanationtraining": "emerging", + "li2026introspective": "emerging", + "chen2025reasoningmodelsdontsay": "emerging", + "joglekar": "emerging", + "foote2023neurongraphinterpretinglanguage": "emerging", + "khan2026can": "emerging", + "marin2026automated": "emerging", "cywinski2025eliciting": "emerging", - "wu2025axbench": "established", - "insufficient": "established", + "simhi-etal-2025-trust": "established", + "aiscientistpitfall": "established", + "marks2025auditinglanguagemodelshidden": "emerging", + "anthropic2025claude-sonnet-4-5": "emerging", + "denison2024sycophancy": "emerging", + "taylor2025school": "emerging", + "shilov2025beyond": "emerging", + "hahami2026detectingdisturbancenuancedview": "emerging", + "baker2025monitoringreasoningmodelsmisbehavior": "emerging", + "haklay2026pitfalls": "emerging", + "lermen2025deceptive": "emerging", + "anil2024manyshot": "established", + "ouyang2022traininglanguagemodelsfollow": "established", + "gurarieh2025preciseinparameterconcepterasure": "established", + "tan2024analysing": "established", "shaham2024multimodal": "established", - "slocum2025believe": "emerging", - "petri2025": "emerging", - "petri2026v2": "emerging", + "find": "established", + "zhao2026chainofthoughthijacking": "emerging", + "bai2022constitutional": "emerging", + "eshuijs2026honestansweraidingllmjudges": "emerging", + "heindrich2025do": "emerging", + "jain2024mechanistically": "established", + "jain2024what": "established", + "insufficient": "established", + "3618408.3618656": "established", + "3692070.3693959": "established", + "wu2025towards": "established", + "oldfield2026beyond": "established", + "du-etal-2026-automated": "established", + "chatbotknows": "established", + "kempermann2026safe": "established", + "xie2023defending": "established", + "zhang-etal-2024-defending": "established", + "11370531": "established", + "laban2026llms": "established", + "simhi2026old": "established", "hu2022lora": "established", - "gurarieh2025preciseinparameterconcepterasure": "established", + "marks2024interpreting": "established", + "kim2025rethinking": "established", + "lo-etal-2024-large": "established", + "dai-etal-2022-knowledge": "established", "meng2023masseditingmemorytransformer": "established", - "tan2024analysing": "established", - "hendrycks2021naturaladversarialexamples": "established", - "hewitt2025position": "established", + "hoelscher-obermaier-etal-2023-detecting": "established", + "alphaedit": "established", + "crispedit": "established", + "zhao2026verifying": "established", + "chaudhary2026beyond": "established", + "inan2023llamaguardllmbasedinputoutput": "emerging", + "jain2023baselinedefensesadversarialattacks": "emerging", + "sharma2025constitutionalclassifiersdefendinguniversal": "emerging", + "bengio2025internationalaisafetyreport": "emerging", + "wallace2024instructionhierarchytrainingllms": "emerging", + "yu2025safety": "emerging", + "agarwal2025context": "emerging", + "fang2026nextguardtrainingfreestreamingsafeguard": "emerging", "szegedy2015going": "established", - "bai2022constitutional": "emerging", - "bai2022training": "emerging" + "tinaz2025emergence": "established", + "paulo2024does": "established", + "sharma2024locating": "established", + "haon2025mechanistic": "established", + "neo2025towards": "established", + "wang2026dlmscopemechanisticinterpretabilitydiffusion": "emerging", + "ensign2024investigating": "emerging", + "gu2024mamba": "established", + "peng-etal-2023-rwkv": "established", + "pmlr-v37-sohl-dickstein15": "established", + "lepikhin2021gshard": "established", + "dai-etal-2024-deepseekmoe": "established", + "zimerman2025explaining": "established", + "dao202mamba2": "established", + "peng2025rwkv": "established", + "ali-etal-2025-hidden": "established", + "jiang2024mixtralexperts": "emerging", + "kimiteam2026kimik2openagentic": "emerging", + "deepseekai2026deepseekv4highlyefficientmilliontoken": "emerging" }, "references": [ { - "key": "adebayo2018sanity", - "text": "Julius Adebayo, Justin Gilmer, Michael Muelly, Ian Goodfellow, Moritz Hardt, and Been Kim. Sanity checks for saliency maps. In Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS’18, pp. 9525–9536, Red Hook, NY, USA, 2018a. Curran Associates Inc.", + "key": "abahana2026rlhf", + "text": "Zelalem Abahana, David Evans, Satish Mahadevan Srinivasan, and Matjaz Gams. When rlhf fails: A mechanistic taxonomy of reward hacking, collapse, and evaluator gaming, 2026. URL https://arxiv.org/abs/2606.03238.", + "tier": "emerging" + }, + { + "key": "achille2019critical", + "text": "Alessandro Achille, Matteo Rovere, and Stefano Soatto. Critical learning periods in deep networks. In International Conference on Learning Representations, 2019.", "tier": "established" }, { "key": "adebayo2020sanitycheckssaliencymaps", - "text": "Julius Adebayo, Justin Gilmer, Michael Muelly, Ian Goodfellow, Moritz Hardt, and Been Kim. Sanity checks for saliency maps. In Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS’18, pp. 9525–9536, Red Hook, NY, USA, 2018b. Curran Associates Inc.", + "text": "Julius Adebayo, Justin Gilmer, Michael Muelly, Ian Goodfellow, Moritz Hardt, and Been Kim. Sanity checks for saliency maps. In Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS’18, pp. 9525–9536, Red Hook, NY, USA, 2018. Curran Associates Inc.", "tier": "established" }, { @@ -2396,24 +2734,49 @@ }, { "key": "adler2025towards", - "text": "Micah Adler, Dan Alistarh, and Nir Shavit. Towards combinatorial interpretability of neural computation. arXiv preprint arXiv:2504.08842, 2025.", + "text": "Micah Adler, Dan Alistarh, and Nir Shavit. Towards combinatorial interpretability of neural computation, 2025. URL https://arxiv.org/abs/2504.08842.", + "tier": "emerging" + }, + { + "key": "agarwal2025context", + "text": "Isha Agarwal, Saharsha Navani, and Fazl Barez. Context matters: Analyzing the generalizability of linear probing and steering across diverse scenarios. In Mechanistic Interpretability Workshop at NeurIPS 2025, 2025. URL https://openreview.net/forum?id=H5sbfvEbTh.", "tier": "emerging" }, { "key": "aharon2006k", "text": "Michal Aharon, Michael Elad, and Alfred Bruckstein. K-svd: An algorithm for designing overcomplete dictionaries for sparse representation. IEEE Transactions on signal processing, 54 (11): 4311–4322, 2006.", - "tier": null + "tier": "established" + }, + { + "key": "gqa", + "text": "Joshua Ainslie, James Lee-Thorp, Michiel de Jong, Yury Zemlyanskiy, Federico Lebron, and Sumit Sanghai. GQA: Training generalized multi-query transformer models from multi-head checkpoints. In Houda Bouamor, Juan Pino, and Kalika Bali (eds.), Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, pp. 4895–4901, Singapore, December 2023. Association for Computational Linguistics. . URL https://aclanthology.org/2023.emnlp-main.298/.", + "tier": "established" }, { "key": "alain2017understanding", "text": "Guillaume Alain and Yoshua Bengio. Understanding intermediate layers using linear classifier probes. The Fifth International Conference on Learning Representations, 2017. URL https://openreview.net/forum?id=ryF7rTqgl.", "tier": "established" }, + { + "key": "ali-etal-2025-hidden", + "text": "Ameen Ali Ali, Itamar Zimerman, and Lior Wolf. The hidden attention of mamba models. In Wanxiang Che, Joyce Nabende, Ekaterina Shutova, and Mohammad Taher Pilehvar (eds.), Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 1516–1534, Vienna, Austria, July 2025. Association for Computational Linguistics. ISBN 979-8-89176-251-0. . URL https://aclanthology.org/2025.acl-long.76/.", + "tier": "established" + }, { "key": "ameisen2025circuit", "text": "Emmanuel Ameisen, Jack Lindsey, Adam Pearce, Wes Gurnee, Nicholas L. Turner, Brian Chen, Craig Citro, David Abrahams, Shan Carter, Basil Hosmer, Jonathan Marcus, Michael Sklar, Adly Templeton, Trenton Bricken, Callum McDougall, Hoagy Cunningham, Thomas Henighan, Adam Jermyn, Andy Jones, Andrew Persic, Zhenyi Qi, T. Ben Thompson, Sam Zimmerman, Kelley Rivoire, Thomas Conerly, Chris Olah, and Joshua Batson. Circuit tracing: Revealing computational graphs in language models. Transformer Circuits Thread, 2025. URL https://transformer-circuits.pub/2025/attribution-graphs/methods.html.", "tier": "emerging" }, + { + "key": "anil2024manyshot", + "text": "Cem Anil, Esin DURMUS, Nina Rimsky, Mrinank Sharma, Joe Benton, Sandipan Kundu, Joshua Batson, Meg Tong, Jesse Mu, Daniel J Ford, Francesco Mosconi, Rajashree Agrawal, Rylan Schaeffer, Naomi Bashkansky, Samuel Svenningsen, Mike Lambert, Ansh Radhakrishnan, Carson Denison, Evan J Hubinger, Yuntao Bai, Trenton Bricken, Timothy Maxwell, Nicholas Schiefer, James Sully, Alex Tamkin, Tamera Lanham, Karina Nguyen, Tomasz Korbak, Jared Kaplan, Deep Ganguli, Samuel R. Bowman, Ethan Perez, Roger Baker Grosse, and David Duvenaud. Many-shot jailbreaking. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024. URL https://openreview.net/forum?id=cw5mgd71jW.", + "tier": "established" + }, + { + "key": "anthropic2024modeldiffing", + "text": "Anthropic. Model diffing. Transformer Circuits, October 2024. URL https://transformer-circuits.pub/2024/model-diffing/index.html.", + "tier": "emerging" + }, { "key": "anthropic2025claude-sonnet-4-5", "text": "Anthropic. Claude sonnet 4.5 system card. System card, Anthropic, September 2025a. URL https://assets.anthropic.com/m/12f214efcc2f457a/original/Claude-Sonnet-4-5-System-Card.pdf.", @@ -2426,8 +2789,8 @@ }, { "key": "aoyama_language_2025", - "text": "Tatsuya Aoyama and Ethan Wilcox. Language models grow less humanlike beyond phase transition, 2025. URL http://arxiv.org/abs/2502.18802.", - "tier": null + "text": "Tatsuya Aoyama and Ethan Wilcox. Language models grow less humanlike beyond phase transition. In Wanxiang Che, Joyce Nabende, Ekaterina Shutova, and Mohammad Taher Pilehvar (eds.), Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 24938–24958, Vienna, Austria, July 2025. Association for Computational Linguistics. ISBN 979-8-89176-251-0. . URL https://aclanthology.org/2025.acl-long.1214/.", + "tier": "established" }, { "key": "apollo2025sonnet37evalaware", @@ -2439,20 +2802,50 @@ "text": "Santiago Aranguri and Tom McGrath. Discovering undesired rare behaviors via model diff amplification. Goodfire, 2025. https://www.goodfire.ai/research/model-diff-amplification.", "tier": "emerging" }, + { + "key": "arcuschin2026chainofthought", + "text": "Iván Arcuschin, Jett Janiak, Robert Krzyzanowski, Senthooran Rajamanoharan, Neel Nanda, and Arthur Conmy. Chain-of-thought reasoning in the wild is not always faithful. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=NUyt4uxzx0.", + "tier": "established" + }, { "key": "arditi2024refusallanguagemodelsmediated", "text": "Andy Arditi, Oscar Balcells Obeso, Aaquib Syed, Daniel Paleka, Nina Rimsky, Wes Gurnee, and Neel Nanda. Refusal in language models is mediated by a single direction. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024.", "tier": "established" }, { - "key": "bae2022influencefunctionsanswerquestion", - "text": "Juhan Bae, Nathan Ng, Alston Lo, Marzyeh Ghassemi, and Roger Grosse. If influence functions are the answer, then what is the question? In Proceedings of the 36th International Conference on Neural Information Processing Systems, NIPS ’22, Red Hook, NY, USA, 2022. Curran Associates Inc. ISBN 9781713871088.", - "tier": null + "key": "arora-etal-2024-causalgym", + "text": "Aryaman Arora, Dan Jurafsky, and Christopher Potts. CausalGym: Benchmarking causal interpretability methods on linguistic tasks. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 14638–14663, Bangkok, Thailand, August 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.785/.", + "tier": "established" + }, + { + "key": "arora2026language", + "text": "Aryaman Arora, Zhengxuan Wu, Jacob Steinhardt, and Sarah Schwettmann. Language model circuits are sparse in the neuron basis. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=OrviwFWcN4.", + "tier": "established" + }, + { + "key": "philosophicalevalframework", + "text": "Kola Ayonrinde and Louis Jaburi. Evaluating explanations: An explanatory virtues framework for mechanistic interpretability – the strange science part i.ii, 2025. URL https://arxiv.org/abs/2505.01372.", + "tier": "emerging" + }, + { + "key": "proofnet", + "text": "Zhangir Azerbayev, Bartosz Piotrowski, Hailey Schoelkopf, Edward W. Ayers, Dragomir Radev, and Jeremy Avigad. Proofnet: Autoformalizing and formally proving undergraduate-level mathematics, 2023. URL https://arxiv.org/abs/2302.12433.", + "tier": "emerging" }, { - "key": "bae2024trainingdataattributionapproximate", - "text": "Juhan Bae, Wu Lin, Jonathan Lorraine, and Roger Baker Grosse. Training data attribution via approximate unrolling. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024. URL https://openreview.net/forum?id=3NaqGg92KZ.", - "tier": null + "key": "ba2016layernormalization", + "text": "Jimmy Lei Ba, Jamie Ryan Kiros, and Geoffrey E. Hinton. Layer normalization, 2016. URL https://arxiv.org/abs/1607.06450.", + "tier": "emerging" + }, + { + "key": "lrp", + "text": "Sebastian Bach, Alexander Binder, Grégoire Montavon, Frederick Klauschen, Klaus-Robert Müller, and Wojciech Samek. On pixel-wise explanations for non-linear classifier decisions by layer-wise relevance propagation. PLOS ONE, 10 (7): 1–46, 07 2015. . URL https://doi.org/10.1371/journal.pone.0130140.", + "tier": "established" + }, + { + "key": "bae2022influencefunctionsanswerquestion", + "text": "Juhan Bae, Nathan Ng, Alston Lo, Marzyeh Ghassemi, and Roger Grosse. If influence functions are the answer, then what is the question? In Proceedings of the 36th International Conference on Neural Information Processing Systems, NIPS ’22, Red Hook, NY, USA, 2022. Curran Associates Inc. ISBN 9781713871088.", + "tier": "established" }, { "key": "baek_towards_2025", @@ -2461,12 +2854,22 @@ }, { "key": "bai2022training", - "text": "Yuntao Bai, Andy Jones, Kamal Ndousse, Amanda Askell, Anna Chen, Nova DasSarma, Dawn Drain, Stanislav Fort, Deep Ganguli, Tom Henighan, et al. Training a helpful and harmless assistant with reinforcement learning from human feedback. arXiv preprint arXiv:2204.05862, 2022a.", + "text": "Yuntao Bai, Andy Jones, Kamal Ndousse, Amanda Askell, Anna Chen, Nova DasSarma, Dawn Drain, Stanislav Fort, Deep Ganguli, Tom Henighan, Nicholas Joseph, Saurav Kadavath, Jackson Kernion, Tom Conerly, Sheer El-Showk, Nelson Elhage, Zac Hatfield-Dodds, Danny Hernandez, Tristan Hume, Scott Johnston, Shauna Kravec, Liane Lovitt, Neel Nanda, Catherine Olsson, Dario Amodei, Tom Brown, Jack Clark, Sam McCandlish, Chris Olah, Ben Mann, and Jared Kaplan. Training a helpful and harmless assistant with reinforcement learning from human feedback, 2022a. URL https://arxiv.org/abs/2204.05862.", "tier": "emerging" }, { "key": "bai2022constitutional", - "text": "Yuntao Bai, Saurav Kadavath, Sandipan Kundu, Amanda Askell, Jackson Kernion, Andy Jones, Anna Chen, Anna Goldie, Azalia Mirhoseini, Cameron McKinnon, et al. Constitutional ai: Harmlessness from ai feedback. arXiv preprint arXiv:2212.08073, 2022b.", + "text": "Yuntao Bai, Saurav Kadavath, Sandipan Kundu, Amanda Askell, Jackson Kernion, Andy Jones, Anna Chen, Anna Goldie, Azalia Mirhoseini, Cameron McKinnon, Carol Chen, Catherine Olsson, Christopher Olah, Danny Hernandez, Dawn Drain, Deep Ganguli, Dustin Li, Eli Tran-Johnson, Ethan Perez, Jamie Kerr, Jared Mueller, Jeffrey Ladish, Joshua Landau, Kamal Ndousse, Kamile Lukosuite, Liane Lovitt, Michael Sellitto, Nelson Elhage, Nicholas Schiefer, Noemi Mercado, Nova DasSarma, Robert Lasenby, Robin Larson, Sam Ringer, Scott Johnston, Shauna Kravec, Sheer El Showk, Stanislav Fort, Tamera Lanham, Timothy Telleen-Lawton, Tom Conerly, Tom Henighan, Tristan Hume, Samuel R. Bowman, Zac Hatfield-Dodds, Ben Mann, Dario Amodei, Nicholas Joseph, Sam McCandlish, Tom Brown, and Jared Kaplan. Constitutional ai: Harmlessness from ai feedback, 2022b. URL https://arxiv.org/abs/2212.08073.", + "tier": "emerging" + }, + { + "key": "bai-etal-2024-longbench", + "text": "Yushi Bai, Xin Lv, Jiajie Zhang, Hongchang Lyu, Jiankai Tang, Zhidian Huang, Zhengxiao Du, Xiao Liu, Aohan Zeng, Lei Hou, Yuxiao Dong, Jie Tang, and Juanzi Li. LongBench: A bilingual, multitask benchmark for long context understanding. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 3119–3137, Bangkok, Thailand, August 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.172/.", + "tier": "established" + }, + { + "key": "baker2025monitoringreasoningmodelsmisbehavior", + "text": "Bowen Baker, Joost Huizinga, Leo Gao, Zehao Dou, Melody Y. Guan, Aleksander Madry, Wojciech Zaremba, Jakub Pachocki, and David Farhi. Monitoring reasoning models for misbehavior and the risks of promoting obfuscation, 2025. URL https://arxiv.org/abs/2503.11926.", "tier": "emerging" }, { @@ -2474,16 +2877,46 @@ "text": "Fazl Barez. Automated interpretability-driven model auditing and control: A research agenda. AI Governance Initiative, University of Oxford, January 2026. URL https://aigi.ox.ac.uk/publications/automated-interpretability-driven-model-auditing-and-control-a-research-agenda/.", "tier": "emerging" }, + { + "key": "barez2025chain", + "text": "Fazl Barez, Tung-Yu Wu, Iván Arcuschin, Michael Lan, Vincent Wang, Noah Siegel, Nicolas Collignon, Clement Neo, Isabelle Lee, Alasdair Paren, et al. Chain-of-thought is not explainability. In The Fortieth Annual Conference on Neural Information Processing Systems Position Paper Track, 2026.", + "tier": "established" + }, + { + "key": "Barlow:1961aa", + "text": "H. B. Barlow. Possible principles underlying the transformations of sensory messages. In Walter A. Rosenblith (ed.), Sensory Communication, pp. 217–234. MIT Press, Cambridge, MA, 1961.", + "tier": "established" + }, + { + "key": "batra2026multimodalmodeldiffingfeature", + "text": "Hunar Batra, Lachin Naghashyar, Ashkan Khakzar, Philip Torr, Christian Schroeder de Witt, Constantin Venhoff, and Ronald Clark. Multimodal model diffing for feature discovery and control, 2026. URL https://arxiv.org/abs/2608.09928.", + "tier": "emerging" + }, { "key": "bau2018identifying", "text": "Anthony Bau, Yonatan Belinkov, Hassan Sajjad, Nadir Durrani, Fahim Dalvi, and James Glass. Identifying and controlling important neurons in neural machine translation. In International Conference on Learning Representations, 2019. URL https://openreview.net/forum?id=H1z-PsR5KX.", - "tier": null + "tier": "established" + }, + { + "key": "netdissect2017", + "text": "David Bau, Bolei Zhou, Aditya Khosla, Aude Oliva, and Antonio Torralba. Network dissection: Quantifying interpretability of deep visual representations. In Computer Vision and Pattern Recognition, 2017.", + "tier": "established" }, { - "key": "bayazit_crosscoding_2025", - "text": "Deniz Bayazit, Aaron Mueller, and Antoine Bosselut. Crosscoding through time: Tracking emergence & consolidation of linguistic representations throughout llm pretraining. arXiv preprint arXiv:2509.05291, 2025.", + "key": "bau2020units", + "text": "David Bau, Jun-Yan Zhu, Hendrik Strobelt, Agata Lapedriza, Bolei Zhou, and Antonio Torralba. Understanding the role of individual units in a deep neural network. Proceedings of the National Academy of Sciences, 2020. ISSN 0027-8424. . URL https://www.pnas.org/content/early/2020/08/31/1907375117.", + "tier": "established" + }, + { + "key": "buildingbetterao", + "text": "Jan Philipp Bauer, Celeste De Schamphelaere, Adam Karvonen, Niclas Luick, and Neel Nanda. Building better activation oracles. In Mechanistic Interpretability Workshop at ICML 2026, 2026. URL https://openreview.net/forum?id=6nkeLpn3cY.", "tier": "emerging" }, + { + "key": "bayazit-etal-2026-crosscoding", + "text": "Deniz Bayazit, Aaron Mueller, and Antoine Bosselut. Crosscoding through time: Tracking emergence & consolidation of linguistic representations throughout LLM pretraining. In Maria Liakata, Viviane P. Moreira, Jiajun Zhang, and David Jurgens (eds.), Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 1353–1377, San Diego, California, United States, July 2026. Association for Computational Linguistics. ISBN 979-8-89176-390-6. . URL https://aclanthology.org/2026.acl-long.60/.", + "tier": "established" + }, { "key": "belinkov-2022-probing", "text": "Yonatan Belinkov. Probing classifiers: Promises, shortcomings, and advances. Computational Linguistics, 48 (1): 207–219, March 2022. . URL https://aclanthology.org/2022.cl-1.7/.", @@ -2499,34 +2932,74 @@ "text": "Yonatan Belinkov, Nadir Durrani, Fahim Dalvi, Hassan Sajjad, and James Glass. What do neural machine translation models learn about morphology? In Regina Barzilay and Min-Yen Kan (eds.), Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 861–872, Vancouver, Canada, July 2017. Association for Computational Linguistics. . URL https://aclanthology.org/P17-1080/.", "tier": "established" }, + { + "key": "belrose2024statistics", + "text": "Nora Belrose, Quintin Pope, Lucia Quirke, Alex Mallen, and Xiaoli Fern. Neural networks learn statistics of increasing complexity. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.", + "tier": "established" + }, { "key": "belrose2025elicitinglatentpredictionstransformers", "text": "Nora Belrose, Igor Ostrovsky, Lev McKinney, Zach Furman, Logan Smith, Danny Halawi, Stella Biderman, and Jacob Steinhardt. Eliciting latent predictions from transformers with the tuned lens, 2025. URL https://arxiv.org/abs/2303.08112.", "tier": "emerging" }, + { + "key": "bengio2013estimatingpropagatinggradientsstochastic", + "text": "Yoshua Bengio, Nicholas Léonard, and Aaron Courville. Estimating or propagating gradients through stochastic neurons for conditional computation, 2013. URL https://arxiv.org/abs/1308.3432.", + "tier": "emerging" + }, + { + "key": "bengio2025internationalaisafetyreport", + "text": "Yoshua Bengio, Sören Mindermann, Daniel Privitera, Tamay Besiroglu, Rishi Bommasani, Stephen Casper, Yejin Choi, Philip Fox, Ben Garfinkel, Danielle Goldfarb, Hoda Heidari, Anson Ho, Sayash Kapoor, Leila Khalatbari, Shayne Longpre, Sam Manning, Vasilios Mavroudis, Mantas Mazeika, Julian Michael, Jessica Newman, Kwan Yee Ng, Chinasa T. Okolo, Deborah Raji, Girish Sastry, Elizabeth Seger, Theodora Skeadas, Tobin South, Emma Strubell, Florian Tramèr, Lucia Velasco, Nicole Wheeler, Daron Acemoglu, Olubayo Adekanmbi, David Dalrymple, Thomas G. Dietterich, Edward W. Felten, Pascale Fung, Pierre-Olivier Gourinchas, Fredrik Heintz, Geoffrey Hinton, Nick Jennings, Andreas Krause, Susan Leavy, Percy Liang, Teresa Ludermir, Vidushi Marda, Helen Margetts, John McDermid, Jane Munga, Arvind Narayanan, Alondra Nelson, Clara Neppel, Alice Oh, Gopal Ramchurn, Stuart Russell, Marietje Schaake, Bernhard Schölkopf, Dawn Song, Alvaro Soto, Lee Tiedrich, Gaël Varoquaux, Andrew Yao, Ya-Qin Zhang, Fahad Albalawi, Marwan Alserkal, Olubunmi Ajala, Guillaume Avrin, Christian Busch, André Carlos Ponce de Leon Ferreira de Carvalho, Bronwyn Fox, Amandeep Singh Gill, Ahmet Halit Hatip, Juha Heikkilä, Gill Jolly, Ziv Katzir, Hiroaki Kitano, Antonio Krüger, Chris Johnson, Saif M. Khan, Kyoung Mu Lee, Dominic Vincent Ligot, Oleksii Molchanovskyi, Andrea Monti, Nusu Mwamanzi, Mona Nemer, Nuria Oliver, José Ramón López Portillo, Balaraman Ravindran, Raquel Pezoa Rivera, Hammam Riza, Crystal Rugege, Ciarán Seoighe, Jerry Sheehan, Haroon Sheikh, Denise Wong, and Yi Zeng. International ai safety report, 2025. URL https://arxiv.org/abs/2501.17805.", + "tier": "emerging" + }, { "key": "bereska2024mechanisticinterpretabilityaisafety", "text": "Leonard Bereska and Stratis Gavves. Mechanistic interpretability for AI safety - a review. Transactions on Machine Learning Research, 2024. ISSN 2835-8856. URL https://openreview.net/forum?id=ePUVetPKu6.", "tier": "established" }, + { + "key": "bhaskar2024finding", + "text": "Adithya Bhaskar, Alexander Wettig, Dan Friedman, and Danqi Chen. Finding transformer circuits with edge pruning. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024. URL https://openreview.net/forum?id=8oSY3rA9jY.", + "tier": "established" + }, { "key": "biderman2023emergent", "text": "Stella Biderman, Usvsn Prashanth, Lintang Sutawika, Hailey Schoelkopf, Quentin Anthony, Shivanshu Purohit, and Edward Raff. Emergent and predictable memorization in large language models. Advances in Neural Information Processing Systems, 36: 28072–28090, 2023a.", - "tier": null + "tier": "established" }, { "key": "biderman2023pythia", "text": "Stella Biderman, Hailey Schoelkopf, Quentin Gregory Anthony, Herbie Bradley, Kyle O’Brien, Eric Hallahan, Mohammad Aflah Khan, Shivanshu Purohit, USVSN Sai Prashanth, Edward Raff, et al. Pythia: A suite for analyzing large language models across training and scaling. In International Conference on Machine Learning, pp. 2397–2430. PMLR, 2023b.", - "tier": null + "tier": "established" + }, + { + "key": "biderman2026position", + "text": "Stella Biderman, Mohammad Aflah Khan, Niloofar Mireshghallah, Catherine Arnett, Fazl Barez, and Naomi Saphra. Position: Don’t just ”fix it in post”: A science of AI must study learning dynamics. In Forty-third International Conference on Machine Learning Position Paper Track, 2026. URL https://openreview.net/forum?id=VbHh05xQ7y.", + "tier": "established" }, { "key": "bills2023language", "text": "Steven Bills, Nick Cammarata, Dan Mossing, Henk Tillman, Leo Gao, Gabriel Goh, Ilya Sutskever, Jan Leike, Jeff Wu, and William Saunders. Language models can explain neurons in language models, 2023.", "tier": "emerging" }, + { + "key": "binder", + "text": "Felix Jedidja Binder, James Chua, Tomek Korbak, Henry Sleight, John Hughes, Robert Long, Ethan Perez, Miles Turpin, and Owain Evans. Looking inward: Language models can learn about themselves by introspection. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=eb5pkwIB5i.", + "tier": "established" + }, + { + "key": "blank2026rlens", + "text": "Camila Blank, Agam Bhatia, and Neel Nanda. R-lens: Making J-lens more faithful on early layers. AI Alignment Forum, August 2026. URL https://www.alignmentforum.org/posts/nv8oedrnLXKRzNEL9/r-lens-making-j-lens-more-faithful-on-early-layers. Blog post.", + "tier": "emerging" + }, + { + "key": "borkar2025privacy", + "text": "Jaydeep Borkar, Matthew Jagielski, Katherine Lee, Niloofar Mireshghallah, David A. Smith, and Christopher A. Choquette-Choo. Privacy ripple effects from adding or removing personal information in language model training. In Findings of the Association for Computational Linguistics: ACL 2025, 2025.", + "tier": "established" + }, { "key": "braun2025interpretability", - "text": "Dan Braun, Lucius Bushnaq, Stefan Heimersheim, Jake Mendel, and Lee Sharkey. Interpretability in parameter space: Minimizing mechanistic description length with attribution-based parameter decomposition. arXiv preprint arXiv:2501.14926, 2025.", + "text": "Dan Braun, Lucius Bushnaq, Stefan Heimersheim, Jake Mendel, and Lee Sharkey. Interpretability in parameter space: Minimizing mechanistic description length with attribution-based parameter decomposition, 2025. URL https://arxiv.org/abs/2501.14926.", "tier": "emerging" }, { @@ -2539,25 +3012,30 @@ "text": "Trenton Bricken, Siddharth Mishra-Sharma, Jonathan Marcus, Adam Jermyn, Christopher Olah, Kelley Rivoire, and Thomas Henighan. Stage-wise model diffing. Transformer Circuits Thread (Anthropic), December 2024. URL https://transformer-circuits.pub/2024/model-diffing/index.html.", "tier": "emerging" }, + { + "key": "bruijns2026pretrainingcurriculaenableselective", + "text": "Sebastian A. Bruijns, Jirko Rubruck, Mia H. Whitefield, Kai J. Sandbrink, Fazl Barez, and Christopher Summerfield. Pretraining curricula enable selective fine-tuning, 2026. URL https://arxiv.org/abs/2607.04846.", + "tier": "emerging" + }, { "key": "burns2023discovering", "text": "Collin Burns, Haotian Ye, Dan Klein, and Jacob Steinhardt. Discovering latent knowledge in language models without supervision. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=ETKGuby0hcs.", "tier": "established" }, - { - "key": "burns2024discoveringlatentknowledgelanguage", - "text": "Collin Burns, Haotian Ye, Dan Klein, and Jacob Steinhardt. Discovering latent knowledge in language models without supervision, 2024. URL https://arxiv.org/abs/2212.03827.", - "tier": "emerging" - }, { "key": "bushnaq2025stochastic", - "text": "Lucius Bushnaq, Dan Braun, and Lee Sharkey. Stochastic parameter decomposition. arXiv preprint arXiv:2506.20790, 2025.", + "text": "Lucius Bushnaq, Dan Braun, and Lee Sharkey. Stochastic parameter decomposition, 2025. URL https://arxiv.org/abs/2506.20790.", "tier": "emerging" }, { "key": "bussmann2024batchtopksparseautoencoders", "text": "Bart Bussmann, Patrick Leask, and Neel Nanda. Batchtopk sparse autoencoders. In NeurIPS 2024 Workshop on Scientific Methods for Understanding Deep Learning, 2024. URL https://openreview.net/forum?id=d4dpOCqybL.", - "tier": null + "tier": "emerging" + }, + { + "key": "cao-etal-2021-low", + "text": "Steven Cao, Victor Sanh, and Alexander Rush. Low-complexity probing via finding subnetworks. In Kristina Toutanova, Anna Rumshisky, Luke Zettlemoyer, Dilek Hakkani-Tur, Iz Beltagy, Steven Bethard, Ryan Cotterell, Tanmoy Chakraborty, and Yichao Zhou (eds.), Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 960–966, Online, June 2021. Association for Computational Linguistics. . URL https://aclanthology.org/2021.naacl-main.74/.", + "tier": "established" }, { "key": "insufficient", @@ -2566,58 +3044,98 @@ }, { "key": "chan2022causal", - "text": "Lawrence Chan, Adrià Garriga-Alonso, Nicholas Goldwosky-Dill, Ryan Greenblatt, Jenny Nitishinskaya, Ansh Radhakrishnan, Buck Shlegeris, and Nate Thomas. Causal scrubbing, a method for rigorously testing interpretability hypotheses. AI Alignment Forum, 2022. https://www.alignmentforum.org/posts/JvZhhzycHu2Yd57RN/causal-scrubbing-a-method-for-rigorously-testing.", + "text": "Lawrence Chan, Adrià Garriga-Alonso, Nicholas Goldwosky-Dill, Ryan Greenblatt, Jenny Nitishinskaya, Ansh Radhakrishnan, Buck Shlegeris, and Nate Thomas. Causal scrubbing, a method for rigorously testing interpretability hypotheses. AI Alignment Forum, 2022a. https://www.alignmentforum.org/posts/JvZhhzycHu2Yd57RN/causal-scrubbing-a-method-for-rigorously-testing.", "tier": "emerging" }, + { + "key": "chan2022data", + "text": "Stephanie C. Y. Chan, Adam Santoro, Andrew K. Lampinen, Jane X. Wang, Aaditya Singh, Pierre H. Richemond, Jay McClelland, and Felix Hill. Data distributional properties drive emergent in-context learning in transformers. In Advances in Neural Information Processing Systems, volume 35, 2022b.", + "tier": "established" + }, { "key": "chang2022word", "text": "Tyler A Chang and Benjamin K Bergen. Word acquisition in neural language models. Transactions of the Association for Computational Linguistics, 10: 1–16, 2022.", - "tier": null + "tier": "established" }, { - "key": "chanin2024absorption", - "text": "David Chanin, James Wilken-Smith, Tomáš Dulka, Hardik Bhatnagar, Satvik Golechha, and Joseph Bloom. A is for absorption: Studying feature splitting and absorption in sparse autoencoders, 2025. URL https://arxiv.org/abs/2409.14507.", - "tier": null + "key": "chang2024characterizing", + "text": "Tyler A Chang, Zhuowen Tu, and Benjamin K Bergen. Characterizing learning curves during language model pre-training: Learning, forgetting, and stability. Transactions of the Association for Computational Linguistics, 12: 1346–1362, 2024.", + "tier": "established" }, { - "key": "chaudhary2025safetynetdetectingharmfuloutputs", - "text": "Maheep Chaudhary and Fazl Barez. Safetynet: Detecting harmful outputs in llms by modeling and monitoring deceptive behaviors, 2025. URL https://arxiv.org/abs/2505.14300.", - "tier": "emerging" + "key": "chanin2025absorption", + "text": "David Chanin, James Wilken-Smith, Tomáš Dulka, Hardik Bhatnagar, Satvik Golechha, and Joseph Isaac Bloom. A is for absorption: Studying feature splitting and absorption in sparse autoencoders. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025. URL https://openreview.net/forum?id=R73ybUciQF.", + "tier": "established" + }, + { + "key": "chaudhary2026beyond", + "text": "Maheep Chaudhary and Fazl Barez. Beyond black-box obfuscation: Mechanistic analysis and defense of white-box monitors. In Third Conference on Language Modeling, 2026. URL https://openreview.net/forum?id=4JrlXy2N6j.", + "tier": "established" }, { "key": "chaudhary2024evaluating", - "text": "Maheep Chaudhary and Atticus Geiger. Evaluating open-source sparse autoencoders on disentangling factual knowledge in gpt-2 small. arXiv preprint arXiv:2409.04478, 2024.", + "text": "Maheep Chaudhary and Atticus Geiger. Evaluating open-source sparse autoencoders on disentangling factual knowledge in gpt-2 small, 2024. URL https://arxiv.org/abs/2409.04478.", "tier": "emerging" }, { "key": "chen_sudden_2024", "text": "Angelica Chen, Ravid Shwartz-Ziv, Kyunghyun Cho, Matthew L. Leavitt, and Naomi Saphra. Sudden drops in the loss: Syntax acquisition, phase transitions, and simplicity bias in MLMs. In The Twelfth International Conference on Learning Representations, 2024a. URL https://openreview.net/forum?id=MO5PiKHELW.", - "tier": null + "tier": "established" }, { "key": "selfie", "text": "Haozhe Chen, Carl Vondrick, and Chengzhi Mao. Selfie: self-interpretation of large language model embeddings. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024b.", - "tier": null + "tier": "established" + }, + { + "key": "chen2021evaluatinglargelanguagemodels", + "text": "Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, and Wojciech Zaremba. Evaluating large language models trained on code, 2021. URL https://arxiv.org/abs/2107.03374.", + "tier": "emerging" }, { "key": "chen2025low", - "text": "Matthew Chen, Joshua Engels, and Max Tegmark. Low-rank adapting models for sparse autoencoders. arXiv preprint arXiv:2501.19406, 2025.", + "text": "Matthew Chen, Joshua Engels, and Max Tegmark. Low-rank adapting models for sparse autoencoders. In Forty-second International Conference on Machine Learning, 2025a. URL https://openreview.net/forum?id=lDPtsCYTwr.", + "tier": "established" + }, + { + "key": "chen2025persona", + "text": "Runjin Chen, Andy Arditi, Henry Sleight, Owain Evans, and Jack Lindsey. Persona vectors: Monitoring and controlling character traits in language models, 2025b. URL https://arxiv.org/abs/2507.21509.", "tier": "emerging" }, { - "key": "christiano2023deepreinforcementlearninghuman", - "text": "Paul F Christiano, Jan Leike, Tom Brown, Miljan Martic, Shane Legg, and Dario Amodei. Deep reinforcement learning from human preferences. In I. Guyon, U. Von Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (eds.), Advances in Neural Information Processing Systems, volume 30. Curran Associates, Inc., 2017. URL https://proceedings.neurips.cc/paper_files/paper/2017/file/d5e2c0adad503c91f91df240d0cd4e49-Paper.pdf.", + "key": "basispursuit", + "text": "Scott Shaobing Chen, David L. Donoho, and Michael A. Saunders. Atomic decomposition by basis pursuit. SIAM Journal on Scientific Computing, 20 (1): 33–61, 1998. . URL https://doi.org/10.1137/S1064827596304010.", + "tier": "established" + }, + { + "key": "chen2025reasoningmodelsdontsay", + "text": "Yanda Chen, Joe Benton, Ansh Radhakrishnan, Jonathan Uesato, Carson Denison, John Schulman, Arushi Somani, Peter Hase, Misha Wagner, Fabien Roger, Vlad Mikulik, Samuel R. Bowman, Jan Leike, Jared Kaplan, and Ethan Perez. Reasoning models don’t always say what they think, 2025c. URL https://arxiv.org/abs/2505.05410.", + "tier": "emerging" + }, + { + "key": "3618408.3618656", + "text": "Bilal Chughtai, Lawrence Chan, and Neel Nanda. A toy model of universality: reverse engineering how networks learn group operations. In Proceedings of the 40th International Conference on Machine Learning, ICML’23. JMLR.org, 2023.", "tier": "established" }, { "key": "clark2019what", "text": "Kevin Clark, Urvashi Khandelwal, Omer Levy, and Christopher D. Manning. What does BERT look at? an analysis of BERT’s attention. In Tal Linzen, Grzegorz Chrupała, Yonatan Belinkov, and Dieuwke Hupkes (eds.), Proceedings of the 2019 ACL Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP, pp. 276–286, Florence, Italy, August 2019. Association for Computational Linguistics. . URL https://aclanthology.org/W19-4828/.", - "tier": "established" + "tier": "emerging" }, { "key": "cloud2024gradient", - "text": "Alex Cloud, Jacob Goldman-Wetzler, Evžen Wybitul, Joseph Miller, and Alexander Matt Turner. Gradient routing: Masking gradients to localize computation in neural networks. arXiv preprint arXiv:2410.04332, 2024.", - "tier": null + "text": "Alex Cloud, Jacob Goldman-Wetzler, Evžen Wybitul, Joseph Miller, and Alexander Matt Turner. Gradient routing: Masking gradients to localize computation in neural networks, 2024. URL https://arxiv.org/abs/2410.04332.", + "tier": "emerging" + }, + { + "key": "cloud2026language", + "text": "Alex Cloud, Minh Le, James Chua, Jan Betley, Anna Sztyber-Betley, Jacob Hilton, Samuel Marks, and Owain Evans. Language models transmit behavioural traits through hidden signals in data. Nature, 652: 615–621, 2026. . URL https://www.nature.com/articles/s41586-026-10319-8.", + "tier": "established" + }, + { + "key": "gsm8k", + "text": "Karl Cobbe, Vineet Kosaraju, Mohammad Bavarian, Mark Chen, Heewoo Jun, Lukasz Kaiser, Matthias Plappert, Jerry Tworek, Jacob Hilton, Reiichiro Nakano, Christopher Hesse, and John Schulman. Training verifiers to solve math word problems, 2021. URL https://arxiv.org/abs/2110.14168.", + "tier": "emerging" }, { "key": "conmy2023automatedcircuitdiscoverymechanistic", @@ -2629,24 +3147,49 @@ "text": "Alexis Conneau, German Kruszewski, Guillaume Lample, Loïc Barrault, and Marco Baroni. What you can cram into a single $&!#* vector: Probing sentence embeddings for linguistic properties. In Iryna Gurevych and Yusuke Miyao (eds.), Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 2126–2136, Melbourne, Australia, July 2018. Association for Computational Linguistics. . URL https://aclanthology.org/P18-1198/.", "tier": "established" }, + { + "key": "cox2008metareasoning", + "text": "Michael T. Cox and Anita Raja. Metareasoning: A manifesto. In Proceedings of the 2008 AAAI Workshop on Metareasoning: Thinking about Thinking, number WS-08-07 in Technical Report, pp. 1–4. AAAI Press, 2008. URL https://aaai.org/papers/ws08-07-001-metareasoning-a-manifesto/.", + "tier": "emerging" + }, + { + "key": "netmodular", + "text": "Róbert Csordás, Sjoerd van Steenkiste, and Jürgen Schmidhuber. Are neural nets modular? inspecting functional modularity through differentiable weight masks. In International Conference on Learning Representations, 2021. URL https://openreview.net/forum?id=7uVcpu-gMD.", + "tier": "established" + }, { "key": "rnnnonlinear", "text": "Róbert Csordás, Christopher Potts, Christopher D Manning, and Atticus Geiger. Recurrent neural networks learn to store and generate sequences using non-linear representations. In Yonatan Belinkov, Najoung Kim, Jaap Jumelet, Hosein Mohebbi, Aaron Mueller, and Hanjie Chen (eds.), Proceedings of the 7th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP, pp. 248–262, Miami, Florida, US, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.blackboxnlp-1.17/.", - "tier": "established" + "tier": "emerging" }, { "key": "cunningham2023sparse", - "text": "Hoagy Cunningham et al. Sparse autoencoders find highly interpretable features. arXiv preprint arXiv:2309.08600, 2023.", + "text": "Hoagy Cunningham, Aidan Ewart, Logan Riggs, Robert Huben, and Lee Sharkey. Sparse autoencoders find highly interpretable features in language models. In The Twelfth International Conference on Learning Representations, 2024. URL https://openreview.net/forum?id=F76bwRSLeK.", "tier": "established" }, { "key": "cywinski2025eliciting", - "text": "Bartosz Cywiński, Emil Ryd, Rowan Wang, Senthooran Rajamanoharan, Neel Nanda, Arthur Conmy, and Samuel Marks. Eliciting secret knowledge from language models. arXiv preprint arXiv:2510.01070, 2025.", + "text": "Bartosz Cywiński, Emil Ryd, Rowan Wang, Senthooran Rajamanoharan, Neel Nanda, Arthur Conmy, and Samuel Marks. Eliciting secret knowledge from language models, 2025. URL https://arxiv.org/abs/2510.01070.", "tier": "emerging" }, + { + "key": "dai-etal-2022-knowledge", + "text": "Damai Dai, Li Dong, Yaru Hao, Zhifang Sui, Baobao Chang, and Furu Wei. Knowledge neurons in pretrained transformers. In Smaranda Muresan, Preslav Nakov, and Aline Villavicencio (eds.), Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 8493–8502, Dublin, Ireland, May 2022. Association for Computational Linguistics. . URL https://aclanthology.org/2022.acl-long.581/.", + "tier": "established" + }, + { + "key": "dai-etal-2024-deepseekmoe", + "text": "Damai Dai, Chengqi Deng, Chenggang Zhao, R.X. Xu, Huazuo Gao, Deli Chen, Jiashi Li, Wangding Zeng, Xingkai Yu, Y. Wu, Zhenda Xie, Y.K. Li, Panpan Huang, Fuli Luo, Chong Ruan, Zhifang Sui, and Wenfeng Liang. DeepSeekMoE: Towards ultimate expert specialization in mixture-of-experts language models. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 1280–1297, Bangkok, Thailand, August 2024a. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.70/.", + "tier": "established" + }, + { + "key": "dai2024safe", + "text": "Josef Dai, Xuehai Pan, Ruiyang Sun, Jiaming Ji, Xinbo Xu, Mickel Liu, Yizhou Wang, and Yaodong Yang. Safe RLHF: Safe reinforcement learning from human feedback. In The Twelfth International Conference on Learning Representations, 2024b. URL https://openreview.net/forum?id=TyFrPOKYXw.", + "tier": "established" + }, { "key": "Dalvi_Durrani_Sajjad_Belinkov_Bau_Glass_2019", - "text": "Fahim Dalvi, Nadir Durrani, Hassan Sajjad, Yonatan Belinkov, Anthony Bau, and James Glass. What is one grain of sand in the desert? analyzing individual neurons in deep nlp models. Proceedings of the AAAI Conference on Artificial Intelligence, 33 (01): 6309–6317, Jul. 2019. . URL https://ojs.aaai.org/index.php/AAAI/article/view/4592.", + "text": "Fahim Dalvi, Nadir Durrani, Hassan Sajjad, Yonatan Belinkov, Anthony Bau, and James Glass. What is one grain of sand in the desert? analyzing individual neurons in deep nlp models. In Proceedings of the Thirty-Third AAAI Conference on Artificial Intelligence and Thirty-First Innovative Applications of Artificial Intelligence Conference and Ninth AAAI Symposium on Educational Advances in Artificial Intelligence, AAAI’19/IAAI’19/EAAI’19. AAAI Press, 2019. ISBN 978-1-57735-809-1. . URL https://doi.org/10.1609/aaai.v33i01.33016309.", "tier": "established" }, { @@ -2655,20 +3198,40 @@ "tier": "established" }, { - "key": "dbm", - "text": "Nicola De Cao, Michael Sejr Schlichtkrull, Wilker Aziz, and Ivan Titov. How do decisions emerge across layers in neural models? interpretation with differentiable masking. In Bonnie Webber, Trevor Cohn, Yulan He, and Yang Liu (eds.), Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 3243–3255, Online, November 2020. Association for Computational Linguistics. . URL https://aclanthology.org/2020.emnlp-main.262/.", - "tier": null + "key": "dao202mamba2", + "text": "Tri Dao and Albert Gu. Transformers are ssms: generalized models and efficient algorithms through structured state space duality. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.", + "tier": "established" }, { "key": "deepseekai2025deepseekv3technicalreport", "text": "DeepSeek-AI. Deepseek-v3 technical report, 2025. URL https://arxiv.org/abs/2412.19437.", "tier": "emerging" }, + { + "key": "deepseekv2", + "text": "DeepSeek-AI, Aixin Liu, Bei Feng, Bin Wang, Bingxuan Wang, Bo Liu, Chenggang Zhao, Chengqi Dengr, Chong Ruan, Damai Dai, Daya Guo, Dejian Yang, Deli Chen, Dongjie Ji, Erhang Li, Fangyun Lin, Fuli Luo, Guangbo Hao, Guanting Chen, Guowei Li, H. Zhang, Hanwei Xu, Hao Yang, Haowei Zhang, Honghui Ding, Huajian Xin, Huazuo Gao, Hui Li, Hui Qu, J. L. Cai, Jian Liang, Jianzhong Guo, Jiaqi Ni, Jiashi Li, Jin Chen, Jingyang Yuan, Junjie Qiu, Junxiao Song, Kai Dong, Kaige Gao, Kang Guan, Lean Wang, Lecong Zhang, Lei Xu, Leyi Xia, Liang Zhao, Liyue Zhang, Meng Li, Miaojun Wang, Mingchuan Zhang, Minghua Zhang, Minghui Tang, Mingming Li, Ning Tian, Panpan Huang, Peiyi Wang, Peng Zhang, Qihao Zhu, Qinyu Chen, Qiushi Du, R. J. Chen, R. L. Jin, Ruiqi Ge, Ruizhe Pan, Runxin Xu, Ruyi Chen, S. S. Li, Shanghao Lu, Shangyan Zhou, Shanhuang Chen, Shaoqing Wu, Shengfeng Ye, Shirong Ma, Shiyu Wang, Shuang Zhou, Shuiping Yu, Shunfeng Zhou, Size Zheng, T. Wang, Tian Pei, Tian Yuan, Tianyu Sun, W. L. Xiao, Wangding Zeng, Wei An, Wen Liu, Wenfeng Liang, Wenjun Gao, Wentao Zhang, X. Q. Li, Xiangyue Jin, Xianzu Wang, Xiao Bi, Xiaodong Liu, Xiaohan Wang, Xiaojin Shen, Xiaokang Chen, Xiaosha Chen, Xiaotao Nie, Xiaowen Sun, Xiaoxiang Wang, Xin Liu, Xin Xie, Xingkai Yu, Xinnan Song, Xinyi Zhou, Xinyu Yang, Xuan Lu, Xuecheng Su, Y. Wu, Y. K. Li, Y. X. Wei, Y. X. Zhu, Yanhong Xu, Yanping Huang, Yao Li, Yao Zhao, Yaofeng Sun, Yaohui Li, Yaohui Wang, Yi Zheng, Yichao Zhang, Yiliang Xiong, Yilong Zhao, Ying He, Ying Tang, Yishi Piao, Yixin Dong, Yixuan Tan, Yiyuan Liu, Yongji Wang, Yongqiang Guo, Yuchen Zhu, Yuduan Wang, Yuheng Zou, Yukun Zha, Yunxian Ma, Yuting Yan, Yuxiang You, Yuxuan Liu, Z. Z. Ren, Zehui Ren, Zhangli Sha, Zhe Fu, Zhen Huang, Zhen Zhang, Zhenda Xie, Zhewen Hao, Zhihong Shao, Zhiniu Wen, Zhipeng Xu, Zhongyu Zhang, Zhuoshu Li, Zihan Wang, Zihui Gu, Zilin Li, and Ziwei Xie. Deepseek-v2: A strong, economical, and efficient mixture-of-experts language model, 2024. URL https://arxiv.org/abs/2405.04434.", + "tier": "emerging" + }, + { + "key": "deepseekai2026deepseekv4highlyefficientmilliontoken", + "text": "DeepSeek-AI, Anyi Xu, Bangcai Lin, Bing Xue, Bingxuan Wang, Bingzheng Xu, Bochao Wu, Bowei Zhang, Chaofan Lin, Chen Dong, Chenchen Ling, Chengda Lu, Chenggang Zhao, Chengqi Deng, Chengyu Hou, Chenhao Xu, Chenze Shao, Chong Ruan, Conner Sun, Damai Dai, Daya Guo, Dejian Yang, Deli Chen, Donghao Li, Dongjie Ji, Erhang Li, Fang Wei, Fangyun Lin, Fangzhou Yuan, Feiyu Xia, Fucong Dai, Guangbo Hao, Guanting Chen, Guoai Cao, Guolai Meng, Guowei Li, Han Yu, Han Zhang, Hanwei Xu, Hao Li, Haofen Liang, Haoling Zhang, Haoming Luo, Haoran Wei, Haotian Yuan, Haowei Zhang, Haowen Luo, Haoyu Chen, Haozhe Ji, Hengqing Zhang, Honghui Ding, Hongxuan Tang, Huanqi Cao, Huazuo Gao, Hui Qu, Hui Zeng, J Yang, JQ Zhu, Jia Luo, Jia Song, Jia Yu, Jialiang Huang, Jialu Cai, Jian Liang, Jiangting Zhou, Jiasheng Ye, Jiashi Li, Jiaxin Xu, Jiewen Hu, Jieyu Yang, Jin Chen, Jin Yan, Jingchang Chen, Jingli Zhou, Jingting Xiang, Jingyang Yuan, Jingyuan Cheng, Jingzi Zhou, Jinhua Zhu, Jiping Yu, Joseph Sun, Jun Ran, Junguang Jiang, Junjie Qiu, Junlong Li, Junmin Zheng, Junxiao Song, Kai Dong, Kaige Gao, Kang Guan, Kexing Zhou, Kezhao Huang, Kuai Yu, Lean Wang, Lecong Zhang, Lei Wang, Leyi Xia, Li Zhang, Liang Zhao, Lihua Guo, Lingxiao Luo, Linwang Ma, Linyan Zhu, Litong Wang, Liyu Cai, Liyue Zhang, Longhao Chen, MS Di, MY Xu, Max Mei, Miaojun Wang, Mingchuan Zhang, Minghua Zhang, Minghui Tang, Mingming Li, Mingxu Zhou, Minmin Han, Ning Wang, Panpan Huang, Panpan Wang, Peixin Cong, Peiyi Wang, Peng Zhang, Qiancheng Wang, Qihao Zhu, Qingyang Li, Qinyu Chen, Qiushi Du, Qiwei Jiang, Rui Tian, Ruifan Xu, Ruijie Lu, Ruiling Xu, Ruiqi Ge, Ruisong Zhang, Ruizhe Pan, Runji Wang, Runqian Chen, Runqiu Yin, Runxin Xu, Ruomeng Shen, Ruoyu Zhang, Ruyi Chen, SH Liu, Shanghao Lu, Shangmian Sun, Shangyan Zhou, Shanhuang Chen, Shaofei Cai, Shaoheng Nie, Shaoqing Wu, Shaoyuan Chen, Shengding Hu, Shengyu Liu, Shiqiang Hu, Shirong Ma, Shiyu Wang, Shuiping Yu, Shunfeng Zhou, Shuting Pan, Shuying Yu, Songyang Zhou, Tao Ni, Tao Yun, Tian Jin, Tian Pei, Tian Ye, Tianle Lin, Tianran Ji, Tianyi Cui, Tianyuan Yue, Tingting Yu, Tun Wang, W Zhang, WL Xiao, Wangding Zeng, Wei An, Weilin Zhao, Wen Liu, Wenfeng Liang, Wenjie Pang, Wenjing Luo, Wenjing Yao, Wenjun Gao, Wenkai Yang, Wenlve Huang, Wenqing Hou, Wentao Zhang, Wenting Ma, Xi Gao, Xiang He, Xiangwen Wang, Xianzu Wang, Xiao Bi, Xiaodong Liu, Xiaohan Wang, Xiaokang Chen, Xiaokang Zhang, Xiaotao Nie, Xiaowen Sun, Xiaoxiang Wang, Xin Cheng, Xin Liu, Xin Xie, Xingchao Liu, Xingchen Liu, Xingkai Yu, Xingyou Li, Xinyu Yang, Xinyu Zhang, Xu Chen, Xuanyu Wang, Xuecheng Su, Xueyin Chen, Xuheng Lin, Xuwei Fu, YC Yan, YQ Wang, YW Ma, Yanfeng Luo, Yang Zhang, Yanhong Xu, Yanru Ma, Yanwen Huang, Yao Li, Yao Li, Yao Xu, Yao Zhao, Yaofeng Sun, Yaohui Wang, Yi Qian, Yi Shao, Yi Yu, Yichao Zhang, Yifan Ding, Yifan Shi, Yijia Wu, Yiliang Xiong, Yiling Ma, Ying He, Ying Tang, Ying Zhou, Yingjia Luo, Yinmin Zhong, Yishi Piao, Yisong Wang, Yixiang Zhang, Yixiao Chen, Yixuan Tan, Yixuan Wei, Yiyang Ma, Yiyuan Liu, Yonglun Yang, Yongqiang Guo, Yongtong Wu, Yu Wu, YuKun Li, Yuan Cheng, Yuan Ou, Yuanfan Xu, Yuanhao Li, Yuduan Wang, Yuehan Yang, Yuer Xu, Yuhan Wu, Yuhao Meng, Yuheng Zou, Yukun Zha, Yunfan Xiong, Yupeng Chen, Yuping Lin, Yuqian Cao, Yuqian Wang, Yushun Zhang, Yuting Yan, Yutong Lin, Yuxian Gu, Yuxiang Luo, Yuxiang You, Yuxuan Liu, Yuxuan Zhou, Yuyang Zhou, Yuzhen Huang, ZF Wu, Zehao Wang, Zehua Zhao, Zehui Ren, Zekai Zhang, Zhangli Sha, Zhe Fu, Zhe Ju, Zhean Xu, Zhenda Xie, Zhengyan Zhang, Zheren Gao, Zhewen Hao, Zhibin Gou, Zhicheng Ma, Zhigang Yan, Zhihong Shao, Zhixian Huang, Zhixuan Chen, Zhiyu Wu, Zhizhou Ren, Zhongyu Wu, Zhuoshu Li, Zhuping Zhang, Zian Xu, Zihao Wang, Zihua Qu, Zihui Gu, Zijia Zhu, Zilin Li, Zipeng Zhang, Ziwei Xie, Ziyi Gao, Ziyi Wan, Zizheng Pan, and Zongqing Yao. Deepseek-v4: Towards highly efficient million-token context intelligence, 2026. URL https://arxiv.org/abs/2606.19348.", + "tier": "emerging" + }, + { + "key": "denison2024sycophancy", + "text": "Carson Denison, Monte MacDiarmid, Fazl Barez, David Duvenaud, Shauna Kravec, Samuel Marks, Nicholas Schiefer, Ryan Soklaski, Alex Tamkin, Jared Kaplan, Buck Shlegeris, Samuel R. Bowman, Ethan Perez, and Evan Hubinger. Sycophancy to subterfuge: Investigating reward-tampering in large language models, 2024. URL https://arxiv.org/abs/2406.10162.", + "tier": "emerging" + }, { "key": "devlin2019bert", "text": "Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. BERT: Pre-training of deep bidirectional transformers for language understanding. In Jill Burstein, Christy Doran, and Thamar Solorio (eds.), Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 4171–4186, Minneapolis, Minnesota, June 2019. Association for Computational Linguistics. . URL https://aclanthology.org/N19-1423/.", "tier": "established" }, + { + "key": "snmf", + "text": "Chris H.Q. Ding, Tao Li, and Michael I. Jordan. Convex and semi-nonnegative matrix factorizations. IEEE Transactions on Pattern Analysis and Machine Intelligence, 32 (1): 45–55, 2010. .", + "tier": "established" + }, { "key": "distill_pub", "text": "Distill. Distill. https://distill.pub/, 2017.", @@ -2682,16 +3245,16 @@ { "key": "NIPS2003_1843e35d", "text": "David Donoho and Victoria Stodden. When does non-negative matrix factorization give a correct decomposition into parts? In S. Thrun, L. Saul, and B. Schölkopf (eds.), Advances in Neural Information Processing Systems, volume 16. MIT Press, 2003. URL https://proceedings.neurips.cc/paper_files/paper/2003/file/1843e35d41ccf6e63273495ba42df3c1-Paper.pdf.", - "tier": null + "tier": "established" }, { "key": "dooms2025compositionality", "text": "Thomas Dooms, Ward Gauderis, Geraint A. Wiggins, and Jose Oramas. Compositionality unlocks deep interpretable models. In AAAI’25 workshop on CoLoRAI - Connecting Low-Rank Representations in AI, 2025. URL https://openreview.net/forum?id=bXAt5iZ69l.", - "tier": "established" + "tier": "emerging" }, { "key": "draye2025spartan", - "text": "Florent Draye, Anson Lei, Ingmar Posner, and Bernhard Schölkopf. Sparse attention post-training for mechanistic interpretability. arXiv preprint arXiv:2512.05865, 2025.", + "text": "Florent Draye, Anson Lei, Hsiao-Ru Pan, Ingmar Posner, and Bernhard Schölkopf. Intrinsically interpretable attention via sparse post-training, 2026. URL https://arxiv.org/abs/2512.05865.", "tier": "emerging" }, { @@ -2699,25 +3262,50 @@ "text": "Mengnan Du, Fengxiang He, Na Zou, Dacheng Tao, and Xia Hu. Shortcut learning of large language models in natural language understanding. Commun. ACM, 67 (1): 110–120, December 2023. ISSN 0001-0782. . URL https://doi.org/10.1145/3596490.", "tier": "established" }, + { + "key": "du-etal-2026-automated", + "text": "Yuntao Du, Zitao Li, Bolin Ding, Yaliang Li, Hanshen Xiao, Jingren Zhou, and Ninghui Li. Automated profile inference with language model agents. In Maria Liakata, Viviane P. Moreira, Jiajun Zhang, and David Jurgens (eds.), Findings of the Association for Computational Linguistics: ACL 2026, pp. 9977–10000, San Diego, California, United States, July 2026. Association for Computational Linguistics. ISBN 979-8-89176-395-1. . URL https://aclanthology.org/2026.findings-acl.485/.", + "tier": "established" + }, + { + "key": "du2024understanding", + "text": "Zhengxiao Du, Aohan Zeng, Yuxiao Dong, and Jie Tang. Understanding emergent abilities of language models from the loss perspective. In Advances in Neural Information Processing Systems, volume 37, 2024. URL https://openreview.net/forum?id=35DAviqMFo.", + "tier": "established" + }, + { + "key": "minder_overcoming_2025", + "text": "Clément Dumas, Julian Minder, Caden Juang, Bilal Chughtai, and Neel Nanda. Overcoming sparsity artifacts in crosscoders to interpret chat-tuning. In Mechanistic Interpretability Workshop at NeurIPS 2025, 2025a. URL https://openreview.net/forum?id=CdiCNA1Y2R.", + "tier": "emerging" + }, { "key": "dumas_what_2025", - "text": "Clément Dumas, Julian Minder, and Neel Nanda. What we learned trying to diff base and chat models (and why it matters). https://www.alignmentforum.org/posts/xmpauEXEerzYcJKNm/what-we-learned-trying-to-diff-base-and-chat-models-and-why, 2025.", + "text": "Clément Dumas, Julian Minder, and Neel Nanda. What we learned trying to diff base and chat models (and why it matters). https://www.alignmentforum.org/posts/xmpauEXEerzYcJKNm/what-we-learned-trying-to-diff-base-and-chat-models-and-why, 2025b.", "tier": "emerging" }, { "key": "dunefsky2025one", "text": "Jacob Dunefsky and Arman Cohan. One-shot optimized steering vectors mediate safety-relevant behaviors in llms. In Second Conference on Language Modeling, 2025.", - "tier": null + "tier": "established" }, { "key": "dunefsky2024transcodersinterpretablellmfeature", "text": "Jacob Dunefsky, Philippe Chlenski, and Neel Nanda. Transcoders find interpretable llm feature circuits. In Proceedings of the 38th International Conference on Neural Information Processing Systems, NIPS ’24, Red Hook, NY, USA, 2024. Curran Associates Inc. ISBN 9798331314385.", "tier": "established" }, + { + "key": "durmus2024towards", + "text": "Esin DURMUS, Karina Nguyen, Thomas Liao, Nicholas Schiefer, Amanda Askell, Anton Bakhtin, Carol Chen, Zac Hatfield-Dodds, Danny Hernandez, Nicholas Joseph, Liane Lovitt, Sam McCandlish, Orowa Sikder, Alex Tamkin, Janel Thamkul, Jared Kaplan, Jack Clark, and Deep Ganguli. Towards measuring the representation of subjective global opinions in language models. In First Conference on Language Modeling, 2024. URL https://openreview.net/forum?id=zl16jLb91v.", + "tier": "established" + }, { "key": "elazar-etal-2021-amnesic", "text": "Yanai Elazar, Shauli Ravfogel, Alon Jacovi, and Yoav Goldberg. Amnesic probing: Behavioral explanation with amnesic counterfactuals. Transactions of the Association for Computational Linguistics, 9: 160–175, 03 2021. ISSN 2307-387X. . URL https://doi.org/10.1162/tacl_a_00359.", - "tier": null + "tier": "established" + }, + { + "key": "elazar2022measuring", + "text": "Yanai Elazar, Nora Kassner, Shauli Ravfogel, Amir Feder, Abhilasha Ravichander, Marius Mosbach, Yonatan Belinkov, Hinrich Schütze, and Yoav Goldberg. Measuring causal effects of data statistics on language model’s ‘factual’ predictions, 2023. URL https://arxiv.org/abs/2207.14251.", + "tier": "emerging" }, { "key": "elhage2021mathematical", @@ -2734,31 +3322,86 @@ "text": "Nelson Elhage, Tristan Hume, Catherine Olsson, Neel Nanda, Tom Henighan, Scott Johnston, Sheer El Showk, Nicholas Joseph, Nova DasSarma, Ben Mann, Danny Hernandez, Amanda Askell, Kamal Ndousse, Andy Jones, Dawn Drain, Anna Chen, Yuntao Bai, Deep Ganguli, Liane Lovitt, Zac Hatfield-Dodds, Jackson Kernion, Tom Conerly, Shauna Kravec, Stanislav Fort, Saurav Kadavath, Josh Jacobson, Eli Tran-Johnson, Jared Kaplan, Jack Clark, Tom Brown, Sam McCandlish, Dario Amodei, and Christopher Olah. Softmax linear units. Transformer Circuits Thread, 2022b.", "tier": "emerging" }, + { + "key": "elman1990finding", + "text": "Jeffrey L. Elman. Finding structure in time. Cognitive Science, 14 (2): 179–211, 1990. .", + "tier": "established" + }, + { + "key": "elman1991distributed", + "text": "Jeffrey L. Elman. Distributed representations, simple recurrent networks, and grammatical structure. Machine Learning, 7 (2): 195–225, 1991. .", + "tier": "established" + }, { "key": "engels2025languagemodelfeaturesonedimensionally", "text": "Joshua Engels, Eric J Michaud, Isaac Liao, Wes Gurnee, and Max Tegmark. Not all language model features are one-dimensionally linear. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=d63a4AM4hb.", "tier": "established" }, + { + "key": "ensign2024investigating", + "text": "Danielle Ensign and Adrià Garriga-Alonso. Investigating the indirect object identification circuit in mamba, 2024. URL https://arxiv.org/abs/2407.14008.", + "tier": "emerging" + }, + { + "key": "eshuijs2026honestansweraidingllmjudges", + "text": "Leon Eshuijs, Archie Chaudhury, Alan McBeth, and Ethan Nguyen. But what is your honest answer? aiding llm-judges with honest alternatives using steering vectors, 2026. URL https://arxiv.org/abs/2505.17760.", + "tier": "emerging" + }, + { + "key": "alphaedit", + "text": "Junfeng Fang, Houcheng Jiang, Kun Wang, Yunshan Ma, Jie Shi, Xiang Wang, Xiangnan He, and Tat-Seng Chua. Alphaedit: Null-space constrained model editing for language models. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=HvSytvg3Jh.", + "tier": "established" + }, + { + "key": "fang2026nextguardtrainingfreestreamingsafeguard", + "text": "Junfeng Fang, Nachuan Chen, Houcheng Jiang, Dan Zhang, Fei Shen, Xiang Wang, Xiangnan He, and Tat-Seng Chua. Next-guard: Training-free streaming safeguard without token-level labels, 2026. URL https://arxiv.org/abs/2603.02219.", + "tier": "emerging" + }, + { + "key": "fanous2025syceval", + "text": "Aaron Fanous, Jacob Goldberg, Ank Agarwal, Joanna Lin, Anson Zhou, Sonnet Xu, Vasiliki Bikia, Roxana Daneshjou, and Sanmi Koyejo. Syceval: Evaluating llm sycophancy. In Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society, pp. 893–900, 2025.", + "tier": "established" + }, { "key": "ferrando2024primer", - "text": "Javier Ferrando, Gabriele Sarti, Arianna Bisazza, and Marta R Costa-Jussà. A primer on the inner workings of transformer-based language models. arXiv preprint arXiv:2405.00208, 2024.", + "text": "Javier Ferrando, Gabriele Sarti, Arianna Bisazza, and Marta R. Costa-jussà. A primer on the inner workings of transformer-based language models, 2024. URL https://arxiv.org/abs/2405.00208.", "tier": "emerging" }, + { + "key": "feucht2026arithmeticwildllamauses", + "text": "Sheridan Feucht, Tal Haklay, Usha Bhalla, Daniel Wurgaft, Can Rager, Raphaël Sarfati, Jack Merullo, Thomas McGrath, Owen Lewis, Ekdeep Singh Lubana, Thomas Fel, and Atticus Geiger. Arithmetic in the wild: Llama uses base-10 addition to reason about cyclic concepts, 2026. URL https://arxiv.org/abs/2605.01148.", + "tier": "emerging" + }, + { + "key": "sparsecoding", + "text": "David J. Field. What is the goal of sensory coding? Neural Computation, 6 (4): 559–601, 1994. .", + "tier": "established" + }, { "key": "foote2023neurongraphinterpretinglanguage", "text": "Alex Foote, Neel Nanda, Esben Kran, Ioannis Konstas, and Fazl Barez. N2g: A SCALABLE APPROACH FOR QUANTIFYING INTERPRETABLE NEURON REPRESENTATION IN LLMS. In ICLR 2023 Workshop on Trustworthy and Reliable Large-Scale Machine Learning Models, 2023.", "tier": "emerging" }, { - "key": "petri2025", - "text": "Kai Fronsdal, Isha Gupta, Abhay Sheshadri, Jonathan Michala, Stephen McAleer, Rowan Wang, Sara Price, and Sam Bowman. Petri: Parallel exploration of risky interactions, 2025. URL https://github.com/safety-research/petri.", + "key": "fornasiere2026languagemodelsrecognizedropout", + "text": "Damiano Fornasiere, Mirko Bronzi, Spencer Kitts, Alessandro Palmas, Yoshua Bengio, and Oliver Richardson. Language models recognize dropout and gaussian noise applied to their activations, 2026. URL https://arxiv.org/abs/2604.17465.", "tier": "emerging" }, { - "key": "petri2026v2", - "text": "Kai Fronsdal, Jonathan Michala, and Sam Bowman. Petri 2.0: New scenarios, new model comparisons, and improved eval-awareness mitigations, 2026. URL https://alignment.anthropic.com/2026/petri-v2/.", + "key": "frankle2020linear", + "text": "Jonathan Frankle, Gintare Karolina Dziugaite, Daniel M. Roy, and Michael Carbin. Linear mode connectivity and the lottery ticket hypothesis. In International Conference on Machine Learning, pp. 3259–3269, 2020.", + "tier": "established" + }, + { + "key": "nla", + "text": "Kit Fraser-Taliente, Subhash Kantamneni, Euan Ong, Dan Mossing, Christina Lu, Paul C. Bogdan, Emmanuel Ameisen, James Chen, Dzmitry Kishylau, Adam Pearce, Julius Tarng, Alex Wu, Jeff Wu, Yang Zhang, Daniel M. Ziegler, Evan Hubinger, Joshua Batson, Jack Lindsey, Samuel Zimmerman, and Samuel Marks. Natural language autoencoders produce unsupervised explanations of llm activations. Transformer Circuits Thread, 2026. URL https://transformer-circuits.pub/2026/nla/index.html.", "tier": "emerging" }, + { + "key": "pmlr-v267-fu25e", + "text": "Tingchen Fu, Mrinank Sharma, Philip Torr, Shay B Cohen, David Krueger, and Fazl Barez. PoisonBench: Assessing language model vulnerability to poisoned preference data. In Aarti Singh, Maryam Fazel, Daniel Hsu, Simon Lacoste-Julien, Felix Berkenkamp, Tegan Maharaj, Kiri Wagstaff, and Jerry Zhu (eds.), Proceedings of the 42nd International Conference on Machine Learning, volume 267 of Proceedings of Machine Learning Research, pp. 17806–17831. PMLR, 13–19 Jul 2025. URL https://proceedings.mlr.press/v267/fu25e.html.", + "tier": "established" + }, { "key": "ganguli2022redteaminglanguagemodels", "text": "Deep Ganguli, Liane Lovitt, Jackson Kernion, Amanda Askell, Yuntao Bai, Saurav Kadavath, Ben Mann, Ethan Perez, Nicholas Schiefer, Kamal Ndousse, Andy Jones, Sam Bowman, Anna Chen, Tom Conerly, Nova DasSarma, Dawn Drain, Nelson Elhage, Sheer El-Showk, Stanislav Fort, Zac Hatfield-Dodds, Tom Henighan, Danny Hernandez, Tristan Hume, Josh Jacobson, Scott Johnston, Shauna Kravec, Catherine Olsson, Sam Ringer, Eli Tran-Johnson, Dario Amodei, Tom Brown, Nicholas Joseph, Sam McCandlish, Chris Olah, Jared Kaplan, and Jack Clark. Red teaming language models to reduce harms: Methods, scaling behaviors, and lessons learned, 2022. URL https://arxiv.org/abs/2209.07858.", @@ -2771,18 +3414,18 @@ }, { "key": "gao2020pile", - "text": "Leo Gao, Stella Biderman, Sid Black, Laurence Golding, Travis Hoppe, Charles Foster, Jason Phang, Horace He, Anish Thite, Noa Nabeshima, et al. The pile: An 800gb dataset of diverse text for language modeling. arXiv preprint arXiv:2101.00027, 2020.", - "tier": null + "text": "Leo Gao, Stella Biderman, Sid Black, Laurence Golding, Travis Hoppe, Charles Foster, Jason Phang, Horace He, Anish Thite, Noa Nabeshima, Shawn Presser, and Connor Leahy. The pile: An 800gb dataset of diverse text for language modeling, 2020. URL https://arxiv.org/abs/2101.00027.", + "tier": "emerging" }, { "key": "gao2024scalingevaluatingsparseautoencoders", - "text": "Leo Gao, Tom Dupre la Tour, Henk Tillman, Gabriel Goh, Rajan Troll, Alec Radford, Ilya Sutskever, Jan Leike, and Jeffrey Wu. Scaling and evaluating sparse autoencoders. In The Thirteenth International Conference on Learning Representations, 2025a. URL https://openreview.net/forum?id=tcsZt9ZNKD.", + "text": "Leo Gao, Tom Dupre la Tour, Henk Tillman, Gabriel Goh, Rajan Troll, Alec Radford, Ilya Sutskever, Jan Leike, and Jeffrey Wu. Scaling and evaluating sparse autoencoders. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=tcsZt9ZNKD.", "tier": "established" }, { "key": "gao2025weight", - "text": "Leo Gao, Achyuta Rajaram, Jacob Coxon, Soham V. Govande, Bowen Baker, and Dan Mossing. Weight-sparse transformers have interpretable circuits, 2025b. URL https://arxiv.org/abs/2511.13653.", - "tier": "emerging" + "text": "Leo Gao, Achyuta Rajaram, Jacob Coxon, Soham V. Govande, Bowen Baker, and Daniel P Mossing. Weight-sparse transformers have interpretable circuits. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=Vw10TpuFCp.", + "tier": "established" }, { "key": "detectvul", @@ -2790,65 +3433,115 @@ "tier": "established" }, { - "key": "garfinkle2016robust", - "text": "Charles J Garfinkle and Christopher J Hillar. Robust identifiability in sparse dictionary learning. arXiv preprint arXiv:1606.06997, 2016.", - "tier": null - }, - { - "key": "ge_evolution_2025", - "text": "Xuyang Ge, Wentao Shu, Jiaxing Wu, Yunhua Zhou, Zhengfu He, and Xipeng Qiu. Evolution of concepts in language model pre-training. arXiv preprint arXiv:2509.17196, 2025.", - "tier": "emerging" + "key": "pruthi2020estimatingtrainingdatainfluence", + "text": "Garima, Frederick Liu, Satyen Kale, and Mukund Sundararajan. Estimating training data influence by tracing gradient descent. In Proceedings of the 34th International Conference on Neural Information Processing Systems, NIPS ’20, Red Hook, NY, USA, 2020. Curran Associates Inc. ISBN 9781713829546.", + "tier": "established" }, { "key": "ge2026evolution", "text": "Xuyang Ge, Wentao Shu, Jiaxing Wu, Yunhua Zhou, Zhengfu He, and Xipeng Qiu. Evolution of concepts in language model pre-training. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=qKmXz1UR0Z.", - "tier": null + "tier": "established" }, { "key": "geiger2021causalabstractionsneuralnetworks", "text": "Atticus Geiger, Hanson Lu, Thomas Icard, and Christopher Potts. Causal abstractions of neural networks. In M. Ranzato, A. Beygelzimer, Y. Dauphin, P.S. Liang, and J. Wortman Vaughan (eds.), Advances in Neural Information Processing Systems, volume 34, pp. 9574–9586. Curran Associates, Inc., 2021. URL https://proceedings.neurips.cc/paper_files/paper/2021/file/4f5c422f4d49a5a807eda27434231040-Paper.pdf.", "tier": "established" }, + { + "key": "pmlr-v162-geiger22a", + "text": "Atticus Geiger, Zhengxuan Wu, Hanson Lu, Josh Rozner, Elisa Kreiss, Thomas Icard, Noah Goodman, and Christopher Potts. Inducing causal structure for interpretable neural networks. In Kamalika Chaudhuri, Stefanie Jegelka, Le Song, Csaba Szepesvari, Gang Niu, and Sivan Sabato (eds.), Proceedings of the 39th International Conference on Machine Learning, volume 162 of Proceedings of Machine Learning Research, pp. 7324–7338. PMLR, 17–23 Jul 2022. URL https://proceedings.mlr.press/v162/geiger22a.html.", + "tier": "established" + }, { "key": "das", "text": "Atticus Geiger, Zhengxuan Wu, Christopher Potts, Thomas Icard, and Noah Goodman. Finding alignments between interpretable causal variables and distributed neural representations. In Francesco Locatello and Vanessa Didelez (eds.), Proceedings of the Third Conference on Causal Learning and Reasoning, volume 236 of Proceedings of Machine Learning Research, pp. 160–187. PMLR, 01–03 Apr 2024. URL https://proceedings.mlr.press/v236/geiger24a.html.", - "tier": null + "tier": "established" + }, + { + "key": "JMLR:v26:23-0058", + "text": "Atticus Geiger, Duligur Ibeling, Amir Zur, Maheep Chaudhary, Sonakshi Chauhan, Jing Huang, Aryaman Arora, Zhengxuan Wu, Noah Goodman, Christopher Potts, and Thomas Icard. Causal abstraction: A theoretical foundation for mechanistic interpretability. Journal of Machine Learning Research, 26 (83): 1–64, 2025. URL http://jmlr.org/papers/v26/23-0058.html.", + "tier": "established" }, { "key": "Geirhos_2020", "text": "Robert Geirhos, Jörn-Henrik Jacobsen, Claudio Michaelis, Richard Zemel, Wieland Brendel, Matthias Bethge, and Felix A. Wichmann. Shortcut learning in deep neural networks. Nature Machine Intelligence, 2 (11): 665–673, November 2020. ISSN 2522-5839. . URL http://dx.doi.org/10.1038/s42256-020-00257-z.", "tier": "established" }, + { + "key": "ekfac", + "text": "Thomas George, César Laurent, Xavier Bouthillier, Nicolas Ballas, and Pascal Vincent. Fast approximate natural gradient descent in a kronecker-factored eigenbasis. In Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS’18, pp. 9573–9583, Red Hook, NY, USA, 2018. Curran Associates Inc.", + "tier": "established" + }, { "key": "georgiev2023journeydestinationdataguides", "text": "Kristian Georgiev, Joshua Vendrow, Hadi Salman, Sung Min Park, and Aleksander Madry. The journey, not the destination: How data guides diffusion models, 2023. URL https://arxiv.org/abs/2312.06205.", - "tier": null + "tier": "emerging" + }, + { + "key": "geva-etal-2021-transformer", + "text": "Mor Geva, Roei Schuster, Jonathan Berant, and Omer Levy. Transformer feed-forward layers are key-value memories. In Marie-Francine Moens, Xuanjing Huang, Lucia Specia, and Scott Wen-tau Yih (eds.), Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing, pp. 5484–5495, Online and Punta Cana, Dominican Republic, November 2021. Association for Computational Linguistics. . URL https://aclanthology.org/2021.emnlp-main.446/.", + "tier": "established" }, { "key": "patchscopes", "text": "Asma Ghandeharioun, Avi Caciularu, Adam Pearce, Lucas Dixon, and Mor Geva. Patchscopes: a unifying framework for inspecting hidden representations of language models. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.", - "tier": null + "tier": "established" + }, + { + "key": "rnnfsa1992", + "text": "C. L. Giles, C. B. Miller, D. Chen, H. H. Chen, G. Z. Sun, and Y. C. Lee. Learning and extracting finite state automata with second-order recurrent neural networks. Neural Comput., 4 (3): 393–405, May 1992. ISSN 0899-7667. . URL https://doi.org/10.1162/neco.1992.4.3.393.", + "tier": "established" }, { "key": "giulianelli-etal-2018-hood", "text": "Mario Giulianelli, Jack Harding, Florian Mohnert, Dieuwke Hupkes, and Willem Zuidema. Under the hood: Using diagnostic classifiers to investigate and improve how language models track agreement information. In Tal Linzen, Grzegorz Chrupała, and Afra Alishahi (eds.), Proceedings of the 2018 EMNLP Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP, pp. 240–248, Brussels, Belgium, November 2018. Association for Computational Linguistics. . URL https://aclanthology.org/W18-5426/.", - "tier": null + "tier": "emerging" + }, + { + "key": "goel", + "text": "Avichal Goel, Yoon Kim, Nir N Shavit, and Tony T. Wang. Learning to interpret weight differences in language models. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=6As4wfTB77.", + "tier": "established" }, { "key": "goldowskydill2023localizingmodelbehaviorpath", "text": "Nicholas Goldowsky-Dill, Chris MacLeod, Lucas Sato, and Aryaman Arora. Localizing model behavior with path patching, 2023. URL https://arxiv.org/abs/2304.05969.", "tier": "emerging" }, + { + "key": "llama3", + "text": "Aaron Grattafiori, Abhimanyu Dubey, Abhinav Jauhri, Abhinav Pandey, Abhishek Kadian, Ahmad Al-Dahle, Aiesha Letman, Akhil Mathur, Alan Schelten, Alex Vaughan, Amy Yang, Angela Fan, Anirudh Goyal, Anthony Hartshorn, Aobo Yang, Archi Mitra, Archie Sravankumar, Artem Korenev, Arthur Hinsvark, Arun Rao, Aston Zhang, Aurelien Rodriguez, Austen Gregerson, Ava Spataru, Baptiste Roziere, Bethany Biron, Binh Tang, Bobbie Chern, Charlotte Caucheteux, Chaya Nayak, Chloe Bi, Chris Marra, Chris McConnell, Christian Keller, Christophe Touret, Chunyang Wu, Corinne Wong, Cristian Canton Ferrer, Cyrus Nikolaidis, Damien Allonsius, Daniel Song, Danielle Pintz, Danny Livshits, Danny Wyatt, David Esiobu, Dhruv Choudhary, Dhruv Mahajan, Diego Garcia-Olano, Diego Perino, Dieuwke Hupkes, Egor Lakomkin, Ehab AlBadawy, Elina Lobanova, Emily Dinan, Eric Michael Smith, Filip Radenovic, Francisco Guzmán, Frank Zhang, Gabriel Synnaeve, Gabrielle Lee, Georgia Lewis Anderson, Govind Thattai, Graeme Nail, Gregoire Mialon, Guan Pang, Guillem Cucurell, Hailey Nguyen, Hannah Korevaar, Hu Xu, Hugo Touvron, Iliyan Zarov, Imanol Arrieta Ibarra, Isabel Kloumann, Ishan Misra, Ivan Evtimov, Jack Zhang, Jade Copet, Jaewon Lee, Jan Geffert, Jana Vranes, Jason Park, Jay Mahadeokar, Jeet Shah, Jelmer van der Linde, Jennifer Billock, Jenny Hong, Jenya Lee, Jeremy Fu, Jianfeng Chi, Jianyu Huang, Jiawen Liu, Jie Wang, Jiecao Yu, Joanna Bitton, Joe Spisak, Jongsoo Park, Joseph Rocca, Joshua Johnstun, Joshua Saxe, Junteng Jia, Kalyan Vasuden Alwala, Karthik Prasad, Kartikeya Upasani, Kate Plawiak, Ke Li, Kenneth Heafield, Kevin Stone, Khalid El-Arini, Krithika Iyer, Kshitiz Malik, Kuenley Chiu, Kunal Bhalla, Kushal Lakhotia, Lauren Rantala-Yeary, Laurens van der Maaten, Lawrence Chen, Liang Tan, Liz Jenkins, Louis Martin, Lovish Madaan, Lubo Malo, Lukas Blecher, Lukas Landzaat, Luke de Oliveira, Madeline Muzzi, Mahesh Pasupuleti, Mannat Singh, Manohar Paluri, Marcin Kardas, Maria Tsimpoukelli, Mathew Oldham, Mathieu Rita, Maya Pavlova, Melanie Kambadur, Mike Lewis, Min Si, Mitesh Kumar Singh, Mona Hassan, Naman Goyal, Narjes Torabi, Nikolay Bashlykov, Nikolay Bogoychev, Niladri Chatterji, Ning Zhang, Olivier Duchenne, Onur Çelebi, Patrick Alrassy, Pengchuan Zhang, Pengwei Li, Petar Vasic, Peter Weng, Prajjwal Bhargava, Pratik Dubal, Praveen Krishnan, Punit Singh Koura, Puxin Xu, Qing He, Qingxiao Dong, Ragavan Srinivasan, Raj Ganapathy, Ramon Calderer, Ricardo Silveira Cabral, Robert Stojnic, Roberta Raileanu, Rohan Maheswari, Rohit Girdhar, Rohit Patel, Romain Sauvestre, Ronnie Polidoro, Roshan Sumbaly, Ross Taylor, Ruan Silva, Rui Hou, Rui Wang, Saghar Hosseini, Sahana Chennabasappa, Sanjay Singh, Sean Bell, Seohyun Sonia Kim, Sergey Edunov, Shaoliang Nie, Sharan Narang, Sharath Raparthy, Sheng Shen, Shengye Wan, Shruti Bhosale, Shun Zhang, Simon Vandenhende, Soumya Batra, Spencer Whitman, Sten Sootla, Stephane Collot, Suchin Gururangan, Sydney Borodinsky, Tamar Herman, Tara Fowler, Tarek Sheasha, Thomas Georgiou, Thomas Scialom, Tobias Speckbacher, Todor Mihaylov, Tong Xiao, Ujjwal Karn, Vedanuj Goswami, Vibhor Gupta, Vignesh Ramanathan, Viktor Kerkez, Vincent Gonguet, Virginie Do, Vish Vogeti, Vítor Albiero, Vladan Petrovic, Weiwei Chu, Wenhan Xiong, Wenyin Fu, Whitney Meers, Xavier Martinet, Xiaodong Wang, Xiaofang Wang, Xiaoqing Ellen Tan, Xide Xia, Xinfeng Xie, Xuchao Jia, Xuewei Wang, Yaelle Goldschlag, Yashesh Gaur, Yasmine Babaei, Yi Wen, Yiwen Song, Yuchen Zhang, Yue Li, Yuning Mao, Zacharie Delpierre Coudert, Zheng Yan, Zhengxing Chen, Zoe Papakipos, Aaditya Singh, Aayushi Srivastava, Abha Jain, Adam Kelsey, Adam Shajnfeld, Adithya Gangidi, Adolfo Victoria, Ahuva Goldstand, Ajay Menon, Ajay Sharma, Alex Boesenberg, Alexei Baevski, Allie Feinstein, Amanda Kallet, Amit Sangani, Amos Teo, Anam Yunus, Andrei Lupu, Andres Alvarado, Andrew Caples, Andrew Gu, Andrew Ho, Andrew Poulton, Andrew Ryan, Ankit Ramchandani, Annie Dong, Annie Franco, Anuj Goyal, Aparajita Saraf, Arkabandhu Chowdhury, Ashley Gabriel, Ashwin Bharambe, Assaf Eisenman, Azadeh Yazdan, Beau James, Ben Maurer, Benjamin Leonhardi, Bernie Huang, Beth Loyd, Beto De Paola, Bhargavi Paranjape, Bing Liu, Bo Wu, Boyu Ni, Braden Hancock, Bram Wasti, Brandon Spence, Brani Stojkovic, Brian Gamido, Britt Montalvo, Carl Parker, Carly Burton, Catalina Mejia, Ce Liu, Changhan Wang, Changkyu Kim, Chao Zhou, Chester Hu, Ching-Hsiang Chu, Chris Cai, Chris Tindal, Christoph Feichtenhofer, Cynthia Gao, Damon Civin, Dana Beaty, Daniel Kreymer, Daniel Li, David Adkins, David Xu, Davide Testuggine, Delia David, Devi Parikh, Diana Liskovich, Didem Foss, Dingkang Wang, Duc Le, Dustin Holland, Edward Dowling, Eissa Jamil, Elaine Montgomery, Eleonora Presani, Emily Hahn, Emily Wood, Eric-Tuan Le, Erik Brinkman, Esteban Arcaute, Evan Dunbar, Evan Smothers, Fei Sun, Felix Kreuk, Feng Tian, Filippos Kokkinos, Firat Ozgenel, Francesco Caggioni, Frank Kanayet, Frank Seide, Gabriela Medina Florez, Gabriella Schwarz, Gada Badeer, Georgia Swee, Gil Halpern, Grant Herman, Grigory Sizov, Guangyi, Zhang, Guna Lakshminarayanan, Hakan Inan, Hamid Shojanazeri, Han Zou, Hannah Wang, Hanwen Zha, Haroun Habeeb, Harrison Rudolph, Helen Suk, Henry Aspegren, Hunter Goldman, Hongyuan Zhan, Ibrahim Damlaj, Igor Molybog, Igor Tufanov, Ilias Leontiadis, Irina-Elena Veliche, Itai Gat, Jake Weissman, James Geboski, James Kohli, Janice Lam, Japhet Asher, Jean-Baptiste Gaya, Jeff Marcus, Jeff Tang, Jennifer Chan, Jenny Zhen, Jeremy Reizenstein, Jeremy Teboul, Jessica Zhong, Jian Jin, Jingyi Yang, Joe Cummings, Jon Carvill, Jon Shepard, Jonathan McPhie, Jonathan Torres, Josh Ginsburg, Junjie Wang, Kai Wu, Kam Hou U, Karan Saxena, Kartikay Khandelwal, Katayoun Zand, Kathy Matosich, Kaushik Veeraraghavan, Kelly Michelena, Keqian Li, Kiran Jagadeesh, Kun Huang, Kunal Chawla, Kyle Huang, Lailin Chen, Lakshya Garg, Lavender A, Leandro Silva, Lee Bell, Lei Zhang, Liangpeng Guo, Licheng Yu, Liron Moshkovich, Luca Wehrstedt, Madian Khabsa, Manav Avalani, Manish Bhatt, Martynas Mankus, Matan Hasson, Matthew Lennie, Matthias Reso, Maxim Groshev, Maxim Naumov, Maya Lathi, Meghan Keneally, Miao Liu, Michael L. Seltzer, Michal Valko, Michelle Restrepo, Mihir Patel, Mik Vyatskov, Mikayel Samvelyan, Mike Clark, Mike Macey, Mike Wang, Miquel Jubert Hermoso, Mo Metanat, Mohammad Rastegari, Munish Bansal, Nandhini Santhanam, Natascha Parks, Natasha White, Navyata Bawa, Nayan Singhal, Nick Egebo, Nicolas Usunier, Nikhil Mehta, Nikolay Pavlovich Laptev, Ning Dong, Norman Cheng, Oleg Chernoguz, Olivia Hart, Omkar Salpekar, Ozlem Kalinli, Parkin Kent, Parth Parekh, Paul Saab, Pavan Balaji, Pedro Rittner, Philip Bontrager, Pierre Roux, Piotr Dollar, Polina Zvyagina, Prashant Ratanchandani, Pritish Yuvraj, Qian Liang, Rachad Alao, Rachel Rodriguez, Rafi Ayub, Raghotham Murthy, Raghu Nayani, Rahul Mitra, Rangaprabhu Parthasarathy, Raymond Li, Rebekkah Hogan, Robin Battey, Rocky Wang, Russ Howes, Ruty Rinott, Sachin Mehta, Sachin Siby, Sai Jayesh Bondu, Samyak Datta, Sara Chugh, Sara Hunt, Sargun Dhillon, Sasha Sidorov, Satadru Pan, Saurabh Mahajan, Saurabh Verma, Seiji Yamamoto, Sharadh Ramaswamy, Shaun Lindsay, Shaun Lindsay, Sheng Feng, Shenghao Lin, Shengxin Cindy Zha, Shishir Patil, Shiva Shankar, Shuqiang Zhang, Shuqiang Zhang, Sinong Wang, Sneha Agarwal, Soji Sajuyigbe, Soumith Chintala, Stephanie Max, Stephen Chen, Steve Kehoe, Steve Satterfield, Sudarshan Govindaprasad, Sumit Gupta, Summer Deng, Sungmin Cho, Sunny Virk, Suraj Subramanian, Sy Choudhury, Sydney Goldman, Tal Remez, Tamar Glaser, Tamara Best, Thilo Koehler, Thomas Robinson, Tianhe Li, Tianjun Zhang, Tim Matthews, Timothy Chou, Tzook Shaked, Varun Vontimitta, Victoria Ajayi, Victoria Montanez, Vijai Mohan, Vinay Satish Kumar, Vishal Mangla, Vlad Ionescu, Vlad Poenaru, Vlad Tiberiu Mihailescu, Vladimir Ivanov, Wei Li, Wenchen Wang, Wenwen Jiang, Wes Bouaziz, Will Constable, Xiaocheng Tang, Xiaojian Wu, Xiaolan Wang, Xilun Wu, Xinbo Gao, Yaniv Kleinman, Yanjun Chen, Ye Hu, Ye Jia, Ye Qi, Yenda Li, Yilin Zhang, Ying Zhang, Yossi Adi, Youngjin Nam, Yu, Wang, Yu Zhao, Yuchen Hao, Yundi Qian, Yunlu Li, Yuzi He, Zach Rait, Zachary DeVito, Zef Rosnbrick, Zhaoduo Wen, Zhenyu Yang, Zhiwei Zhao, and Zhiyu Ma. The llama 3 herd of models, 2024. URL https://arxiv.org/abs/2407.21783.", + "tier": "emerging" + }, { "key": "greenblatt2024alignmentfakinglargelanguage", "text": "Ryan Greenblatt, Carson Denison, Benjamin Wright, Fabien Roger, Monte MacDiarmid, Sam Marks, Johannes Treutlein, Tim Belonax, Jack Chen, David Duvenaud, Akbir Khan, Julian Michael, Sören Mindermann, Ethan Perez, Linda Petrini, Jonathan Uesato, Jared Kaplan, Buck Shlegeris, Samuel R. Bowman, and Evan Hubinger. Alignment faking in large language models, 2024. URL https://arxiv.org/abs/2412.14093.", "tier": "emerging" }, + { + "key": "groeneveld2024olmo", + "text": "Dirk Groeneveld, Iz Beltagy, Evan Walsh, Akshita Bhagia, Rodney Kinney, Oyvind Tafjord, Ananya Jha, Hamish Ivison, Ian Magnusson, Yizhong Wang, et al. Olmo: Accelerating the science of language models. In Proceedings of the 62nd annual meeting of the association for computational linguistics (volume 1: long papers), pp. 15789–15809, 2024.", + "tier": "established" + }, + { + "key": "groger2026revisiting", + "text": "Fabian Gröger, Shuo Wen, and Maria Brbic. Revisiting the platonic representation hypothesis: An aristotelian view. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=uz0gAAYydl.", + "tier": "established" + }, { "key": "grosse2023studyinglargelanguagemodel", "text": "Roger Grosse, Juhan Bae, Cem Anil, Nelson Elhage, Alex Tamkin, Amirhossein Tajdini, Benoit Steiner, Dustin Li, Esin Durmus, Ethan Perez, Evan Hubinger, Kamilė Lukošiūtė, Karina Nguyen, Nicholas Joseph, Sam McCandlish, Jared Kaplan, and Samuel R. Bowman. Studying large language model generalization with influence functions, 2023. URL https://arxiv.org/abs/2308.03296.", "tier": "emerging" }, + { + "key": "gu2024mamba", + "text": "Albert Gu and Tri Dao. Mamba: Linear-time sequence modeling with selective state spaces. In First Conference on Language Modeling, 2024. URL https://openreview.net/forum?id=tEYskw1VY2.", + "tier": "established" + }, + { + "key": "guo2026introspectivecouplingselfexplanationtraining", + "text": "Zifan Carl Guo, Laura Ruis, Jacob Andreas, and Belinda Z. Li. Introspective coupling: Self-explanation training tracks behavioral change despite fixed supervision, 2026. URL https://arxiv.org/abs/2606.32038.", + "tier": "emerging" + }, { "key": "gupta-etal-2015-distributional", "text": "Abhijeet Gupta, Gemma Boleda, Marco Baroni, and Sebastian Padó. Distributional vectors encode referential attributes. In Lluís Màrquez, Chris Callison-Burch, and Jian Su (eds.), Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing, pp. 12–21, Lisbon, Portugal, September 2015. Association for Computational Linguistics. . URL https://aclanthology.org/D15-1002/.", @@ -2857,7 +3550,7 @@ { "key": "gur-arieh-etal-2025-enhancing", "text": "Yoav Gur-Arieh, Roy Mayan, Chen Agassy, Atticus Geiger, and Mor Geva. Enhancing automated interpretability with output-centric feature descriptions. In Wanxiang Che, Joyce Nabende, Ekaterina Shutova, and Mohammad Taher Pilehvar (eds.), Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 5757–5778, Vienna, Austria, July 2025a. Association for Computational Linguistics. ISBN 979-8-89176-251-0. . URL https://aclanthology.org/2025.acl-long.288/.", - "tier": null + "tier": "established" }, { "key": "gurarieh2025preciseinparameterconcepterasure", @@ -2865,15 +3558,40 @@ "tier": "established" }, { - "key": "haider2025multi", - "text": "Muhammad Umair Haider, Hammad Rizwan, Hassan Sajjad, and AB Siddique. Multi-granular node pruning for circuit discovery. arXiv preprint arXiv:2512.10903, 2025.", + "key": "jlens", + "text": "Wes Gurnee, Nicholas Sofroniew, Adam Pearce, Mateusz Piotrowski, Isaac Kauvar, Runjin Chen, Anna Soligo, Paul Bogdan, Euan Ong, Rowan Wang, Ben Thompson, David Abrahams, Subhash Kantamneni, Emmanuel Ameisen, Joshua Batson, and Jack Lindsey. Verbalizable representations form a global workspace in language models. Transformer Circuits Thread, 2026. URL https://transformer-circuits.pub/2026/workspace/index.html.", + "tier": "emerging" + }, + { + "key": "habibi2026symbolicmechanisticdataattribution", + "text": "Reza Habibi, Darian Lee, and Magy Seif El-Nasr. Symbolic mechanistic data attribution: Tracing training influence to learned behavioral policies, 2026. URL https://arxiv.org/abs/2606.29171.", + "tier": "emerging" + }, + { + "key": "hahami2026detectingdisturbancenuancedview", + "text": "Ely Hahami, Ishaan Sinha, Lavik Jain, Josh Kaplan, and Jon Hahami. Detecting the disturbance: A nuanced view of introspective abilities in llms, 2026. URL https://arxiv.org/abs/2512.12411.", "tier": "emerging" }, + { + "key": "haider2026multigranular", + "text": "Muhammad Umair Haider, Hammad Rizwan, Hassan Sajjad, and A.B. Siddique. Multi-granular node pruning for causal circuit discovery. In Third Conference on Language Modeling, 2026. URL https://openreview.net/forum?id=Ol96ETTo6k.", + "tier": "established" + }, { "key": "haklay-etal-2025-position", "text": "Tal Haklay, Hadas Orgad, David Bau, Aaron Mueller, and Yonatan Belinkov. Position-aware automatic circuit discovery. In Wanxiang Che, Joyce Nabende, Ekaterina Shutova, and Mohammad Taher Pilehvar (eds.), Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 2792–2817, Vienna, Austria, July 2025. Association for Computational Linguistics. ISBN 979-8-89176-251-0. . URL https://aclanthology.org/2025.acl-long.141/.", "tier": "established" }, + { + "key": "haklay2026pitfalls", + "text": "Tal Haklay, Nikhil Prakash, Sana Pandey, Antonio Torralba, Aaron Mueller, Jacob Andreas, Tamar Rott Shaham, and Yonatan Belinkov. Pitfalls in evaluating interpretability agents, 2026. URL https://arxiv.org/abs/2603.20101.", + "tier": "emerging" + }, + { + "key": "han2026sage", + "text": "Jiaojiao Han, Wujiang Xu, Mingyu Jin, and Mengnan Du. Sage: An agentic explainer framework for interpreting sae features in language models. In Proceedings of the 19th Conference of the European Chapter of the Association for Computational Linguistics (Volume 5: Industry Track), pp. 483–495, 2026.", + "tier": "established" + }, { "key": "hanna2023how", "text": "Michael Hanna, Ollie Liu, and Alexandre Variengien. How does GPT-2 compute greater-than?: Interpreting mathematical abilities in a pre-trained language model. In Thirty-seventh Conference on Neural Information Processing Systems, 2023. URL https://openreview.net/forum?id=p4PckNQR8k.", @@ -2885,14 +3603,24 @@ "tier": "established" }, { - "key": "harrasse2025tracing", - "text": "Abir Harrasse, Florent Draye, Zhijing Jin, and Bernhard Schölkopf. Tracing multilingual representations in llms with cross-layer transcoders. arXiv preprint arXiv:2511.10840, 2025a.", - "tier": null + "key": "haon2025mechanistic", + "text": "Bear Häon, Kaylene Caswell Stocking, Ian Chuang, and Claire Tomlin. Mechanistic interpretability for steering vision-language-action models. In 9th Annual Conference on Robot Learning, 2025. URL https://openreview.net/forum?id=YvsUD8C9QS.", + "tier": "established" }, { "key": "harrasse2025tinysqlprogressivetexttosqldataset", - "text": "Abir Harrasse, Philip Quirke, Clement Neo, Dhruv Nathawani, Luke Marks, and Amir Abdullah. TinySQL: A progressive text-to-SQL dataset for mechanistic interpretability research. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing, pp. 29256–29284, Suzhou, China, November 2025b. Association for Computational Linguistics. ISBN 979-8-89176-332-6. . URL https://aclanthology.org/2025.emnlp-main.1489/.", - "tier": null + "text": "Abir Harrasse, Philip Quirke, Clement Neo, Dhruv Nathawani, Luke Marks, and Amir Abdullah. TinySQL: A progressive text-to-SQL dataset for mechanistic interpretability research. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing, pp. 29256–29284, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-332-6. . URL https://aclanthology.org/2025.emnlp-main.1489/.", + "tier": "established" + }, + { + "key": "harrasse2025tracing", + "text": "Abir Harrasse, Florent Draye, Punya Syon Pandey, Zhijing Jin, and Bernhard Schölkopf. Tracing multilingual representations in llms with cross-layer transcoders, 2026. URL https://arxiv.org/abs/2511.10840.", + "tier": "emerging" + }, + { + "key": "localinform", + "text": "Peter Hase, Mohit Bansal, Been Kim, and Asma Ghandeharioun. Does localization inform editing? surprising differences in causality-based localization vs. knowledge editing in language models. In Proceedings of the 37th International Conference on Neural Information Processing Systems, NIPS ’23, Red Hook, NY, USA, 2023. Curran Associates Inc.", + "tier": "established" }, { "key": "he-etal-2025-red", @@ -2900,30 +3628,80 @@ "tier": "established" }, { - "key": "heimersheim2024useinterpretactivationpatching", - "text": "Stefan Heimersheim and Neel Nanda. How to use and interpret activation patching. arXiv preprint arXiv:2404.15255, 2024.", + "key": "he2024llamascopeextractingmillions", + "text": "Zhengfu He, Wentao Shu, Xuyang Ge, Lingjie Chen, Junxuan Wang, Yunhua Zhou, Frances Liu, Qipeng Guo, Xuanjing Huang, Zuxuan Wu, Yu-Gang Jiang, and Xipeng Qiu. Llama scope: Extracting millions of features from llama-3.1-8b with sparse autoencoders, 2024. URL https://arxiv.org/abs/2410.20526.", + "tier": "emerging" + }, + { + "key": "heap2026automated", + "text": "Thomas Heap, Tim Lawson, Lucy Farnik, and Laurence Aitchison. Automated interpretability metrics do not distinguish trained and random transformers. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=USyGD0eUod.", + "tier": "established" + }, + { + "key": "heindrich2025do", + "text": "Lovis Heindrich, Philip Torr, Fazl Barez, and Veronika Thost. Do sparse autoencoders generalize? a case study of answerability. In ICML 2025 Workshop on Reliable and Responsible Foundation Models, 2025. URL https://openreview.net/forum?id=rs3alQ5BV8.", "tier": "emerging" }, { - "key": "hendrycks2021naturaladversarialexamples", - "text": "Dan Hendrycks, Kevin Zhao, Steven Basart, Jacob Steinhardt, and Dawn Song. Natural adversarial examples. In Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pp. 15262–15271, June 2021.", + "key": "helwe-etal-2025-navigating", + "text": "Chadi Helwe, Oana Balalau, and Davide Ceolin. Navigating the political compass: Evaluating multilingual LLMs across languages and nationalities. In Wanxiang Che, Joyce Nabende, Ekaterina Shutova, and Mohammad Taher Pilehvar (eds.), Findings of the Association for Computational Linguistics: ACL 2025, pp. 17179–17204, Vienna, Austria, July 2025. Association for Computational Linguistics. ISBN 979-8-89176-256-5. . URL https://aclanthology.org/2025.findings-acl.883/.", + "tier": "established" + }, + { + "key": "mmlu", + "text": "Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. Measuring massive multitask language understanding. In International Conference on Learning Representations, 2021. URL https://openreview.net/forum?id=d7KBjmI3GmQ.", + "tier": "established" + }, + { + "key": "hernandez2022natural", + "text": "Evan Hernandez, Sarah Schwettmann, David Bau, Teona Bagashvili, Antonio Torralba, and Jacob Andreas. Natural language descriptions of deep features. In International Conference on Learning Representations, 2022. URL https://openreview.net/forum?id=NudBMY-tzDr.", "tier": "established" }, { "key": "hestness2017deeplearningscalingpredictable", "text": "Joel Hestness, Sharan Narang, Newsha Ardalani, Gregory Diamos, Heewoo Jun, Hassan Kianinejad, Md. Mostofa Ali Patwary, Yang Yang, and Yanqi Zhou. Deep learning scaling is predictable, empirically, 2017. URL https://arxiv.org/abs/1712.00409.", - "tier": null + "tier": "emerging" }, { "key": "hewitt-liang-2019-designing", "text": "John Hewitt and Percy Liang. Designing and interpreting probes with control tasks. In Kentaro Inui, Jing Jiang, Vincent Ng, and Xiaojun Wan (eds.), Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pp. 2733–2743, Hong Kong, China, November 2019. Association for Computational Linguistics. . URL https://aclanthology.org/D19-1275/.", - "tier": null + "tier": "established" + }, + { + "key": "hewitt2019structural", + "text": "John Hewitt and Christopher D Manning. A structural probe for finding syntax in word representations. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 4129–4138, 2019.", + "tier": "established" }, { "key": "hewitt2025position", "text": "John Hewitt, Robert Geirhos, and Been Kim. Position: We can’t understand AI using our existing vocabulary. In Forty-second International Conference on Machine Learning Position Paper Track, 2025. URL https://openreview.net/forum?id=asQJx56NqB.", "tier": "established" }, + { + "key": "hinton1986learning", + "text": "Geoffrey E. Hinton. Learning distributed representations of concepts. In Proceedings of the Eighth Annual Conference of the Cognitive Science Society, pp. 1–12, Amherst, MA, 1986.", + "tier": "established" + }, + { + "key": "hinton1986distributed", + "text": "Geoffrey E. Hinton, James L. McClelland, and David E. Rumelhart. Distributed representations. In David E. Rumelhart, James L. McClelland, and PDP Research Group (eds.), Parallel Distributed Processing: Explorations in the Microstructure of Cognition, Volume 1: Foundations, pp. 77–109. MIT Press, Cambridge, MA, 1986. ISBN 026268053X. .", + "tier": "established" + }, + { + "key": "hoelscher-obermaier-etal-2023-detecting", + "text": "Jason Hoelscher-Obermaier, Julia Persson, Esben Kran, Ioannis Konstas, and Fazl Barez. Detecting edit failures in large language models: An improved specificity benchmark. In Anna Rogers, Jordan Boyd-Graber, and Naoaki Okazaki (eds.), Findings of the Association for Computational Linguistics: ACL 2023, pp. 11548–11559, Toronto, Canada, July 2023. Association for Computational Linguistics. . URL https://aclanthology.org/2023.findings-acl.733/.", + "tier": "established" + }, + { + "key": "hoffmann2022training", + "text": "Jordan Hoffmann, Sebastian Borgeaud, Arthur Mensch, Elena Buchatskaya, Trevor Cai, Eliza Rutherford, Diego de Las Casas, Lisa Anne Hendricks, Johannes Welbl, Aidan Clark, Tom Hennigan, Eric Noland, Katie Millican, George van den Driessche, Bogdan Damoc, Aurelia Guy, Simon Osindero, Karen Simonyan, Erich Elsen, Oriol Vinyals, Jack W. Rae, and Laurent Sifre. Training compute-optimal large language models. In Proceedings of the 36th International Conference on Neural Information Processing Systems, NIPS ’22, Red Hook, NY, USA, 2022. Curran Associates Inc. ISBN 9781713871088.", + "tier": "established" + }, + { + "key": "hong-etal-2025-measuring", + "text": "Jiseung Hong, Grace Byun, Seungone Kim, and Kai Shu. Measuring sycophancy of language models in multi-turn dialogues. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Findings of the Association for Computational Linguistics: EMNLP 2025, pp. 2239–2259, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-335-7. . URL https://aclanthology.org/2025.findings-emnlp.121/.", + "tier": "established" + }, { "key": "hoogland2023towards", "text": "Jesse Hoogland, Alexander Gietelink Oldenziel, Daniel Murfet, and Stan van Wingerden. Towards developmental interpretability. https://www.lesswrong.com/posts/TjaeCWvLZtEDAS5Ex/towards-developmental-interpretability, July 12 2023.", @@ -2931,8 +3709,13 @@ }, { "key": "hoogland2025losslandscapedegeneracystagewise", - "text": "Jesse Hoogland, George Wang, Matthew Farrugia-Roberts, Liam Carroll, Susan Wei, and Daniel Murfet. Loss landscape degeneracy and stagewise development in transformers, 2025. URL https://arxiv.org/abs/2402.02364.", - "tier": null + "text": "Jesse Hoogland, George Wang, Matthew Farrugia-Roberts, Liam Carroll, Susan Wei, and Daniel Murfet. Loss landscape degeneracy and stagewise development in transformers. Transactions on Machine Learning Research, 2025. ISSN 2835-8856. URL https://openreview.net/forum?id=45qJyBG8Oj.", + "tier": "established" + }, + { + "key": "hotelling1933analysis", + "text": "Harold Hotelling. Analysis of a complex of statistical variables into principal components. Journal of Educational Psychology, 24 (6): 417–441, 1933. .", + "tier": "established" }, { "key": "hu2022lora", @@ -2940,74 +3723,124 @@ "tier": "established" }, { - "key": "hu2026transcoderadaptersreasoningmodeldiffing", - "text": "Nathan Hu, Jake Ward, Thomas Icard, and Christopher Potts. Transcoder adapters for reasoning-model diffing, 2026. URL https://arxiv.org/abs/2602.20904.", - "tier": "emerging" - }, - { - "key": "huang-etal-2024-ravel", - "text": "Jing Huang, Zhengxuan Wu, Christopher Potts, Mor Geva, and Atticus Geiger. RAVEL: Evaluating interpretability methods on disentangling language model representations. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 8669–8687, Bangkok, Thailand, August 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.470/.", + "key": "hu2023latent", + "text": "Michael Y. Hu, Angelica Chen, Naomi Saphra, and Kyunghyun Cho. Latent state models of training dynamics. Transactions on Machine Learning Research, 2023.", + "tier": "established" + }, + { + "key": "hu2026transcoderadaptersreasoningmodeldiffing", + "text": "Nathan Hu, Jake Ward, Thomas Icard, and Christopher Potts. Transcoder adapters for reasoning-model diffing, 2026. URL https://arxiv.org/abs/2602.20904.", + "tier": "emerging" + }, + { + "key": "huang-etal-2024-ravel", + "text": "Jing Huang, Zhengxuan Wu, Christopher Potts, Mor Geva, and Atticus Geiger. RAVEL: Evaluating interpretability methods on disentangling language model representations. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 8669–8687, Bangkok, Thailand, August 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.470/.", + "tier": "established" + }, + { + "key": "pcd", + "text": "Vincent Huang, Dami Choi, Daniel D. Johnson, Sarah Schwettmann, and Jacob Steinhardt. Predictive concept decoders: Training scalable end-to-end interpretability assistants. In Mechanistic Interpretability Workshop at ICML 2026, 2026. URL https://openreview.net/forum?id=s9IaugZ0En.", + "tier": "emerging" + }, + { + "key": "hubinger2024sleeperagentstrainingdeceptive", + "text": "Evan Hubinger, Carson Denison, Jesse Mu, Mike Lambert, Meg Tong, Monte MacDiarmid, Tamera Lanham, Daniel M. Ziegler, Tim Maxwell, Newton Cheng, Adam Jermyn, Amanda Askell, Ansh Radhakrishnan, Cem Anil, David Duvenaud, Deep Ganguli, Fazl Barez, Jack Clark, Kamal Ndousse, Kshitij Sachan, Michael Sellitto, Mrinank Sharma, Nova DasSarma, Roger Grosse, Shauna Kravec, Yuntao Bai, Zachary Witten, Marina Favaro, Jan Brauner, Holden Karnofsky, Paul Christiano, Samuel R. Bowman, Logan Graham, Jared Kaplan, Sören Mindermann, Ryan Greenblatt, Buck Shlegeris, Nicholas Schiefer, and Ethan Perez. Sleeper agents: Training deceptive llms that persist through safety training, 2024. URL https://arxiv.org/abs/2401.05566.", + "tier": "emerging" + }, + { + "key": "huh2024platonicrepresentationhypothesis", + "text": "Minyoung Huh, Brian Cheung, Tongzhou Wang, and Phillip Isola. Position: The platonic representation hypothesis. In Ruslan Salakhutdinov, Zico Kolter, Katherine Heller, Adrian Weller, Nuria Oliver, Jonathan Scarlett, and Felix Berkenkamp (eds.), Proceedings of the 41st International Conference on Machine Learning, volume 235 of Proceedings of Machine Learning Research, pp. 20617–20642. PMLR, 21–27 Jul 2024. URL https://proceedings.mlr.press/v235/huh24a.html.", + "tier": "established" + }, + { + "key": "hyvarinen2000independent", + "text": "Aapo Hyvärinen and Erkki Oja. Independent component analysis: algorithms and applications. Neural networks, 13 (4-5): 411–430, 2000.", + "tier": "established" + }, + { + "key": "crispedit", + "text": "Zarif Ikram, Arad Firouzkouhi, Stephen Tu, Mahdi Soltanolkotabi, and Paria Rashidinejad. Crispedit: Low-curvature projections for scalable non-destructive LLM editing. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=gTHlB3WK0l.", + "tier": "established" + }, + { + "key": "ilyas2022datamodelspredictingpredictionstraining", + "text": "Andrew Ilyas, Sung Min Park, Logan Engstrom, Guillaume Leclerc, and Aleksander Madry. Datamodels: Understanding predictions with data and data with predictions. In Kamalika Chaudhuri, Stefanie Jegelka, Le Song, Csaba Szepesvari, Gang Niu, and Sivan Sabato (eds.), Proceedings of the 39th International Conference on Machine Learning, volume 162 of Proceedings of Machine Learning Research, pp. 9525–9587. PMLR, 17–23 Jul 2022. URL https://proceedings.mlr.press/v162/ilyas22a.html.", + "tier": "established" + }, + { + "key": "inaba2025howllmslearn", + "text": "Tatsuro Inaba, Go Kamoda, Kentaro Inui, Masaru Isonuma, Yusuke Miyao, Yohei Oseki, Yu Takagi, and Benjamin Heinzerling. How a bilingual LM becomes bilingual: Tracing internal representations with sparse autoencoders. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Findings of the Association for Computational Linguistics: EMNLP 2025, pp. 13458–13470, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-335-7. . URL https://aclanthology.org/2025.findings-emnlp.725/.", "tier": "established" }, { - "key": "pcd", - "text": "Vincent Huang, Dami Choi, Daniel D Johnson, Sarah Schwettmann, and Jacob Steinhardt. Predictive concept decoders: Training scalable end-to-end interpretability assistants. arXiv preprint arXiv:2512.15712, 2025.", + "key": "inan2023llamaguardllmbasedinputoutput", + "text": "Hakan Inan, Kartikeya Upasani, Jianfeng Chi, Rashi Rungta, Krithika Iyer, Yuning Mao, Michael Tontchev, Qing Hu, Brian Fuller, Davide Testuggine, and Madian Khabsa. Llama guard: Llm-based input-output safeguard for human-ai conversations, 2023. URL https://arxiv.org/abs/2312.06674.", "tier": "emerging" }, { - "key": "hubinger2021riskslearnedoptimizationadvanced", - "text": "Evan Hubinger, Chris van Merwijk, Vladimir Mikulik, Joar Skalse, and Scott Garrabrant. Risks from learned optimization in advanced machine learning systems, 2021. URL https://arxiv.org/abs/1906.01820.", - "tier": "emerging" + "key": "jacovi-goldberg-2020-towards", + "text": "Alon Jacovi and Yoav Goldberg. Towards faithfully interpretable NLP systems: How should we define and evaluate faithfulness? In Dan Jurafsky, Joyce Chai, Natalie Schluter, and Joel Tetreault (eds.), Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 4198–4205, Online, July 2020. Association for Computational Linguistics. . URL https://aclanthology.org/2020.acl-main.386/.", + "tier": "established" }, { - "key": "hubinger2024sleeperagentstrainingdeceptive", - "text": "Evan Hubinger, Carson Denison, Jesse Mu, Mike Lambert, Meg Tong, Monte MacDiarmid, Tamera Lanham, Daniel M. Ziegler, Tim Maxwell, Newton Cheng, Adam Jermyn, Amanda Askell, Ansh Radhakrishnan, Cem Anil, David Duvenaud, Deep Ganguli, Fazl Barez, Jack Clark, Kamal Ndousse, Kshitij Sachan, Michael Sellitto, Mrinank Sharma, Nova DasSarma, Roger Grosse, Shauna Kravec, Yuntao Bai, Zachary Witten, Marina Favaro, Jan Brauner, Holden Karnofsky, Paul Christiano, Samuel R. Bowman, Logan Graham, Jared Kaplan, Sören Mindermann, Ryan Greenblatt, Buck Shlegeris, Nicholas Schiefer, and Ethan Perez. Sleeper agents: Training deceptive llms that persist through safety training, 2024. URL https://arxiv.org/abs/2401.05566.", + "key": "jain2023baselinedefensesadversarialattacks", + "text": "Neel Jain, Avi Schwarzschild, Yuxin Wen, Gowthami Somepalli, John Kirchenbauer, Ping yeh Chiang, Micah Goldblum, Aniruddha Saha, Jonas Geiping, and Tom Goldstein. Baseline defenses for adversarial attacks against aligned language models, 2023. URL https://arxiv.org/abs/2309.00614.", "tier": "emerging" }, { - "key": "huh2024platonicrepresentationhypothesis", - "text": "Minyoung Huh, Brian Cheung, Tongzhou Wang, and Phillip Isola. Position: The platonic representation hypothesis. In Ruslan Salakhutdinov, Zico Kolter, Katherine Heller, Adrian Weller, Nuria Oliver, Jonathan Scarlett, and Felix Berkenkamp (eds.), Proceedings of the 41st International Conference on Machine Learning, volume 235 of Proceedings of Machine Learning Research, pp. 20617–20642. PMLR, 21–27 Jul 2024. URL https://proceedings.mlr.press/v235/huh24a.html.", + "key": "jain2024mechanistically", + "text": "Samyak Jain, Robert Kirk, Ekdeep Singh Lubana, Robert P. Dick, Hidenori Tanaka, Tim Rocktäschel, Edward Grefenstette, and David Krueger. Mechanistically analyzing the effects of fine-tuning on procedurally defined tasks. In The Twelfth International Conference on Learning Representations, 2024a. URL https://openreview.net/forum?id=A0HKeKl4Nl.", "tier": "established" }, { - "key": "hyvarinen2000independent", - "text": "Aapo Hyvärinen and Erkki Oja. Independent component analysis: algorithms and applications. Neural networks, 13 (4-5): 411–430, 2000.", - "tier": null + "key": "jain2024what", + "text": "Samyak Jain, Ekdeep Singh Lubana, Kemal Oksuz, Tom Joy, Philip Torr, Amartya Sanyal, and Puneet K. Dokania. What makes and breaks safety fine-tuning? a mechanistic study. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024b. URL https://openreview.net/forum?id=JEflV4nRlH.", + "tier": "established" }, { - "key": "hyvarinen2023nonlinear", - "text": "Aapo Hyvärinen, Ilyes Khemakhem, and Hiroshi Morioka. Nonlinear independent component analysis for principled disentanglement in unsupervised deep learning. Patterns, 4 (10), 2023.", - "tier": null + "key": "jain2019attention", + "text": "Sarthak Jain and Byron C. Wallace. Attention is not Explanation. In Jill Burstein, Christy Doran, and Thamar Solorio (eds.), Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 3543–3556, Minneapolis, Minnesota, June 2019. Association for Computational Linguistics. . URL https://aclanthology.org/N19-1357/.", + "tier": "established" }, { - "key": "ilyas2022datamodelspredictingpredictionstraining", - "text": "Andrew Ilyas, Sung Min Park, Logan Engstrom, Guillaume Leclerc, and Aleksander Madry. Datamodels: Understanding predictions with data and data with predictions. In Kamalika Chaudhuri, Stefanie Jegelka, Le Song, Csaba Szepesvari, Gang Niu, and Sivan Sabato (eds.), Proceedings of the 39th International Conference on Machine Learning, volume 162 of Proceedings of Machine Learning Research, pp. 9525–9587. PMLR, 17–23 Jul 2022. URL https://proceedings.mlr.press/v162/ilyas22a.html.", - "tier": null + "key": "jakkli2026current", + "text": "Arya Jakkli, Senthooran Rajamanoharan, and Neel Nanda. Current activation oracles are hard to use on safety-relevant tasks. In Mechanistic Interpretability Workshop at ICML 2026, 2026. URL https://openreview.net/forum?id=7nRmqgz3Wv.", + "tier": "emerging" }, { - "key": "jacovi2020faithfullyinterpretablenlpsystems", - "text": "Alon Jacovi and Yoav Goldberg. Towards faithfully interpretable nlp systems: How should we define and evaluate faithfulness?, 2020. URL https://arxiv.org/abs/2004.03685.", - "tier": null + "key": "jiang2024mixtralexperts", + "text": "Albert Q. Jiang, Alexandre Sablayrolles, Antoine Roux, Arthur Mensch, Blanche Savary, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Emma Bou Hanna, Florian Bressand, Gianna Lengyel, Guillaume Bour, Guillaume Lample, Lélio Renard Lavaud, Lucile Saulnier, Marie-Anne Lachaux, Pierre Stock, Sandeep Subramanian, Sophia Yang, Szymon Antoniak, Teven Le Scao, Théophile Gervet, Thibaut Lavril, Thomas Wang, Timothée Lacroix, and William El Sayed. Mixtral of experts, 2024. URL https://arxiv.org/abs/2401.04088.", + "tier": "emerging" }, { - "key": "jain2019attention", - "text": "Sarthak Jain and Byron C. Wallace. Attention is not Explanation. In Jill Burstein, Christy Doran, and Thamar Solorio (eds.), Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 3543–3556, Minneapolis, Minnesota, June 2019. Association for Computational Linguistics. . URL https://aclanthology.org/N19-1357/.", + "key": "jimenez2024swebench", + "text": "Carlos E Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, and Karthik R Narasimhan. SWE-bench: Can language models resolve real-world github issues? In The Twelfth International Conference on Learning Representations, 2024. URL https://openreview.net/forum?id=VTF8yNQM66.", "tier": "established" }, { "key": "jiralerspong_cross-architecture_2025", "text": "Thomas Jiralerspong and Trenton Bricken. Cross-Architecture Model Diffing with Crosscoders: Unsupervised Discovery of Differences Between LLMs. In Workshop on Mechanistic Interpretability, September 2025. URL https://openreview.net/forum?id=ZB84SvrZB8.", - "tier": null + "tier": "emerging" + }, + { + "key": "joglekar", + "text": "Manas Joglekar, Jeremy Chen, Gabriel Wu, Jason Yosinski, Jasmine Wang, Boaz Barak, and Amelia Glaese. Training llms for honesty via confessions, 2025. URL https://arxiv.org/abs/2512.08093.", + "tier": "emerging" }, { - "key": "joshi2026causalitykeyinterpretabilityclaims", - "text": "Shruti Joshi, Aaron Mueller, David Klindt, Wieland Brendel, Patrik Reizinger, and Dhanya Sridhar. Causality is key for interpretability claims to generalise, 2026. URL https://arxiv.org/abs/2602.16698.", + "key": "jones2025forecasting", + "text": "Erik Jones, Meg Tong, Jesse Mu, Mohammed Mahfoud, Jan Leike, Roger Grosse, Jared Kaplan, William Fithian, Ethan Perez, and Mrinank Sharma. Forecasting rare language model behaviors, 2025. URL https://arxiv.org/abs/2502.16797.", "tier": "emerging" }, { "key": "jumelet2024black", - "text": "Jaap Jumelet, Lisa Bylinina, Willem Zuidema, and Jakub Szymanik. Black big boxes: Do language models hide a theory of adjective order? arXiv preprint arXiv:2407.02136, 2024.", - "tier": null + "text": "Jaap Jumelet, Lisa Bylinina, Willem Zuidema, and Jakub Szymanik. Black big boxes: Tracing adjective order preferences in large language models, 2026. URL https://arxiv.org/abs/2407.02136.", + "tier": "emerging" + }, + { + "key": "juneja2023linear", + "text": "Jeevesh Juneja, Rachit Bansal, Kyunghyun Cho, João Sedoc, and Naomi Saphra. Linear connectivity reveals generalization strategies. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=hY6M0JHl3uL.", + "tier": "established" }, { "key": "kamath2025tracing", @@ -3015,29 +3848,54 @@ "tier": "emerging" }, { - "key": "kangaslahti2026hiddenbreakthroughslanguagemodel", - "text": "Sara Kangaslahti, Elan Rosenfeld, and Naomi Saphra. Hidden breakthroughs in language model training, 2026. URL https://arxiv.org/abs/2506.15872.", - "tier": null + "key": "kangaslahti2026hidden", + "text": "Sara Kangaslahti, Elan Rosenfeld, and Naomi Saphra. Hidden breakthroughs in language model training. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=eiEIKGuqaf.", + "tier": "established" }, { "key": "kaplan2020scaling", "text": "Jared Kaplan, Sam McCandlish, Tom Henighan, Tom B. Brown, Benjamin Chess, Rewon Child, Scott Gray, Alec Radford, Jeffrey Wu, and Dario Amodei. Scaling laws for neural language models. arXiv preprint arXiv:2001.08361, 2020. URL https://arxiv.org/abs/2001.08361.", - "tier": null + "tier": "emerging" }, { "key": "karpathy2015visualizing", - "text": "Andrej Karpathy, Justin Johnson, and Li Fei-Fei. Visualizing and understanding recurrent networks. arXiv preprint arXiv:1506.02078, 2015.", + "text": "Andrej Karpathy, Justin Johnson, and Li Fei-Fei. Visualizing and understanding recurrent networks, 2015. URL https://arxiv.org/abs/1506.02078.", "tier": "emerging" }, + { + "key": "karvonen2025saebench", + "text": "Adam Karvonen, Can Rager, Johnny Lin, Curt Tigges, Joseph Isaac Bloom, David Chanin, Yeu-Tong Lau, Eoin Farrell, Callum Stuart McDougall, Kola Ayonrinde, Demian Till, Matthew Wearden, Arthur Conmy, Samuel Marks, and Neel Nanda. SAEBench: A comprehensive benchmark for sparse autoencoders in language model interpretability. In Forty-second International Conference on Machine Learning, 2025.", + "tier": "established" + }, { "key": "ao", - "text": "Adam Karvonen, James Chua, Clément Dumas, Kit Fraser-Taliente, Subhash Kantamneni, Julian Minder, Euan Ong, Arnab Sen Sharma, Daniel Wen, Owain Evans, et al. Activation oracles: Training and evaluating llms as general-purpose activation explainers. arXiv preprint arXiv:2512.15674, 2025.", + "text": "Adam Karvonen, James Chua, Clément Dumas, Kit Fraser-Taliente, Subhash Kantamneni, Julian Minder, Euan Ong, Arnab Sen Sharma, Daniel Wen, Owain Evans, and Samuel Marks. Activation oracles: Training and evaluating LLMs as general-purpose activation explainers. In Forty-third International Conference on Machine Learning, 2026a. URL https://openreview.net/forum?id=DbZjxkZrZm.", + "tier": "established" + }, + { + "key": "chive", + "text": "Adam Karvonen, Euan Ong, Subhash Kantamneni, and Samuel Marks. Would this change your answer? evaluating explanations of llm behavior in the wild with counterfactual experiments, 2026b. URL https://arxiv.org/abs/2608.16747.", "tier": "emerging" }, + { + "key": "kempermann2026safe", + "text": "Maximilian Kempermann, Sai Sarathi Macharla Vasu, Mathusan Raveenthiran, Tracie Farrell, and Ingmar Weber. Safe for whom? Rethinking how we evaluate the safety of LLMs for real users. Proceedings of the IASEAI Conference, 2 (1): 305–317, 2026. URL https://ojs.aaai.org/index.php/IASEAI/article/view/43033.", + "tier": "established" + }, { "key": "kempf2026simplellmbaselinescompetitive", "text": "Elias Kempf, Simon Schrodi, Bartosz Cywiński, Thomas Brox, Neel Nanda, and Arthur Conmy. Simple llm baselines are competitive for model diffing, 2026. URL https://arxiv.org/abs/2602.10371.", - "tier": null + "tier": "emerging" + }, + { + "key": "khan2026can", + "text": "Ayan Antik Khan, Harsh Kohli, Yuekun Yao, Huan Sun, and Ziyu Yao. Can language model agents be helpful circuit explainers in mechanistic interpretability?, 2026. URL https://arxiv.org/abs/2606.24026.", + "tier": "emerging" + }, + { + "key": "kharlapenko2025scaling", + "text": "Dmitrii Kharlapenko, Stepan Shabalin, Arthur Conmy, and Neel Nanda. Scaling sparse feature circuits for studying in-context learning. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=Mp8Og8Rpop.", + "tier": "established" }, { "key": "kim2018tcav", @@ -3045,13 +3903,13 @@ "tier": "established" }, { - "key": "kim2025llmspursueagenticinterpretability", - "text": "Been Kim, John Hewitt, Neel Nanda, Noah Fiedel, and Oyvind Tafjord. Because we have llms, we can and should pursue agentic interpretability, 2025. URL https://arxiv.org/abs/2506.12152.", - "tier": "emerging" + "key": "kim2025rethinking", + "text": "Minseon Kim, Jin Myung Kwak, Lama Alssum, Bernard Ghanem, Philip Torr, David Krueger, Fazl Barez, and Adel Bibi. Rethinking safety in LLM fine-tuning: An optimization perspective. In Second Conference on Language Modeling, 2025. URL https://openreview.net/forum?id=ZnOoEA2nDn.", + "tier": "established" }, { - "key": "kim2023probabilistic", - "text": "Eunji Kim, Dahuin Jung, Sangha Park, Siwon Kim, and Sungroh Yoon. Probabilistic concept bottleneck models. In Proceedings of the 40th International Conference on Machine Learning, ICML’23. JMLR.org, 2023.", + "key": "kindermans2017unreliabilitysaliencymethods", + "text": "Pieter-Jan Kindermans, Sara Hooker, Julius Adebayo, Maximilian Alber, Kristof T. Schütt, Sven Dähne, Dumitru Erhan, and Been Kim. The (Un)reliability of Saliency Methods, pp. 267–280. Springer-Verlag, Berlin, Heidelberg, 2022. ISBN 978-3-030-28953-9. URL https://doi.org/10.1007/978-3-030-28954-6_14.", "tier": "established" }, { @@ -3069,10 +3927,15 @@ "text": "David Klindt, Charles O’Neill, Patrik Reizinger, Harald Maurer, and Nina Miolane. From superposition to sparse codes: interpretable representations in neural networks, 2025. URL https://arxiv.org/abs/2503.01824.", "tier": "emerging" }, + { + "key": "koepke2026platoscaveexaminingcrossmodal", + "text": "A. Sophia Koepke, Daniil Zverev, Shiry Ginosar, and Alexei A. Efros. Back into plato’s cave: Examining cross-modal representational convergence at scale, 2026. URL https://arxiv.org/abs/2604.18572.", + "tier": "emerging" + }, { "key": "koh2020understandingblackboxpredictionsinfluence", "text": "Pang Wei Koh and Percy Liang. Understanding black-box predictions via influence functions. In Proceedings of the 34th International Conference on Machine Learning - Volume 70, ICML’17, pp. 1885–1894. JMLR.org, 2017.", - "tier": null + "tier": "established" }, { "key": "koh2020conceptbottleneckmodels", @@ -3084,6 +3947,11 @@ "text": "Arne Köhn. What’s in an embedding? analyzing word embeddings through multilingual evaluation. In Lluís Màrquez, Chris Callison-Burch, and Jian Su (eds.), Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing, pp. 2067–2073, Lisbon, Portugal, September 2015. Association for Computational Linguistics. . URL https://aclanthology.org/D15-1246/.", "tier": "established" }, + { + "key": "korbak2025chainthoughtmonitorabilitynew", + "text": "Tomek Korbak, Mikita Balesni, Elizabeth Barnes, Yoshua Bengio, Joe Benton, Joseph Bloom, Mark Chen, Alan Cooney, Allan Dafoe, Anca Dragan, Scott Emmons, Owain Evans, David Farhi, Ryan Greenblatt, Dan Hendrycks, Marius Hobbhahn, Evan Hubinger, Geoffrey Irving, Erik Jenner, Daniel Kokotajlo, Victoria Krakovna, Shane Legg, David Lindner, David Luan, Aleksander Mądry, Julian Michael, Neel Nanda, Dave Orr, Jakub Pachocki, Ethan Perez, Mary Phuong, Fabien Roger, Joshua Saxe, Buck Shlegeris, Martín Soto, Eric Steinberger, Jasmine Wang, Wojciech Zaremba, Bowen Baker, Rohin Shah, and Vlad Mikulik. Chain of thought monitorability: A new and fragile opportunity for ai safety, 2025. URL https://arxiv.org/abs/2507.11473.", + "tier": "emerging" + }, { "key": "kornblith2019similarityneuralnetworkrepresentations", "text": "Simon Kornblith, Mohammad Norouzi, Honglak Lee, and Geoffrey Hinton. Similarity of neural network representations revisited. In Kamalika Chaudhuri and Ruslan Salakhutdinov (eds.), Proceedings of the 36th International Conference on Machine Learning, volume 97 of Proceedings of Machine Learning Research, pp. 3519–3529. PMLR, 09–15 Jun 2019. URL https://proceedings.mlr.press/v97/kornblith19a.html.", @@ -3096,38 +3964,63 @@ }, { "key": "kowal2026concept", - "text": "Matthew Kowal, Goncalo Paulo, Louis Jaburi, Tom Tseng, Lev E McKinney, Stefan Heimersheim, Aaron David Tucker, Adam Gleave, and Kellin Pelrine. Concept influence: Leveraging interpretability to improve performance and efficiency in training data attribution. arXiv preprint arXiv:2602.14869, 2026.", + "text": "Matthew Kowal, Goncalo Paulo, Louis Jaburi, Tom Tseng, Lev E McKinney, Stefan Heimersheim, Aaron David Tucker, Adam Gleave, and Kellin Pelrine. Concept influence: Leveraging interpretability to improve performance and efficiency in training data attribution, 2026. URL https://arxiv.org/abs/2602.14869.", "tier": "emerging" }, { "key": "krishna2025disagreementproblemexplainablemachine", "text": "Satyapriya Krishna, Tessa Han, Alex Gu, Steven Wu, Shahin Jabbari, and Himabindu Lakkaraju. The disagreement problem in explainable machine learning: A practitioner’s perspective. Transactions on Machine Learning Research, 2024. ISSN 2835-8856. URL https://openreview.net/forum?id=jESY2WTZCe.", - "tier": null + "tier": "established" + }, + { + "key": "NIPS2012_c399862d", + "text": "Alex Krizhevsky, Ilya Sutskever, and Geoffrey E Hinton. Imagenet classification with deep convolutional neural networks. In F. Pereira, C.J. Burges, L. Bottou, and K. Weinberger (eds.), Advances in Neural Information Processing Systems, volume 25. Curran Associates, Inc., 2012. URL https://proceedings.neurips.cc/paper_files/paper/2012/file/c399862d3b9d6b76c8436e924a68c45b-Paper.pdf.", + "tier": "established" + }, + { + "key": "laban2026llms", + "text": "Philippe Laban, Hiroaki Hayashi, Yingbo Zhou, and Jennifer Neville. LLMs get lost in multi-turn conversation. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=VKGTGGcwl6.", + "tier": "established" }, { - "key": "krizhevsky2012imagenet", - "text": "Alex Krizhevsky, Ilya Sutskever, and Geoffrey Hinton. Imagenet classification with deep convolutional neural networks. In Advances in Neural Information Processing Systems, 2012.", + "key": "lampinen2019analytic", + "text": "Andrew K. Lampinen and Surya Ganguli. An analytic theory of generalization dynamics and transfer learning in deep linear networks. In International Conference on Learning Representations, 2019. URL https://openreview.net/forum?id=ryfMLoCqtQ.", "tier": "established" }, { - "key": "lanham2023measuringfaithfulnesschainofthoughtreasoning", - "text": "Tamera Lanham, Anna Chen, Ansh Radhakrishnan, Benoit Steiner, Carson Denison, Danny Hernandez, Dustin Li, Esin Durmus, Evan Hubinger, Jackson Kernion, Kamilė Lukošiūtė, Karina Nguyen, Newton Cheng, Nicholas Joseph, Nicholas Schiefer, Oliver Rausch, Robin Larson, Sam McCandlish, Sandipan Kundu, Saurav Kadavath, Shannon Yang, Thomas Henighan, Timothy Maxwell, Timothy Telleen-Lawton, Tristan Hume, Zac Hatfield-Dodds, Jared Kaplan, Jan Brauner, Samuel R. Bowman, and Ethan Perez. Measuring faithfulness in chain-of-thought reasoning, 2023. URL https://arxiv.org/abs/2307.13702.", + "key": "lan-etal-2024-towards", + "text": "Michael Lan, Philip Torr, and Fazl Barez. Towards interpretable sequence continuation: Analyzing shared circuits in large language models. In Yaser Al-Onaizan, Mohit Bansal, and Yun-Nung Chen (eds.), Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, pp. 12576–12601, Miami, Florida, USA, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.emnlp-main.699/.", + "tier": "established" + }, + { + "key": "lan2025quantifyingfeaturespaceuniversality", + "text": "Michael Lan, Philip Torr, Austin Meek, Ashkan Khakzar, David Krueger, and Fazl Barez. Quantifying feature space universality across large language models via sparse autoencoders, 2025. URL https://arxiv.org/abs/2410.06981.", "tier": "emerging" }, { - "key": "lau2023local", - "text": "Edmund Lau, Zach Furman, George Wang, Daniel Murfet, and Susan Wei. The local learning coefficient: A singularity-aware complexity measure. arXiv preprint arXiv:2308.12108, 2023.", - "tier": null + "key": "lan-etal-2026-make", + "text": "Michael Lan, Narmeen Fatimah Oozeer, Chaithanya Bandi, Philip Quirke, Austin Meek, Fazl Barez, and Amir Abdullah. Make mechanistic interpretability auditable: A call to develop guidelines via continuous collaborative reviewing. In Maria Liakata, Viviane P. Moreira, Jiajun Zhang, and David Jurgens (eds.), Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 3516–3540, San Diego, California, United States, July 2026. Association for Computational Linguistics. ISBN 979-8-89176-390-6. . URL https://aclanthology.org/2026.acl-long.159/.", + "tier": "established" }, { - "key": "le2013building", - "text": "Quoc V Le. Building high-level features using large scale unsupervised learning. In 2013 IEEE international conference on acoustics, speech and signal processing, pp. 8595–8598. IEEE, 2013.", - "tier": null + "key": "lau2023local", + "text": "Edmund Lau, Zach Furman, George Wang, Daniel Murfet, and Susan Wei. The local learning coefficient: A singularity-aware complexity measure. In The 28th International Conference on Artificial Intelligence and Statistics, 2025. URL https://openreview.net/forum?id=1av51ZlsuL.", + "tier": "established" }, { "key": "leask2025sparse", - "text": "Patrick Leask, Bart Bussmann, Michael Pearce, Joseph Bloom, Curt Tigges, Noura Al Moubayed, Lee Sharkey, and Neel Nanda. Sparse autoencoders do not find canonical units of analysis. arXiv preprint arXiv:2502.04878, 2025.", - "tier": null + "text": "Patrick Leask, Bart Bussmann, Michael T Pearce, Joseph Isaac Bloom, Curt Tigges, Noura Al Moubayed, Lee Sharkey, and Neel Nanda. Sparse autoencoders do not find canonical units of analysis. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=9ca9eHNrdH.", + "tier": "established" + }, + { + "key": "lecomte2024what", + "text": "Victor Lecomte, Kushal Thaman, Rylan Schaeffer, Naomi Bashkansky, Trevor Chow, and Sanmi Koyejo. What causes polysemanticity? an alternative origin story of mixed selectivity from incidental causes. In ICLR 2024 Workshop on Representational Alignment, 2024. URL https://openreview.net/forum?id=AHfE6WeJLQ.", + "tier": "emerging" + }, + { + "key": "lee1999learning", + "text": "Daniel D. Lee and H. Sebastian Seung. Learning the parts of objects by non-negative matrix factorization. Nature, 401 (6755): 788–791, 1999. .", + "tier": "established" }, { "key": "lee2006efficient", @@ -3136,39 +4029,69 @@ }, { "key": "lee2026rigorous", - "text": "Isabelle Lee, Emmy Liu, Cathy Jiao, Brihi Joshi, Dani Yogatama, Fazl Barez, and Michael Saxon. Rigorous interpretation is a form of evaluation, 2026. URL https://arxiv.org/abs/2605.05508.", + "text": "Isabelle Lee, Emmy Liu, Cathy Jiao, Brihi Joshi, Dani Yogatama, Fazl Barez, and Michael Saxon. Rigorous interpretation is a form of evaluation, 2026a. URL https://arxiv.org/abs/2605.05508.", "tier": "emerging" }, { - "key": "lee2025influence", - "text": "Jin Hwa Lee, Matthew Smith, Maxwell Adam, and Jesse Hoogland. Influence dynamics and stagewise data attribution. arXiv preprint arXiv:2510.12071, 2025.", - "tier": null + "key": "lee2026steps", + "text": "Isabelle Lee, Emmy Liu, Kaiser Sun, Stella Biderman, and David Alvarez-Melis. Workshop on the science of training: Emergence, prediction, and steering (steps). https://science-of-training.github.io/assets/proposal.pdf, 2026b. Workshop proposal, NeurIPS 2026.", + "tier": "emerging" }, { "key": "lei2025spartan", "text": "Anson Lei, Bernhard Schölkopf, and Ingmar Posner. Spartan: A sparse transformer world model attending to what matters. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025.", - "tier": null + "tier": "established" + }, + { + "key": "lepikhin2021gshard", + "text": "Dmitry Lepikhin, HyoukJoong Lee, Yuanzhong Xu, Dehao Chen, Orhan Firat, Yanping Huang, Maxim Krikun, Noam Shazeer, and Zhifeng Chen. {GS}hard: Scaling giant models with conditional computation and automatic sharding. In International Conference on Learning Representations, 2021. URL https://openreview.net/forum?id=qrwe7XHTmYb.", + "tier": "established" + }, + { + "key": "lermen2025deceptive", + "text": "Simon Lermen, Mateusz Dziemian, and Natalia Pérez-Campanero Antolín. Deceptive automated interpretability: Language models coordinating to fool oversight systems, 2025. URL https://arxiv.org/abs/2504.07831.", + "tier": "emerging" }, { "key": "lewicki2000learning", "text": "Michael S. Lewicki and Terrence J. Sejnowski. Learning overcomplete representations. Neural Comput., 12 (2): 337–365, February 2000. ISSN 0899-7667. . URL https://doi.org/10.1162/089976600300015826.", "tier": "established" }, + { + "key": "li2026introspective", + "text": "Belinda Z. Li. Introspective interpretability: a definition, motivation, and open problems. https://belindal.github.io/introspection, February 2026. Blog post.", + "tier": "emerging" + }, { "key": "li2025traininglanguagemodelsexplain", "text": "Belinda Z. Li, Zifan Carl Guo, Vincent Huang, Jacob Steinhardt, and Jacob Andreas. Training language models to explain their own computations, 2025a. URL https://arxiv.org/abs/2511.08579.", "tier": "emerging" }, { - "key": "li2024simulateeliminaterevokebackdoors", - "text": "Haoran Li, Yulin Chen, Zihao Zheng, Qi Hu, Chunkit Chan, Heshan Liu, and Yangqiu Song. Simulate and eliminate: revoke backdoors for generative large language models. In Proceedings of the Thirty-Ninth AAAI Conference on Artificial Intelligence and Thirty-Seventh Conference on Innovative Applications of Artificial Intelligence and Fifteenth Symposium on Educational Advances in Artificial Intelligence, AAAI’25/IAAI’25/EAAI’25. AAAI Press, 2025b. ISBN 978-1-57735-897-8. . URL https://doi.org/10.1609/aaai.v39i1.32018.", - "tier": null + "key": "li2026autocontrol", + "text": "Changyi Li, Pengfei Lu, Xudong Pan, Fazl Barez, and Min Yang. Autocontrol arena: Synthesizing executable test environments for frontier AI risk evaluation. In Forty-third International Conference on Machine Learning, 2026a. URL https://openreview.net/forum?id=XkDpZusDTK.", + "tier": "established" + }, + { + "key": "chloe", + "text": "Chloe Li, Mary Phuong, and Daniel Tan. Spilling the beans: Teaching LLMs to self-report their hidden objectives. In The Fourteenth International Conference on Learning Representations, 2026b. URL https://openreview.net/forum?id=sWs0cCuM8I.", + "tier": "established" }, { "key": "li-etal-2016-visualizing", "text": "Jiwei Li, Xinlei Chen, Eduard Hovy, and Dan Jurafsky. Visualizing and understanding neural models in NLP. In Kevin Knight, Ani Nenkova, and Owen Rambow (eds.), Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 681–691, San Diego, California, June 2016. Association for Computational Linguistics. . URL https://aclanthology.org/N16-1082/.", "tier": "established" }, + { + "key": "li2023emergent", + "text": "Kenneth Li, Aspen K Hopkins, David Bau, Fernanda Viégas, Hanspeter Pfister, and Martin Wattenberg. Emergent world representations: Exploring a sequence model trained on a synthetic task. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=DeG07_TcZvT.", + "tier": "established" + }, + { + "key": "li2026do", + "text": "Millicent Li, Alberto Mario Ceballos Arroyo, Giordano Rogers, Naomi Saphra, and Byron C Wallace. Do activation verbalization methods convey privileged information? In Forty-third International Conference on Machine Learning, 2026c. URL https://openreview.net/forum?id=IFGTKwHwK6.", + "tier": "established" + }, { "key": "li2024wmdp", "text": "Nathaniel Li, Alexander Pan, Anjali Gopal, Summer Yue, Daniel Berrios, Alice Gatti, Justin D. Li, Ann-Kathrin Dombrowski, Shashwat Goel, Gabriel Mukobi, Nathan Helm-Burger, Rassin Lababidi, Lennart Justen, Andrew Bo Liu, Michael Chen, Isabelle Barrass, Oliver Zhang, Xiaoyuan Zhu, Rishub Tamirisa, Bhrugu Bharathi, Ariel Herbert-Voss, Cort B Breuer, Andy Zou, Mantas Mazeika, Zifan Wang, Palash Oswal, Weiran Lin, Adam Alfred Hunt, Justin Tienken-Harder, Kevin Y. Shih, Kemper Talley, John Guan, Ian Steneker, David Campbell, Brad Jokubaitis, Steven Basart, Stephen Fitz, Ponnurangam Kumaraguru, Kallol Krishna Karmakar, Uday Tupakula, Vijay Varadharajan, Yan Shoshitaishvili, Jimmy Ba, Kevin M. Esvelt, Alexandr Wang, and Dan Hendrycks. The WMDP benchmark: Measuring and reducing malicious use with unlearning. In Forty-first International Conference on Machine Learning, 2024. URL https://openreview.net/forum?id=xlr6AUDuJz.", @@ -3177,12 +4100,27 @@ { "key": "pmlr-v44-li15convergent", "text": "Yixuan Li, Jason Yosinski, Jeff Clune, Hod Lipson, and John Hopcroft. Convergent learning: Do different neural networks learn the same representations? In Dmitry Storcheus, Afshin Rostamizadeh, and Sanjiv Kumar (eds.), Proceedings of the 1st International Workshop on Feature Extraction: Modern Questions and Challenges at NIPS 2015, volume 44 of Proceedings of Machine Learning Research, pp. 196–212, Montreal, Canada, 11 Dec 2015. PMLR. URL https://proceedings.mlr.press/v44/li15convergent.html.", - "tier": null + "tier": "emerging" }, { "key": "li2025geometry", - "text": "Yuxiao Li, Eric J Michaud, David D Baek, Joshua Engels, Xiaoqing Sun, and Max Tegmark. The geometry of concepts: Sparse autoencoder feature structure. Entropy, 27 (4): 344, 2025c.", - "tier": null + "text": "Yuxiao Li, Eric J Michaud, David D Baek, Joshua Engels, Xiaoqing Sun, and Max Tegmark. The geometry of concepts: Sparse autoencoder feature structure. Entropy, 27 (4): 344, 2025b.", + "tier": "established" + }, + { + "key": "lieberum2024gemmascopeopensparse", + "text": "Tom Lieberum, Senthooran Rajamanoharan, Arthur Conmy, Lewis Smith, Nicolas Sonnerat, Vikrant Varma, Janos Kramar, Anca Dragan, Rohin Shah, and Neel Nanda. Gemma scope: Open sparse autoencoders everywhere all at once on gemma 2. In Yonatan Belinkov, Najoung Kim, Jaap Jumelet, Hosein Mohebbi, Aaron Mueller, and Hanjie Chen (eds.), Proceedings of the 7th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP, pp. 278–300, Miami, Florida, US, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.blackboxnlp-1.19/.", + "tier": "emerging" + }, + { + "key": "neuronpedia", + "text": "Johnny Lin. Neuronpedia: Interactive reference and tooling for analyzing neural networks, 2023. URL https://www.neuronpedia.org. Software available from neuronpedia.org.", + "tier": "emerging" + }, + { + "key": "lindsey2026emergentintrospectiveawarenesslarge", + "text": "Jack Lindsey. Emergent introspective awareness in large language models, 2026. URL https://arxiv.org/abs/2601.01828.", + "tier": "emerging" }, { "key": "lindsey2024sparsecrosscoders", @@ -3199,11 +4137,6 @@ "text": "Jack Lindsey, Brian Chen, Adam Pearce, Sasha Hydrie, Thomas Conerly, and Jeff Wu. Sparse mixtures of linear transforms. https://transformer-circuits.pub/2025/bulk-update/index.html, 2025b.", "tier": "emerging" }, - { - "key": "lindsey2025biology", - "text": "Jack Lindsey, Wes Gurnee, Emmanuel Ameisen, Brian Chen, Adam Pearce, Nicholas L. Turner, Craig Citro, David Abrahams, Shan Carter, Basil Hosmer, Jonathan Marcus, Michael Sklar, Adly Templeton, Trenton Bricken, Callum McDougall, Hoagy Cunningham, Thomas Henighan, Adam Jermyn, Andy Jones, Andrew Persic, Zhenyi Qi, T. Ben Thompson, Sam Zimmerman, Kelley Rivoire, Thomas Conerly, Chris Olah, and Joshua Batson. On the biology of a large language model. Transformer Circuits Thread, 2025c. URL https://transformer-circuits.pub/2025/attribution-graphs/biology.html.", - "tier": "emerging" - }, { "key": "linzen-etal-2016-assessing", "text": "Tal Linzen, Emmanuel Dupoux, and Yoav Goldberg. Assessing the ability of LSTMs to learn syntax-sensitive dependencies. Transactions of the Association for Computational Linguistics, 4: 521–535, 2016. . URL https://aclanthology.org/Q16-1037/.", @@ -3211,24 +4144,64 @@ }, { "key": "liu2025infinigramscalingunboundedngram", - "text": "Jiacheng Liu, Sewon Min, Luke Zettlemoyer, Yejin Choi, and Hannaneh Hajishirzi. Infini-gram: Scaling unbounded n-gram language models to a trillion tokens. In First Conference on Language Modeling, 2024. URL https://openreview.net/forum?id=u2vAyMeLMm.", - "tier": null + "text": "Jiacheng Liu, Sewon Min, Luke Zettlemoyer, Yejin Choi, and Hannaneh Hajishirzi. Infini-gram: Scaling unbounded n-gram language models to a trillion tokens. In First Conference on Language Modeling, 2024a. URL https://openreview.net/forum?id=u2vAyMeLMm.", + "tier": "established" }, { "key": "liu2025olmotrace", "text": "Jiacheng Liu, Taylor Blanton, Yanai Elazar, Sewon Min, Yen-Sung Chen, Arnavi Chheda-Kothary, Huy Tran, Byron Bischoff, Eric Marsh, Michael Schmitz, et al. Olmotrace: Tracing language model outputs back to trillions of training tokens. In Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 3: System Demonstrations), pp. 178–188, 2025a.", "tier": "established" }, + { + "key": "liu2021probing", + "text": "Leo Z. Liu, Yizhong Wang, Jungo Kasai, Hannaneh Hajishirzi, and Noah A. Smith. Probing across time: What does RoBERTa know and when? In Findings of the Association for Computational Linguistics: EMNLP 2021, pp. 820–842, 2021.", + "tier": "established" + }, + { + "key": "liu2026icalensinterpretinglanguage", + "text": "Sida Liu and Feijiang Han. Ica lens: Interpreting language models without training another dictionary, 2026. URL https://arxiv.org/abs/2606.11722.", + "tier": "emerging" + }, + { + "key": "liu2024repobench", + "text": "Tianyang Liu, Canwen Xu, and Julian McAuley. Repobench: Benchmarking repository-level code auto-completion systems. In The Twelfth International Conference on Learning Representations, 2024b. URL https://openreview.net/forum?id=pPjZIOuQuF.", + "tier": "established" + }, + { + "key": "liu2023omnigrok", + "text": "Ziming Liu, Eric J Michaud, and Max Tegmark. Omnigrok: Grokking beyond algorithmic data. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=zDiHoIWa0q1.", + "tier": "established" + }, { "key": "liu2025kan", "text": "Ziming Liu, Yixuan Wang, Sachin Vaidya, Fabian Ruehle, James Halverson, Marin Soljacic, Thomas Y. Hou, and Max Tegmark. KAN: Kolmogorovarnold networks. In The Thirteenth International Conference on Learning Representations, 2025b. URL https://openreview.net/forum?id=Ozo7qJ5vZi.", - "tier": null + "tier": "established" + }, + { + "key": "lo-etal-2024-large", + "text": "Michelle Lo, Fazl Barez, and Shay B. Cohen. Large language models relearn removed concepts. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Findings of the Association for Computational Linguistics: ACL 2024, pp. 8306–8323, Bangkok, Thailand, August 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.findings-acl.492/.", + "tier": "established" + }, + { + "key": "lotfi2026position", + "text": "Sobhan Lotfi, Ava Iranmanesh, Lachin Naghashyar, Ali Shirali, Fateme Nateghi Haredasht, Sanmi Koyejo, Philip Torr, Yong Suk Lee, Fazl Barez, Joel Lehman, Peter Norvig, and Arvind Narayanan. Position: There are futures that benchmark-driven AI cannot see. In Forty-third International Conference on Machine Learning Position Paper Track, 2026. URL https://openreview.net/forum?id=RzWlUbmRJi.", + "tier": "established" + }, + { + "key": "lubana2023mechanistic", + "text": "Ekdeep Singh Lubana, Eric J. Bigelow, Robert P. Dick, David Krueger, and Hidenori Tanaka. Mechanistic mode connectivity. In International Conference on Machine Learning, pp. 22965–23004, 2023.", + "tier": "established" }, { "key": "lundberg2017unifiedapproachinterpretingmodel", "text": "Scott M. Lundberg and Su-In Lee. A unified approach to interpreting model predictions. In Proceedings of the 31st International Conference on Neural Information Processing Systems, NIPS’17, pp. 4768–4777, Red Hook, NY, USA, 2017. Curran Associates Inc. ISBN 9781510860964.", "tier": "established" }, + { + "key": "aiscientistpitfall", + "text": "Ziming Luo, Atoosa Kasirzadeh, and Nihar B. Shah. The more you automate, the less you see: Hidden pitfalls of autonomous ai scientists. Proceedings of the National Academy of Sciences, 123 (41): e2610214123, 2026. . URL https://www.pnas.org/doi/abs/10.1073/pnas.2610214123.", + "tier": "established" + }, { "key": "macdiarmid2024sleeperagentprobes", "text": "Monte MacDiarmid, Timothy Maxwell, Nicholas Schiefer, Jesse Mu, Jared Kaplan, David Duvenaud, Sam Bowman, Alex Tamkin, Ethan Perez, Mrinank Sharma, Carson Denison, and Evan Hubinger. Simple probes can catch sleeper agents. https://www.anthropic.com/news/probes-catch-sleeper-agents, 2024.", @@ -3236,9 +4209,39 @@ }, { "key": "macdiarmid_natural_2025", - "text": "Monte MacDiarmid, Benjamin Wright, Jonathan Uesato, Joe Benton, Jon Kutasov, Sara Price, Naia Bouscal, Sam Bowman, Trenton Bricken, Alex Cloud, Carson Denison, Johannes Gasteiger, Ryan Greenblatt, Jan Leike, Jack Lindsey, Vlad Mikulik, Ethan Perez, Alex Rodrigues, Drake Thomas, Albert Webson, Daniel Ziegler, and Evan Hubinger. Natural Emergent Misalignment from Reward Hacking in Production RL, November 2025. URL http://arxiv.org/abs/2511.18397. arXiv:2511.18397 [cs].", + "text": "Monte MacDiarmid, Benjamin Wright, Jonathan Uesato, Joe Benton, Jon Kutasov, Sara Price, Naia Bouscal, Sam Bowman, Trenton Bricken, Alex Cloud, Carson Denison, Johannes Gasteiger, Ryan Greenblatt, Jan Leike, Jack Lindsey, Vlad Mikulik, Ethan Perez, Alex Rodrigues, Drake Thomas, Albert Webson, Daniel Ziegler, and Evan Hubinger. Natural emergent misalignment from reward hacking in production rl, 2025. URL https://arxiv.org/abs/2511.18397.", + "tier": "emerging" + }, + { + "key": "makelov2024is", + "text": "Aleksandar Makelov, Georg Lange, Atticus Geiger, and Neel Nanda. Is this the subspace you are looking for? an interpretability illusion for subspace activation patching. In The Twelfth International Conference on Learning Representations, 2024. URL https://openreview.net/forum?id=Ebt7JgMHv1.", + "tier": "established" + }, + { + "key": "matchingpursuit", + "text": "S.G. Mallat and Zhifeng Zhang. Matching pursuits with time-frequency dictionaries. IEEE Transactions on Signal Processing, 41 (12): 3397–3415, 1993. .", + "tier": "established" + }, + { + "key": "marin2026automated", + "text": "Arnau Marin-Llobet and Javier Ferrando. Automated interpretability and feature discovery in language models with agents, 2026. URL https://arxiv.org/abs/2605.01555.", "tier": "emerging" }, + { + "key": "marks2024interpreting", + "text": "Luke Marks, Amir Abdullah, Clement Neo, Rauno Arike, David Krueger, Philip Torr, and Fazl Barez. Interpreting learned feedback patterns in large language models. In Proceedings of the 38th International Conference on Neural Information Processing Systems, NIPS ’24, Red Hook, NY, USA, 2024a. Curran Associates Inc. ISBN 9798331314385.", + "tier": "established" + }, + { + "key": "marks2024enhancingneuralnetworkinterpretability", + "text": "Luke Marks, Alasdair Paren, David Krueger, and Fazl Barez. Enhancing neural network interpretability with feature-aligned sparse autoencoders, 2024b. URL https://arxiv.org/abs/2411.01220.", + "tier": "emerging" + }, + { + "key": "marks2024the", + "text": "Samuel Marks and Max Tegmark. The geometry of truth: Emergent linear structure in large language model representations of true/false datasets. In First Conference on Language Modeling, 2024. URL https://openreview.net/forum?id=aajyHYjjsk.", + "tier": "established" + }, { "key": "marks2025sparse", "text": "Samuel Marks, Can Rager, Eric J Michaud, Yonatan Belinkov, David Bau, and Aaron Mueller. Sparse feature circuits: Discovering and editing interpretable causal graphs in language models. In The Thirteenth International Conference on Learning Representations, 2025a. URL https://openreview.net/forum?id=I4e82CIDxv.", @@ -3247,7 +4250,12 @@ { "key": "marks2025auditinglanguagemodelshidden", "text": "Samuel Marks, Johannes Treutlein, Trenton Bricken, Jack Lindsey, Jonathan Marcus, Siddharth Mishra-Sharma, Daniel Ziegler, Emmanuel Ameisen, Joshua Batson, Tim Belonax, Samuel R. Bowman, Shan Carter, Brian Chen, Hoagy Cunningham, Carson Denison, Florian Dietz, Satvik Golechha, Akbir Khan, Jan Kirchner, Jan Leike, Austin Meek, Kei Nishimura-Gasparian, Euan Ong, Christopher Olah, Adam Pearce, Fabien Roger, Jeanne Salle, Andy Shih, Meg Tong, Drake Thomas, Kelley Rivoire, Adam Jermyn, Monte MacDiarmid, Tom Henighan, and Evan Hubinger. Auditing language models for hidden objectives, 2025b. URL https://arxiv.org/abs/2503.10965.", - "tier": null + "tier": "emerging" + }, + { + "key": "kfac", + "text": "James Martens and Roger Grosse. Optimizing neural networks with kronecker-factored approximate curvature. In Francis Bach and David Blei (eds.), Proceedings of the 32nd International Conference on Machine Learning, volume 37 of Proceedings of Machine Learning Research, pp. 2408–2417, Lille, France, 07–09 Jul 2015. PMLR. URL https://proceedings.mlr.press/v37/martens15.html.", + "tier": "established" }, { "key": "marvin-linzen-2018-targeted", @@ -3259,11 +4267,26 @@ "text": "R. Thomas McCoy, Ellie Pavlick, and Tal Linzen. Right for the wrong reasons: Diagnosing syntactic heuristics in natural language inference. In Anna Korhonen, David Traum, and Lluís Màrquez (eds.), Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pp. 3428–3448, Florence, Italy, July 2019. Association for Computational Linguistics. . URL https://aclanthology.org/P19-1334/.", "tier": "established" }, + { + "key": "mcdougall2025gemmascope2", + "text": "Callum McDougall, Arthur Conmy, János Kramár, Tom Lieberum, Senthooran Rajamanoharan, and Neel Nanda. Gemma scope 2: Technical paper. Google DeepMind technical report, 2025. URL https://storage.googleapis.com/deepmind-media/DeepMind.com/Blog/gemma-scope-2-helping-the-ai-safety-community-deepen-understanding-of-complex-language-model-behavior/Gemma_Scope_2_Technical_Paper.pdf.", + "tier": "emerging" + }, { "key": "mcgrath2023hydraeffectemergentselfrepair", "text": "Thomas McGrath, Matthew Rahtz, Janos Kramar, Vladimir Mikulik, and Shane Legg. The hydra effect: Emergent self-repair in language model computations, 2023. URL https://arxiv.org/abs/2307.15771.", "tier": "emerging" }, + { + "key": "mckenzie2023inverse", + "text": "Ian R. McKenzie, Alexander Lyzhov, Michael Pieler, Alicia Parrish, Aaron Mueller, Ameya Prabhu, Euan McLean, Aaron Kirtland, Alexis Ross, Alisa Liu, Andrew Gritsevskiy, Daniel Wurgaft, Derik Kauffman, Gabriel Recchia, Jiacheng Liu, Joe Cavanagh, Max Weiss, Sicong Huang, The Floating Droid, Tom Tseng, Tomasz Korbak, Xudong Shen, Yuhui Zhang, Zhengping Zhou, Najoung Kim, Samuel R. Bowman, and Ethan Perez. Inverse scaling: When bigger isn’t better. Transactions on Machine Learning Research, 2023. URL https://arxiv.org/abs/2306.09479.", + "tier": "established" + }, + { + "key": "meloux2025everything", + "text": "Maxime Méloux, Silviu Maniu, François Portet, and Maxime Peyrard. Everything, everywhere, all at once: Is mechanistic interpretability identifiable? In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=5IWJBStfU7.", + "tier": "established" + }, { "key": "meng2023locatingeditingfactualassociations", "text": "Kevin Meng, David Bau, Alex Andonian, and Yonatan Belinkov. Locating and editing factual associations in gpt. Advances in neural information processing systems, 35: 17359–17372, 2022.", @@ -3279,45 +4302,60 @@ "text": "Jack Merullo, Connor Watts, Max Loeffler, Liv Gorton, Elana Simon, Tom McGrath, and Owen Lewis. Replicating circuit tracing for a simple known mechanism. https://www.goodfire.ai/research/replicating-circuit-tracing-for-a-simple-mechanism, 2025.", "tier": "emerging" }, + { + "key": "michaud2023quantization", + "text": "Eric J. Michaud, Ziming Liu, Uzay Girit, and Max Tegmark. The quantization model of neural scaling. In Advances in Neural Information Processing Systems, volume 36, 2023. URL https://openreview.net/forum?id=3tbTw2ga8K.", + "tier": "established" + }, + { + "key": "miller2024transformer", + "text": "Joseph Miller, Bilal Chughtai, and William Saunders. Transformer circuit evaluation metrics are not robust. In First Conference on Language Modeling, 2024. URL https://openreview.net/forum?id=zSf8PJyQb2.", + "tier": "established" + }, { "key": "miller2026identifyingintervenableinterpretablefeatures", "text": "Moritz Miller, Florent Draye, and Bernhard Schölkopf. Identifying intervenable and interpretable features via orthogonality regularization, 2026. URL https://arxiv.org/abs/2602.04718.", "tier": "emerging" }, { - "key": "minder_overcoming_2025", - "text": "Julian Minder, Clément Dumas, Caden Juang, Bilal Chugtai, and Neel Nanda. Overcoming Sparsity Artifacts in Crosscoders to Interpret Chat-Tuning, October 2025. URL http://arxiv.org/abs/2504.02922. arXiv:2504.02922 [cs].", + "key": "minsky1969perceptrons", + "text": "Marvin L. Minsky and Seymour A. Papert. Perceptrons: An Introduction to Computational Geometry. MIT Press, Cambridge, MA, 1969.", "tier": "established" }, { "key": "mlodozeniec2025distributionaltrainingdataattribution", "text": "Bruno Kacper Mlodozeniec, Isaac Reid, Samuel Power, David Krueger, Murat A Erdogdu, Richard E. Turner, and Roger Baker Grosse. Distributional training data attribution: What do influence functions sample? In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025. URL https://openreview.net/forum?id=UBRFn7YKMe.", - "tier": null + "tier": "established" }, { - "key": "morcos2018insightsrepresentationalsimilarityneural", - "text": "Ari S. Morcos, Maithra Raghu, and Samy Bengio. Insights on representational similarity in neural networks with canonical correlation, 2018a. URL https://arxiv.org/abs/1806.05759.", - "tier": null + "key": "modarressi2025nolima", + "text": "Ali Modarressi, Hanieh Deilamsalehy, Franck Dernoncourt, Trung Bui, Ryan A. Rossi, Seunghyun Yoon, and Hinrich Schuetze. Nolima: Long-context evaluation beyond literal matching. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=0OshX1hiSa.", + "tier": "established" }, { - "key": "morcos_insights_2018", - "text": "Ari S. Morcos, Maithra Raghu, and Samy Bengio. Insights on representational similarity in neural networks with canonical correlation. In Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS’18, pp. 5732–5741, Red Hook, NY, USA, 2018b. Curran Associates Inc. URL https://dl.acm.org/doi/10.5555/3327345.3327475.", - "tier": null + "key": "morcos2018insightsrepresentationalsimilarityneural", + "text": "Ari S. Morcos, Maithra Raghu, and Samy Bengio. Insights on representational similarity in neural networks with canonical correlation. In Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS’18, pp. 5732–5741, Red Hook, NY, USA, 2018. Curran Associates Inc.", + "tier": "established" }, { - "key": "moschella2023relativerepresentationsenablezeroshot", - "text": "Luca Moschella, Valentino Maiorca, Marco Fumero, Antonio Norelli, Francesco Locatello, and Emanuele Rodolà. Relative representations enable zero-shot latent space communication, 2023. URL https://arxiv.org/abs/2209.15430.", - "tier": "emerging" + "key": "moschella2023relative", + "text": "Luca Moschella, Valentino Maiorca, Marco Fumero, Antonio Norelli, Francesco Locatello, and Emanuele Rodolà. Relative representations enable zero-shot latent space communication. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=SrC-nwieGJ.", + "tier": "established" }, { "key": "mueller2025mib", - "text": "Aaron Mueller, Atticus Geiger, Sarah Wiegreffe, Dana Arad, Iván Arcuschin, Adam Belfki, Yik Siu Chan, Jaden Fried Fiotto-Kaufman, Tal Haklay, Michael Hanna, Jing Huang, Rohan Gupta, Yaniv Nikankin, Hadas Orgad, Nikhil Prakash, Anja Reusch, Aruna Sankaranarayanan, Shun Shao, Alessandro Stolfo, Martin Tutek, Amir Zur, David Bau, and Yonatan Belinkov. MIB: A mechanistic interpretability benchmark. In Forty-second International Conference on Machine Learning, 2025a. URL https://openreview.net/forum?id=sSrOwve6vb.", - "tier": "emerging" + "text": "Aaron Mueller, Atticus Geiger, Sarah Wiegreffe, Dana Arad, Iván Arcuschin, Adam Belfki, Yik Siu Chan, Jaden Fried Fiotto-Kaufman, Tal Haklay, Michael Hanna, Jing Huang, Rohan Gupta, Yaniv Nikankin, Hadas Orgad, Nikhil Prakash, Anja Reusch, Aruna Sankaranarayanan, Shun Shao, Alessandro Stolfo, Martin Tutek, Amir Zur, David Bau, and Yonatan Belinkov. MIB: A mechanistic interpretability benchmark. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=sSrOwve6vb.", + "tier": "established" }, { - "key": "mueller2025isolationentanglementinterpretabilitymethods", - "text": "Aaron Mueller, Andrew Lee, Shruti Joshi, Ekdeep Singh Lubana, Dhanya Sridhar, and Patrik Reizinger. From isolation to entanglement: When do interpretability methods identify and disentangle known concepts?, 2025b. URL https://arxiv.org/abs/2512.15134.", - "tier": "emerging" + "key": "relu", + "text": "Vinod Nair and Geoffrey E. Hinton. Rectified linear units improve restricted boltzmann machines. In Proceedings of the 27th International Conference on International Conference on Machine Learning, ICML’10, pp. 807–814, Madison, WI, USA, 2010. Omnipress. ISBN 9781605589077.", + "tier": "established" + }, + { + "key": "nakkiran2019sgd", + "text": "Preetum Nakkiran, Gal Kaplun, Dimitris Kalimeris, Tristan Yang, Benjamin L. Edelman, Fred Zhang, and Boaz Barak. SGD on neural networks learns functions of increasing complexity. Curran Associates Inc., Red Hook, NY, USA, 2019.", + "tier": "established" }, { "key": "nanda2023progressmeasuresgrokkingmechanistic", @@ -3332,6 +4370,16 @@ { "key": "needham2025largelanguagemodelsknow", "text": "Joe Needham, Giles Edkins, Govind Pimpale, Henning Bartsch, and Marius Hobbhahn. Large language models often know when they are being evaluated, 2025. URL https://arxiv.org/abs/2505.23836.", + "tier": "emerging" + }, + { + "key": "neo-etal-2024-interpreting", + "text": "Clement Neo, Shay B. Cohen, and Fazl Barez. Interpreting context look-ups in transformers: Investigating attention-MLP interactions. In Yaser Al-Onaizan, Mohit Bansal, and Yun-Nung Chen (eds.), Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, pp. 16681–16697, Miami, Florida, USA, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.emnlp-main.930/.", + "tier": "established" + }, + { + "key": "neo2025towards", + "text": "Clement Neo, Luke Ong, Philip Torr, Mor Geva, David Krueger, and Fazl Barez. Towards interpreting visual information processing in vision-language models. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=chanJGoa7f.", "tier": "established" }, { @@ -3339,16 +4387,36 @@ "text": "Andrew Ng. Sparse autoencoder. Stanford CS294A Lecture Notes, 2011.", "tier": "emerging" }, + { + "key": "ICLR2025_8c5f3029", + "text": "Yaniv Nikankin, Anja Reusch, Aaron Mueller, and Yonatan Belinkov. Arithmetic without algorithms: Language models solve math with a bag of heuristics. In Y. Yue, A. Garg, N. Peng, F. Sha, and R. Yu (eds.), International Conference on Learning Representations, volume 2025, pp. 55939–55965, 2025. URL https://proceedings.iclr.cc/paper_files/paper/2025/file/8c5f30296296d2ae402ebbd09aaa9c12-Paper-Conference.pdf.", + "tier": "established" + }, { "key": "nostalgebraist2020interpretinggpt", "text": "nostalgebraist. Interpreting gpt: The logit lens. LessWrong post, 2020. URL https://www.lesswrong.com/posts/AcKRB8wDpdaN6v6ru/interpreting-gpt-the-logit-lens.", "tier": "emerging" }, + { + "key": "obrien2026deep", + "text": "Kyle O’Brien, Stephen Casper, Quentin Gregory Anthony, Tomek Korbak, Robert Kirk, Xander Davies, Ishan Mishra, Geoffrey Irving, Yarin Gal, and Stella Biderman. Deep ignorance: Filtering pretraining data builds tamper-resistant safeguards into open-weight LLMs. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=xcf0QcTcGS.", + "tier": "established" + }, + { + "key": "oikarinen2023clipdissect", + "text": "Tuomas Oikarinen and Tsui-Wei Weng. CLIP-dissect: Automatic description of neuron representations in deep vision networks. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=iPWiwWHc1V.", + "tier": "established" + }, { "key": "oikarinen2023labelfree", "text": "Tuomas Oikarinen, Subhro Das, Lam M. Nguyen, and Tsui-Wei Weng. Label-free concept bottleneck models. In The Eleventh International Conference on Learning Representations, 2023.", "tier": "established" }, + { + "key": "olah2017feature", + "text": "Chris Olah, Alexander Mordvintsev, and Ludwig Schubert. Feature visualization. Distill, 2017. . https://distill.pub/2017/feature-visualization.", + "tier": "emerging" + }, { "key": "olah2020zoom", "text": "Chris Olah, Nick Cammarata, Ludwig Schubert, Gabriel Goh, Michael Petrov, and Shan Carter. Zoom in: An introduction to circuits. Distill, 2020. . https://distill.pub/2020/circuits/zoom-in.", @@ -3364,30 +4432,40 @@ "text": "James Oldfield, Shawn Im, Sharon Li, Mihalis Nicolaou, Ioannis Patras, and Grigorios Chrysos. Towards interpretability without sacrifice: Faithful dense layer decomposition with mixture of decoders. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025. URL https://openreview.net/forum?id=jcvX8XFNqX.", "tier": "established" }, + { + "key": "oldfield2026beyond", + "text": "James Oldfield, Philip Torr, Ioannis Patras, Adel Bibi, and Fazl Barez. Beyond linear probes: Dynamic safety monitoring for language models. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=AGWa8whf92.", + "tier": "established" + }, { "key": "olshausen1996emergence", - "text": "Bruno A. Olshausen and David J. Field. Emergence of simple-cell receptive field properties by learning a sparse code for natural images. Nature, 381: 607–609, 1996. .", + "text": "Bruno A. Olshausen and David J. Field. Emergence of simple-cell receptive field properties by learning a sparse code for natural images. Nature, 381 (6583): 607–609, June 1996. .", + "tier": "established" + }, + { + "key": "olshausen1997sparse", + "text": "Bruno A. Olshausen and David J. Field. Sparse coding with an overcomplete basis set: A strategy employed by V1? Vision Research, 37 (23): 3311–3325, December 1997. .", "tier": "established" }, { "key": "olsson2022incontextlearninginductionheads", - "text": "Catherine Olsson, Nelson Elhage, Neel Nanda, Nicholas Joseph, Nova DasSarma, Tom Henighan, Ben Mann, Amanda Askell, Yuntao Bai, Anna Chen, Tom Conerly, Dawn Drain, Deep Ganguli, Zac Hatfield-Dodds, Danny Hernandez, Scott Johnston, Andy Jones, Jackson Kernion, Liane Lovitt, Kamal Ndousse, Dario Amodei, Tom Brown, Jack Clark, Jared Kaplan, Sam McCandlish, and Chris Olah. In-context learning and induction heads. Transformer Circuits Thread, 2022a. https://transformer-circuits.pub/2022/in-context-learning-and-induction-heads/index.html.", + "text": "Catherine Olsson, Nelson Elhage, Neel Nanda, Nicholas Joseph, Nova DasSarma, Tom Henighan, Ben Mann, Amanda Askell, Yuntao Bai, Anna Chen, Tom Conerly, Dawn Drain, Deep Ganguli, Zac Hatfield-Dodds, Danny Hernandez, Scott Johnston, Andy Jones, Jackson Kernion, Liane Lovitt, Kamal Ndousse, Dario Amodei, Tom Brown, Jack Clark, Jared Kaplan, Sam McCandlish, and Chris Olah. In-context learning and induction heads. Transformer Circuits Thread, 2022. https://transformer-circuits.pub/2022/in-context-learning-and-induction-heads/index.html.", "tier": "emerging" }, { - "key": "olsson2022context", - "text": "Catherine Olsson et al. In-context learning and induction heads. arXiv preprint arXiv:2209.11895, 2022b.", - "tier": "emerging" + "key": "OMLIN199641", + "text": "Christian W. Omlin and C.Lee Giles. Extraction of rules from discrete-time recurrent neural networks. Neural Networks, 9 (1): 41–52, 1996. ISSN 0893-6080. . URL https://www.sciencedirect.com/science/article/pii/0893608095000860.", + "tier": "established" }, { - "key": "oozeer2025beyond", - "text": "Narmeen Fatimah Oozeer, Luke Marks, Fazl Barez, and Amir Abdullah. Beyond linear steering: Unified multi-attribute control for language models. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Findings of the Association for Computational Linguistics: EMNLP 2025, pp. 23513–23557, Suzhou, China, November 2025a. Association for Computational Linguistics. ISBN 979-8-89176-335-7. . URL https://aclanthology.org/2025.findings-emnlp.1278/.", - "tier": null + "key": "oneill2025compute", + "text": "Charles O’Neill, Alim Gumran, and David Klindt. Compute optimal inference and provable amortisation gap in sparse autoencoders. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=8forr1FkvC.", + "tier": "established" }, { - "key": "oozeer2025activation", - "text": "Narmeen Fatimah Oozeer, Dhruv Nathawani, Nirmalendu Prakash, Michael Lan, Abir HARRASSE, and Amir Abdullah. Activation space interventions can be transferred between large language models. In Forty-second International Conference on Machine Learning, 2025b. URL https://openreview.net/forum?id=HXOicJsmMQ.", - "tier": null + "key": "oozeer2025beyond", + "text": "Narmeen Fatimah Oozeer, Luke Marks, Fazl Barez, and Amir Abdullah. Beyond linear steering: Unified multi-attribute control for language models. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Findings of the Association for Computational Linguistics: EMNLP 2025, pp. 23513–23557, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-335-7. . URL https://aclanthology.org/2025.findings-emnlp.1278/.", + "tier": "established" }, { "key": "openai2024gpt4technicalreport", @@ -3395,8 +4473,18 @@ "tier": "emerging" }, { - "key": "orgad2026interpretabilityactionable", - "text": "Hadas Orgad, Fazl Barez, Tal Haklay, Isabelle Lee, Marius Mosbach, Anja Reusch, Naomi Saphra, Byron Wallace, Sarah Wiegreffe, Eric Wong, Ian Tenney, and Mor Geva. Interpretability can be actionable, 2026. URL https://arxiv.org/abs/2605.11161.", + "key": "openai2025gptoss120bgptoss20bmodel", + "text": "OpenAI, :, Sandhini Agarwal, Lama Ahmad, Jason Ai, Sam Altman, Andy Applebaum, Edwin Arbus, Rahul K. Arora, Yu Bai, Bowen Baker, Haiming Bao, Boaz Barak, Ally Bennett, Tyler Bertao, Nivedita Brett, Eugene Brevdo, Greg Brockman, Sebastien Bubeck, Che Chang, Kai Chen, Mark Chen, Enoch Cheung, Aidan Clark, Dan Cook, Marat Dukhan, Casey Dvorak, Kevin Fives, Vlad Fomenko, Timur Garipov, Kristian Georgiev, Mia Glaese, Tarun Gogineni, Adam Goucher, Lukas Gross, Katia Gil Guzman, John Hallman, Jackie Hehir, Johannes Heidecke, Alec Helyar, Haitang Hu, Romain Huet, Jacob Huh, Saachi Jain, Zach Johnson, Chris Koch, Irina Kofman, Dominik Kundel, Jason Kwon, Volodymyr Kyrylov, Elaine Ya Le, Guillaume Leclerc, James Park Lennon, Scott Lessans, Mario Lezcano-Casado, Yuanzhi Li, Zhuohan Li, Ji Lin, Jordan Liss, Lily, Liu, Jiancheng Liu, Kevin Lu, Chris Lu, Zoran Martinovic, Lindsay McCallum, Josh McGrath, Scott McKinney, Aidan McLaughlin, Song Mei, Steve Mostovoy, Tong Mu, Gideon Myles, Alexander Neitz, Alex Nichol, Jakub Pachocki, Alex Paino, Dana Palmie, Ashley Pantuliano, Giambattista Parascandolo, Jongsoo Park, Leher Pathak, Carolina Paz, Ludovic Peran, Dmitry Pimenov, Michelle Pokrass, Elizabeth Proehl, Huida Qiu, Gaby Raila, Filippo Raso, Hongyu Ren, Kimmy Richardson, David Robinson, Bob Rotsted, Hadi Salman, Suvansh Sanjeev, Max Schwarzer, D. Sculley, Harshit Sikchi, Kendal Simon, Karan Singhal, Yang Song, Dane Stuckey, Zhiqing Sun, Philippe Tillet, Sam Toizer, Foivos Tsimpourlas, Nikhil Vyas, Eric Wallace, Xin Wang, Miles Wang, Olivia Watkins, Kevin Weil, Amy Wendling, Kevin Whinnery, Cedric Whitney, Hannah Wong, Lin Yang, Yu Yang, Michihiro Yasunaga, Kristen Ying, Wojciech Zaremba, Wenting Zhan, Cyril Zhang, Brian Zhang, Eddie Zhang, and Shengjia Zhao. gpt-oss-120b & gpt-oss-20b model card, 2025. URL https://arxiv.org/abs/2508.10925.", + "tier": "emerging" + }, + { + "key": "orgad2026position", + "text": "Hadas Orgad, Fazl Barez, Tal Haklay, Isabelle Lee, Marius Mosbach, Anja Reusch, Naomi Saphra, Byron C Wallace, Sarah Wiegreffe, Eric Wong, Ian Tenney, and Mor Geva. Position: Interpretability can be actionable. In Forty-third International Conference on Machine Learning Position Paper Track, 2026. URL https://openreview.net/forum?id=yRUy3w5y56.", + "tier": "established" + }, + { + "key": "oshea2015introductionconvolutionalneuralnetworks", + "text": "Keiron O’Shea and Ryan Nash. An introduction to convolutional neural networks, 2015. URL https://arxiv.org/abs/1511.08458.", "tier": "emerging" }, { @@ -3407,46 +4495,76 @@ { "key": "ouyang2022traininglanguagemodelsfollow", "text": "Long Ouyang, Jeffrey Wu, Xu Jiang, Diogo Almeida, Carroll Wainwright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, John Schulman, Jacob Hilton, Fraser Kelton, Luke Miller, Maddie Simens, Amanda Askell, Peter Welinder, Paul F Christiano, Jan Leike, and Ryan Lowe. Training language models to follow instructions with human feedback. In S. Koyejo, S. Mohamed, A. Agarwal, D. Belgrave, K. Cho, and A. Oh (eds.), Advances in Neural Information Processing Systems, volume 35, pp. 27730–27744. Curran Associates, Inc., 2022. URL https://proceedings.neurips.cc/paper_files/paper/2022/file/b1efde53be364a73914f58805a001731-Paper-Conference.pdf.", - "tier": "emerging" + "tier": "established" }, { "key": "pan2024latentqateachingllmsdecode", - "text": "Alexander Pan, Lijie Chen, and Jacob Steinhardt. Latentqa: Teaching llms to decode activations into natural language, 2024. URL https://arxiv.org/abs/2412.08686.", - "tier": "emerging" + "text": "Alexander Pan, Lijie Chen, and Jacob Steinhardt. LatentQA: Teaching LLMs to decode activations into natural language. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=niUroX9EOd.", + "tier": "established" }, { "key": "park2025monet", "text": "Jungwoo Park, Ahn Young Jin, Kee-Eung Kim, and Jaewoo Kang. Monet: Mixture of monosemantic experts for transformers. In The Thirteenth International Conference on Learning Representations, 2025.", "tier": "established" }, + { + "key": "lrh", + "text": "Kiho Park, Yo Joong Choe, and Victor Veitch. The linear representation hypothesis and the geometry of large language models. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.", + "tier": "established" + }, { "key": "park2023trakattributingmodelbehavior", "text": "Sung Min Park, Kristian Georgiev, Andrew Ilyas, Guillaume Leclerc, and Aleksander Madry. TRAK: Attributing model behavior at scale. In Andreas Krause, Emma Brunskill, Kyunghyun Cho, Barbara Engelhardt, Sivan Sabato, and Jonathan Scarlett (eds.), Proceedings of the 40th International Conference on Machine Learning, volume 202 of Proceedings of Machine Learning Research, pp. 27074–27113. PMLR, 23–29 Jul 2023. URL https://proceedings.mlr.press/v202/park23c.html.", "tier": "established" }, { - "key": "paulo2025sparse", - "text": "Gonçalo Paulo and Nora Belrose. Sparse autoencoders trained on the same data learn different features. arXiv preprint arXiv:2501.16615, 2025.", - "tier": null + "key": "paulo2025sparse", + "text": "Gonçalo Paulo and Nora Belrose. Sparse autoencoders trained on the same data learn different features. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=EjInprGpk9.", + "tier": "established" + }, + { + "key": "paulo2024does", + "text": "Gonçalo Paulo, Thomas Marshall, and Nora Belrose. Do transformer interpretability methods transfer to rnns? In Proceedings of the Thirty-Ninth AAAI Conference on Artificial Intelligence and Thirty-Seventh Conference on Innovative Applications of Artificial Intelligence and Fifteenth Symposium on Educational Advances in Artificial Intelligence, AAAI’25/IAAI’25/EAAI’25. AAAI Press, 2025a. ISBN 978-1-57735-897-8. . URL https://doi.org/10.1609/aaai.v39i26.34969.", + "tier": "established" + }, + { + "key": "paulo2025automatically", + "text": "Gonçalo Santos Paulo, Alex Troy Mallen, Caden Juang, and Nora Belrose. Automatically interpreting millions of features in large language models. In Forty-second International Conference on Machine Learning, 2025b. URL https://openreview.net/forum?id=EemtbhJOXc.", + "tier": "established" + }, + { + "key": "paulo2025transcodersbeatsparseautoencoders", + "text": "Gonçalo Paulo, Stepan Shabalin, and Nora Belrose. Transcoders beat sparse autoencoders for interpretability, 2025c. URL https://arxiv.org/abs/2501.18823.", + "tier": "emerging" + }, + { + "key": "pearce2025bilinear", + "text": "Michael Pearce, Thomas Dooms, Alice Rigg, Jose Oramas, and Lee Sharkey. Bilinear mlps enable weight-based mechanistic interpretability. In Y. Yue, A. Garg, N. Peng, F. Sha, and R. Yu (eds.), International Conference on Learning Representations, volume 2025, pp. 47283–47310, 2025. URL https://proceedings.iclr.cc/paper_files/paper/2025/file/7504142a20a3e1fe9dd7de42f475828c-Paper-Conference.pdf.", + "tier": "established" + }, + { + "key": "pearson1901lines", + "text": "Karl Pearson. On lines and planes of closest fit to systems of points in space. The London, Edinburgh, and Dublin Philosophical Magazine and Journal of Science, 2 (11): 559–572, 1901. .", + "tier": "established" }, { - "key": "paulo2025automatically", - "text": "Gonçalo Santos Paulo, Alex Troy Mallen, Caden Juang, and Nora Belrose. Automatically interpreting millions of features in large language models. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=EemtbhJOXc.", + "key": "peng-etal-2023-rwkv", + "text": "Bo Peng, Eric Alcaide, Quentin Anthony, Alon Albalak, Samuel Arcadinho, Stella Biderman, Huanqi Cao, Xin Cheng, Michael Chung, Leon Derczynski, Xingjian Du, Matteo Grella, Kranthi Gv, Xuzheng He, Haowen Hou, Przemyslaw Kazienko, Jan Kocon, Jiaming Kong, Bartłomiej Koptyra, Hayden Lau, Jiaju Lin, Krishna Sri Ipsit Mantri, Ferdinand Mom, Atsushi Saito, Guangyu Song, Xiangru Tang, Johan Wind, Stanisław Woźniak, Zhenyuan Zhang, Qinghua Zhou, Jian Zhu, and Rui-Jie Zhu. RWKV: Reinventing RNNs for the transformer era. In Houda Bouamor, Juan Pino, and Kalika Bali (eds.), Findings of the Association for Computational Linguistics: EMNLP 2023, pp. 14048–14077, Singapore, December 2023. Association for Computational Linguistics. . URL https://aclanthology.org/2023.findings-emnlp.936/.", "tier": "established" }, { - "key": "pearce2025bilinear", - "text": "Michael Pearce, Thomas Dooms, Alice Rigg, Jose Oramas, and Lee Sharkey. Bilinear mlps enable weight-based mechanistic interpretability. In Y. Yue, A. Garg, N. Peng, F. Sha, and R. Yu (eds.), International Conference on Learning Representations, volume 2025, pp. 47283–47310, 2025. URL https://proceedings.iclr.cc/paper_files/paper/2025/file/7504142a20a3e1fe9dd7de42f475828c-Paper-Conference.pdf.", + "key": "peng2025rwkv", + "text": "Bo Peng, Ruichong Zhang, Daniel Goldstein, Eric Alcaide, Xingjian Du, Haowen Hou, Jiaju Lin, Jiaxing Liu, Janna Lu, William Merrill, Guangyu Song, Kaifeng Tan, Saiteja Utpala, Nathan Wilce, Johan S. Wind, Tianyi Wu, Daniel Wuttke, and Christian Zhou-Zheng. RWKV-7 ”goose” with expressive dynamic state evolution. In Second Conference on Language Modeling, 2025. URL https://openreview.net/forum?id=ayB1PACN5j.", "tier": "established" }, { "key": "perez2022redteaminglanguagemodels", - "text": "Ethan Perez, Saffron Huang, Francis Song, Trevor Cai, Roman Ring, John Aslanides, Amelia Glaese, Nat McAleese, and Geoffrey Irving. Red teaming language models with language models. In Yoav Goldberg, Zornitsa Kozareva, and Yue Zhang (eds.), Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, pp. 3419–3448, Abu Dhabi, United Arab Emirates, December 2022a. Association for Computational Linguistics. . URL https://aclanthology.org/2022.emnlp-main.225/.", + "text": "Ethan Perez, Saffron Huang, Francis Song, Trevor Cai, Roman Ring, John Aslanides, Amelia Glaese, Nat McAleese, and Geoffrey Irving. Red teaming language models with language models. In Yoav Goldberg, Zornitsa Kozareva, and Yue Zhang (eds.), Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, pp. 3419–3448, Abu Dhabi, United Arab Emirates, December 2022. Association for Computational Linguistics. . URL https://aclanthology.org/2022.emnlp-main.225/.", "tier": "established" }, { - "key": "Ethan2022redteam", - "text": "Ethan Perez, Saffron Huang, H. Francis Song, Trevor Cai, Roman Ring, John Aslanides, Amelia Glaese, Nat McAleese, and Geoffrey Irving. Red teaming language models with language models. In Yoav Goldberg, Zornitsa Kozareva, and Yue Zhang (eds.), Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, EMNLP 2022, Abu Dhabi, United Arab Emirates, December 7-11, 2022, pp. 3419–3448. Association for Computational Linguistics, 2022b. . URL https://doi.org/10.18653/v1/2022.emnlp-main.225.", + "key": "petrov2026brokenmath", + "text": "Ivo Petrov, Jasper Dekoninck, and Martin Vechev. Brokenmath: A benchmark for sycophancy in theorem proving with LLMs. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=Y5bEI4cuMd.", "tier": "established" }, { @@ -3457,28 +4575,33 @@ { "key": "piktus-etal-2023-roots", "text": "Aleksandra Piktus, Christopher Akiki, Paulo Villegas, Hugo Laurençon, Gérard Dupont, Sasha Luccioni, Yacine Jernite, and Anna Rogers. The ROOTS search tool: Data transparency for LLMs. In Danushka Bollegala, Ruihong Huang, and Alan Ritter (eds.), Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 3: System Demonstrations), pp. 304–314, Toronto, Canada, July 2023a. Association for Computational Linguistics. . URL https://aclanthology.org/2023.acl-demo.29/.", - "tier": null + "tier": "established" }, { "key": "piktus2023gaia", "text": "Aleksandra Piktus, Odunayo Ogundepo, Christopher Akiki, Akintunde Oladipo, Xinyu Zhang, Hailey Schoelkopf, Stella Biderman, Martin Potthast, and Jimmy Lin. Gaia search: Hugging face and pyserini interoperability for nlp training data exploration. In Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 3: System Demonstrations), pp. 588–598, 2023b.", + "tier": "established" + }, + { + "key": "plunkett", + "text": "Dillon Plunkett, Adam Morris, Keerthi Reddy, and Jorge Morales. Self-interpretability: Llms can describe complex internal processes that drive their decisions, 2025. URL https://arxiv.org/abs/2505.17120.", "tier": "emerging" }, { "key": "power2022grokkinggeneralizationoverfittingsmall", "text": "Alethea Power, Yuri Burda, Harri Edwards, Igor Babuschkin, and Vedant Misra. Grokking: Generalization beyond overfitting on small algorithmic datasets, 2022. URL https://arxiv.org/abs/2201.02177.", - "tier": null - }, - { - "key": "pruthi2020estimatingtrainingdatainfluence", - "text": "Garima Pruthi, Frederick Liu, Mukund Sundararajan, and Satyen Kale. Estimating training data influence by tracing gradient descent, 2020. URL https://arxiv.org/abs/2002.08484.", - "tier": null + "tier": "emerging" }, { "key": "quirke2024understandingadditiontransformers", "text": "Philip Quirke and Fazl Barez. Understanding addition in transformers. In The Twelfth International Conference on Learning Representations, 2024. URL https://openreview.net/forum?id=rIx1YXVWZb.", "tier": "established" }, + { + "key": "quirke2026understandingadditionsubtractiontransformers", + "text": "Philip Quirke, Clement Neo, and Fazl Barez. Understanding addition and subtraction in transformers, 2026. URL https://arxiv.org/abs/2402.02619.", + "tier": "emerging" + }, { "key": "yang2025qwen3technicalreport", "text": "QwenAI. Qwen3 technical report, 2025. URL https://arxiv.org/abs/2505.09388.", @@ -3487,37 +4610,57 @@ { "key": "raghu_svcca_2017", "text": "Maithra Raghu, Justin Gilmer, Jason Yosinski, and Jascha Sohl-Dickstein. SVCCA: Singular Vector Canonical Correlation Analysis for Deep Learning Dynamics and Interpretability. In Advances in Neural Information Processing Systems, volume 30. Curran Associates, Inc., 2017. URL https://proceedings.neurips.cc/paper_files/paper/2017/hash/dc6a7e655d7e5840e66733e9ee67cc69-Abstract.html.", - "tier": null + "tier": "established" + }, + { + "key": "rahaman2019spectral", + "text": "Nasim Rahaman, Aristide Baratin, Devansh Arpit, Felix Draxler, Min Lin, Fred Hamprecht, Yoshua Bengio, and Aaron Courville. On the spectral bias of neural networks. In Kamalika Chaudhuri and Ruslan Salakhutdinov (eds.), Proceedings of the 36th International Conference on Machine Learning, volume 97 of Proceedings of Machine Learning Research, pp. 5301–5310. PMLR, 09–15 Jun 2019. URL https://proceedings.mlr.press/v97/rahaman19a.html.", + "tier": "established" }, { "key": "rai2024practical", - "text": "Daking Rai, Yilun Zhou, Shi Feng, Abulhair Saparov, and Ziyu Yao. A practical review of mechanistic interpretability for transformer-based language models. arXiv preprint arXiv:2407.02646, 2024.", + "text": "Daking Rai, Yilun Zhou, Shi Feng, Abulhair Saparov, and Ziyu Yao. A practical review of mechanistic interpretability for transformer-based language models, 2025. URL https://arxiv.org/abs/2407.02646.", "tier": "emerging" }, { "key": "rajamanoharan2024improvingdictionarylearninggated", - "text": "Senthooran Rajamanoharan, Arthur Conmy, Lewis Smith, Tom Lieberum, Vikrant Varma, János Kramár, Rohin Shah, and Neel Nanda. Improving dictionary learning with gated sparse autoencoders, 2024a. URL https://arxiv.org/abs/2404.16014.", - "tier": "emerging" + "text": "Senthooran Rajamanoharan, Arthur Conmy, Lewis Smith, Tom Lieberum, Vikrant Varma, Janos Kramar, Rohin Shah, and Neel Nanda. Improving sparse decomposition of language model activations with gated sparse autoencoders. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024a. URL https://openreview.net/forum?id=zLBlin2zvW.", + "tier": "established" }, { "key": "rajamanoharan2024jumpingaheadimprovingreconstruction", "text": "Senthooran Rajamanoharan, Tom Lieberum, Nicolas Sonnerat, Arthur Conmy, Vikrant Varma, János Kramár, and Neel Nanda. Jumping ahead: Improving reconstruction fidelity with jumprelu sparse autoencoders, 2024b. URL https://arxiv.org/abs/2407.14435.", - "tier": null + "tier": "emerging" }, { - "key": "rajani2020explainingimprovingmodelbehavior", - "text": "Nazneen Fatema Rajani, Ben Krause, Wengpeng Yin, Tong Niu, Richard Socher, and Caiming Xiong. Explaining and improving model behavior with k nearest neighbor representations, 2020. URL https://arxiv.org/abs/2010.09030.", - "tier": "established" + "key": "rathi2026shaping", + "text": "Neil Rathi and Alec Radford. Shaping capabilities with token-level data filtering, 2026. URL https://arxiv.org/abs/2601.21571.", + "tier": "emerging" }, { "key": "Raukur2022TowardTA", - "text": "Tilman Raukur, An Chang Ho, Stephen Casper, and Dylan Hadfield-Menell. Toward transparent ai: A survey on interpreting the inner structures of deep neural networks. 2023 IEEE Conference on Secure and Trustworthy Machine Learning (SaTML), pp. 464–483, 2022.", + "text": "Tilman Rauker, Anson Ho, Stephen Casper, and Dylan Hadfield-Menell. Toward Transparent AI: A Survey on Interpreting the Inner Structures of Deep Neural Networks . In 2023 IEEE Conference on Secure and Trustworthy Machine Learning (SaTML), pp. 464–483, Los Alamitos, CA, USA, February 2023. IEEE Computer Society. . URL https://doi.ieeecomputersociety.org/10.1109/SaTML54575.2023.00039.", "tier": "established" }, + { + "key": "raval2026curveballsteeringrightdirection", + "text": "Shivam Raval, Hae Jin Song, Linlin Wu, Abir Harrasse, Jeff M. Phillips, Fazl Barez, and Amirali Abdullah. Curveball steering: The right direction to steer isn’t always linear, 2026. URL https://arxiv.org/abs/2603.09313.", + "tier": "emerging" + }, { "key": "ravfogel-etal-2020-null", "text": "Shauli Ravfogel, Yanai Elazar, Hila Gonen, Michael Twiton, and Yoav Goldberg. Null it out: Guarding protected attributes by iterative nullspace projection. In Dan Jurafsky, Joyce Chai, Natalie Schluter, and Joel Tetreault (eds.), Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 7237–7256, Online, July 2020. Association for Computational Linguistics. . URL https://aclanthology.org/2020.acl-main.647/.", - "tier": null + "tier": "established" + }, + { + "key": "razeghi2022impact", + "text": "Yasaman Razeghi, Robert L Logan IV, Matt Gardner, and Sameer Singh. Impact of pretraining term frequencies on few-shot numerical reasoning. In Findings of the Association for Computational Linguistics: EMNLP 2022, pp. 840–854, 2022.", + "tier": "established" + }, + { + "key": "refinetti2023neural", + "text": "Maria Refinetti, Alessandro Ingrosso, and Sebastian Goldt. Neural networks trained with sgd learn distributions of increasing complexity. In International Conference on Machine Learning, 2023.", + "tier": "established" }, { "key": "ribeiro2016whyitrustyou", @@ -3529,15 +4672,20 @@ "text": "Nina Rimsky, Nick Gabrieli, Julian Schulz, Meg Tong, Evan Hubinger, and Alexander Turner. Steering llama 2 via contrastive activation addition. In Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 15504–15522, Bangkok, Thailand, August 2024. Association for Computational Linguistics.", "tier": "established" }, + { + "key": "sgd", + "text": "Herbert Robbins and Sutton Monro. A Stochastic Approximation Method. The Annals of Mathematical Statistics, 22 (3): 400 – 407, 1951. . URL https://doi.org/10.1214/aoms/1177729586.", + "tier": "established" + }, { "key": "rogers2020primer", "text": "Anna Rogers, Olga Kovaleva, and Anna Rumshisky. A primer in BERTology: What we know about how BERT works. Transactions of the Association for Computational Linguistics, 8: 842–866, 2020. . URL https://aclanthology.org/2020.tacl-1.54/.", "tier": "established" }, { - "key": "rosenfeld2019constructivepredictiongeneralizationerror", - "text": "Jonathan S. Rosenfeld, Amir Rosenfeld, Yonatan Belinkov, and Nir Shavit. A constructive prediction of the generalization error across scales, 2019. URL https://arxiv.org/abs/1909.12673.", - "tier": null + "key": "rosenfeld2020a", + "text": "Jonathan S. Rosenfeld, Amir Rosenfeld, Yonatan Belinkov, and Nir Shavit. A constructive prediction of the generalization error across scales. In International Conference on Learning Representations, 2020. URL https://openreview.net/forum?id=ryenvpEKDr.", + "tier": "established" }, { "key": "shaham2024multimodal", @@ -3552,61 +4700,156 @@ { "key": "saphra_interpretability_2023", "text": "Naomi Saphra. Interpretability creationism. The Gradient, 2023. URL https://thegradient.pub/interpretability-creationism/.", - "tier": null + "tier": "emerging" }, { "key": "saphra2024mechanistic", - "text": "Naomi Saphra and Sarah Wiegreffe. Mechanistic? arXiv preprint arXiv:2404.02698, 2024.", + "text": "Naomi Saphra and Sarah Wiegreffe. Mechanistic?, 2024. URL https://arxiv.org/abs/2410.09087.", + "tier": "emerging" + }, + { + "key": "sawmya2025birth", + "text": "Shashata Sawmya, Micah Adler, and Nir Shavit. The birth of knowledge: Emergent features across time, space, and scale in large language models, 2025. URL https://arxiv.org/abs/2505.19440.", "tier": "emerging" }, + { + "key": "saxe2014exact", + "text": "Andrew M Saxe, James L McClelland, and Surya Ganguli. Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. In The Twelfth International Conference on Learning Representations, 2014.", + "tier": "established" + }, { "key": "saxe2019mathematical", "text": "Andrew M Saxe, James L McClelland, and Surya Ganguli. A mathematical theory of semantic development in deep neural networks. Proceedings of the National Academy of Sciences, 116 (23): 11537–11546, 2019.", - "tier": null + "tier": "established" + }, + { + "key": "schaeffer2023emergent", + "text": "Rylan Schaeffer, Brando Miranda, and Sanmi Koyejo. Are emergent abilities of large language models a mirage? In Advances in Neural Information Processing Systems, volume 36, 2023. URL https://arxiv.org/abs/2304.15004.", + "tier": "established" + }, + { + "key": "schaeffer2025elusive", + "text": "Rylan Schaeffer, Hailey Schoelkopf, Brando Miranda, Gabriel Mukobi, Varun Madan, Adam Ibrahim, Herbie Bradley, Stella Biderman, and Sanmi Koyejo. Why has predicting downstream capabilities of frontier AI models with scale remained elusive? In Proceedings of the 42nd International Conference on Machine Learning, 2025. URL https://arxiv.org/abs/2406.04391.", + "tier": "established" }, { "key": "schaeffer2026quantifyingeffecttestset", "text": "Rylan Schaeffer, Joshua Kazdan, Baber Abbasi, Ken Ziyu Liu, Brando Miranda, Ahmed Ahmed, Abhay Puri, Niloofar Mireshghallah, and Sanmi Koyejo. Quantifying the effect of test set contamination on generative evaluations, 2026. URL https://arxiv.org/abs/2601.04301.", - "tier": "established" + "tier": "emerging" }, { "key": "schnass2008dictionary", "text": "Karin Schnass and Pierre Vandergheynst. Dictionary preconditioning for greedy algorithms. IEEE Transactions on Signal Processing, 56 (5): 1994–2002, 2008.", - "tier": null + "tier": "established" }, { - "key": "seo2018noise", - "text": "Junghoon Seo, Jeongyeol Choe, Jamyoung Koo, Seunghyeon Jeon, Beomsu Kim, and Taegyun Jeon. Noise-adding methods of saliency map as series of higher order partial derivative. arXiv preprint arXiv:1806.03000, 2018.", - "tier": "emerging" + "key": "schrodi2026towards", + "text": "Simon Schrodi, Elias Kempf, Fazl Barez, and Thomas Brox. Towards understanding subliminal learning: When and how hidden biases transfer. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=IelhmYSjPt.", + "tier": "established" }, { - "key": "snmf", - "text": "Or Shafran, Atticus Geiger, and Mor Geva. Decomposing mlp activations into interpretable features via semi-nonnegative matrix factorization. arXiv preprint arXiv:2506.10920, 2025.", - "tier": "emerging" + "key": "conceptdef", + "text": "Lisa Schut, Nenad Tomašev, Thomas McGrath, Demis Hassabis, Ulrich Paquet, and Been Kim. Bridging the human–ai knowledge gap through concept discovery and transfer in alphazero. Proceedings of the National Academy of Sciences, 122 (13): e2406675122, 2025. . URL https://www.pnas.org/doi/abs/10.1073/pnas.2406675122.", + "tier": "established" + }, + { + "key": "find", + "text": "Sarah Schwettmann, Tamar Shaham, Joanna Materzynska, Neil Chowdhury, Shuang Li, Jacob Andreas, David Bau, and Antonio Torralba. Find: A function description benchmark for evaluating interpretability methods. In A. Oh, T. Naumann, A. Globerson, K. Saenko, M. Hardt, and S. Levine (eds.), Advances in Neural Information Processing Systems, volume 36, pp. 75688–75715. Curran Associates, Inc., 2023. . URL https://proceedings.neurips.cc/paper_files/paper/2023/file/ef0164c1112f56246224af540857348f-Paper-Datasets_and_Benchmarks.pdf.", + "tier": "established" + }, + { + "key": "sellam2021multiberts", + "text": "Thibault Sellam, Steve Yadlowsky, Ian Tenney, Jason Wei, Naomi Saphra, Alexander D’Amour, Tal Linzen, Jasmijn Bastings, Iulia Raluca Turc, Jacob Eisenstein, Dipanjan Das, and Ellie Pavlick. The multiBERTs: BERT reproductions for robustness analysis. In International Conference on Learning Representations, 2022. URL https://openreview.net/forum?id=K0E_F0gFDgA.", + "tier": "established" + }, + { + "key": "gradcam", + "text": "Ramprasaath R Selvaraju, Michael Cogswell, Abhishek Das, Ramakrishna Vedantam, Devi Parikh, and Dhruv Batra. Grad-cam: Visual explanations from deep networks via gradient-based localization. In Proceedings of the IEEE International Conference on Computer Vision, pp. 618–626, 2017.", + "tier": "established" + }, + { + "key": "shafran-etal-2026-constructing", + "text": "Or David Shafran, Atticus Geiger, and Mor Geva. Constructing interpretable features from compositional neuron groups. In Maria Liakata, Viviane P. Moreira, Jiajun Zhang, and David Jurgens (eds.), Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 42326–42348, San Diego, California, United States, July 2026. Association for Computational Linguistics. ISBN 979-8-89176-390-6. . URL https://aclanthology.org/2026.acl-long.1959/.", + "tier": "established" }, { "key": "sharkey2025open", "text": "Lee Sharkey, Bilal Chughtai, Joshua Batson, Jack Lindsey, Jeffrey Wu, Lucius Bushnaq, Nicholas Goldowsky-Dill, Stefan Heimersheim, Alejandro Ortega, Joseph Isaac Bloom, Stella Biderman, Adrià Garriga-Alonso, Arthur Conmy, Neel Nanda, Jessica Mary Rumbelow, Martin Wattenberg, Nandi Schoots, Joseph Miller, William Saunders, Eric J Michaud, Stephen Casper, Max Tegmark, David Bau, Eric Todd, Atticus Geiger, Mor Geva, Jesse Hoogland, Daniel Murfet, and Thomas McGrath. Open problems in mechanistic interpretability. Transactions on Machine Learning Research, 2025. ISSN 2835-8856. URL https://openreview.net/forum?id=91H76m9Z94.", "tier": "established" }, + { + "key": "sharma2024locating", + "text": "Arnab Sen Sharma, David Atkinson, and David Bau. Locating and editing factual associations in mamba. In First Conference on Language Modeling, 2024a. URL https://openreview.net/forum?id=yoVRyrEgix.", + "tier": "established" + }, { "key": "sharma_towards_2025", "text": "Mrinank Sharma, Meg Tong, Tomasz Korbak, David Duvenaud, Amanda Askell, Samuel R. Bowman, Esin DURMUS, Zac Hatfield-Dodds, Scott R Johnston, Shauna M Kravec, Timothy Maxwell, Sam McCandlish, Kamal Ndousse, Oliver Rausch, Nicholas Schiefer, Da Yan, Miranda Zhang, and Ethan Perez. Towards understanding sycophancy in language models. In The Twelfth International Conference on Learning Representations, 2024b. URL https://openreview.net/forum?id=tvhaxkMKAn.", "tier": "established" }, + { + "key": "sharma2025constitutionalclassifiersdefendinguniversal", + "text": "Mrinank Sharma, Meg Tong, Jesse Mu, Jerry Wei, Jorrit Kruthoff, Scott Goodfriend, Euan Ong, Alwin Peng, Raj Agarwal, Cem Anil, Amanda Askell, Nathan Bailey, Joe Benton, Emma Bluemke, Samuel R. Bowman, Eric Christiansen, Hoagy Cunningham, Andy Dau, Anjali Gopal, Rob Gilson, Logan Graham, Logan Howard, Nimit Kalra, Taesung Lee, Kevin Lin, Peter Lofgren, Francesco Mosconi, Clare O’Hara, Catherine Olsson, Linda Petrini, Samir Rajani, Nikhil Saxena, Alex Silverstein, Tanya Singh, Theodore Sumers, Leonard Tang, Kevin K. Troy, Constantin Weisser, Ruiqi Zhong, Giulio Zhou, Jan Leike, Jared Kaplan, and Ethan Perez. Constitutional classifiers: Defending against universal jailbreaks across thousands of hours of red teaming, 2025. URL https://arxiv.org/abs/2501.18837.", + "tier": "emerging" + }, + { + "key": "sharma2024the", + "text": "Pratyusha Sharma, Jordan T. Ash, and Dipendra Misra. The truth is in there: Improving reasoning in language models with layer-selective rank reduction. In The Twelfth International Conference on Learning Representations, 2024c. URL https://openreview.net/forum?id=ozX92bu8VA.", + "tier": "established" + }, + { + "key": "shazeer2020gluvariantsimprovetransformer", + "text": "Noam Shazeer. Glu variants improve transformer, 2020. URL https://arxiv.org/abs/2002.05202.", + "tier": "emerging" + }, { "key": "shi-etal-2016-string", "text": "Xing Shi, Inkit Padhi, and Kevin Knight. Does string-based neural MT learn source syntax? In Jian Su, Kevin Duh, and Xavier Carreras (eds.), Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing, pp. 1526–1534, Austin, Texas, November 2016. Association for Computational Linguistics. . URL https://aclanthology.org/D16-1159/.", "tier": "established" }, + { + "key": "shilov2025beyond", + "text": "Igor Shilov, Alex Cloud, Aryo Pradipta Gema, Jacob Goldman-Wetzler, Nina Panickssery, Henry Sleight, Erik Jones, and Cem Anil. Beyond data filtering: Knowledge localization for capability removal in llms, 2025. URL https://arxiv.org/abs/2512.05648.", + "tier": "emerging" + }, + { + "key": "shortliffe1977mycin", + "text": "Edward H. Shortliffe. MYCIN: A knowledge-based computer program applied to infectious diseases. In Proceedings of the Annual Symposium on Computer Application in Medical Care, pp. 66–69, 1977. PMCID: PMC2464549.", + "tier": "established" + }, + { + "key": "deeplift", + "text": "Avanti Shrikumar, Peyton Greenside, and Anshul Kundaje. Learning important features through propagating activation differences. In Proceedings of the 34th International Conference on Machine Learning - Volume 70, ICML’17, pp. 3145–3153. JMLR.org, 2017.", + "tier": "established" + }, { "key": "shu-etal-2025-survey", "text": "Dong Shu, Xuansheng Wu, Haiyan Zhao, Daking Rai, Ziyu Yao, Ninghao Liu, and Mengnan Du. A survey on sparse autoencoders: Interpreting the internal mechanisms of large language models. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Findings of the Association for Computational Linguistics: EMNLP 2025, pp. 1690–1712, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-335-7. . URL https://aclanthology.org/2025.findings-emnlp.89/.", "tier": "established" }, + { + "key": "simhi-etal-2025-trust", + "text": "Adi Simhi, Itay Itzhak, Fazl Barez, Gabriel Stanovsky, and Yonatan Belinkov. Trust me, I’m wrong: LLMs hallucinate with certainty despite knowing the answer. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Findings of the Association for Computational Linguistics: EMNLP 2025, pp. 14665–14688, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-335-7. . URL https://aclanthology.org/2025.findings-emnlp.792/.", + "tier": "established" + }, + { + "key": "simhi2026old", + "text": "Adi Simhi, Fazl Barez, Martin Tutek, Yonatan Belinkov, and Shay B Cohen. Old habits die hard: How conversational history geometrically traps LLMs. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=jNhEZteJfq.", + "tier": "established" + }, + { + "key": "simon2026there", + "text": "Jamie Simon, Daniel Kunin, Alexander Atanasov, Enric Boix-Adserà, Blake Bordelon, Jeremy Cohen, Nikhil Ghosh, Florentin Guth, Arthur Jacot, Mason Kamb, Dhruva Karkada, Eric J. Michaud, Berkan Ottlik, and Joseph Turnbull. There will be a scientific theory of deep learning, 2026. URL https://arxiv.org/abs/2604.21691.", + "tier": "emerging" + }, { "key": "simonyan2014deepinsideconvolutionalnetworks", "text": "Karen Simonyan, Andrea Vedaldi, and Andrew Zisserman. Deep inside convolutional networks: Visualising image classification models and saliency maps. In Yoshua Bengio and Yann LeCun (eds.), 2nd International Conference on Learning Representations, ICLR 2014, Banff, AB, Canada, April 14-16, 2014, Workshop Track Proceedings, 2014.", + "tier": "emerging" + }, + { + "key": "singh2023transient", + "text": "Aaditya Singh, Stephanie C. Y. Chan, Ted Moskovitz, Erin Grant, Andrew Saxe, and Felix Hill. The transient nature of emergent in-context learning in transformers. In Advances in Neural Information Processing Systems, volume 36, 2023.", "tier": "established" }, { @@ -3615,8 +4858,8 @@ "tier": "established" }, { - "key": "slocum2025believe", - "text": "Stewart Slocum, Julian Minder, Clément Dumas, Henry Sleight, Ryan Greenblatt, Samuel Marks, and Rowan Wang. Believe it or not: How deeply do llms believe implanted facts? arXiv preprint arXiv:2510.17941, 2025.", + "key": "smilkov2017smoothgradremovingnoiseadding", + "text": "Daniel Smilkov, Nikhil Thorat, Been Kim, Fernanda Viégas, and Martin Wattenberg. Smoothgrad: removing noise by adding noise, 2017. URL https://arxiv.org/abs/1706.03825.", "tier": "emerging" }, { @@ -3624,24 +4867,54 @@ "text": "Lewis Smith. The ’strong’ feature hypothesis could be wrong. LessWrong, August 2024. URL https://www.lesswrong.com/posts/tojtPCCRpKLSHBdpn/the-strong-feature-hypothesis-could-be-wrong.", "tier": "emerging" }, + { + "key": "snell2024predicting", + "text": "Charlie Snell, Eric Wallace, Dan Klein, and Sergey Levine. Predicting emergent capabilities by finetuning, 2024. URL https://arxiv.org/abs/2411.16035.", + "tier": "emerging" + }, + { + "key": "pmlr-v37-sohl-dickstein15", + "text": "Jascha Sohl-Dickstein, Eric Weiss, Niru Maheswaranathan, and Surya Ganguli. Deep unsupervised learning using nonequilibrium thermodynamics. In Francis Bach and David Blei (eds.), Proceedings of the 32nd International Conference on Machine Learning, volume 37 of Proceedings of Machine Learning Research, pp. 2256–2265, Lille, France, 07–09 Jul 2015. PMLR. URL https://proceedings.mlr.press/v37/sohl-dickstein15.html.", + "tier": "established" + }, + { + "key": "song", + "text": "Siyuan Song, Harvey Lederman, Jennifer Hu, and Kyle Mahowald. Privileged self-access matters for introspection in AI. In ICML 2026 Workshop: Philosophy Meets Machine Learning, 2026. URL https://openreview.net/forum?id=ZcqCJHOWAA.", + "tier": "emerging" + }, + { + "key": "bigbench", + "text": "Aarohi Srivastava, Abhinav Rastogi, Abhishek Rao, Abu Awal Md Shoeb, Abubakar Abid, Adam Fisch, Adam R. Brown, Adam Santoro, Aditya Gupta, Adrià Garriga-Alonso, Agnieszka Kluska, Aitor Lewkowycz, Akshat Agarwal, Alethea Power, Alex Ray, Alex Warstadt, Alexander W. Kocurek, Ali Safaya, Ali Tazarv, Alice Xiang, Alicia Parrish, Allen Nie, Aman Hussain, Amanda Askell, Amanda Dsouza, Ambrose Slone, Ameet Rahane, Anantharaman S. Iyer, Anders Johan Andreassen, Andrea Madotto, Andrea Santilli, Andreas Stuhlmüller, Andrew M. Dai, Andrew La, Andrew Kyle Lampinen, Andy Zou, Angela Jiang, Angelica Chen, Anh Vuong, Animesh Gupta, Anna Gottardi, Antonio Norelli, Anu Venkatesh, Arash Gholamidavoodi, Arfa Tabassum, Arul Menezes, Arun Kirubarajan, Asher Mullokandov, Ashish Sabharwal, Austin Herrick, Avia Efrat, Aykut Erdem, Ayla Karakaş, B. Ryan Roberts, Bao Sheng Loe, Barret Zoph, Bartłomiej Bojanowski, Batuhan Özyurt, Behnam Hedayatnia, Behnam Neyshabur, Benjamin Inden, Benno Stein, Berk Ekmekci, Bill Yuchen Lin, Blake Howald, Bryan Orinion, Cameron Diao, Cameron Dour, Catherine Stinson, Cedrick Argueta, Cesar Ferri, Chandan Singh, Charles Rathkopf, Chenlin Meng, Chitta Baral, Chiyu Wu, Chris Callison-Burch, Christopher Waites, Christian Voigt, Christopher D Manning, Christopher Potts, Cindy Ramirez, Clara E. Rivera, Clemencia Siro, Colin Raffel, Courtney Ashcraft, Cristina Garbacea, Damien Sileo, Dan Garrette, Dan Hendrycks, Dan Kilman, Dan Roth, C. Daniel Freeman, Daniel Khashabi, Daniel Levy, Daniel Moseguı́ González, Danielle Perszyk, Danny Hernandez, Danqi Chen, Daphne Ippolito, Dar Gilboa, David Dohan, David Drakard, David Jurgens, Debajyoti Datta, Deep Ganguli, Denis Emelin, Denis Kleyko, Deniz Yuret, Derek Chen, Derek Tam, Dieuwke Hupkes, Diganta Misra, Dilyar Buzan, Dimitri Coelho Mollo, Diyi Yang, Dong-Ho Lee, Dylan Schrader, Ekaterina Shutova, Ekin Dogus Cubuk, Elad Segal, Eleanor Hagerman, Elizabeth Barnes, Elizabeth Donoway, Ellie Pavlick, Emanuele Rodolà, Emma Lam, Eric Chu, Eric Tang, Erkut Erdem, Ernie Chang, Ethan A Chi, Ethan Dyer, Ethan Jerzak, Ethan Kim, Eunice Engefu Manyasi, Evgenii Zheltonozhskii, Fanyue Xia, Fatemeh Siar, Fernando Martı́nez-Plumed, Francesca Happé, Francois Chollet, Frieda Rong, Gaurav Mishra, Genta Indra Winata, Gerard de Melo, Germàn Kruszewski, Giambattista Parascandolo, Giorgio Mariani, Gloria Xinyue Wang, Gonzalo Jaimovitch-Lopez, Gregor Betz, Guy Gur-Ari, Hana Galijasevic, Hannah Kim, Hannah Rashkin, Hannaneh Hajishirzi, Harsh Mehta, Hayden Bogar, Henry Francis Anthony Shevlin, Hinrich Schuetze, Hiromu Yakura, Hongming Zhang, Hugh Mee Wong, Ian Ng, Isaac Noble, Jaap Jumelet, Jack Geissinger, Jackson Kernion, Jacob Hilton, Jaehoon Lee, Jaime Fernández Fisac, James B Simon, James Koppel, James Zheng, James Zou, Jan Kocon, Jana Thompson, Janelle Wingfield, Jared Kaplan, Jarema Radom, Jascha Sohl-Dickstein, Jason Phang, Jason Wei, Jason Yosinski, Jekaterina Novikova, Jelle Bosscher, Jennifer Marsh, Jeremy Kim, Jeroen Taal, Jesse Engel, Jesujoba Alabi, Jiacheng Xu, Jiaming Song, Jillian Tang, Joan Waweru, John Burden, John Miller, John U. Balis, Jonathan Batchelder, Jonathan Berant, Jörg Frohberg, Jos Rozen, Jose Hernandez-Orallo, Joseph Boudeman, Joseph Guerr, Joseph Jones, Joshua B. Tenenbaum, Joshua S. Rule, Joyce Chua, Kamil Kanclerz, Karen Livescu, Karl Krauth, Karthik Gopalakrishnan, Katerina Ignatyeva, Katja Markert, Kaustubh Dhole, Kevin Gimpel, Kevin Omondi, Kory Wallace Mathewson, Kristen Chiafullo, Ksenia Shkaruta, Kumar Shridhar, Kyle McDonell, Kyle Richardson, Laria Reynolds, Leo Gao, Li Zhang, Liam Dugan, Lianhui Qin, Lidia Contreras-Ochando, Louis-Philippe Morency, Luca Moschella, Lucas Lam, Lucy Noble, Ludwig Schmidt, Luheng He, Luis Oliveros-Colón, Luke Metz, Lütfi Kerem Senel, Maarten Bosma, Maarten Sap, Maartje Ter Hoeve, Maheen Farooqi, Manaal Faruqui, Mantas Mazeika, Marco Baturan, Marco Marelli, Marco Maru, Maria Jose Ramirez-Quintana, Marie Tolkiehn, Mario Giulianelli, Martha Lewis, Martin Potthast, Matthew L Leavitt, Matthias Hagen, Mátyás Schubert, Medina Orduna Baitemirova, Melody Arnaud, Melvin McElrath, Michael Andrew Yee, Michael Cohen, Michael Gu, Michael Ivanitskiy, Michael Starritt, Michael Strube, Michał Swędrowski, Michele Bevilacqua, Michihiro Yasunaga, Mihir Kale, Mike Cain, Mimee Xu, Mirac Suzgun, Mitch Walker, Mo Tiwari, Mohit Bansal, Moin Aminnaseri, Mor Geva, Mozhdeh Gheini, Mukund Varma T, Nanyun Peng, Nathan Andrew Chi, Nayeon Lee, Neta Gur-Ari Krakover, Nicholas Cameron, Nicholas Roberts, Nick Doiron, Nicole Martinez, Nikita Nangia, Niklas Deckers, Niklas Muennighoff, Nitish Shirish Keskar, Niveditha S. Iyer, Noah Constant, Noah Fiedel, Nuan Wen, Oliver Zhang, Omar Agha, Omar Elbaghdadi, Omer Levy, Owain Evans, Pablo Antonio Moreno Casares, Parth Doshi, Pascale Fung, Paul Pu Liang, Paul Vicol, Pegah Alipoormolabashi, Peiyuan Liao, Percy Liang, Peter W Chang, Peter Eckersley, Phu Mon Htut, Pinyu Hwang, Piotr Miłkowski, Piyush Patil, Pouya Pezeshkpour, Priti Oli, Qiaozhu Mei, Qing Lyu, Qinlang Chen, Rabin Banjade, Rachel Etta Rudolph, Raefer Gabriel, Rahel Habacker, Ramon Risco, Raphaël Millière, Rhythm Garg, Richard Barnes, Rif A. Saurous, Riku Arakawa, Robbe Raymaekers, Robert Frank, Rohan Sikand, Roman Novak, Roman Sitelew, Ronan Le Bras, Rosanne Liu, Rowan Jacobs, Rui Zhang, Russ Salakhutdinov, Ryan Andrew Chi, Seungjae Ryan Lee, Ryan Stovall, Ryan Teehan, Rylan Yang, Sahib Singh, Saif M. Mohammad, Sajant Anand, Sam Dillavou, Sam Shleifer, Sam Wiseman, Samuel Gruetter, Samuel R. Bowman, Samuel Stern Schoenholz, Sanghyun Han, Sanjeev Kwatra, Sarah A. Rous, Sarik Ghazarian, Sayan Ghosh, Sean Casey, Sebastian Bischoff, Sebastian Gehrmann, Sebastian Schuster, Sepideh Sadeghi, Shadi Hamdan, Sharon Zhou, Shashank Srivastava, Sherry Shi, Shikhar Singh, Shima Asaadi, Shixiang Shane Gu, Shubh Pachchigar, Shubham Toshniwal, Shyam Upadhyay, Shyamolima Shammie Debnath, Siamak Shakeri, Simon Thormeyer, Simone Melzi, Siva Reddy, Sneha Priscilla Makini, Soo-Hwan Lee, Spencer Torene, Sriharsha Hatwar, Stanislas Dehaene, Stefan Divic, Stefano Ermon, Stella Biderman, Stephanie Lin, Stephen Prasad, Steven Piantadosi, Stuart Shieber, Summer Misherghi, Svetlana Kiritchenko, Swaroop Mishra, Tal Linzen, Tal Schuster, Tao Li, Tao Yu, Tariq Ali, Tatsunori Hashimoto, Te-Lin Wu, Théo Desbordes, Theodore Rothschild, Thomas Phan, Tianle Wang, Tiberius Nkinyili, Timo Schick, Timofei Kornev, Titus Tunduny, Tobias Gerstenberg, Trenton Chang, Trishala Neeraj, Tushar Khot, Tyler Shultz, Uri Shaham, Vedant Misra, Vera Demberg, Victoria Nyamai, Vikas Raunak, Vinay Venkatesh Ramasesh, vinay uday prabhu, Vishakh Padmakumar, Vivek Srikumar, William Fedus, William Saunders, William Zhang, Wout Vossen, Xiang Ren, Xiaoyu Tong, Xinran Zhao, Xinyi Wu, Xudong Shen, Yadollah Yaghoobzadeh, Yair Lakretz, Yangqiu Song, Yasaman Bahri, Yejin Choi, Yichi Yang, Sophie Hao, Yifu Chen, Yonatan Belinkov, Yu Hou, Yufang Hou, Yuntao Bai, Zachary Seid, Zhuoye Zhao, Zijian Wang, Zijie J. Wang, Zirui Wang, and Ziyi Wu. Beyond the imitation game: Quantifying and extrapolating the capabilities of language models. Transactions on Machine Learning Research, 2023. ISSN 2835-8856. URL https://openreview.net/forum?id=uyTL5Bvosj. Featured Certification.", + "tier": "established" + }, + { + "key": "3692070.3693959", + "text": "Dashiell Stander, Qinan Yu, Honglu Fan, and Stella Biderman. Grokking group multiplication with cosets. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.", + "tier": "established" + }, { "key": "ig", - "text": "Mukund Sundararajan, Ankur Taly, and Qiqi Yan. Axiomatic attribution for deep networks. In Proceedings of the 34th International Conference on Machine Learning, pp. 3319–3328. JMLR.org, 2017.", + "text": "Mukund Sundararajan, Ankur Taly, and Qiqi Yan. Axiomatic attribution for deep networks. In Proceedings of the 34th International Conference on Machine Learning - Volume 70, ICML’17, pp. 3319–3328. JMLR.org, 2017.", "tier": "established" }, { - "key": "szegedy2015going", - "text": "Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, and Andrew Rabinovich. Going deeper with convolutions. In Proceedings of the IEEE conference on computer vision and pattern recognition, pp. 1–9, 2015.", + "key": "sutter2025the", + "text": "Denis Sutter, Julian Minder, Thomas Hofmann, and Tiago Pimentel. The non-linear representation dilemma: Is causal abstraction enough for mechanistic interpretability? In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025. URL https://openreview.net/forum?id=ZYXTLo7kCi.", "tier": "established" }, { - "key": "tan2025analyzinggeneralizationreliabilitysteering", - "text": "Daniel Tan, David Chanin, Aengus Lynch, Brooks Paige, Dimitrios Kanoulas, Adrià Garriga-Alonso, and Robert Kirk. Analysing the generalisation and reliability of steering vectors. In Proceedings of the 38th International Conference on Neural Information Processing Systems, NIPS ’24, Red Hook, NY, USA, 2024a. Curran Associates Inc. ISBN 9798331314385.", + "key": "syed2023attributionpatchingoutperformsautomated", + "text": "Aaquib Syed, Can Rager, and Arthur Conmy. Attribution patching outperforms automated circuit discovery. In Yonatan Belinkov, Najoung Kim, Jaap Jumelet, Hosein Mohebbi, Aaron Mueller, and Hanjie Chen (eds.), Proceedings of the 7th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP, pp. 407–416, Miami, Florida, US, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.blackboxnlp-1.25/.", + "tier": "emerging" + }, + { + "key": "szegedy2015going", + "text": "Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, and Andrew Rabinovich. Going deeper with convolutions. In Proceedings of the IEEE conference on computer vision and pattern recognition, pp. 1–9, 2015.", "tier": "established" }, { "key": "tan2024analysing", - "text": "Daniel Chee Hian Tan, David Chanin, Aengus Lynch, Brooks Paige, Dimitrios Kanoulas, Adrià Garriga-Alonso, and Robert Kirk. Analysing the generalisation and reliability of steering vectors. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024b.", + "text": "Daniel Chee Hian Tan, David Chanin, Aengus Lynch, Brooks Paige, Dimitrios Kanoulas, Adrià Garriga-Alonso, and Robert Kirk. Analysing the generalisation and reliability of steering vectors. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024.", "tier": "established" }, { @@ -3649,9 +4922,24 @@ "text": "Ruixiang Tang, Dehan Kong, Longtao Huang, and Hui Xue. Large language models can be lazy learners: Analyze shortcuts in in-context learning. In Anna Rogers, Jordan Boyd-Graber, and Naoaki Okazaki (eds.), Findings of the Association for Computational Linguistics: ACL 2023, pp. 4645–4657, Toronto, Canada, July 2023. Association for Computational Linguistics. . URL https://aclanthology.org/2023.findings-acl.284/.", "tier": "established" }, + { + "key": "taylor2025school", + "text": "Mia Taylor, James Chua, Jan Betley, Johannes Treutlein, and Owain Evans. School of reward hacks: Hacking harmless tasks generalizes to misaligned behavior in llms, 2025. URL https://arxiv.org/abs/2508.17511.", + "tier": "emerging" + }, { "key": "gemmateam2024gemmaopenmodelsbased", - "text": "Gemma Team, Thomas Mesnard, Cassidy Hardin, Robert Dadashi, Surya Bhupatiraju, Shreya Pathak, Laurent Sifre, Morgane Rivière, Mihir Sanjay Kale, Juliette Love, Pouya Tafti, Léonard Hussenot, Pier Giuseppe Sessa, Aakanksha Chowdhery, Adam Roberts, Aditya Barua, Alex Botev, Alex Castro-Ros, Ambrose Slone, Amélie Héliou, Andrea Tacchetti, Anna Bulanova, Antonia Paterson, Beth Tsai, Bobak Shahriari, Charline Le Lan, Christopher A. Choquette-Choo, Clément Crepy, Daniel Cer, Daphne Ippolito, David Reid, Elena Buchatskaya, Eric Ni, Eric Noland, Geng Yan, George Tucker, George-Christian Muraru, Grigory Rozhdestvenskiy, Henryk Michalewski, Ian Tenney, Ivan Grishchenko, Jacob Austin, James Keeling, Jane Labanowski, Jean-Baptiste Lespiau, Jeff Stanway, Jenny Brennan, Jeremy Chen, Johan Ferret, Justin Chiu, Justin Mao-Jones, Katherine Lee, Kathy Yu, Katie Millican, Lars Lowe Sjoesund, Lisa Lee, Lucas Dixon, Machel Reid, Maciej Mikuła, Mateo Wirth, Michael Sharman, Nikolai Chinaev, Nithum Thain, Olivier Bachem, Oscar Chang, Oscar Wahltinez, Paige Bailey, Paul Michel, Petko Yotov, Rahma Chaabouni, Ramona Comanescu, Reena Jana, Rohan Anil, Ross McIlroy, Ruibo Liu, Ryan Mullins, Samuel L Smith, Sebastian Borgeaud, Sertan Girgin, Sholto Douglas, Shree Pandya, Siamak Shakeri, Soham De, Ted Klimenko, Tom Hennigan, Vlad Feinberg, Wojciech Stokowiec, Yu hui Chen, Zafarali Ahmed, Zhitao Gong, Tris Warkentin, Ludovic Peran, Minh Giang, Clément Farabet, Oriol Vinyals, Jeff Dean, Koray Kavukcuoglu, Demis Hassabis, Zoubin Ghahramani, Douglas Eck, Joelle Barral, Fernando Pereira, Eli Collins, Armand Joulin, Noah Fiedel, Evan Senter, Alek Andreev, and Kathleen Kenealy. Gemma: Open models based on gemini research and technology, 2024. URL https://arxiv.org/abs/2403.08295.", + "text": "Gemma Team, Thomas Mesnard, Cassidy Hardin, Robert Dadashi, Surya Bhupatiraju, Shreya Pathak, Laurent Sifre, Morgane Rivière, Mihir Sanjay Kale, Juliette Love, Pouya Tafti, Léonard Hussenot, Pier Giuseppe Sessa, Aakanksha Chowdhery, Adam Roberts, Aditya Barua, Alex Botev, Alex Castro-Ros, Ambrose Slone, Amélie Héliou, Andrea Tacchetti, Anna Bulanova, Antonia Paterson, Beth Tsai, Bobak Shahriari, Charline Le Lan, Christopher A. Choquette-Choo, Clément Crepy, Daniel Cer, Daphne Ippolito, David Reid, Elena Buchatskaya, Eric Ni, Eric Noland, Geng Yan, George Tucker, George-Christian Muraru, Grigory Rozhdestvenskiy, Henryk Michalewski, Ian Tenney, Ivan Grishchenko, Jacob Austin, James Keeling, Jane Labanowski, Jean-Baptiste Lespiau, Jeff Stanway, Jenny Brennan, Jeremy Chen, Johan Ferret, Justin Chiu, Justin Mao-Jones, Katherine Lee, Kathy Yu, Katie Millican, Lars Lowe Sjoesund, Lisa Lee, Lucas Dixon, Machel Reid, Maciej Mikuła, Mateo Wirth, Michael Sharman, Nikolai Chinaev, Nithum Thain, Olivier Bachem, Oscar Chang, Oscar Wahltinez, Paige Bailey, Paul Michel, Petko Yotov, Rahma Chaabouni, Ramona Comanescu, Reena Jana, Rohan Anil, Ross McIlroy, Ruibo Liu, Ryan Mullins, Samuel L Smith, Sebastian Borgeaud, Sertan Girgin, Sholto Douglas, Shree Pandya, Siamak Shakeri, Soham De, Ted Klimenko, Tom Hennigan, Vlad Feinberg, Wojciech Stokowiec, Yu hui Chen, Zafarali Ahmed, Zhitao Gong, Tris Warkentin, Ludovic Peran, Minh Giang, Clément Farabet, Oriol Vinyals, Jeff Dean, Koray Kavukcuoglu, Demis Hassabis, Zoubin Ghahramani, Douglas Eck, Joelle Barral, Fernando Pereira, Eli Collins, Armand Joulin, Noah Fiedel, Evan Senter, Alek Andreev, and Kathleen Kenealy. Gemma: Open models based on gemini research and technology, 2024a. URL https://arxiv.org/abs/2403.08295.", + "tier": "emerging" + }, + { + "key": "gemmateam2024gemma2improvingopen", + "text": "Gemma Team, Morgane Riviere, Shreya Pathak, Pier Giuseppe Sessa, Cassidy Hardin, Surya Bhupatiraju, Léonard Hussenot, Thomas Mesnard, Bobak Shahriari, Alexandre Ramé, Johan Ferret, Peter Liu, Pouya Tafti, Abe Friesen, Michelle Casbon, Sabela Ramos, Ravin Kumar, Charline Le Lan, Sammy Jerome, Anton Tsitsulin, Nino Vieillard, Piotr Stanczyk, Sertan Girgin, Nikola Momchev, Matt Hoffman, Shantanu Thakoor, Jean-Bastien Grill, Behnam Neyshabur, Olivier Bachem, Alanna Walton, Aliaksei Severyn, Alicia Parrish, Aliya Ahmad, Allen Hutchison, Alvin Abdagic, Amanda Carl, Amy Shen, Andy Brock, Andy Coenen, Anthony Laforge, Antonia Paterson, Ben Bastian, Bilal Piot, Bo Wu, Brandon Royal, Charlie Chen, Chintu Kumar, Chris Perry, Chris Welty, Christopher A. Choquette-Choo, Danila Sinopalnikov, David Weinberger, Dimple Vijaykumar, Dominika Rogozińska, Dustin Herbison, Elisa Bandy, Emma Wang, Eric Noland, Erica Moreira, Evan Senter, Evgenii Eltyshev, Francesco Visin, Gabriel Rasskin, Gary Wei, Glenn Cameron, Gus Martins, Hadi Hashemi, Hanna Klimczak-Plucińska, Harleen Batra, Harsh Dhand, Ivan Nardini, Jacinda Mein, Jack Zhou, James Svensson, Jeff Stanway, Jetha Chan, Jin Peng Zhou, Joana Carrasqueira, Joana Iljazi, Jocelyn Becker, Joe Fernandez, Joost van Amersfoort, Josh Gordon, Josh Lipschultz, Josh Newlan, Ju yeong Ji, Kareem Mohamed, Kartikeya Badola, Kat Black, Katie Millican, Keelin McDonell, Kelvin Nguyen, Kiranbir Sodhia, Kish Greene, Lars Lowe Sjoesund, Lauren Usui, Laurent Sifre, Lena Heuermann, Leticia Lago, Lilly McNealus, Livio Baldini Soares, Logan Kilpatrick, Lucas Dixon, Luciano Martins, Machel Reid, Manvinder Singh, Mark Iverson, Martin Görner, Mat Velloso, Mateo Wirth, Matt Davidow, Matt Miller, Matthew Rahtz, Matthew Watson, Meg Risdal, Mehran Kazemi, Michael Moynihan, Ming Zhang, Minsuk Kahng, Minwoo Park, Mofi Rahman, Mohit Khatwani, Natalie Dao, Nenshad Bardoliwalla, Nesh Devanathan, Neta Dumai, Nilay Chauhan, Oscar Wahltinez, Pankil Botarda, Parker Barnes, Paul Barham, Paul Michel, Pengchong Jin, Petko Georgiev, Phil Culliton, Pradeep Kuppala, Ramona Comanescu, Ramona Merhej, Reena Jana, Reza Ardeshir Rokni, Rishabh Agarwal, Ryan Mullins, Samaneh Saadat, Sara Mc Carthy, Sarah Cogan, Sarah Perrin, Sébastien M. R. Arnold, Sebastian Krause, Shengyang Dai, Shruti Garg, Shruti Sheth, Sue Ronstrom, Susan Chan, Timothy Jordan, Ting Yu, Tom Eccles, Tom Hennigan, Tomas Kocisky, Tulsee Doshi, Vihan Jain, Vikas Yadav, Vilobh Meshram, Vishal Dharmadhikari, Warren Barkley, Wei Wei, Wenming Ye, Woohyun Han, Woosuk Kwon, Xiang Xu, Zhe Shen, Zhitao Gong, Zichuan Wei, Victor Cotruta, Phoebe Kirk, Anand Rao, Minh Giang, Ludovic Peran, Tris Warkentin, Eli Collins, Joelle Barral, Zoubin Ghahramani, Raia Hadsell, D. Sculley, Jeanine Banks, Anca Dragan, Slav Petrov, Oriol Vinyals, Jeff Dean, Demis Hassabis, Koray Kavukcuoglu, Clement Farabet, Elena Buchatskaya, Sebastian Borgeaud, Noah Fiedel, Armand Joulin, Kathleen Kenealy, Robert Dadashi, and Alek Andreev. Gemma 2: Improving open language models at a practical size, 2024b. URL https://arxiv.org/abs/2408.00118.", + "tier": "emerging" + }, + { + "key": "kimiteam2026kimik2openagentic", + "text": "Kimi Team, Yifan Bai, Yiping Bao, Y. Charles, Cheng Chen, Guanduo Chen, Haiting Chen, Huarong Chen, Jiahao Chen, Ningxin Chen, Ruijue Chen, Yanru Chen, Yuankun Chen, Yutian Chen, Zhuofu Chen, Jialei Cui, Hao Ding, Mengnan Dong, Angang Du, Chenzhuang Du, Dikang Du, Yulun Du, Yu Fan, Yichen Feng, Kelin Fu, Bofei Gao, Chenxiao Gao, Hongcheng Gao, Peizhong Gao, Tong Gao, Yuyao Ge, Shangyi Geng, Qizheng Gu, Xinran Gu, Longyu Guan, Haiqing Guo, Jianhang Guo, Xiaoru Hao, Tianhong He, Weiran He, Wenyang He, Yunjia He, Chao Hong, Hao Hu, Yangyang Hu, Zhenxing Hu, Weixiao Huang, Zhiqi Huang, Zihao Huang, Tao Jiang, Zhejun Jiang, Xinyi Jin, Yongsheng Kang, Guokun Lai, Cheng Li, Fang Li, Haoyang Li, Ming Li, Wentao Li, Yang Li, Yanhao Li, Yiwei Li, Zhaowei Li, Zheming Li, Hongzhan Lin, Xiaohan Lin, Zongyu Lin, Chengyin Liu, Chenyu Liu, Hongzhang Liu, Jingyuan Liu, Junqi Liu, Liang Liu, Shaowei Liu, T. Y. Liu, Tianwei Liu, Weizhou Liu, Yangyang Liu, Yibo Liu, Yiping Liu, Yue Liu, Zhengying Liu, Enzhe Lu, Haoyu Lu, Lijun Lu, Yashuo Luo, Shengling Ma, Xinyu Ma, Yingwei Ma, Shaoguang Mao, Jie Mei, Xin Men, Yibo Miao, Siyuan Pan, Yebo Peng, Ruoyu Qin, Zeyu Qin, Bowen Qu, Zeyu Shang, Lidong Shi, Shengyuan Shi, Feifan Song, Jianlin Su, Zhengyuan Su, Lin Sui, Xinjie Sun, Flood Sung, Yunpeng Tai, Heyi Tang, Jiawen Tao, Qifeng Teng, Chaoran Tian, Chensi Wang, Dinglu Wang, Feng Wang, Hailong Wang, Haiming Wang, Jianzhou Wang, Jiaxing Wang, Jinhong Wang, Shengjie Wang, Shuyi Wang, Si Wang, Xinyuan Wang, Yao Wang, Yejie Wang, Yiqin Wang, Yuxin Wang, Yuzhi Wang, Zhaoji Wang, Zhengtao Wang, Zhengtao Wang, Zhexu Wang, Chu Wei, Qianqian Wei, Haoning Wu, Wenhao Wu, Xingzhe Wu, Yuxin Wu, Chenjun Xiao, Jin Xie, Xiaotong Xie, Weimin Xiong, Boyu Xu, Jinjing Xu, L. H. Xu, Lin Xu, Suting Xu, Weixin Xu, Xinran Xu, Yangchuan Xu, Ziyao Xu, Jing Xu, Jing Xu, Junjie Yan, Yuzi Yan, Hao Yang, Xiaofei Yang, Yi Yang, Ying Yang, Zhen Yang, Zhilin Yang, Zonghan Yang, Haotian Yao, Xingcheng Yao, Wenjie Ye, Zhuorui Ye, Bohong Yin, Longhui Yu, Enming Yuan, Hongbang Yuan, Mengjie Yuan, Siyu Yuan, Haobing Zhan, Dehao Zhang, Hao Zhang, Wanlu Zhang, Xiaobin Zhang, Yadong Zhang, Yangkun Zhang, Yichi Zhang, Yizhi Zhang, Yongting Zhang, Yu Zhang, Yutao Zhang, Yutong Zhang, Zheng Zhang, Haotian Zhao, Yikai Zhao, Zijia Zhao, Huabin Zheng, Shaojie Zheng, Longguang Zhong, Jianren Zhou, Xinyu Zhou, Zaida Zhou, Jinguo Zhu, Zhen Zhu, Weiyu Zhuang, and Xinxing Zu. Kimi k2: Open agentic intelligence, 2026. URL https://arxiv.org/abs/2507.20534.", "tier": "emerging" }, { @@ -3661,23 +4949,28 @@ }, { "key": "tenney2019bertrediscoversclassicalnlp", - "text": "Ian Tenney, Dipanjan Das, and Ellie Pavlick. Bert rediscovers the classical nlp pipeline, 2019. URL https://arxiv.org/abs/1905.05950.", - "tier": "emerging" + "text": "Ian Tenney, Dipanjan Das, and Ellie Pavlick. BERT rediscovers the classical NLP pipeline. In Anna Korhonen, David Traum, and Lluís Màrquez (eds.), Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pp. 4593–4601, Florence, Italy, July 2019. Association for Computational Linguistics. . URL https://aclanthology.org/P19-1452/.", + "tier": "established" }, { "key": "thasarathan_universal_2025", "text": "Harrish Thasarathan, Julian Forsyth, Thomas Fel, Matthew Kowal, and Konstantinos G. Derpanis. Universal sparse autoencoders: Interpretable cross-model concept alignment. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=UoaxRN88oR.", - "tier": null + "tier": "established" }, { "key": "smds", - "text": "Federico Tiblias, Irina Bigoulaeva, Jingcheng Niu, Simone Balloccu, and Iryna Gurevych. Shape happens: Automatic feature manifold discovery in llms via supervised multi-dimensional scaling. arXiv preprint arXiv:2510.01025, 2025.", - "tier": "emerging" + "text": "Federico Tiblias, Irina Bigoulaeva, Jingcheng Niu, Simone Balloccu, and Iryna Gurevych. Hypothesis-driven feature manifold analysis in LLMs via supervised multi-dimensional scaling. Transactions on Machine Learning Research, 2026. ISSN 2835-8856. URL https://openreview.net/forum?id=vCKZ40YYPr.", + "tier": "established" }, { "key": "tigges2024llm", "text": "Curt Tigges, Michael Hanna, Qinan Yu, and Stella Biderman. Llm circuit analyses are consistent across training and scale. Advances in Neural Information Processing Systems, 37: 40699–40731, 2024.", - "tier": null + "tier": "established" + }, + { + "key": "tinaz2025emergence", + "text": "Berk Tinaz, Zalan Fabian, and Mahdi Soltanolkotabi. Emergence and evolution of interpretable concepts in diffusion models. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025. URL https://openreview.net/forum?id=C7LxkebvVW.", + "tier": "established" }, { "key": "touvron2023llamaopenefficientfoundation", @@ -3692,22 +4985,27 @@ { "key": "turpin2023languagemodelsdontsay", "text": "Miles Turpin, Julian Michael, Ethan Perez, and Samuel R. Bowman. Language models don’t always say what they think: unfaithful explanations in chain-of-thought prompting. In Proceedings of the 37th International Conference on Neural Information Processing Systems, NIPS ’23, Red Hook, NY, USA, 2023. Curran Associates Inc.", - "tier": null + "tier": "established" }, { - "key": "moritz2024sthocastic", - "text": "Moritz Vandenhirtz, Sonia Laguna, Ričards Marcinkevičs, and Julia E. Vogt. Stochastic concept bottleneck models. In A. Globerson, L. Mackey, D. Belgrave, A. Fan, U. Paquet, J. Tomczak, and C. Zhang (eds.), Advances in Neural Information Processing Systems, volume 37, pp. 51787–51810. Curran Associates, Inc., 2024. . URL https://proceedings.neurips.cc/paper_files/paper/2024/file/5c7894ac8788555f1cecf536f1e0fd35-Paper-Conference.pdf.", + "key": "vanderwal2025polypythias", + "text": "Oskar van der Wal, Pietro Lesci, Max Müller-Eberstein, Naomi Saphra, Hailey Schoelkopf, Willem Zuidema, and Stella Biderman. PolyPythias: Stability and outliers across fifty language model pre-training runs. In The Thirteenth International Conference on Learning Representations, 2025.", "tier": "established" }, { - "key": "vaswani2017attention", - "text": "Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Ł ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In I. Guyon, U. Von Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (eds.), Advances in Neural Information Processing Systems, volume 30. Curran Associates, Inc., 2017a. URL https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf.", + "key": "vandenhirtz2024stochastic", + "text": "Moritz Vandenhirtz, Sonia Laguna, Ričards Marcinkevičs, and Julia E Vogt. Stochastic concept bottleneck models. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024. URL https://openreview.net/forum?id=iSjqTQ5S1f.", "tier": "established" }, { - "key": "vaswani2023attentionneed", - "text": "Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Ł ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In I. Guyon, U. Von Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (eds.), Advances in Neural Information Processing Systems, volume 30. Curran Associates, Inc., 2017b. URL https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf.", - "tier": null + "key": "varma2023explaining", + "text": "Vikrant Varma, Rohin Shah, Zachary Kenton, János Kramár, and Ramana Kumar. Explaining grokking through circuit efficiency, 2023. URL https://arxiv.org/abs/2309.02390.", + "tier": "emerging" + }, + { + "key": "vaswani2017attention", + "text": "Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Ł ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In I. Guyon, U. Von Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (eds.), Advances in Neural Information Processing Systems, volume 30. Curran Associates, Inc., 2017. URL https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf.", + "tier": "established" }, { "key": "vig2019multiscale", @@ -3715,29 +5013,39 @@ "tier": "established" }, { - "key": "vig2020causalmediationanalysisinterpreting", - "text": "Jesse Vig, Sebastian Gehrmann, Yonatan Belinkov, Sharon Qian, Daniel Nevo, Yaron Singer, and Stuart Shieber. Investigating gender bias in language models using causal mediation analysis. In H. Larochelle, M. Ranzato, R. Hadsell, M.F. Balcan, and H. Lin (eds.), Advances in Neural Information Processing Systems, volume 33, pp. 12388–12401. Curran Associates, Inc., 2020. URL https://proceedings.neurips.cc/paper_files/paper/2020/file/92650b2e92217715fe312e6fa7b90d82-Paper.pdf.", - "tier": "established" + "key": "vig-belinkov-2019-analyzing", + "text": "Jesse Vig and Yonatan Belinkov. Analyzing the structure of attention in a transformer language model. In Tal Linzen, Grzegorz Chrupała, Yonatan Belinkov, and Dieuwke Hupkes (eds.), Proceedings of the 2019 ACL Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP, pp. 63–76, Florence, Italy, August 2019. Association for Computational Linguistics. . URL https://aclanthology.org/W19-4808/.", + "tier": "emerging" }, { "key": "voita-titov-2020-information", "text": "Elena Voita and Ivan Titov. Information-theoretic probing with minimum description length. In Bonnie Webber, Trevor Cohn, Yulan He, and Yang Liu (eds.), Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 183–196, Online, November 2020. Association for Computational Linguistics. . URL https://aclanthology.org/2020.emnlp-main.14/.", - "tier": null + "tier": "established" }, { "key": "voita_analyzing_2019", - "text": "Elena Voita, David Talbot, Fedor Moiseev, Rico Sennrich, and Ivan Titov. Analyzing multi-head self-attention: Specialized heads do the heavy lifting, the rest can be pruned. In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, 2019.", - "tier": null + "text": "Elena Voita, David Talbot, Fedor Moiseev, Rico Sennrich, and Ivan Titov. Analyzing multi-head self-attention: Specialized heads do the heavy lifting, the rest can be pruned. In Anna Korhonen, David Traum, and Lluís Màrquez (eds.), Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pp. 5797–5808, Florence, Italy, July 2019. Association for Computational Linguistics. . URL https://aclanthology.org/P19-1580/.", + "tier": "established" + }, + { + "key": "wallace2024instructionhierarchytrainingllms", + "text": "Eric Wallace, Kai Xiao, Reimar Leike, Lilian Weng, Johannes Heidecke, and Alex Beutel. The instruction hierarchy: Training llms to prioritize privileged instructions, 2024. URL https://arxiv.org/abs/2404.13208.", + "tier": "emerging" }, { "key": "SuperGLUE", - "text": "Alex Wang, Yada Pruksachatkun, Nikita Nangia, Amanpreet Singh, Julian Michael, Felix Hill, Omer Levy, and Samuel Bowman. Superglue: A stickier benchmark for general-purpose language understanding systems. In H. Wallach, H. Larochelle, A. Beygelzimer, F. d' Alché-Buc, E. Fox, and R. Garnett (eds.), Advances in Neural Information Processing Systems, volume 32. Curran Associates, Inc., 2019. URL https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf.", + "text": "Alex Wang, Yada Pruksachatkun, Nikita Nangia, Amanpreet Singh, Julian Michael, Felix Hill, Omer Levy, and Samuel Bowman. Superglue: A stickier benchmark for general-purpose language understanding systems. In H. Wallach, H. Larochelle, A. Beygelzimer, F. d'Alché-Buc, E. Fox, and R. Garnett (eds.), Advances in Neural Information Processing Systems, volume 32. Curran Associates, Inc., 2019. URL https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf.", "tier": "established" }, { "key": "wang2025better", "text": "Andrew Wang, Elisa Nguyen, Runshi Yang, Juhan Bae, Sheila A. McIlraith, and Roger Baker Grosse. Better training data attribution via better inverse hessian-vector products. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025a. URL https://openreview.net/forum?id=7LTTzYXyJ1.", - "tier": null + "tier": "established" + }, + { + "key": "wang2025differentiation", + "text": "George Wang, Jesse Hoogland, Stan van Wingerden, Zach Furman, and Daniel Murfet. Differentiation and specialization of attention heads via the refined local learning coefficient. In International Conference on Learning Representations, 2025b.", + "tier": "established" }, { "key": "wang2022interpretabilitywildcircuitindirect", @@ -3747,11 +5055,21 @@ { "key": "wang_towards_2018", "text": "Liwei Wang, Lunjia Hu, Jiayuan Gu, Zhiqiang Hu, Yue Wu, Kun He, and John Hopcroft. Towards Understanding Learning Representations: To What Extent Do Different Neural Networks Learn the Same Representation. In Advances in Neural Information Processing Systems, volume 31. Curran Associates, Inc., 2018. URL https://proceedings.neurips.cc/paper_files/paper/2018/hash/5fc34ed307aac159a30d81181c99847e-Abstract.html.", - "tier": null + "tier": "established" + }, + { + "key": "wang2026persona", + "text": "Miles Wang, Tom Dupré la Tour, Olivia Watkins, Aleksandar Makelov, Ryan Chi, Samuel Miserendino, Jeffrey Wang, Achyuta Rajaram, Johannes Heidecke, Tejal Patwardhan, and Daniel Mossing. Persona features control emergent misalignment. In The Fourteenth International Conference on Learning Representations, 2026a.", + "tier": "established" + }, + { + "key": "chatbotknows", + "text": "Wenxuan Wang, Zirui Liu, Haoxuan Kou, Xuefeng Liu, and Jiaxing Shen. The chatbot knows it’s you: Dialogue attribution in unauthenticated human–llm sessions. In Proceedings of the ACM Web Conference 2026, WWW ’26, pp. 8797–8807, New York, NY, USA, 2026b. Association for Computing Machinery. ISBN 9798400723070. . URL https://doi.org/10.1145/3774904.3793048.", + "tier": "established" }, { - "key": "wang_persona_2025", - "text": "Miles Wang, Tom Dupré la Tour, Olivia Watkins, Alex Makelov, Ryan A. Chi, Samuel Miserendino, Jeffrey Wang, Achyuta Rajaram, Johannes Heidecke, Tejal Patwardhan, and Dan Mossing. Persona Features Control Emergent Misalignment, October 2025b. URL http://arxiv.org/abs/2506.19823. arXiv:2506.19823 [cs].", + "key": "wang2026dlmscopemechanisticinterpretabilitydiffusion", + "text": "Xu Wang, Bingqing Jiang, Yu Wan, Baosong Yang, Lingpeng Kong, and Difan Zou. Dlm-scope: Mechanistic interpretability of diffusion language models via sparse autoencoders, 2026c. URL https://arxiv.org/abs/2602.05859.", "tier": "emerging" }, { @@ -3759,10 +5077,25 @@ "text": "Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, and Wenhu Chen. MMLU-pro: A more robust and challenging multi-task language understanding benchmark. In The Thirty-eight Conference on Neural Information Processing Systems Datasets and Benchmarks Track, 2024. URL https://openreview.net/forum?id=y10DM6R2r3.", "tier": "established" }, + { + "key": "wani-etal-2026-answer", + "text": "Farooq Ahmad Wani, Alessandro Suglia, Rohit Saxena, Aryo Pradipta Gema, Wai-Chung Kwan, Fazl Barez, Maria Sofia Bucarelli, Fabrizio Silvestri, and Pasquale Minervini. Same answer, different representations: Hidden instability in VLMs. In Hung-Yi Lee, Sunipa Dev, Junyi Jessy Li, and Desmond Elliott (eds.), Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pp. 41059–41087, Budapest, Hungary, October 2026. Association for Computational Linguistics. ISBN 979-8-89176-459-0. . URL https://aclanthology.org/2026.emnlp-main.1803/.", + "tier": "established" + }, { "key": "watanabe2009algebraic", "text": "Sumio Watanabe. Algebraic Geometry and Statistical Learning Theory. Cambridge University Press, 2009. ISBN 9780521864671.", - "tier": null + "tier": "established" + }, + { + "key": "wei2022emergent", + "text": "Jason Wei, Yi Tay, Rishi Bommasani, Colin Raffel, Barret Zoph, Sebastian Borgeaud, Dani Yogatama, Maarten Bosma, Denny Zhou, Donald Metzler, Ed H. Chi, Tatsunori Hashimoto, Oriol Vinyals, Percy Liang, Jeff Dean, and William Fedus. Emergent abilities of large language models. Transactions on Machine Learning Research, 2022. URL https://openreview.net/forum?id=yzkSU5zdwD.", + "tier": "established" + }, + { + "key": "11370531", + "text": "Zeming Wei, Yifei Wang, Ang Li, Yichuan Mo, and Yisen Wang. Jailbreak and guard aligned language models with only few in-context demonstrations. IEEE Transactions on Pattern Analysis and Machine Intelligence, 48 (6): 6835–6846, 2026. .", + "tier": "established" }, { "key": "wiegreffe2019attention", @@ -3770,19 +5103,54 @@ "tier": "established" }, { - "key": "querycircuit", - "text": "Tung-Yu Wu and Fazl Barez. Query circuits: Explaining how language models answer user prompts. arXiv preprint arXiv:2509.24808, 2025.", + "key": "icml2025actionable", + "text": "Workshop Organizers. Workshop on actionable interpretability. ICML 2025, 2025. URL https://actionable-interpretability.github.io/.", "tier": "emerging" }, + { + "key": "querycircuit", + "text": "Tung-Yu Wu and Fazl Barez. Query circuits: Explaining how language models answer user prompts. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=7F0sragazb.", + "tier": "established" + }, + { + "key": "wu2025ushaped", + "text": "Tung-Yu Wu and Pei-Yu Lo. U-shaped and inverted-u scaling behind emergent abilities of large language models. In International Conference on Learning Representations, 2025. URL https://arxiv.org/abs/2410.01692.", + "tier": "established" + }, + { + "key": "and", + "text": "Tung-Yu Wu, Yu-Xiang Lin, and Tsui-Wei Weng. And: audio network dissection for interpreting deep acoustic models. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.", + "tier": "established" + }, + { + "key": "wu2025towards", + "text": "Wilson Wu, Louis Jaburi, jacob drori, and Jason Gross. Towards a unified and verified understanding of group-operation networks. In The Thirteenth International Conference on Learning Representations, 2025a. URL https://openreview.net/forum?id=8xxEBAtD7y.", + "tier": "established" + }, + { + "key": "wu-etal-2022-causal", + "text": "Zhengxuan Wu, Atticus Geiger, Joshua Rozner, Elisa Kreiss, Hanson Lu, Thomas Icard, Christopher Potts, and Noah Goodman. Causal distillation for language models. In Marine Carpuat, Marie-Catherine de Marneffe, and Ivan Vladimir Meza Ruiz (eds.), Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 4288–4295, Seattle, United States, July 2022. Association for Computational Linguistics. . URL https://aclanthology.org/2022.naacl-main.318/.", + "tier": "established" + }, { "key": "wu2025axbench", - "text": "Zhengxuan Wu, Aryaman Arora, Atticus Geiger, Zheng Wang, Jing Huang, Dan Jurafsky, Christopher D Manning, and Christopher Potts. Axbench: Steering LLMs? even simple baselines outperform sparse autoencoders. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=K2CckZjNy0.", + "text": "Zhengxuan Wu, Aryaman Arora, Atticus Geiger, Zheng Wang, Jing Huang, Dan Jurafsky, Christopher D Manning, and Christopher Potts. Axbench: Steering LLMs? even simple baselines outperform sparse autoencoders. In Forty-second International Conference on Machine Learning, 2025b. URL https://openreview.net/forum?id=K2CckZjNy0.", + "tier": "established" + }, + { + "key": "wurgaft2026manifoldsteeringrevealsshared", + "text": "Daniel Wurgaft, Can Rager, Matthew Kowal, Vasudev Shyam, Sheridan Feucht, Usha Bhalla, Tal Haklay, Eric Bigelow, Raphael Sarfati, Thomas McGrath, Owen Lewis, Jack Merullo, Noah Goodman, Thomas Fel, Atticus Geiger, and Ekdeep Singh Lubana. Manifold steering reveals the shared geometry of neural network representation and behavior, 2026. URL https://arxiv.org/abs/2605.05115.", + "tier": "emerging" + }, + { + "key": "xie2023defending", + "text": "Yueqi Xie, Jingwei Yi, Jiawei Shao, Justin Curl, Lingjuan Lyu, Qifeng Chen, Xing Xie, and Fangzhao Wu. Defending ChatGPT against jailbreak attack via self-reminders. Nature Machine Intelligence, 5 (12): 1486–1496, 2023. .", "tier": "established" }, { - "key": "xu2025infinigramminiexactngram", - "text": "Hao Xu, Jiacheng Liu, Yejin Choi, Noah A. Smith, and Hannaneh Hajishirzi. Infini-gram mini: Exact n-gram search at the Internet scale with FM-index. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing, pp. 24944–24969, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-332-6. . URL https://aclanthology.org/2025.emnlp-main.1268/.", - "tier": null + "key": "xu2024saetrack", + "text": "Yang Xu, Yi Wang, Hengguan Huang, and Hao Wang. Tracking the feature dynamics in llm training: A mechanistic study, 2025b. URL https://arxiv.org/abs/2412.17626.", + "tier": "emerging" }, { "key": "yang2025mixture", @@ -3799,19 +5167,29 @@ "text": "Wenqian Ye, Luyang Jiang, Eric Xie, Guangtao Zheng, Yunsheng Ma, Xu Cao, Dongliang Guo, Daiqing Qi, Zeyu He, Yijun Tian, Christopher W. Porter, Megan Coffee, Zhe Zeng, Sheng Li, Ziran Wang, Ting-Hao Kenneth Huang, James Matthew Rehg, Henry Kautz, and Aidong Zhang. The clever hans mirage: A comprehensive survey on spurious correlations in machine learning. Transactions on Machine Learning Research, 2026. ISSN 2835-8856. URL https://openreview.net/forum?id=kIuqPmS1b1.", "tier": "established" }, + { + "key": "yin2025which", + "text": "Kayo Yin and Jacob Steinhardt. Which attention heads matter for in-context learning? In Aarti Singh, Maryam Fazel, Daniel Hsu, Simon Lacoste-Julien, Felix Berkenkamp, Tegan Maharaj, Kiri Wagstaff, and Jerry Zhu (eds.), Proceedings of the 42nd International Conference on Machine Learning, volume 267 of Proceedings of Machine Learning Research, pp. 72428–72461. PMLR, 13–19 Jul 2025. URL https://proceedings.mlr.press/v267/yin25e.html.", + "tier": "established" + }, + { + "key": "yu2025safety", + "text": "Cheng Yu and Orestis Papakyriakopoulos. Safety devolution in AI agents. In ICLR 2025 Workshop on Human-AI Coevolution, 2025. URL https://openreview.net/forum?id=7nJmuFFkWd.", + "tier": "emerging" + }, { "key": "yu-etal-2024-latent", "text": "Xuemin Yu, Fahim Dalvi, Nadir Durrani, Marzia Nouri, and Hassan Sajjad. Latent concept-based explanation of NLP models. In Yaser Al-Onaizan, Mohit Bansal, and Yun-Nung Chen (eds.), Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, pp. 12435–12459, Miami, Florida, USA, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.emnlp-main.692/.", "tier": "established" }, { - "key": "yun-etal-2021-transformer", - "text": "Zeyu Yun, Yubei Chen, Bruno Olshausen, and Yann LeCun. Transformer visualization via dictionary learning: contextualized embedding as a linear superposition of transformer factors. In Eneko Agirre, Marianna Apidianaki, and Ivan Vulić (eds.), Proceedings of Deep Learning Inside Out (DeeLIO): The 2nd Workshop on Knowledge Extraction and Integration for Deep Learning Architectures, pp. 1–10, Online, June 2021. Association for Computational Linguistics. . URL https://aclanthology.org/2021.deelio-1.1/.", - "tier": null + "key": "yu2026vectorquantizedlatentconcepts", + "text": "Xuemin Yu, Ankur Garg, Samira Ebrahimi Kahou, and Hassan Sajjad. Vector quantized latent concepts: A scalable alternative to clustering-based concept discovery, 2026. URL https://arxiv.org/abs/2602.02726.", + "tier": "emerging" }, { - "key": "zanbaghi2025detectingsleeperagentslarge", - "text": "Shahin Zanbaghi, Ryan Rostampour, Farhan Abid, and Salim Al Jarmakani. Detecting sleeper agents in large language models via semantic drift analysis, 2025. URL https://arxiv.org/abs/2511.15992.", + "key": "yun-etal-2021-transformer", + "text": "Zeyu Yun, Yubei Chen, Bruno Olshausen, and Yann LeCun. Transformer visualization via dictionary learning: contextualized embedding as a linear superposition of transformer factors. In Eneko Agirre, Marianna Apidianaki, and Ivan Vulić (eds.), Proceedings of Deep Learning Inside Out (DeeLIO): The 2nd Workshop on Knowledge Extraction and Integration for Deep Learning Architectures, pp. 1–10, Online, June 2021. Association for Computational Linguistics. . URL https://aclanthology.org/2021.deelio-1.1/.", "tier": "emerging" }, { @@ -3826,28 +5204,53 @@ }, { "key": "zhang2025cybench", - "text": "Andy K Zhang, Neil Perry, Riya Dulepet, Joey Ji, Celeste Menders, Justin W Lin, Eliot Jones, Gashon Hussein, Samantha Liu, Donovan Julian Jasper, Pura Peetathawatchai, Ari Glenn, Vikram Sivashankar, Daniel Zamoshchin, Leo Glikbarg, Derek Askaryar, Haoxiang Yang, Aolin Zhang, Rishi Alluri, Nathan Tran, Rinnara Sangpisit, Kenny O Oseleononmen, Dan Boneh, Daniel E. Ho, and Percy Liang. Cybench: A framework for evaluating cybersecurity capabilities and risks of language models. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=tc90LV0yRL.", + "text": "Andy K Zhang, Neil Perry, Riya Dulepet, Joey Ji, Celeste Menders, Justin W Lin, Eliot Jones, Gashon Hussein, Samantha Liu, Donovan Julian Jasper, Pura Peetathawatchai, Ari Glenn, Vikram Sivashankar, Daniel Zamoshchin, Leo Glikbarg, Derek Askaryar, Haoxiang Yang, Aolin Zhang, Rishi Alluri, Nathan Tran, Rinnara Sangpisit, Kenny O Oseleononmen, Dan Boneh, Daniel E. Ho, and Percy Liang. Cybench: A framework for evaluating cybersecurity capabilities and risks of language models. In The Thirteenth International Conference on Learning Representations, 2025a. URL https://openreview.net/forum?id=tc90LV0yRL.", "tier": "established" }, { "key": "zhang2024towards", "text": "Fred Zhang and Neel Nanda. Towards best practices of activation patching in language models: Metrics and methods. In The Twelfth International Conference on Learning Representations, 2024. URL https://openreview.net/forum?id=Hf17y6u9BC.", - "tier": "emerging" + "tier": "established" }, { "key": "zhang2026locatesteerimprovepractical", - "text": "Hengyuan Zhang, Zhihao Zhang, Mingyang Wang, Zunhai Su, Yiwei Wang, Qianli Wang, Shuzhou Yuan, Ercong Nie, Xufeng Duan, Qibo Xue, Zeping Yu, Chenming Shang, Xiao Liang, Jing Xiong, Hui Shen, Chaofan Tao, Zhengwu Liu, Senjie Jin, Zhiheng Xi, Dongdong Zhang, Sophia Ananiadou, Tao Gui, Ruobing Xie, Hayden Kwok-Hay So, Hinrich Schütze, Xuanjing Huang, Qi Zhang, and Ngai Wong. Locate, steer, and improve: A practical survey of actionable mechanistic interpretability in large language models, 2026. URL https://arxiv.org/abs/2601.14004.", + "text": "Hengyuan Zhang, Zhihao Zhang, Ercong Nie, Mingyang Wang, Zunhai Su, Yiwei Wang, Qianli Wang, Shuzhou Yuan, Xufeng Duan, Qibo Xue, Zeping Yu, Chenming Shang, Xiao Liang, Jing Xiong, Hui Shen, Chaofan Tao, Zhengwu Liu, Senjie Jin, Zhiheng Xi, Dongdong Zhang, Sophia Ananiadou, Tao Gui, Ruobing Xie, Hayden Kwok-Hay So, Hinrich Schuetze, Xuanjing Huang, Qi Zhang, and Ngai Wong. Locate, steer, and improve: A practical survey of actionable mechanistic interpretability in large language models. In Maria Liakata, Viviane P. Moreira, Jiajun Zhang, and David Jurgens (eds.), Findings of the Association for Computational Linguistics: ACL 2026, pp. 10317–10362, San Diego, California, United States, July 2026a. Association for Computational Linguistics. ISBN 979-8-89176-395-1. . URL https://aclanthology.org/2026.findings-acl.502/.", + "tier": "established" + }, + { + "key": "eapgp", + "text": "Lin Zhang, Wenshuo Dong, Zhuoran Zhang, Shu Yang, Lijie Hu, Ninghao Liu, Pan Zhou, and Di Wang. EAP-GP: Mitigating saturation effect in gradient-based automated circuit identification. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025b. URL https://openreview.net/forum?id=lGyXq0LOeQ.", + "tier": "established" + }, + { + "key": "zhang-etal-2024-bench", + "text": "Xinrong Zhang, Yingfa Chen, Shengding Hu, Zihang Xu, Junhao Chen, Moo Hao, Xu Han, Zhen Thai, Shuo Wang, Zhiyuan Liu, and Maosong Sun. \\inftyBench: Extending long context evaluation beyond 100K tokens. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 15262–15277, Bangkok, Thailand, August 2024a. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.814/.", + "tier": "established" + }, + { + "key": "zhang2026whatshapes", + "text": "Yuchen Zhang, Anietta Weckauff, Diego Garcia-Olano, and Maksym Andriushchenko. What shapes emergent misalignment? insights from training dynamics, model priors, and data, 2026b. URL https://arxiv.org/abs/2606.20814.", "tier": "emerging" }, + { + "key": "zhang-etal-2024-defending", + "text": "Zhexin Zhang, Junxiao Yang, Pei Ke, Fei Mi, Hongning Wang, and Minlie Huang. Defending large language models against jailbreaking attacks through goal prioritization. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 8865–8887, Bangkok, Thailand, August 2024b. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.481/.", + "tier": "established" + }, { "key": "llmexplainsurvey", "text": "Haiyan Zhao, Hanjie Chen, Fan Yang, Ninghao Liu, Huiqi Deng, Hengyi Cai, Shuaiqiang Wang, Dawei Yin, and Mengnan Du. Explainability for large language models: A survey. ACM Trans. Intell. Syst. Technol., 15 (2), February 2024a. ISSN 2157-6904. . URL https://doi.org/10.1145/3639372.", "tier": "established" }, + { + "key": "zhao2026chainofthoughthijacking", + "text": "Jianli Zhao, Tingchen Fu, Rylan Schaeffer, Mrinank Sharma, and Fazl Barez. Chain-of-thought hijacking, 2026a. URL https://arxiv.org/abs/2510.26418.", + "tier": "emerging" + }, { "key": "zhao2024distributional", "text": "Rosie Zhao, Naomi Saphra, and Sham M Kakade. Distributional scaling laws for emergent capabilities. In NeurIPS 2024 Workshop on Scientific Methods for Understanding Deep Learning, 2024b.", - "tier": null + "tier": "emerging" }, { "key": "zhao-etal-2024-explaining", @@ -3859,45 +5262,67 @@ "text": "Yu Zhao, Alessio Devoto, Giwon Hong, Xiaotang Du, Aryo Pradipta Gema, Hongru Wang, Xuanli He, Kam-Fai Wong, and Pasquale Minervini. Steering knowledge selection behaviours in llms via sae-based representation engineering. In Proceedings of the 2025 Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers), pp. 5117–5136, 2025.", "tier": "established" }, + { + "key": "zhao2026verifying", + "text": "Zheng Zhao, Yeskendir Koishekenov, Xianjun Yang, Naila Murray, and Nicola Cancedda. Verifying chain-of-thought reasoning via its computational graph. In The Fourteenth International Conference on Learning Representations, 2026b. URL https://openreview.net/forum?id=CxiNICq0Rr.", + "tier": "established" + }, + { + "key": "agieval", + "text": "Wanjun Zhong, Ruixiang Cui, Yiduo Guo, Yaobo Liang, Shuai Lu, Yanlin Wang, Amin Saied, Weizhu Chen, and Nan Duan. AGIEval: A human-centric benchmark for evaluating foundation models. In Kevin Duh, Helena Gomez, and Steven Bethard (eds.), Findings of the Association for Computational Linguistics: NAACL 2024, pp. 2299–2314, Mexico City, Mexico, June 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.findings-naacl.149/.", + "tier": "established" + }, + { + "key": "cam", + "text": "Bolei Zhou, Aditya Khosla, Agata Lapedriza, Aude Oliva, and Antonio Torralba. Learning deep features for discriminative localization. In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 2921–2929, 2016.", + "tier": "established" + }, + { + "key": "zhu2025establishing", + "text": "Yuxuan Zhu, Tengjun Jin, Yada Pruksachatkun, Andy K Zhang, Shu Liu, Sasha Cui, Sayash Kapoor, Shayne Longpre, Kevin Meng, Rebecca Weiss, Fazl Barez, Rahul Gupta, Jwala Dhamala, Jacob Merizian, Mario Giulianelli, Harry Coppock, Cozmin Ududec, Antony Kellermann, Jasjeet S Sekhon, Jacob Steinhardt, Sarah Schwettmann, Arvind Narayanan, Matei Zaharia, Ion Stoica, Percy Liang, and Daniel Kang. Establishing best practices in building rigorous agentic benchmarks. In The Thirty-ninth Annual Conference on Neural Information Processing Systems Datasets and Benchmarks Track, 2025. URL https://openreview.net/forum?id=E58HNCqoaA.", + "tier": "established" + }, + { + "key": "zimerman2025explaining", + "text": "Itamar Zimerman, Ameen Ali Ali, and Lior Wolf. Explaining modern gated-linear RNNs via a unified implicit attention formulation. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=wnT8bfJCDx.", + "tier": "established" + }, { "key": "zucchet2025language", - "text": "Nicolas Zucchet, Jörg Bornschein, Stephanie Chan, Andrew Lampinen, Razvan Pascanu, and Soham De. How do language models learn facts? dynamics, curricula and hallucinations. arXiv preprint arXiv:2503.21676, 2025a.", - "tier": null + "text": "Nicolas Zucchet, Jörg Bornschein, Stephanie Chan, Andrew Lampinen, Razvan Pascanu, and Soham De. How do language models learn facts? dynamics, curricula and hallucinations, 2025a. URL https://arxiv.org/abs/2503.21676.", + "tier": "emerging" }, { - "key": "zucchet2025emergence", - "text": "Nicolas Zucchet, Francesco d’Angelo, Andrew K Lampinen, and Stephanie CY Chan. The emergence of sparse attention: impact of data distribution and benefits of repetition. arXiv preprint arXiv:2505.17863, 2025b.", - "tier": null + "key": "zucchet2025the", + "text": "Nicolas Zucchet, Francesco D’Angelo, Andrew Kyle Lampinen, and Stephanie C.Y. Chan. The emergence of sparse attention: impact of data distribution and benefits of repetition. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025b. URL https://openreview.net/forum?id=jMhRbV47pS.", + "tier": "established" } ], "figures": { - "figs/1_General summary.pdf": "1_general-summary.svg", + "figs/1_General_summary.pdf": "1_general_summary.svg", + "figs/7_Circuit.pdf": "7_circuit.svg", "figs/5_Priviledge.pdf": "5_priviledge.svg", - "figs/5_Superposition and Polysemanticity.pdf": "5_superposition-and-polysemanticity.svg", + "figs/5_Superposition_and_Polysemanticity.pdf": "5_superposition_and_polysemanticity.svg", "figs/6_SAE.pdf": "6_sae.svg", - "figs/JumpRELU.png": "jumprelu.png", + "figs/JumpReLU.pdf": "jumprelu.svg", "figs/6_Transcoders.pdf": "6_transcoders.svg", "figs/6_CLT.pdf": "6_clt.svg", - "figs/clt_size_vs_depth_e32_clean_v2.png": "clt_size_vs_depth_e32_clean_v2.png", + "figs/clt_size_vs_depth_e32_clean_v3.pdf": "clt_size_vs_depth_e32_clean_v3.svg", "figs/6_Crosscoders.pdf": "6_crosscoders.svg", - "figs/7_Circuit.pdf": "7_circuit.svg", "figs/8_Logit_Lens.pdf": "8_logit_lens.svg", "figs/8_Patchscopes.pdf": "8_patchscopes.svg", - "figs/8_Latent QA.pdf": "8_latent-qa.svg", + "figs/8_Latent_QA.pdf": "8_latent_qa.svg", "figs/8_Probing.pdf": "8_probing.svg", - "figs/9_Grokking.pdf": "9_grokking.svg", "figs/11_ModelDiffing.pdf": "11_modeldiffing.svg", "figs/11_Steering.pdf": "11_steering.svg", - "figs/12_Directions_compressed.png": "12_directions_compressed.png" + "figs/9_Grokking.pdf": "9_grokking.svg", + "figs/research_directions_compressed.pdf": "research_directions_compressed.png" }, "audit": { - "removed_comment_macros": 94, + "removed_comment_macros": 0, "dropped_environments": [], - "warnings": [ - "citation key not in bibliography: binder", - "citation key not in bibliography: song" - ], - "numbered_equations": 15, - "citations_rendered_from_bbl": 426 + "warnings": [], + "numbered_equations": 6, + "citations_rendered_from_bbl": 818 } } \ No newline at end of file diff --git a/src/generated/references.html b/src/generated/references.html index 6ca053ba09..27ebcefaa5 100644 --- a/src/generated/references.html +++ b/src/generated/references.html @@ -1,300 +1,519 @@
-
Julius Adebayo, Justin Gilmer, Michael Muelly, Ian Goodfellow, Moritz Hardt, and Been Kim. Sanity checks for saliency maps. In Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS’18, pp. 9525–9536, Red Hook, NY, USA, 2018a. Curran Associates Inc.
-
Julius Adebayo, Justin Gilmer, Michael Muelly, Ian Goodfellow, Moritz Hardt, and Been Kim. Sanity checks for saliency maps. In Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS’18, pp. 9525–9536, Red Hook, NY, USA, 2018b. Curran Associates Inc.
+
Zelalem Abahana, David Evans, Satish Mahadevan Srinivasan, and Matjaz Gams. When rlhf fails: A mechanistic taxonomy of reward hacking, collapse, and evaluator gaming, 2026. URL https://arxiv.org/abs/2606.03238.
+
Alessandro Achille, Matteo Rovere, and Stefano Soatto. Critical learning periods in deep networks. In International Conference on Learning Representations, 2019.
+
Julius Adebayo, Justin Gilmer, Michael Muelly, Ian Goodfellow, Moritz Hardt, and Been Kim. Sanity checks for saliency maps. In Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS’18, pp. 9525–9536, Red Hook, NY, USA, 2018. Curran Associates Inc.
Yossi Adi, Einat Kermany, Yonatan Belinkov, Ofer Lavi, and Yoav Goldberg. Fine-grained analysis of sentence embeddings using auxiliary prediction tasks. In International Conference on Learning Representations, 2017. URL https://openreview.net/forum?id=BJh6Ztuxl.
-
Micah Adler, Dan Alistarh, and Nir Shavit. Towards combinatorial interpretability of neural computation. arXiv preprint arXiv:2504.08842, 2025.
-
Michal Aharon, Michael Elad, and Alfred Bruckstein. K-svd: An algorithm for designing overcomplete dictionaries for sparse representation. IEEE Transactions on signal processing, 54 (11): 4311–4322, 2006.
+
Micah Adler, Dan Alistarh, and Nir Shavit. Towards combinatorial interpretability of neural computation, 2025. URL https://arxiv.org/abs/2504.08842.
+
Isha Agarwal, Saharsha Navani, and Fazl Barez. Context matters: Analyzing the generalizability of linear probing and steering across diverse scenarios. In Mechanistic Interpretability Workshop at NeurIPS 2025, 2025. URL https://openreview.net/forum?id=H5sbfvEbTh.
+
Michal Aharon, Michael Elad, and Alfred Bruckstein. K-svd: An algorithm for designing overcomplete dictionaries for sparse representation. IEEE Transactions on signal processing, 54 (11): 4311–4322, 2006.
+
Joshua Ainslie, James Lee-Thorp, Michiel de Jong, Yury Zemlyanskiy, Federico Lebron, and Sumit Sanghai. GQA: Training generalized multi-query transformer models from multi-head checkpoints. In Houda Bouamor, Juan Pino, and Kalika Bali (eds.), Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, pp. 4895–4901, Singapore, December 2023. Association for Computational Linguistics. . URL https://aclanthology.org/2023.emnlp-main.298/.
Guillaume Alain and Yoshua Bengio. Understanding intermediate layers using linear classifier probes. The Fifth International Conference on Learning Representations, 2017. URL https://openreview.net/forum?id=ryF7rTqgl.
+
Ameen Ali Ali, Itamar Zimerman, and Lior Wolf. The hidden attention of mamba models. In Wanxiang Che, Joyce Nabende, Ekaterina Shutova, and Mohammad Taher Pilehvar (eds.), Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 1516–1534, Vienna, Austria, July 2025. Association for Computational Linguistics. ISBN 979-8-89176-251-0. . URL https://aclanthology.org/2025.acl-long.76/.
Emmanuel Ameisen, Jack Lindsey, Adam Pearce, Wes Gurnee, Nicholas L. Turner, Brian Chen, Craig Citro, David Abrahams, Shan Carter, Basil Hosmer, Jonathan Marcus, Michael Sklar, Adly Templeton, Trenton Bricken, Callum McDougall, Hoagy Cunningham, Thomas Henighan, Adam Jermyn, Andy Jones, Andrew Persic, Zhenyi Qi, T. Ben Thompson, Sam Zimmerman, Kelley Rivoire, Thomas Conerly, Chris Olah, and Joshua Batson. Circuit tracing: Revealing computational graphs in language models. Transformer Circuits Thread, 2025. URL https://transformer-circuits.pub/2025/attribution-graphs/methods.html.
+
Cem Anil, Esin DURMUS, Nina Rimsky, Mrinank Sharma, Joe Benton, Sandipan Kundu, Joshua Batson, Meg Tong, Jesse Mu, Daniel J Ford, Francesco Mosconi, Rajashree Agrawal, Rylan Schaeffer, Naomi Bashkansky, Samuel Svenningsen, Mike Lambert, Ansh Radhakrishnan, Carson Denison, Evan J Hubinger, Yuntao Bai, Trenton Bricken, Timothy Maxwell, Nicholas Schiefer, James Sully, Alex Tamkin, Tamera Lanham, Karina Nguyen, Tomasz Korbak, Jared Kaplan, Deep Ganguli, Samuel R. Bowman, Ethan Perez, Roger Baker Grosse, and David Duvenaud. Many-shot jailbreaking. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024. URL https://openreview.net/forum?id=cw5mgd71jW.
+
Anthropic. Model diffing. Transformer Circuits, October 2024. URL https://transformer-circuits.pub/2024/model-diffing/index.html.
Anthropic. Claude sonnet 4.5 system card. System card, Anthropic, September 2025a. URL https://assets.anthropic.com/m/12f214efcc2f457a/original/Claude-Sonnet-4-5-System-Card.pdf.
Anthropic. System card: Claude opus 4 & claude sonnet 4. Technical report, Anthropic, May 2025b. URL https://www-cdn.anthropic.com/6d8a8055020700718b0c49369f60816ba2a7c285/Claude%204%20System%20Card.pdf.
-
Tatsuya Aoyama and Ethan Wilcox. Language models grow less humanlike beyond phase transition, 2025. URL http://arxiv.org/abs/2502.18802.
+
Tatsuya Aoyama and Ethan Wilcox. Language models grow less humanlike beyond phase transition. In Wanxiang Che, Joyce Nabende, Ekaterina Shutova, and Mohammad Taher Pilehvar (eds.), Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 24938–24958, Vienna, Austria, July 2025. Association for Computational Linguistics. ISBN 979-8-89176-251-0. . URL https://aclanthology.org/2025.acl-long.1214/.
Apollo Research. Claude sonnet 3.7 (often) knows when it’s in alignment evaluations. Apollo Research Blog, March 2025. URL https://www.apolloresearch.ai/science/claude-sonnet-37-often-knows-when-its-in-alignment-evaluations/.
Santiago Aranguri and Tom McGrath. Discovering undesired rare behaviors via model diff amplification. Goodfire, 2025. https://www.goodfire.ai/research/model-diff-amplification.
+
Iván Arcuschin, Jett Janiak, Robert Krzyzanowski, Senthooran Rajamanoharan, Neel Nanda, and Arthur Conmy. Chain-of-thought reasoning in the wild is not always faithful. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=NUyt4uxzx0.
Andy Arditi, Oscar Balcells Obeso, Aaquib Syed, Daniel Paleka, Nina Rimsky, Wes Gurnee, and Neel Nanda. Refusal in language models is mediated by a single direction. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024.
-
Juhan Bae, Nathan Ng, Alston Lo, Marzyeh Ghassemi, and Roger Grosse. If influence functions are the answer, then what is the question? In Proceedings of the 36th International Conference on Neural Information Processing Systems, NIPS ’22, Red Hook, NY, USA, 2022. Curran Associates Inc. ISBN 9781713871088.
-
Juhan Bae, Wu Lin, Jonathan Lorraine, and Roger Baker Grosse. Training data attribution via approximate unrolling. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024. URL https://openreview.net/forum?id=3NaqGg92KZ.
+
Aryaman Arora, Dan Jurafsky, and Christopher Potts. CausalGym: Benchmarking causal interpretability methods on linguistic tasks. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 14638–14663, Bangkok, Thailand, August 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.785/.
+
Aryaman Arora, Zhengxuan Wu, Jacob Steinhardt, and Sarah Schwettmann. Language model circuits are sparse in the neuron basis. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=OrviwFWcN4.
+
Kola Ayonrinde and Louis Jaburi. Evaluating explanations: An explanatory virtues framework for mechanistic interpretability – the strange science part i.ii, 2025. URL https://arxiv.org/abs/2505.01372.
+
Zhangir Azerbayev, Bartosz Piotrowski, Hailey Schoelkopf, Edward W. Ayers, Dragomir Radev, and Jeremy Avigad. Proofnet: Autoformalizing and formally proving undergraduate-level mathematics, 2023. URL https://arxiv.org/abs/2302.12433.
+
Jimmy Lei Ba, Jamie Ryan Kiros, and Geoffrey E. Hinton. Layer normalization, 2016. URL https://arxiv.org/abs/1607.06450.
+
Sebastian Bach, Alexander Binder, Grégoire Montavon, Frederick Klauschen, Klaus-Robert Müller, and Wojciech Samek. On pixel-wise explanations for non-linear classifier decisions by layer-wise relevance propagation. PLOS ONE, 10 (7): 1–46, 07 2015. . URL https://doi.org/10.1371/journal.pone.0130140.
+
Juhan Bae, Nathan Ng, Alston Lo, Marzyeh Ghassemi, and Roger Grosse. If influence functions are the answer, then what is the question? In Proceedings of the 36th International Conference on Neural Information Processing Systems, NIPS ’22, Red Hook, NY, USA, 2022. Curran Associates Inc. ISBN 9781713871088.
David D. Baek and Max Tegmark. Towards understanding distilled reasoning models: A representational approach. In ICLR 2025 Workshop on Building Trust in Language Models and Applications, 2025. URL https://openreview.net/forum?id=UYZCcnwgc4.
-
Yuntao Bai, Andy Jones, Kamal Ndousse, Amanda Askell, Anna Chen, Nova DasSarma, Dawn Drain, Stanislav Fort, Deep Ganguli, Tom Henighan, et al. Training a helpful and harmless assistant with reinforcement learning from human feedback. arXiv preprint arXiv:2204.05862, 2022a.
-
Yuntao Bai, Saurav Kadavath, Sandipan Kundu, Amanda Askell, Jackson Kernion, Andy Jones, Anna Chen, Anna Goldie, Azalia Mirhoseini, Cameron McKinnon, et al. Constitutional ai: Harmlessness from ai feedback. arXiv preprint arXiv:2212.08073, 2022b.
+
Yuntao Bai, Andy Jones, Kamal Ndousse, Amanda Askell, Anna Chen, Nova DasSarma, Dawn Drain, Stanislav Fort, Deep Ganguli, Tom Henighan, Nicholas Joseph, Saurav Kadavath, Jackson Kernion, Tom Conerly, Sheer El-Showk, Nelson Elhage, Zac Hatfield-Dodds, Danny Hernandez, Tristan Hume, Scott Johnston, Shauna Kravec, Liane Lovitt, Neel Nanda, Catherine Olsson, Dario Amodei, Tom Brown, Jack Clark, Sam McCandlish, Chris Olah, Ben Mann, and Jared Kaplan. Training a helpful and harmless assistant with reinforcement learning from human feedback, 2022a. URL https://arxiv.org/abs/2204.05862.
+
Yuntao Bai, Saurav Kadavath, Sandipan Kundu, Amanda Askell, Jackson Kernion, Andy Jones, Anna Chen, Anna Goldie, Azalia Mirhoseini, Cameron McKinnon, Carol Chen, Catherine Olsson, Christopher Olah, Danny Hernandez, Dawn Drain, Deep Ganguli, Dustin Li, Eli Tran-Johnson, Ethan Perez, Jamie Kerr, Jared Mueller, Jeffrey Ladish, Joshua Landau, Kamal Ndousse, Kamile Lukosuite, Liane Lovitt, Michael Sellitto, Nelson Elhage, Nicholas Schiefer, Noemi Mercado, Nova DasSarma, Robert Lasenby, Robin Larson, Sam Ringer, Scott Johnston, Shauna Kravec, Sheer El Showk, Stanislav Fort, Tamera Lanham, Timothy Telleen-Lawton, Tom Conerly, Tom Henighan, Tristan Hume, Samuel R. Bowman, Zac Hatfield-Dodds, Ben Mann, Dario Amodei, Nicholas Joseph, Sam McCandlish, Tom Brown, and Jared Kaplan. Constitutional ai: Harmlessness from ai feedback, 2022b. URL https://arxiv.org/abs/2212.08073.
+
Yushi Bai, Xin Lv, Jiajie Zhang, Hongchang Lyu, Jiankai Tang, Zhidian Huang, Zhengxiao Du, Xiao Liu, Aohan Zeng, Lei Hou, Yuxiao Dong, Jie Tang, and Juanzi Li. LongBench: A bilingual, multitask benchmark for long context understanding. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 3119–3137, Bangkok, Thailand, August 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.172/.
+
Bowen Baker, Joost Huizinga, Leo Gao, Zehao Dou, Melody Y. Guan, Aleksander Madry, Wojciech Zaremba, Jakub Pachocki, and David Farhi. Monitoring reasoning models for misbehavior and the risks of promoting obfuscation, 2025. URL https://arxiv.org/abs/2503.11926.
Fazl Barez. Automated interpretability-driven model auditing and control: A research agenda. AI Governance Initiative, University of Oxford, January 2026. URL https://aigi.ox.ac.uk/publications/automated-interpretability-driven-model-auditing-and-control-a-research-agenda/.
-
Anthony Bau, Yonatan Belinkov, Hassan Sajjad, Nadir Durrani, Fahim Dalvi, and James Glass. Identifying and controlling important neurons in neural machine translation. In International Conference on Learning Representations, 2019. URL https://openreview.net/forum?id=H1z-PsR5KX.
-
Deniz Bayazit, Aaron Mueller, and Antoine Bosselut. Crosscoding through time: Tracking emergence & consolidation of linguistic representations throughout llm pretraining. arXiv preprint arXiv:2509.05291, 2025.
+
Fazl Barez, Tung-Yu Wu, Iván Arcuschin, Michael Lan, Vincent Wang, Noah Siegel, Nicolas Collignon, Clement Neo, Isabelle Lee, Alasdair Paren, et al. Chain-of-thought is not explainability. In The Fortieth Annual Conference on Neural Information Processing Systems Position Paper Track, 2026.
+
H. B. Barlow. Possible principles underlying the transformations of sensory messages. In Walter A. Rosenblith (ed.), Sensory Communication, pp. 217–234. MIT Press, Cambridge, MA, 1961.
+
Hunar Batra, Lachin Naghashyar, Ashkan Khakzar, Philip Torr, Christian Schroeder de Witt, Constantin Venhoff, and Ronald Clark. Multimodal model diffing for feature discovery and control, 2026. URL https://arxiv.org/abs/2608.09928.
+
Anthony Bau, Yonatan Belinkov, Hassan Sajjad, Nadir Durrani, Fahim Dalvi, and James Glass. Identifying and controlling important neurons in neural machine translation. In International Conference on Learning Representations, 2019. URL https://openreview.net/forum?id=H1z-PsR5KX.
+
David Bau, Bolei Zhou, Aditya Khosla, Aude Oliva, and Antonio Torralba. Network dissection: Quantifying interpretability of deep visual representations. In Computer Vision and Pattern Recognition, 2017.
+
David Bau, Jun-Yan Zhu, Hendrik Strobelt, Agata Lapedriza, Bolei Zhou, and Antonio Torralba. Understanding the role of individual units in a deep neural network. Proceedings of the National Academy of Sciences, 2020. ISSN 0027-8424. . URL https://www.pnas.org/content/early/2020/08/31/1907375117.
+
Jan Philipp Bauer, Celeste De Schamphelaere, Adam Karvonen, Niclas Luick, and Neel Nanda. Building better activation oracles. In Mechanistic Interpretability Workshop at ICML 2026, 2026. URL https://openreview.net/forum?id=6nkeLpn3cY.
+
Deniz Bayazit, Aaron Mueller, and Antoine Bosselut. Crosscoding through time: Tracking emergence & consolidation of linguistic representations throughout LLM pretraining. In Maria Liakata, Viviane P. Moreira, Jiajun Zhang, and David Jurgens (eds.), Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 1353–1377, San Diego, California, United States, July 2026. Association for Computational Linguistics. ISBN 979-8-89176-390-6. . URL https://aclanthology.org/2026.acl-long.60/.
Yonatan Belinkov. Probing classifiers: Promises, shortcomings, and advances. Computational Linguistics, 48 (1): 207–219, March 2022. . URL https://aclanthology.org/2022.cl-1.7/.
Yonatan Belinkov and James Glass. Analysis methods in neural language processing: A survey. Transactions of the Association for Computational Linguistics, 7: 49–72, 2019. . URL https://aclanthology.org/Q19-1004/.
Yonatan Belinkov, Nadir Durrani, Fahim Dalvi, Hassan Sajjad, and James Glass. What do neural machine translation models learn about morphology? In Regina Barzilay and Min-Yen Kan (eds.), Proceedings of the 55th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 861–872, Vancouver, Canada, July 2017. Association for Computational Linguistics. . URL https://aclanthology.org/P17-1080/.
+
Nora Belrose, Quintin Pope, Lucia Quirke, Alex Mallen, and Xiaoli Fern. Neural networks learn statistics of increasing complexity. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.
Nora Belrose, Igor Ostrovsky, Lev McKinney, Zach Furman, Logan Smith, Danny Halawi, Stella Biderman, and Jacob Steinhardt. Eliciting latent predictions from transformers with the tuned lens, 2025. URL https://arxiv.org/abs/2303.08112.
+
Yoshua Bengio, Nicholas Léonard, and Aaron Courville. Estimating or propagating gradients through stochastic neurons for conditional computation, 2013. URL https://arxiv.org/abs/1308.3432.
+
Yoshua Bengio, Sören Mindermann, Daniel Privitera, Tamay Besiroglu, Rishi Bommasani, Stephen Casper, Yejin Choi, Philip Fox, Ben Garfinkel, Danielle Goldfarb, Hoda Heidari, Anson Ho, Sayash Kapoor, Leila Khalatbari, Shayne Longpre, Sam Manning, Vasilios Mavroudis, Mantas Mazeika, Julian Michael, Jessica Newman, Kwan Yee Ng, Chinasa T. Okolo, Deborah Raji, Girish Sastry, Elizabeth Seger, Theodora Skeadas, Tobin South, Emma Strubell, Florian Tramèr, Lucia Velasco, Nicole Wheeler, Daron Acemoglu, Olubayo Adekanmbi, David Dalrymple, Thomas G. Dietterich, Edward W. Felten, Pascale Fung, Pierre-Olivier Gourinchas, Fredrik Heintz, Geoffrey Hinton, Nick Jennings, Andreas Krause, Susan Leavy, Percy Liang, Teresa Ludermir, Vidushi Marda, Helen Margetts, John McDermid, Jane Munga, Arvind Narayanan, Alondra Nelson, Clara Neppel, Alice Oh, Gopal Ramchurn, Stuart Russell, Marietje Schaake, Bernhard Schölkopf, Dawn Song, Alvaro Soto, Lee Tiedrich, Gaël Varoquaux, Andrew Yao, Ya-Qin Zhang, Fahad Albalawi, Marwan Alserkal, Olubunmi Ajala, Guillaume Avrin, Christian Busch, André Carlos Ponce de Leon Ferreira de Carvalho, Bronwyn Fox, Amandeep Singh Gill, Ahmet Halit Hatip, Juha Heikkilä, Gill Jolly, Ziv Katzir, Hiroaki Kitano, Antonio Krüger, Chris Johnson, Saif M. Khan, Kyoung Mu Lee, Dominic Vincent Ligot, Oleksii Molchanovskyi, Andrea Monti, Nusu Mwamanzi, Mona Nemer, Nuria Oliver, José Ramón López Portillo, Balaraman Ravindran, Raquel Pezoa Rivera, Hammam Riza, Crystal Rugege, Ciarán Seoighe, Jerry Sheehan, Haroon Sheikh, Denise Wong, and Yi Zeng. International ai safety report, 2025. URL https://arxiv.org/abs/2501.17805.
Leonard Bereska and Stratis Gavves. Mechanistic interpretability for AI safety - a review. Transactions on Machine Learning Research, 2024. ISSN 2835-8856. URL https://openreview.net/forum?id=ePUVetPKu6.
-
Stella Biderman, Usvsn Prashanth, Lintang Sutawika, Hailey Schoelkopf, Quentin Anthony, Shivanshu Purohit, and Edward Raff. Emergent and predictable memorization in large language models. Advances in Neural Information Processing Systems, 36: 28072–28090, 2023a.
-
Stella Biderman, Hailey Schoelkopf, Quentin Gregory Anthony, Herbie Bradley, Kyle O’Brien, Eric Hallahan, Mohammad Aflah Khan, Shivanshu Purohit, USVSN Sai Prashanth, Edward Raff, et al. Pythia: A suite for analyzing large language models across training and scaling. In International Conference on Machine Learning, pp. 2397–2430. PMLR, 2023b.
+
Adithya Bhaskar, Alexander Wettig, Dan Friedman, and Danqi Chen. Finding transformer circuits with edge pruning. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024. URL https://openreview.net/forum?id=8oSY3rA9jY.
+
Stella Biderman, Usvsn Prashanth, Lintang Sutawika, Hailey Schoelkopf, Quentin Anthony, Shivanshu Purohit, and Edward Raff. Emergent and predictable memorization in large language models. Advances in Neural Information Processing Systems, 36: 28072–28090, 2023a.
+
Stella Biderman, Hailey Schoelkopf, Quentin Gregory Anthony, Herbie Bradley, Kyle O’Brien, Eric Hallahan, Mohammad Aflah Khan, Shivanshu Purohit, USVSN Sai Prashanth, Edward Raff, et al. Pythia: A suite for analyzing large language models across training and scaling. In International Conference on Machine Learning, pp. 2397–2430. PMLR, 2023b.
+
Stella Biderman, Mohammad Aflah Khan, Niloofar Mireshghallah, Catherine Arnett, Fazl Barez, and Naomi Saphra. Position: Don’t just ”fix it in post”: A science of AI must study learning dynamics. In Forty-third International Conference on Machine Learning Position Paper Track, 2026. URL https://openreview.net/forum?id=VbHh05xQ7y.
Steven Bills, Nick Cammarata, Dan Mossing, Henk Tillman, Leo Gao, Gabriel Goh, Ilya Sutskever, Jan Leike, Jeff Wu, and William Saunders. Language models can explain neurons in language models, 2023.
-
Dan Braun, Lucius Bushnaq, Stefan Heimersheim, Jake Mendel, and Lee Sharkey. Interpretability in parameter space: Minimizing mechanistic description length with attribution-based parameter decomposition. arXiv preprint arXiv:2501.14926, 2025.
+
Felix Jedidja Binder, James Chua, Tomek Korbak, Henry Sleight, John Hughes, Robert Long, Ethan Perez, Miles Turpin, and Owain Evans. Looking inward: Language models can learn about themselves by introspection. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=eb5pkwIB5i.
+
Camila Blank, Agam Bhatia, and Neel Nanda. R-lens: Making J-lens more faithful on early layers. AI Alignment Forum, August 2026. URL https://www.alignmentforum.org/posts/nv8oedrnLXKRzNEL9/r-lens-making-j-lens-more-faithful-on-early-layers. Blog post.
+
Jaydeep Borkar, Matthew Jagielski, Katherine Lee, Niloofar Mireshghallah, David A. Smith, and Christopher A. Choquette-Choo. Privacy ripple effects from adding or removing personal information in language model training. In Findings of the Association for Computational Linguistics: ACL 2025, 2025.
+
Dan Braun, Lucius Bushnaq, Stefan Heimersheim, Jake Mendel, and Lee Sharkey. Interpretability in parameter space: Minimizing mechanistic description length with attribution-based parameter decomposition, 2025. URL https://arxiv.org/abs/2501.14926.
Trenton Bricken, Adly Templeton, Joshua Batson, Brian Chen, Adam Jermyn, Tom Conerly, Nick Turner, Cem Anil, Carson Denison, Amanda Askell, Robert Lasenby, Yifan Wu, Shauna Kravec, Nicholas Schiefer, Tim Maxwell, Nicholas Joseph, Zac Hatfield-Dodds, Alex Tamkin, Karina Nguyen, Brayden McLean, Josiah E Burke, Tristan Hume, Shan Carter, Tom Henighan, and Christopher Olah. Towards monosemanticity: Decomposing language models with dictionary learning. Transformer Circuits Thread, 2023. https://transformer-circuits.pub/2023/monosemantic-features/index.html.
Trenton Bricken, Siddharth Mishra-Sharma, Jonathan Marcus, Adam Jermyn, Christopher Olah, Kelley Rivoire, and Thomas Henighan. Stage-wise model diffing. Transformer Circuits Thread (Anthropic), December 2024. URL https://transformer-circuits.pub/2024/model-diffing/index.html.
+
Sebastian A. Bruijns, Jirko Rubruck, Mia H. Whitefield, Kai J. Sandbrink, Fazl Barez, and Christopher Summerfield. Pretraining curricula enable selective fine-tuning, 2026. URL https://arxiv.org/abs/2607.04846.
Collin Burns, Haotian Ye, Dan Klein, and Jacob Steinhardt. Discovering latent knowledge in language models without supervision. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=ETKGuby0hcs.
-
Collin Burns, Haotian Ye, Dan Klein, and Jacob Steinhardt. Discovering latent knowledge in language models without supervision, 2024. URL https://arxiv.org/abs/2212.03827.
-
Lucius Bushnaq, Dan Braun, and Lee Sharkey. Stochastic parameter decomposition. arXiv preprint arXiv:2506.20790, 2025.
-
Bart Bussmann, Patrick Leask, and Neel Nanda. Batchtopk sparse autoencoders. In NeurIPS 2024 Workshop on Scientific Methods for Understanding Deep Learning, 2024. URL https://openreview.net/forum?id=d4dpOCqybL.
+
Lucius Bushnaq, Dan Braun, and Lee Sharkey. Stochastic parameter decomposition, 2025. URL https://arxiv.org/abs/2506.20790.
+
Bart Bussmann, Patrick Leask, and Neel Nanda. Batchtopk sparse autoencoders. In NeurIPS 2024 Workshop on Scientific Methods for Understanding Deep Learning, 2024. URL https://openreview.net/forum?id=d4dpOCqybL.
+
Steven Cao, Victor Sanh, and Alexander Rush. Low-complexity probing via finding subnetworks. In Kristina Toutanova, Anna Rumshisky, Luke Zettlemoyer, Dilek Hakkani-Tur, Iz Beltagy, Steven Bethard, Ryan Cotterell, Tanmoy Chakraborty, and Yichao Zhou (eds.), Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 960–966, Online, June 2021. Association for Computational Linguistics. . URL https://aclanthology.org/2021.naacl-main.74/.
Stephen Casper, Carson Ezell, Charlotte Siegmann, Noam Kolt, Taylor Lynn Curtis, Benjamin Bucknall, Andreas Haupt, Kevin Wei, Jérémy Scheurer, Marius Hobbhahn, Lee Sharkey, Satyapriya Krishna, Marvin Von Hagen, Silas Alberti, Alan Chan, Qinyi Sun, Michael Gerovitch, David Bau, Max Tegmark, David Krueger, and Dylan Hadfield-Menell. Black-box access is insufficient for rigorous ai audits. In Proceedings of the 2024 ACM Conference on Fairness, Accountability, and Transparency, FAccT ’24, pp. 2254–2272, New York, NY, USA, 2024. Association for Computing Machinery. ISBN 9798400704505. . URL https://doi.org/10.1145/3630106.3659037.
-
Lawrence Chan, Adrià Garriga-Alonso, Nicholas Goldwosky-Dill, Ryan Greenblatt, Jenny Nitishinskaya, Ansh Radhakrishnan, Buck Shlegeris, and Nate Thomas. Causal scrubbing, a method for rigorously testing interpretability hypotheses. AI Alignment Forum, 2022. https://www.alignmentforum.org/posts/JvZhhzycHu2Yd57RN/causal-scrubbing-a-method-for-rigorously-testing.
-
Tyler A Chang and Benjamin K Bergen. Word acquisition in neural language models. Transactions of the Association for Computational Linguistics, 10: 1–16, 2022.
-
David Chanin, James Wilken-Smith, Tomáš Dulka, Hardik Bhatnagar, Satvik Golechha, and Joseph Bloom. A is for absorption: Studying feature splitting and absorption in sparse autoencoders, 2025. URL https://arxiv.org/abs/2409.14507.
-
Maheep Chaudhary and Fazl Barez. Safetynet: Detecting harmful outputs in llms by modeling and monitoring deceptive behaviors, 2025. URL https://arxiv.org/abs/2505.14300.
-
Maheep Chaudhary and Atticus Geiger. Evaluating open-source sparse autoencoders on disentangling factual knowledge in gpt-2 small. arXiv preprint arXiv:2409.04478, 2024.
-
Angelica Chen, Ravid Shwartz-Ziv, Kyunghyun Cho, Matthew L. Leavitt, and Naomi Saphra. Sudden drops in the loss: Syntax acquisition, phase transitions, and simplicity bias in MLMs. In The Twelfth International Conference on Learning Representations, 2024a. URL https://openreview.net/forum?id=MO5PiKHELW.
-
Haozhe Chen, Carl Vondrick, and Chengzhi Mao. Selfie: self-interpretation of large language model embeddings. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024b.
-
Matthew Chen, Joshua Engels, and Max Tegmark. Low-rank adapting models for sparse autoencoders. arXiv preprint arXiv:2501.19406, 2025.
-
Paul F Christiano, Jan Leike, Tom Brown, Miljan Martic, Shane Legg, and Dario Amodei. Deep reinforcement learning from human preferences. In I. Guyon, U. Von Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (eds.), Advances in Neural Information Processing Systems, volume 30. Curran Associates, Inc., 2017. URL https://proceedings.neurips.cc/paper_files/paper/2017/file/d5e2c0adad503c91f91df240d0cd4e49-Paper.pdf.
-
Kevin Clark, Urvashi Khandelwal, Omer Levy, and Christopher D. Manning. What does BERT look at? an analysis of BERT’s attention. In Tal Linzen, Grzegorz Chrupała, Yonatan Belinkov, and Dieuwke Hupkes (eds.), Proceedings of the 2019 ACL Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP, pp. 276–286, Florence, Italy, August 2019. Association for Computational Linguistics. . URL https://aclanthology.org/W19-4828/.
-
Alex Cloud, Jacob Goldman-Wetzler, Evžen Wybitul, Joseph Miller, and Alexander Matt Turner. Gradient routing: Masking gradients to localize computation in neural networks. arXiv preprint arXiv:2410.04332, 2024.
+
Lawrence Chan, Adrià Garriga-Alonso, Nicholas Goldwosky-Dill, Ryan Greenblatt, Jenny Nitishinskaya, Ansh Radhakrishnan, Buck Shlegeris, and Nate Thomas. Causal scrubbing, a method for rigorously testing interpretability hypotheses. AI Alignment Forum, 2022a. https://www.alignmentforum.org/posts/JvZhhzycHu2Yd57RN/causal-scrubbing-a-method-for-rigorously-testing.
+
Stephanie C. Y. Chan, Adam Santoro, Andrew K. Lampinen, Jane X. Wang, Aaditya Singh, Pierre H. Richemond, Jay McClelland, and Felix Hill. Data distributional properties drive emergent in-context learning in transformers. In Advances in Neural Information Processing Systems, volume 35, 2022b.
+
Tyler A Chang and Benjamin K Bergen. Word acquisition in neural language models. Transactions of the Association for Computational Linguistics, 10: 1–16, 2022.
+
Tyler A Chang, Zhuowen Tu, and Benjamin K Bergen. Characterizing learning curves during language model pre-training: Learning, forgetting, and stability. Transactions of the Association for Computational Linguistics, 12: 1346–1362, 2024.
+
David Chanin, James Wilken-Smith, Tomáš Dulka, Hardik Bhatnagar, Satvik Golechha, and Joseph Isaac Bloom. A is for absorption: Studying feature splitting and absorption in sparse autoencoders. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025. URL https://openreview.net/forum?id=R73ybUciQF.
+
Maheep Chaudhary and Fazl Barez. Beyond black-box obfuscation: Mechanistic analysis and defense of white-box monitors. In Third Conference on Language Modeling, 2026. URL https://openreview.net/forum?id=4JrlXy2N6j.
+
Maheep Chaudhary and Atticus Geiger. Evaluating open-source sparse autoencoders on disentangling factual knowledge in gpt-2 small, 2024. URL https://arxiv.org/abs/2409.04478.
+
Angelica Chen, Ravid Shwartz-Ziv, Kyunghyun Cho, Matthew L. Leavitt, and Naomi Saphra. Sudden drops in the loss: Syntax acquisition, phase transitions, and simplicity bias in MLMs. In The Twelfth International Conference on Learning Representations, 2024a. URL https://openreview.net/forum?id=MO5PiKHELW.
+
Haozhe Chen, Carl Vondrick, and Chengzhi Mao. Selfie: self-interpretation of large language model embeddings. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024b.
+
Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, Nick Ryder, Mikhail Pavlov, Alethea Power, Lukasz Kaiser, Mohammad Bavarian, Clemens Winter, Philippe Tillet, Felipe Petroski Such, Dave Cummings, Matthias Plappert, Fotios Chantzis, Elizabeth Barnes, Ariel Herbert-Voss, William Hebgen Guss, Alex Nichol, Alex Paino, Nikolas Tezak, Jie Tang, Igor Babuschkin, Suchir Balaji, Shantanu Jain, William Saunders, Christopher Hesse, Andrew N. Carr, Jan Leike, Josh Achiam, Vedant Misra, Evan Morikawa, Alec Radford, Matthew Knight, Miles Brundage, Mira Murati, Katie Mayer, Peter Welinder, Bob McGrew, Dario Amodei, Sam McCandlish, Ilya Sutskever, and Wojciech Zaremba. Evaluating large language models trained on code, 2021. URL https://arxiv.org/abs/2107.03374.
+
Matthew Chen, Joshua Engels, and Max Tegmark. Low-rank adapting models for sparse autoencoders. In Forty-second International Conference on Machine Learning, 2025a. URL https://openreview.net/forum?id=lDPtsCYTwr.
+
Runjin Chen, Andy Arditi, Henry Sleight, Owain Evans, and Jack Lindsey. Persona vectors: Monitoring and controlling character traits in language models, 2025b. URL https://arxiv.org/abs/2507.21509.
+
Scott Shaobing Chen, David L. Donoho, and Michael A. Saunders. Atomic decomposition by basis pursuit. SIAM Journal on Scientific Computing, 20 (1): 33–61, 1998. . URL https://doi.org/10.1137/S1064827596304010.
+
Yanda Chen, Joe Benton, Ansh Radhakrishnan, Jonathan Uesato, Carson Denison, John Schulman, Arushi Somani, Peter Hase, Misha Wagner, Fabien Roger, Vlad Mikulik, Samuel R. Bowman, Jan Leike, Jared Kaplan, and Ethan Perez. Reasoning models don’t always say what they think, 2025c. URL https://arxiv.org/abs/2505.05410.
+
Bilal Chughtai, Lawrence Chan, and Neel Nanda. A toy model of universality: reverse engineering how networks learn group operations. In Proceedings of the 40th International Conference on Machine Learning, ICML’23. JMLR.org, 2023.
+
Kevin Clark, Urvashi Khandelwal, Omer Levy, and Christopher D. Manning. What does BERT look at? an analysis of BERT’s attention. In Tal Linzen, Grzegorz Chrupała, Yonatan Belinkov, and Dieuwke Hupkes (eds.), Proceedings of the 2019 ACL Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP, pp. 276–286, Florence, Italy, August 2019. Association for Computational Linguistics. . URL https://aclanthology.org/W19-4828/.
+
Alex Cloud, Jacob Goldman-Wetzler, Evžen Wybitul, Joseph Miller, and Alexander Matt Turner. Gradient routing: Masking gradients to localize computation in neural networks, 2024. URL https://arxiv.org/abs/2410.04332.
+
Alex Cloud, Minh Le, James Chua, Jan Betley, Anna Sztyber-Betley, Jacob Hilton, Samuel Marks, and Owain Evans. Language models transmit behavioural traits through hidden signals in data. Nature, 652: 615–621, 2026. . URL https://www.nature.com/articles/s41586-026-10319-8.
+
Karl Cobbe, Vineet Kosaraju, Mohammad Bavarian, Mark Chen, Heewoo Jun, Lukasz Kaiser, Matthias Plappert, Jerry Tworek, Jacob Hilton, Reiichiro Nakano, Christopher Hesse, and John Schulman. Training verifiers to solve math word problems, 2021. URL https://arxiv.org/abs/2110.14168.
Arthur Conmy, Augustine N. Mavor-Parker, Aengus Lynch, Stefan Heimersheim, and Adrià Garriga-Alonso. Towards automated circuit discovery for mechanistic interpretability. In Proceedings of the 37th International Conference on Neural Information Processing Systems, NIPS ’23, Red Hook, NY, USA, 2023. Curran Associates Inc.
Alexis Conneau, German Kruszewski, Guillaume Lample, Loïc Barrault, and Marco Baroni. What you can cram into a single $&!#* vector: Probing sentence embeddings for linguistic properties. In Iryna Gurevych and Yusuke Miyao (eds.), Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 2126–2136, Melbourne, Australia, July 2018. Association for Computational Linguistics. . URL https://aclanthology.org/P18-1198/.
-
Róbert Csordás, Christopher Potts, Christopher D Manning, and Atticus Geiger. Recurrent neural networks learn to store and generate sequences using non-linear representations. In Yonatan Belinkov, Najoung Kim, Jaap Jumelet, Hosein Mohebbi, Aaron Mueller, and Hanjie Chen (eds.), Proceedings of the 7th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP, pp. 248–262, Miami, Florida, US, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.blackboxnlp-1.17/.
-
Hoagy Cunningham et al. Sparse autoencoders find highly interpretable features. arXiv preprint arXiv:2309.08600, 2023.
-
Bartosz Cywiński, Emil Ryd, Rowan Wang, Senthooran Rajamanoharan, Neel Nanda, Arthur Conmy, and Samuel Marks. Eliciting secret knowledge from language models. arXiv preprint arXiv:2510.01070, 2025.
-
Fahim Dalvi, Nadir Durrani, Hassan Sajjad, Yonatan Belinkov, Anthony Bau, and James Glass. What is one grain of sand in the desert? analyzing individual neurons in deep nlp models. Proceedings of the AAAI Conference on Artificial Intelligence, 33 (01): 6309–6317, Jul. 2019. . URL https://ojs.aaai.org/index.php/AAAI/article/view/4592.
+
Michael T. Cox and Anita Raja. Metareasoning: A manifesto. In Proceedings of the 2008 AAAI Workshop on Metareasoning: Thinking about Thinking, number WS-08-07 in Technical Report, pp. 1–4. AAAI Press, 2008. URL https://aaai.org/papers/ws08-07-001-metareasoning-a-manifesto/.
+
Róbert Csordás, Sjoerd van Steenkiste, and Jürgen Schmidhuber. Are neural nets modular? inspecting functional modularity through differentiable weight masks. In International Conference on Learning Representations, 2021. URL https://openreview.net/forum?id=7uVcpu-gMD.
+
Róbert Csordás, Christopher Potts, Christopher D Manning, and Atticus Geiger. Recurrent neural networks learn to store and generate sequences using non-linear representations. In Yonatan Belinkov, Najoung Kim, Jaap Jumelet, Hosein Mohebbi, Aaron Mueller, and Hanjie Chen (eds.), Proceedings of the 7th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP, pp. 248–262, Miami, Florida, US, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.blackboxnlp-1.17/.
+
Hoagy Cunningham, Aidan Ewart, Logan Riggs, Robert Huben, and Lee Sharkey. Sparse autoencoders find highly interpretable features in language models. In The Twelfth International Conference on Learning Representations, 2024. URL https://openreview.net/forum?id=F76bwRSLeK.
+
Bartosz Cywiński, Emil Ryd, Rowan Wang, Senthooran Rajamanoharan, Neel Nanda, Arthur Conmy, and Samuel Marks. Eliciting secret knowledge from language models, 2025. URL https://arxiv.org/abs/2510.01070.
+
Damai Dai, Li Dong, Yaru Hao, Zhifang Sui, Baobao Chang, and Furu Wei. Knowledge neurons in pretrained transformers. In Smaranda Muresan, Preslav Nakov, and Aline Villavicencio (eds.), Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 8493–8502, Dublin, Ireland, May 2022. Association for Computational Linguistics. . URL https://aclanthology.org/2022.acl-long.581/.
+
Damai Dai, Chengqi Deng, Chenggang Zhao, R.X. Xu, Huazuo Gao, Deli Chen, Jiashi Li, Wangding Zeng, Xingkai Yu, Y. Wu, Zhenda Xie, Y.K. Li, Panpan Huang, Fuli Luo, Chong Ruan, Zhifang Sui, and Wenfeng Liang. DeepSeekMoE: Towards ultimate expert specialization in mixture-of-experts language models. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 1280–1297, Bangkok, Thailand, August 2024a. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.70/.
+
Josef Dai, Xuehai Pan, Ruiyang Sun, Jiaming Ji, Xinbo Xu, Mickel Liu, Yizhou Wang, and Yaodong Yang. Safe RLHF: Safe reinforcement learning from human feedback. In The Twelfth International Conference on Learning Representations, 2024b. URL https://openreview.net/forum?id=TyFrPOKYXw.
+
Fahim Dalvi, Nadir Durrani, Hassan Sajjad, Yonatan Belinkov, Anthony Bau, and James Glass. What is one grain of sand in the desert? analyzing individual neurons in deep nlp models. In Proceedings of the Thirty-Third AAAI Conference on Artificial Intelligence and Thirty-First Innovative Applications of Artificial Intelligence Conference and Ninth AAAI Symposium on Educational Advances in Artificial Intelligence, AAAI’19/IAAI’19/EAAI’19. AAAI Press, 2019. ISBN 978-1-57735-809-1. . URL https://doi.org/10.1609/aaai.v33i01.33016309.
Fahim Dalvi, Abdul Rafae Khan, Firoj Alam, Nadir Durrani, Jia Xu, and Hassan Sajjad. Discovering latent concepts learned in BERT. In International Conference on Learning Representations, 2022. URL https://openreview.net/forum?id=POTMtpYI1xH.
-
Nicola De Cao, Michael Sejr Schlichtkrull, Wilker Aziz, and Ivan Titov. How do decisions emerge across layers in neural models? interpretation with differentiable masking. In Bonnie Webber, Trevor Cohn, Yulan He, and Yang Liu (eds.), Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 3243–3255, Online, November 2020. Association for Computational Linguistics. . URL https://aclanthology.org/2020.emnlp-main.262/.
+
Tri Dao and Albert Gu. Transformers are ssms: generalized models and efficient algorithms through structured state space duality. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.
DeepSeek-AI. Deepseek-v3 technical report, 2025. URL https://arxiv.org/abs/2412.19437.
+
DeepSeek-AI, Aixin Liu, Bei Feng, Bin Wang, Bingxuan Wang, Bo Liu, Chenggang Zhao, Chengqi Dengr, Chong Ruan, Damai Dai, Daya Guo, Dejian Yang, Deli Chen, Dongjie Ji, Erhang Li, Fangyun Lin, Fuli Luo, Guangbo Hao, Guanting Chen, Guowei Li, H. Zhang, Hanwei Xu, Hao Yang, Haowei Zhang, Honghui Ding, Huajian Xin, Huazuo Gao, Hui Li, Hui Qu, J. L. Cai, Jian Liang, Jianzhong Guo, Jiaqi Ni, Jiashi Li, Jin Chen, Jingyang Yuan, Junjie Qiu, Junxiao Song, Kai Dong, Kaige Gao, Kang Guan, Lean Wang, Lecong Zhang, Lei Xu, Leyi Xia, Liang Zhao, Liyue Zhang, Meng Li, Miaojun Wang, Mingchuan Zhang, Minghua Zhang, Minghui Tang, Mingming Li, Ning Tian, Panpan Huang, Peiyi Wang, Peng Zhang, Qihao Zhu, Qinyu Chen, Qiushi Du, R. J. Chen, R. L. Jin, Ruiqi Ge, Ruizhe Pan, Runxin Xu, Ruyi Chen, S. S. Li, Shanghao Lu, Shangyan Zhou, Shanhuang Chen, Shaoqing Wu, Shengfeng Ye, Shirong Ma, Shiyu Wang, Shuang Zhou, Shuiping Yu, Shunfeng Zhou, Size Zheng, T. Wang, Tian Pei, Tian Yuan, Tianyu Sun, W. L. Xiao, Wangding Zeng, Wei An, Wen Liu, Wenfeng Liang, Wenjun Gao, Wentao Zhang, X. Q. Li, Xiangyue Jin, Xianzu Wang, Xiao Bi, Xiaodong Liu, Xiaohan Wang, Xiaojin Shen, Xiaokang Chen, Xiaosha Chen, Xiaotao Nie, Xiaowen Sun, Xiaoxiang Wang, Xin Liu, Xin Xie, Xingkai Yu, Xinnan Song, Xinyi Zhou, Xinyu Yang, Xuan Lu, Xuecheng Su, Y. Wu, Y. K. Li, Y. X. Wei, Y. X. Zhu, Yanhong Xu, Yanping Huang, Yao Li, Yao Zhao, Yaofeng Sun, Yaohui Li, Yaohui Wang, Yi Zheng, Yichao Zhang, Yiliang Xiong, Yilong Zhao, Ying He, Ying Tang, Yishi Piao, Yixin Dong, Yixuan Tan, Yiyuan Liu, Yongji Wang, Yongqiang Guo, Yuchen Zhu, Yuduan Wang, Yuheng Zou, Yukun Zha, Yunxian Ma, Yuting Yan, Yuxiang You, Yuxuan Liu, Z. Z. Ren, Zehui Ren, Zhangli Sha, Zhe Fu, Zhen Huang, Zhen Zhang, Zhenda Xie, Zhewen Hao, Zhihong Shao, Zhiniu Wen, Zhipeng Xu, Zhongyu Zhang, Zhuoshu Li, Zihan Wang, Zihui Gu, Zilin Li, and Ziwei Xie. Deepseek-v2: A strong, economical, and efficient mixture-of-experts language model, 2024. URL https://arxiv.org/abs/2405.04434.
+
DeepSeek-AI, Anyi Xu, Bangcai Lin, Bing Xue, Bingxuan Wang, Bingzheng Xu, Bochao Wu, Bowei Zhang, Chaofan Lin, Chen Dong, Chenchen Ling, Chengda Lu, Chenggang Zhao, Chengqi Deng, Chengyu Hou, Chenhao Xu, Chenze Shao, Chong Ruan, Conner Sun, Damai Dai, Daya Guo, Dejian Yang, Deli Chen, Donghao Li, Dongjie Ji, Erhang Li, Fang Wei, Fangyun Lin, Fangzhou Yuan, Feiyu Xia, Fucong Dai, Guangbo Hao, Guanting Chen, Guoai Cao, Guolai Meng, Guowei Li, Han Yu, Han Zhang, Hanwei Xu, Hao Li, Haofen Liang, Haoling Zhang, Haoming Luo, Haoran Wei, Haotian Yuan, Haowei Zhang, Haowen Luo, Haoyu Chen, Haozhe Ji, Hengqing Zhang, Honghui Ding, Hongxuan Tang, Huanqi Cao, Huazuo Gao, Hui Qu, Hui Zeng, J Yang, JQ Zhu, Jia Luo, Jia Song, Jia Yu, Jialiang Huang, Jialu Cai, Jian Liang, Jiangting Zhou, Jiasheng Ye, Jiashi Li, Jiaxin Xu, Jiewen Hu, Jieyu Yang, Jin Chen, Jin Yan, Jingchang Chen, Jingli Zhou, Jingting Xiang, Jingyang Yuan, Jingyuan Cheng, Jingzi Zhou, Jinhua Zhu, Jiping Yu, Joseph Sun, Jun Ran, Junguang Jiang, Junjie Qiu, Junlong Li, Junmin Zheng, Junxiao Song, Kai Dong, Kaige Gao, Kang Guan, Kexing Zhou, Kezhao Huang, Kuai Yu, Lean Wang, Lecong Zhang, Lei Wang, Leyi Xia, Li Zhang, Liang Zhao, Lihua Guo, Lingxiao Luo, Linwang Ma, Linyan Zhu, Litong Wang, Liyu Cai, Liyue Zhang, Longhao Chen, MS Di, MY Xu, Max Mei, Miaojun Wang, Mingchuan Zhang, Minghua Zhang, Minghui Tang, Mingming Li, Mingxu Zhou, Minmin Han, Ning Wang, Panpan Huang, Panpan Wang, Peixin Cong, Peiyi Wang, Peng Zhang, Qiancheng Wang, Qihao Zhu, Qingyang Li, Qinyu Chen, Qiushi Du, Qiwei Jiang, Rui Tian, Ruifan Xu, Ruijie Lu, Ruiling Xu, Ruiqi Ge, Ruisong Zhang, Ruizhe Pan, Runji Wang, Runqian Chen, Runqiu Yin, Runxin Xu, Ruomeng Shen, Ruoyu Zhang, Ruyi Chen, SH Liu, Shanghao Lu, Shangmian Sun, Shangyan Zhou, Shanhuang Chen, Shaofei Cai, Shaoheng Nie, Shaoqing Wu, Shaoyuan Chen, Shengding Hu, Shengyu Liu, Shiqiang Hu, Shirong Ma, Shiyu Wang, Shuiping Yu, Shunfeng Zhou, Shuting Pan, Shuying Yu, Songyang Zhou, Tao Ni, Tao Yun, Tian Jin, Tian Pei, Tian Ye, Tianle Lin, Tianran Ji, Tianyi Cui, Tianyuan Yue, Tingting Yu, Tun Wang, W Zhang, WL Xiao, Wangding Zeng, Wei An, Weilin Zhao, Wen Liu, Wenfeng Liang, Wenjie Pang, Wenjing Luo, Wenjing Yao, Wenjun Gao, Wenkai Yang, Wenlve Huang, Wenqing Hou, Wentao Zhang, Wenting Ma, Xi Gao, Xiang He, Xiangwen Wang, Xianzu Wang, Xiao Bi, Xiaodong Liu, Xiaohan Wang, Xiaokang Chen, Xiaokang Zhang, Xiaotao Nie, Xiaowen Sun, Xiaoxiang Wang, Xin Cheng, Xin Liu, Xin Xie, Xingchao Liu, Xingchen Liu, Xingkai Yu, Xingyou Li, Xinyu Yang, Xinyu Zhang, Xu Chen, Xuanyu Wang, Xuecheng Su, Xueyin Chen, Xuheng Lin, Xuwei Fu, YC Yan, YQ Wang, YW Ma, Yanfeng Luo, Yang Zhang, Yanhong Xu, Yanru Ma, Yanwen Huang, Yao Li, Yao Li, Yao Xu, Yao Zhao, Yaofeng Sun, Yaohui Wang, Yi Qian, Yi Shao, Yi Yu, Yichao Zhang, Yifan Ding, Yifan Shi, Yijia Wu, Yiliang Xiong, Yiling Ma, Ying He, Ying Tang, Ying Zhou, Yingjia Luo, Yinmin Zhong, Yishi Piao, Yisong Wang, Yixiang Zhang, Yixiao Chen, Yixuan Tan, Yixuan Wei, Yiyang Ma, Yiyuan Liu, Yonglun Yang, Yongqiang Guo, Yongtong Wu, Yu Wu, YuKun Li, Yuan Cheng, Yuan Ou, Yuanfan Xu, Yuanhao Li, Yuduan Wang, Yuehan Yang, Yuer Xu, Yuhan Wu, Yuhao Meng, Yuheng Zou, Yukun Zha, Yunfan Xiong, Yupeng Chen, Yuping Lin, Yuqian Cao, Yuqian Wang, Yushun Zhang, Yuting Yan, Yutong Lin, Yuxian Gu, Yuxiang Luo, Yuxiang You, Yuxuan Liu, Yuxuan Zhou, Yuyang Zhou, Yuzhen Huang, ZF Wu, Zehao Wang, Zehua Zhao, Zehui Ren, Zekai Zhang, Zhangli Sha, Zhe Fu, Zhe Ju, Zhean Xu, Zhenda Xie, Zhengyan Zhang, Zheren Gao, Zhewen Hao, Zhibin Gou, Zhicheng Ma, Zhigang Yan, Zhihong Shao, Zhixian Huang, Zhixuan Chen, Zhiyu Wu, Zhizhou Ren, Zhongyu Wu, Zhuoshu Li, Zhuping Zhang, Zian Xu, Zihao Wang, Zihua Qu, Zihui Gu, Zijia Zhu, Zilin Li, Zipeng Zhang, Ziwei Xie, Ziyi Gao, Ziyi Wan, Zizheng Pan, and Zongqing Yao. Deepseek-v4: Towards highly efficient million-token context intelligence, 2026. URL https://arxiv.org/abs/2606.19348.
+
Carson Denison, Monte MacDiarmid, Fazl Barez, David Duvenaud, Shauna Kravec, Samuel Marks, Nicholas Schiefer, Ryan Soklaski, Alex Tamkin, Jared Kaplan, Buck Shlegeris, Samuel R. Bowman, Ethan Perez, and Evan Hubinger. Sycophancy to subterfuge: Investigating reward-tampering in large language models, 2024. URL https://arxiv.org/abs/2406.10162.
Jacob Devlin, Ming-Wei Chang, Kenton Lee, and Kristina Toutanova. BERT: Pre-training of deep bidirectional transformers for language understanding. In Jill Burstein, Christy Doran, and Thamar Solorio (eds.), Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 4171–4186, Minneapolis, Minnesota, June 2019. Association for Computational Linguistics. . URL https://aclanthology.org/N19-1423/.
+
Chris H.Q. Ding, Tao Li, and Michael I. Jordan. Convex and semi-nonnegative matrix factorizations. IEEE Transactions on Pattern Analysis and Machine Intelligence, 32 (1): 45–55, 2010. .
Distill. Distill. https://distill.pub/, 2017.
Gabriele Dominici, Pietro Barbiero, Mateo Espinosa Zarlenga, Alberto Termine, Martin Gjoreski, Giuseppe Marra, and Marc Langheinrich. Causal concept graph models: Beyond causal opacity in deep learning. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=lmKJ1b6PaL.
-
David Donoho and Victoria Stodden. When does non-negative matrix factorization give a correct decomposition into parts? In S. Thrun, L. Saul, and B. Schölkopf (eds.), Advances in Neural Information Processing Systems, volume 16. MIT Press, 2003. URL https://proceedings.neurips.cc/paper_files/paper/2003/file/1843e35d41ccf6e63273495ba42df3c1-Paper.pdf.
-
Thomas Dooms, Ward Gauderis, Geraint A. Wiggins, and Jose Oramas. Compositionality unlocks deep interpretable models. In AAAI’25 workshop on CoLoRAI - Connecting Low-Rank Representations in AI, 2025. URL https://openreview.net/forum?id=bXAt5iZ69l.
-
Florent Draye, Anson Lei, Ingmar Posner, and Bernhard Schölkopf. Sparse attention post-training for mechanistic interpretability. arXiv preprint arXiv:2512.05865, 2025.
+
David Donoho and Victoria Stodden. When does non-negative matrix factorization give a correct decomposition into parts? In S. Thrun, L. Saul, and B. Schölkopf (eds.), Advances in Neural Information Processing Systems, volume 16. MIT Press, 2003. URL https://proceedings.neurips.cc/paper_files/paper/2003/file/1843e35d41ccf6e63273495ba42df3c1-Paper.pdf.
+
Thomas Dooms, Ward Gauderis, Geraint A. Wiggins, and Jose Oramas. Compositionality unlocks deep interpretable models. In AAAI’25 workshop on CoLoRAI - Connecting Low-Rank Representations in AI, 2025. URL https://openreview.net/forum?id=bXAt5iZ69l.
+
Florent Draye, Anson Lei, Hsiao-Ru Pan, Ingmar Posner, and Bernhard Schölkopf. Intrinsically interpretable attention via sparse post-training, 2026. URL https://arxiv.org/abs/2512.05865.
Mengnan Du, Fengxiang He, Na Zou, Dacheng Tao, and Xia Hu. Shortcut learning of large language models in natural language understanding. Commun. ACM, 67 (1): 110–120, December 2023. ISSN 0001-0782. . URL https://doi.org/10.1145/3596490.
-
Clément Dumas, Julian Minder, and Neel Nanda. What we learned trying to diff base and chat models (and why it matters). https://www.alignmentforum.org/posts/xmpauEXEerzYcJKNm/what-we-learned-trying-to-diff-base-and-chat-models-and-why, 2025.
-
Jacob Dunefsky and Arman Cohan. One-shot optimized steering vectors mediate safety-relevant behaviors in llms. In Second Conference on Language Modeling, 2025.
+
Yuntao Du, Zitao Li, Bolin Ding, Yaliang Li, Hanshen Xiao, Jingren Zhou, and Ninghui Li. Automated profile inference with language model agents. In Maria Liakata, Viviane P. Moreira, Jiajun Zhang, and David Jurgens (eds.), Findings of the Association for Computational Linguistics: ACL 2026, pp. 9977–10000, San Diego, California, United States, July 2026. Association for Computational Linguistics. ISBN 979-8-89176-395-1. . URL https://aclanthology.org/2026.findings-acl.485/.
+
Zhengxiao Du, Aohan Zeng, Yuxiao Dong, and Jie Tang. Understanding emergent abilities of language models from the loss perspective. In Advances in Neural Information Processing Systems, volume 37, 2024. URL https://openreview.net/forum?id=35DAviqMFo.
+
Clément Dumas, Julian Minder, Caden Juang, Bilal Chughtai, and Neel Nanda. Overcoming sparsity artifacts in crosscoders to interpret chat-tuning. In Mechanistic Interpretability Workshop at NeurIPS 2025, 2025a. URL https://openreview.net/forum?id=CdiCNA1Y2R.
+
Clément Dumas, Julian Minder, and Neel Nanda. What we learned trying to diff base and chat models (and why it matters). https://www.alignmentforum.org/posts/xmpauEXEerzYcJKNm/what-we-learned-trying-to-diff-base-and-chat-models-and-why, 2025b.
+
Jacob Dunefsky and Arman Cohan. One-shot optimized steering vectors mediate safety-relevant behaviors in llms. In Second Conference on Language Modeling, 2025.
Jacob Dunefsky, Philippe Chlenski, and Neel Nanda. Transcoders find interpretable llm feature circuits. In Proceedings of the 38th International Conference on Neural Information Processing Systems, NIPS ’24, Red Hook, NY, USA, 2024. Curran Associates Inc. ISBN 9798331314385.
-
Yanai Elazar, Shauli Ravfogel, Alon Jacovi, and Yoav Goldberg. Amnesic probing: Behavioral explanation with amnesic counterfactuals. Transactions of the Association for Computational Linguistics, 9: 160–175, 03 2021. ISSN 2307-387X. . URL https://doi.org/10.1162/tacl_a_00359.
+
Esin DURMUS, Karina Nguyen, Thomas Liao, Nicholas Schiefer, Amanda Askell, Anton Bakhtin, Carol Chen, Zac Hatfield-Dodds, Danny Hernandez, Nicholas Joseph, Liane Lovitt, Sam McCandlish, Orowa Sikder, Alex Tamkin, Janel Thamkul, Jared Kaplan, Jack Clark, and Deep Ganguli. Towards measuring the representation of subjective global opinions in language models. In First Conference on Language Modeling, 2024. URL https://openreview.net/forum?id=zl16jLb91v.
+
Yanai Elazar, Shauli Ravfogel, Alon Jacovi, and Yoav Goldberg. Amnesic probing: Behavioral explanation with amnesic counterfactuals. Transactions of the Association for Computational Linguistics, 9: 160–175, 03 2021. ISSN 2307-387X. . URL https://doi.org/10.1162/tacl_a_00359.
+
Yanai Elazar, Nora Kassner, Shauli Ravfogel, Amir Feder, Abhilasha Ravichander, Marius Mosbach, Yonatan Belinkov, Hinrich Schütze, and Yoav Goldberg. Measuring causal effects of data statistics on language model’s ‘factual’ predictions, 2023. URL https://arxiv.org/abs/2207.14251.
Nelson Elhage, Neel Nanda, Catherine Olsson, Tom Henighan, Nicholas Joseph, Ben Mann, Amanda Askell, Yuntao Bai, Anna Chen, Tom Conerly, Nova DasSarma, Dawn Drain, Deep Ganguli, Zac Hatfield-Dodds, Danny Hernandez, Andy Jones, Jackson Kernion, Liane Lovitt, Kamal Ndousse, Dario Amodei, Tom Brown, Jack Clark, Jared Kaplan, Sam McCandlish, and Chris Olah. A mathematical framework for transformer circuits. Transformer Circuits Thread, 2021.
Nelson Elhage, Tristan Hume, Catherine Olsson, Neel Nanda, Tom Henighan, Yuntao Bai, Ben Mann, Amanda Askell, and Chris Olah. Toy models of superposition. Transformer Circuits Thread, 2022a. URL https://transformer-circuits.pub/2022/toy_model/.
Nelson Elhage, Tristan Hume, Catherine Olsson, Neel Nanda, Tom Henighan, Scott Johnston, Sheer El Showk, Nicholas Joseph, Nova DasSarma, Ben Mann, Danny Hernandez, Amanda Askell, Kamal Ndousse, Andy Jones, Dawn Drain, Anna Chen, Yuntao Bai, Deep Ganguli, Liane Lovitt, Zac Hatfield-Dodds, Jackson Kernion, Tom Conerly, Shauna Kravec, Stanislav Fort, Saurav Kadavath, Josh Jacobson, Eli Tran-Johnson, Jared Kaplan, Jack Clark, Tom Brown, Sam McCandlish, Dario Amodei, and Christopher Olah. Softmax linear units. Transformer Circuits Thread, 2022b.
+
Jeffrey L. Elman. Finding structure in time. Cognitive Science, 14 (2): 179–211, 1990. .
+
Jeffrey L. Elman. Distributed representations, simple recurrent networks, and grammatical structure. Machine Learning, 7 (2): 195–225, 1991. .
Joshua Engels, Eric J Michaud, Isaac Liao, Wes Gurnee, and Max Tegmark. Not all language model features are one-dimensionally linear. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=d63a4AM4hb.
-
Javier Ferrando, Gabriele Sarti, Arianna Bisazza, and Marta R Costa-Jussà. A primer on the inner workings of transformer-based language models. arXiv preprint arXiv:2405.00208, 2024.
+
Danielle Ensign and Adrià Garriga-Alonso. Investigating the indirect object identification circuit in mamba, 2024. URL https://arxiv.org/abs/2407.14008.
+
Leon Eshuijs, Archie Chaudhury, Alan McBeth, and Ethan Nguyen. But what is your honest answer? aiding llm-judges with honest alternatives using steering vectors, 2026. URL https://arxiv.org/abs/2505.17760.
+
Junfeng Fang, Houcheng Jiang, Kun Wang, Yunshan Ma, Jie Shi, Xiang Wang, Xiangnan He, and Tat-Seng Chua. Alphaedit: Null-space constrained model editing for language models. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=HvSytvg3Jh.
+
Junfeng Fang, Nachuan Chen, Houcheng Jiang, Dan Zhang, Fei Shen, Xiang Wang, Xiangnan He, and Tat-Seng Chua. Next-guard: Training-free streaming safeguard without token-level labels, 2026. URL https://arxiv.org/abs/2603.02219.
+
Aaron Fanous, Jacob Goldberg, Ank Agarwal, Joanna Lin, Anson Zhou, Sonnet Xu, Vasiliki Bikia, Roxana Daneshjou, and Sanmi Koyejo. Syceval: Evaluating llm sycophancy. In Proceedings of the AAAI/ACM Conference on AI, Ethics, and Society, pp. 893–900, 2025.
+
Javier Ferrando, Gabriele Sarti, Arianna Bisazza, and Marta R. Costa-jussà. A primer on the inner workings of transformer-based language models, 2024. URL https://arxiv.org/abs/2405.00208.
+
Sheridan Feucht, Tal Haklay, Usha Bhalla, Daniel Wurgaft, Can Rager, Raphaël Sarfati, Jack Merullo, Thomas McGrath, Owen Lewis, Ekdeep Singh Lubana, Thomas Fel, and Atticus Geiger. Arithmetic in the wild: Llama uses base-10 addition to reason about cyclic concepts, 2026. URL https://arxiv.org/abs/2605.01148.
+
David J. Field. What is the goal of sensory coding? Neural Computation, 6 (4): 559–601, 1994. .
Alex Foote, Neel Nanda, Esben Kran, Ioannis Konstas, and Fazl Barez. N2g: A SCALABLE APPROACH FOR QUANTIFYING INTERPRETABLE NEURON REPRESENTATION IN LLMS. In ICLR 2023 Workshop on Trustworthy and Reliable Large-Scale Machine Learning Models, 2023.
-
Kai Fronsdal, Isha Gupta, Abhay Sheshadri, Jonathan Michala, Stephen McAleer, Rowan Wang, Sara Price, and Sam Bowman. Petri: Parallel exploration of risky interactions, 2025. URL https://github.com/safety-research/petri.
-
Kai Fronsdal, Jonathan Michala, and Sam Bowman. Petri 2.0: New scenarios, new model comparisons, and improved eval-awareness mitigations, 2026. URL https://alignment.anthropic.com/2026/petri-v2/.
+
Damiano Fornasiere, Mirko Bronzi, Spencer Kitts, Alessandro Palmas, Yoshua Bengio, and Oliver Richardson. Language models recognize dropout and gaussian noise applied to their activations, 2026. URL https://arxiv.org/abs/2604.17465.
+
Jonathan Frankle, Gintare Karolina Dziugaite, Daniel M. Roy, and Michael Carbin. Linear mode connectivity and the lottery ticket hypothesis. In International Conference on Machine Learning, pp. 3259–3269, 2020.
+
Kit Fraser-Taliente, Subhash Kantamneni, Euan Ong, Dan Mossing, Christina Lu, Paul C. Bogdan, Emmanuel Ameisen, James Chen, Dzmitry Kishylau, Adam Pearce, Julius Tarng, Alex Wu, Jeff Wu, Yang Zhang, Daniel M. Ziegler, Evan Hubinger, Joshua Batson, Jack Lindsey, Samuel Zimmerman, and Samuel Marks. Natural language autoencoders produce unsupervised explanations of llm activations. Transformer Circuits Thread, 2026. URL https://transformer-circuits.pub/2026/nla/index.html.
+
Tingchen Fu, Mrinank Sharma, Philip Torr, Shay B Cohen, David Krueger, and Fazl Barez. PoisonBench: Assessing language model vulnerability to poisoned preference data. In Aarti Singh, Maryam Fazel, Daniel Hsu, Simon Lacoste-Julien, Felix Berkenkamp, Tegan Maharaj, Kiri Wagstaff, and Jerry Zhu (eds.), Proceedings of the 42nd International Conference on Machine Learning, volume 267 of Proceedings of Machine Learning Research, pp. 17806–17831. PMLR, 13–19 Jul 2025. URL https://proceedings.mlr.press/v267/fu25e.html.
Deep Ganguli, Liane Lovitt, Jackson Kernion, Amanda Askell, Yuntao Bai, Saurav Kadavath, Ben Mann, Ethan Perez, Nicholas Schiefer, Kamal Ndousse, Andy Jones, Sam Bowman, Anna Chen, Tom Conerly, Nova DasSarma, Dawn Drain, Nelson Elhage, Sheer El-Showk, Stanislav Fort, Zac Hatfield-Dodds, Tom Henighan, Danny Hernandez, Tristan Hume, Josh Jacobson, Scott Johnston, Shauna Kravec, Catherine Olsson, Sam Ringer, Eli Tran-Johnson, Dario Amodei, Tom Brown, Nicholas Joseph, Sam McCandlish, Chris Olah, Jared Kaplan, and Jack Clark. Red teaming language models to reduce harms: Methods, scaling behaviors, and lessons learned, 2022. URL https://arxiv.org/abs/2209.07858.
Leo Gao. An ambitious vision for interpretability. LessWrong, 2025. URL https://www.lesswrong.com/posts/Hy6PX43HGgmfiTaKu/an-ambitious-vision-for-interpretability.
-
Leo Gao, Stella Biderman, Sid Black, Laurence Golding, Travis Hoppe, Charles Foster, Jason Phang, Horace He, Anish Thite, Noa Nabeshima, et al. The pile: An 800gb dataset of diverse text for language modeling. arXiv preprint arXiv:2101.00027, 2020.
-
Leo Gao, Tom Dupre la Tour, Henk Tillman, Gabriel Goh, Rajan Troll, Alec Radford, Ilya Sutskever, Jan Leike, and Jeffrey Wu. Scaling and evaluating sparse autoencoders. In The Thirteenth International Conference on Learning Representations, 2025a. URL https://openreview.net/forum?id=tcsZt9ZNKD.
-
Leo Gao, Achyuta Rajaram, Jacob Coxon, Soham V. Govande, Bowen Baker, and Dan Mossing. Weight-sparse transformers have interpretable circuits, 2025b. URL https://arxiv.org/abs/2511.13653.
+
Leo Gao, Stella Biderman, Sid Black, Laurence Golding, Travis Hoppe, Charles Foster, Jason Phang, Horace He, Anish Thite, Noa Nabeshima, Shawn Presser, and Connor Leahy. The pile: An 800gb dataset of diverse text for language modeling, 2020. URL https://arxiv.org/abs/2101.00027.
+
Leo Gao, Tom Dupre la Tour, Henk Tillman, Gabriel Goh, Rajan Troll, Alec Radford, Ilya Sutskever, Jan Leike, and Jeffrey Wu. Scaling and evaluating sparse autoencoders. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=tcsZt9ZNKD.
+
Leo Gao, Achyuta Rajaram, Jacob Coxon, Soham V. Govande, Bowen Baker, and Daniel P Mossing. Weight-sparse transformers have interpretable circuits. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=Vw10TpuFCp.
Jorge Garcı́a-Carrasco, Alejandro Maté, and Juan Trujillo. Detecting and understanding vulnerabilities in language models via mechanistic interpretability. In Proceedings of the Thirty-Third International Joint Conference on Artificial Intelligence, IJCAI ’24, 2024. ISBN 978-1-956792-04-1. . URL https://doi.org/10.24963/ijcai.2024/43.
-
Charles J Garfinkle and Christopher J Hillar. Robust identifiability in sparse dictionary learning. arXiv preprint arXiv:1606.06997, 2016.
-
Xuyang Ge, Wentao Shu, Jiaxing Wu, Yunhua Zhou, Zhengfu He, and Xipeng Qiu. Evolution of concepts in language model pre-training. arXiv preprint arXiv:2509.17196, 2025.
-
Xuyang Ge, Wentao Shu, Jiaxing Wu, Yunhua Zhou, Zhengfu He, and Xipeng Qiu. Evolution of concepts in language model pre-training. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=qKmXz1UR0Z.
+
Garima, Frederick Liu, Satyen Kale, and Mukund Sundararajan. Estimating training data influence by tracing gradient descent. In Proceedings of the 34th International Conference on Neural Information Processing Systems, NIPS ’20, Red Hook, NY, USA, 2020. Curran Associates Inc. ISBN 9781713829546.
+
Xuyang Ge, Wentao Shu, Jiaxing Wu, Yunhua Zhou, Zhengfu He, and Xipeng Qiu. Evolution of concepts in language model pre-training. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=qKmXz1UR0Z.
Atticus Geiger, Hanson Lu, Thomas Icard, and Christopher Potts. Causal abstractions of neural networks. In M. Ranzato, A. Beygelzimer, Y. Dauphin, P.S. Liang, and J. Wortman Vaughan (eds.), Advances in Neural Information Processing Systems, volume 34, pp. 9574–9586. Curran Associates, Inc., 2021. URL https://proceedings.neurips.cc/paper_files/paper/2021/file/4f5c422f4d49a5a807eda27434231040-Paper.pdf.
-
Atticus Geiger, Zhengxuan Wu, Christopher Potts, Thomas Icard, and Noah Goodman. Finding alignments between interpretable causal variables and distributed neural representations. In Francesco Locatello and Vanessa Didelez (eds.), Proceedings of the Third Conference on Causal Learning and Reasoning, volume 236 of Proceedings of Machine Learning Research, pp. 160–187. PMLR, 01–03 Apr 2024. URL https://proceedings.mlr.press/v236/geiger24a.html.
+
Atticus Geiger, Zhengxuan Wu, Hanson Lu, Josh Rozner, Elisa Kreiss, Thomas Icard, Noah Goodman, and Christopher Potts. Inducing causal structure for interpretable neural networks. In Kamalika Chaudhuri, Stefanie Jegelka, Le Song, Csaba Szepesvari, Gang Niu, and Sivan Sabato (eds.), Proceedings of the 39th International Conference on Machine Learning, volume 162 of Proceedings of Machine Learning Research, pp. 7324–7338. PMLR, 17–23 Jul 2022. URL https://proceedings.mlr.press/v162/geiger22a.html.
+
Atticus Geiger, Zhengxuan Wu, Christopher Potts, Thomas Icard, and Noah Goodman. Finding alignments between interpretable causal variables and distributed neural representations. In Francesco Locatello and Vanessa Didelez (eds.), Proceedings of the Third Conference on Causal Learning and Reasoning, volume 236 of Proceedings of Machine Learning Research, pp. 160–187. PMLR, 01–03 Apr 2024. URL https://proceedings.mlr.press/v236/geiger24a.html.
+
Atticus Geiger, Duligur Ibeling, Amir Zur, Maheep Chaudhary, Sonakshi Chauhan, Jing Huang, Aryaman Arora, Zhengxuan Wu, Noah Goodman, Christopher Potts, and Thomas Icard. Causal abstraction: A theoretical foundation for mechanistic interpretability. Journal of Machine Learning Research, 26 (83): 1–64, 2025. URL http://jmlr.org/papers/v26/23-0058.html.
Robert Geirhos, Jörn-Henrik Jacobsen, Claudio Michaelis, Richard Zemel, Wieland Brendel, Matthias Bethge, and Felix A. Wichmann. Shortcut learning in deep neural networks. Nature Machine Intelligence, 2 (11): 665–673, November 2020. ISSN 2522-5839. . URL http://dx.doi.org/10.1038/s42256-020-00257-z.
-
Kristian Georgiev, Joshua Vendrow, Hadi Salman, Sung Min Park, and Aleksander Madry. The journey, not the destination: How data guides diffusion models, 2023. URL https://arxiv.org/abs/2312.06205.
-
Asma Ghandeharioun, Avi Caciularu, Adam Pearce, Lucas Dixon, and Mor Geva. Patchscopes: a unifying framework for inspecting hidden representations of language models. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.
-
Mario Giulianelli, Jack Harding, Florian Mohnert, Dieuwke Hupkes, and Willem Zuidema. Under the hood: Using diagnostic classifiers to investigate and improve how language models track agreement information. In Tal Linzen, Grzegorz Chrupała, and Afra Alishahi (eds.), Proceedings of the 2018 EMNLP Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP, pp. 240–248, Brussels, Belgium, November 2018. Association for Computational Linguistics. . URL https://aclanthology.org/W18-5426/.
+
Thomas George, César Laurent, Xavier Bouthillier, Nicolas Ballas, and Pascal Vincent. Fast approximate natural gradient descent in a kronecker-factored eigenbasis. In Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS’18, pp. 9573–9583, Red Hook, NY, USA, 2018. Curran Associates Inc.
+
Kristian Georgiev, Joshua Vendrow, Hadi Salman, Sung Min Park, and Aleksander Madry. The journey, not the destination: How data guides diffusion models, 2023. URL https://arxiv.org/abs/2312.06205.
+
Mor Geva, Roei Schuster, Jonathan Berant, and Omer Levy. Transformer feed-forward layers are key-value memories. In Marie-Francine Moens, Xuanjing Huang, Lucia Specia, and Scott Wen-tau Yih (eds.), Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing, pp. 5484–5495, Online and Punta Cana, Dominican Republic, November 2021. Association for Computational Linguistics. . URL https://aclanthology.org/2021.emnlp-main.446/.
+
Asma Ghandeharioun, Avi Caciularu, Adam Pearce, Lucas Dixon, and Mor Geva. Patchscopes: a unifying framework for inspecting hidden representations of language models. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.
+
C. L. Giles, C. B. Miller, D. Chen, H. H. Chen, G. Z. Sun, and Y. C. Lee. Learning and extracting finite state automata with second-order recurrent neural networks. Neural Comput., 4 (3): 393–405, May 1992. ISSN 0899-7667. . URL https://doi.org/10.1162/neco.1992.4.3.393.
+
Mario Giulianelli, Jack Harding, Florian Mohnert, Dieuwke Hupkes, and Willem Zuidema. Under the hood: Using diagnostic classifiers to investigate and improve how language models track agreement information. In Tal Linzen, Grzegorz Chrupała, and Afra Alishahi (eds.), Proceedings of the 2018 EMNLP Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP, pp. 240–248, Brussels, Belgium, November 2018. Association for Computational Linguistics. . URL https://aclanthology.org/W18-5426/.
+
Avichal Goel, Yoon Kim, Nir N Shavit, and Tony T. Wang. Learning to interpret weight differences in language models. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=6As4wfTB77.
Nicholas Goldowsky-Dill, Chris MacLeod, Lucas Sato, and Aryaman Arora. Localizing model behavior with path patching, 2023. URL https://arxiv.org/abs/2304.05969.
+
Aaron Grattafiori, Abhimanyu Dubey, Abhinav Jauhri, Abhinav Pandey, Abhishek Kadian, Ahmad Al-Dahle, Aiesha Letman, Akhil Mathur, Alan Schelten, Alex Vaughan, Amy Yang, Angela Fan, Anirudh Goyal, Anthony Hartshorn, Aobo Yang, Archi Mitra, Archie Sravankumar, Artem Korenev, Arthur Hinsvark, Arun Rao, Aston Zhang, Aurelien Rodriguez, Austen Gregerson, Ava Spataru, Baptiste Roziere, Bethany Biron, Binh Tang, Bobbie Chern, Charlotte Caucheteux, Chaya Nayak, Chloe Bi, Chris Marra, Chris McConnell, Christian Keller, Christophe Touret, Chunyang Wu, Corinne Wong, Cristian Canton Ferrer, Cyrus Nikolaidis, Damien Allonsius, Daniel Song, Danielle Pintz, Danny Livshits, Danny Wyatt, David Esiobu, Dhruv Choudhary, Dhruv Mahajan, Diego Garcia-Olano, Diego Perino, Dieuwke Hupkes, Egor Lakomkin, Ehab AlBadawy, Elina Lobanova, Emily Dinan, Eric Michael Smith, Filip Radenovic, Francisco Guzmán, Frank Zhang, Gabriel Synnaeve, Gabrielle Lee, Georgia Lewis Anderson, Govind Thattai, Graeme Nail, Gregoire Mialon, Guan Pang, Guillem Cucurell, Hailey Nguyen, Hannah Korevaar, Hu Xu, Hugo Touvron, Iliyan Zarov, Imanol Arrieta Ibarra, Isabel Kloumann, Ishan Misra, Ivan Evtimov, Jack Zhang, Jade Copet, Jaewon Lee, Jan Geffert, Jana Vranes, Jason Park, Jay Mahadeokar, Jeet Shah, Jelmer van der Linde, Jennifer Billock, Jenny Hong, Jenya Lee, Jeremy Fu, Jianfeng Chi, Jianyu Huang, Jiawen Liu, Jie Wang, Jiecao Yu, Joanna Bitton, Joe Spisak, Jongsoo Park, Joseph Rocca, Joshua Johnstun, Joshua Saxe, Junteng Jia, Kalyan Vasuden Alwala, Karthik Prasad, Kartikeya Upasani, Kate Plawiak, Ke Li, Kenneth Heafield, Kevin Stone, Khalid El-Arini, Krithika Iyer, Kshitiz Malik, Kuenley Chiu, Kunal Bhalla, Kushal Lakhotia, Lauren Rantala-Yeary, Laurens van der Maaten, Lawrence Chen, Liang Tan, Liz Jenkins, Louis Martin, Lovish Madaan, Lubo Malo, Lukas Blecher, Lukas Landzaat, Luke de Oliveira, Madeline Muzzi, Mahesh Pasupuleti, Mannat Singh, Manohar Paluri, Marcin Kardas, Maria Tsimpoukelli, Mathew Oldham, Mathieu Rita, Maya Pavlova, Melanie Kambadur, Mike Lewis, Min Si, Mitesh Kumar Singh, Mona Hassan, Naman Goyal, Narjes Torabi, Nikolay Bashlykov, Nikolay Bogoychev, Niladri Chatterji, Ning Zhang, Olivier Duchenne, Onur Çelebi, Patrick Alrassy, Pengchuan Zhang, Pengwei Li, Petar Vasic, Peter Weng, Prajjwal Bhargava, Pratik Dubal, Praveen Krishnan, Punit Singh Koura, Puxin Xu, Qing He, Qingxiao Dong, Ragavan Srinivasan, Raj Ganapathy, Ramon Calderer, Ricardo Silveira Cabral, Robert Stojnic, Roberta Raileanu, Rohan Maheswari, Rohit Girdhar, Rohit Patel, Romain Sauvestre, Ronnie Polidoro, Roshan Sumbaly, Ross Taylor, Ruan Silva, Rui Hou, Rui Wang, Saghar Hosseini, Sahana Chennabasappa, Sanjay Singh, Sean Bell, Seohyun Sonia Kim, Sergey Edunov, Shaoliang Nie, Sharan Narang, Sharath Raparthy, Sheng Shen, Shengye Wan, Shruti Bhosale, Shun Zhang, Simon Vandenhende, Soumya Batra, Spencer Whitman, Sten Sootla, Stephane Collot, Suchin Gururangan, Sydney Borodinsky, Tamar Herman, Tara Fowler, Tarek Sheasha, Thomas Georgiou, Thomas Scialom, Tobias Speckbacher, Todor Mihaylov, Tong Xiao, Ujjwal Karn, Vedanuj Goswami, Vibhor Gupta, Vignesh Ramanathan, Viktor Kerkez, Vincent Gonguet, Virginie Do, Vish Vogeti, Vítor Albiero, Vladan Petrovic, Weiwei Chu, Wenhan Xiong, Wenyin Fu, Whitney Meers, Xavier Martinet, Xiaodong Wang, Xiaofang Wang, Xiaoqing Ellen Tan, Xide Xia, Xinfeng Xie, Xuchao Jia, Xuewei Wang, Yaelle Goldschlag, Yashesh Gaur, Yasmine Babaei, Yi Wen, Yiwen Song, Yuchen Zhang, Yue Li, Yuning Mao, Zacharie Delpierre Coudert, Zheng Yan, Zhengxing Chen, Zoe Papakipos, Aaditya Singh, Aayushi Srivastava, Abha Jain, Adam Kelsey, Adam Shajnfeld, Adithya Gangidi, Adolfo Victoria, Ahuva Goldstand, Ajay Menon, Ajay Sharma, Alex Boesenberg, Alexei Baevski, Allie Feinstein, Amanda Kallet, Amit Sangani, Amos Teo, Anam Yunus, Andrei Lupu, Andres Alvarado, Andrew Caples, Andrew Gu, Andrew Ho, Andrew Poulton, Andrew Ryan, Ankit Ramchandani, Annie Dong, Annie Franco, Anuj Goyal, Aparajita Saraf, Arkabandhu Chowdhury, Ashley Gabriel, Ashwin Bharambe, Assaf Eisenman, Azadeh Yazdan, Beau James, Ben Maurer, Benjamin Leonhardi, Bernie Huang, Beth Loyd, Beto De Paola, Bhargavi Paranjape, Bing Liu, Bo Wu, Boyu Ni, Braden Hancock, Bram Wasti, Brandon Spence, Brani Stojkovic, Brian Gamido, Britt Montalvo, Carl Parker, Carly Burton, Catalina Mejia, Ce Liu, Changhan Wang, Changkyu Kim, Chao Zhou, Chester Hu, Ching-Hsiang Chu, Chris Cai, Chris Tindal, Christoph Feichtenhofer, Cynthia Gao, Damon Civin, Dana Beaty, Daniel Kreymer, Daniel Li, David Adkins, David Xu, Davide Testuggine, Delia David, Devi Parikh, Diana Liskovich, Didem Foss, Dingkang Wang, Duc Le, Dustin Holland, Edward Dowling, Eissa Jamil, Elaine Montgomery, Eleonora Presani, Emily Hahn, Emily Wood, Eric-Tuan Le, Erik Brinkman, Esteban Arcaute, Evan Dunbar, Evan Smothers, Fei Sun, Felix Kreuk, Feng Tian, Filippos Kokkinos, Firat Ozgenel, Francesco Caggioni, Frank Kanayet, Frank Seide, Gabriela Medina Florez, Gabriella Schwarz, Gada Badeer, Georgia Swee, Gil Halpern, Grant Herman, Grigory Sizov, Guangyi, Zhang, Guna Lakshminarayanan, Hakan Inan, Hamid Shojanazeri, Han Zou, Hannah Wang, Hanwen Zha, Haroun Habeeb, Harrison Rudolph, Helen Suk, Henry Aspegren, Hunter Goldman, Hongyuan Zhan, Ibrahim Damlaj, Igor Molybog, Igor Tufanov, Ilias Leontiadis, Irina-Elena Veliche, Itai Gat, Jake Weissman, James Geboski, James Kohli, Janice Lam, Japhet Asher, Jean-Baptiste Gaya, Jeff Marcus, Jeff Tang, Jennifer Chan, Jenny Zhen, Jeremy Reizenstein, Jeremy Teboul, Jessica Zhong, Jian Jin, Jingyi Yang, Joe Cummings, Jon Carvill, Jon Shepard, Jonathan McPhie, Jonathan Torres, Josh Ginsburg, Junjie Wang, Kai Wu, Kam Hou U, Karan Saxena, Kartikay Khandelwal, Katayoun Zand, Kathy Matosich, Kaushik Veeraraghavan, Kelly Michelena, Keqian Li, Kiran Jagadeesh, Kun Huang, Kunal Chawla, Kyle Huang, Lailin Chen, Lakshya Garg, Lavender A, Leandro Silva, Lee Bell, Lei Zhang, Liangpeng Guo, Licheng Yu, Liron Moshkovich, Luca Wehrstedt, Madian Khabsa, Manav Avalani, Manish Bhatt, Martynas Mankus, Matan Hasson, Matthew Lennie, Matthias Reso, Maxim Groshev, Maxim Naumov, Maya Lathi, Meghan Keneally, Miao Liu, Michael L. Seltzer, Michal Valko, Michelle Restrepo, Mihir Patel, Mik Vyatskov, Mikayel Samvelyan, Mike Clark, Mike Macey, Mike Wang, Miquel Jubert Hermoso, Mo Metanat, Mohammad Rastegari, Munish Bansal, Nandhini Santhanam, Natascha Parks, Natasha White, Navyata Bawa, Nayan Singhal, Nick Egebo, Nicolas Usunier, Nikhil Mehta, Nikolay Pavlovich Laptev, Ning Dong, Norman Cheng, Oleg Chernoguz, Olivia Hart, Omkar Salpekar, Ozlem Kalinli, Parkin Kent, Parth Parekh, Paul Saab, Pavan Balaji, Pedro Rittner, Philip Bontrager, Pierre Roux, Piotr Dollar, Polina Zvyagina, Prashant Ratanchandani, Pritish Yuvraj, Qian Liang, Rachad Alao, Rachel Rodriguez, Rafi Ayub, Raghotham Murthy, Raghu Nayani, Rahul Mitra, Rangaprabhu Parthasarathy, Raymond Li, Rebekkah Hogan, Robin Battey, Rocky Wang, Russ Howes, Ruty Rinott, Sachin Mehta, Sachin Siby, Sai Jayesh Bondu, Samyak Datta, Sara Chugh, Sara Hunt, Sargun Dhillon, Sasha Sidorov, Satadru Pan, Saurabh Mahajan, Saurabh Verma, Seiji Yamamoto, Sharadh Ramaswamy, Shaun Lindsay, Shaun Lindsay, Sheng Feng, Shenghao Lin, Shengxin Cindy Zha, Shishir Patil, Shiva Shankar, Shuqiang Zhang, Shuqiang Zhang, Sinong Wang, Sneha Agarwal, Soji Sajuyigbe, Soumith Chintala, Stephanie Max, Stephen Chen, Steve Kehoe, Steve Satterfield, Sudarshan Govindaprasad, Sumit Gupta, Summer Deng, Sungmin Cho, Sunny Virk, Suraj Subramanian, Sy Choudhury, Sydney Goldman, Tal Remez, Tamar Glaser, Tamara Best, Thilo Koehler, Thomas Robinson, Tianhe Li, Tianjun Zhang, Tim Matthews, Timothy Chou, Tzook Shaked, Varun Vontimitta, Victoria Ajayi, Victoria Montanez, Vijai Mohan, Vinay Satish Kumar, Vishal Mangla, Vlad Ionescu, Vlad Poenaru, Vlad Tiberiu Mihailescu, Vladimir Ivanov, Wei Li, Wenchen Wang, Wenwen Jiang, Wes Bouaziz, Will Constable, Xiaocheng Tang, Xiaojian Wu, Xiaolan Wang, Xilun Wu, Xinbo Gao, Yaniv Kleinman, Yanjun Chen, Ye Hu, Ye Jia, Ye Qi, Yenda Li, Yilin Zhang, Ying Zhang, Yossi Adi, Youngjin Nam, Yu, Wang, Yu Zhao, Yuchen Hao, Yundi Qian, Yunlu Li, Yuzi He, Zach Rait, Zachary DeVito, Zef Rosnbrick, Zhaoduo Wen, Zhenyu Yang, Zhiwei Zhao, and Zhiyu Ma. The llama 3 herd of models, 2024. URL https://arxiv.org/abs/2407.21783.
Ryan Greenblatt, Carson Denison, Benjamin Wright, Fabien Roger, Monte MacDiarmid, Sam Marks, Johannes Treutlein, Tim Belonax, Jack Chen, David Duvenaud, Akbir Khan, Julian Michael, Sören Mindermann, Ethan Perez, Linda Petrini, Jonathan Uesato, Jared Kaplan, Buck Shlegeris, Samuel R. Bowman, and Evan Hubinger. Alignment faking in large language models, 2024. URL https://arxiv.org/abs/2412.14093.
+
Dirk Groeneveld, Iz Beltagy, Evan Walsh, Akshita Bhagia, Rodney Kinney, Oyvind Tafjord, Ananya Jha, Hamish Ivison, Ian Magnusson, Yizhong Wang, et al. Olmo: Accelerating the science of language models. In Proceedings of the 62nd annual meeting of the association for computational linguistics (volume 1: long papers), pp. 15789–15809, 2024.
+
Fabian Gröger, Shuo Wen, and Maria Brbic. Revisiting the platonic representation hypothesis: An aristotelian view. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=uz0gAAYydl.
Roger Grosse, Juhan Bae, Cem Anil, Nelson Elhage, Alex Tamkin, Amirhossein Tajdini, Benoit Steiner, Dustin Li, Esin Durmus, Ethan Perez, Evan Hubinger, Kamilė Lukošiūtė, Karina Nguyen, Nicholas Joseph, Sam McCandlish, Jared Kaplan, and Samuel R. Bowman. Studying large language model generalization with influence functions, 2023. URL https://arxiv.org/abs/2308.03296.
+
Albert Gu and Tri Dao. Mamba: Linear-time sequence modeling with selective state spaces. In First Conference on Language Modeling, 2024. URL https://openreview.net/forum?id=tEYskw1VY2.
+
Zifan Carl Guo, Laura Ruis, Jacob Andreas, and Belinda Z. Li. Introspective coupling: Self-explanation training tracks behavioral change despite fixed supervision, 2026. URL https://arxiv.org/abs/2606.32038.
Abhijeet Gupta, Gemma Boleda, Marco Baroni, and Sebastian Padó. Distributional vectors encode referential attributes. In Lluís Màrquez, Chris Callison-Burch, and Jian Su (eds.), Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing, pp. 12–21, Lisbon, Portugal, September 2015. Association for Computational Linguistics. . URL https://aclanthology.org/D15-1002/.
-
Yoav Gur-Arieh, Roy Mayan, Chen Agassy, Atticus Geiger, and Mor Geva. Enhancing automated interpretability with output-centric feature descriptions. In Wanxiang Che, Joyce Nabende, Ekaterina Shutova, and Mohammad Taher Pilehvar (eds.), Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 5757–5778, Vienna, Austria, July 2025a. Association for Computational Linguistics. ISBN 979-8-89176-251-0. . URL https://aclanthology.org/2025.acl-long.288/.
+
Yoav Gur-Arieh, Roy Mayan, Chen Agassy, Atticus Geiger, and Mor Geva. Enhancing automated interpretability with output-centric feature descriptions. In Wanxiang Che, Joyce Nabende, Ekaterina Shutova, and Mohammad Taher Pilehvar (eds.), Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 5757–5778, Vienna, Austria, July 2025a. Association for Computational Linguistics. ISBN 979-8-89176-251-0. . URL https://aclanthology.org/2025.acl-long.288/.
Yoav Gur-Arieh, Clara Haya Suslik, Yihuai Hong, Fazl Barez, and Mor Geva. Precise in-parameter concept erasure in large language models. In Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing, pp. 18986–19006, Suzhou, China, November 2025b. Association for Computational Linguistics.
-
Muhammad Umair Haider, Hammad Rizwan, Hassan Sajjad, and AB Siddique. Multi-granular node pruning for circuit discovery. arXiv preprint arXiv:2512.10903, 2025.
+
Wes Gurnee, Nicholas Sofroniew, Adam Pearce, Mateusz Piotrowski, Isaac Kauvar, Runjin Chen, Anna Soligo, Paul Bogdan, Euan Ong, Rowan Wang, Ben Thompson, David Abrahams, Subhash Kantamneni, Emmanuel Ameisen, Joshua Batson, and Jack Lindsey. Verbalizable representations form a global workspace in language models. Transformer Circuits Thread, 2026. URL https://transformer-circuits.pub/2026/workspace/index.html.
+
Reza Habibi, Darian Lee, and Magy Seif El-Nasr. Symbolic mechanistic data attribution: Tracing training influence to learned behavioral policies, 2026. URL https://arxiv.org/abs/2606.29171.
+
Ely Hahami, Ishaan Sinha, Lavik Jain, Josh Kaplan, and Jon Hahami. Detecting the disturbance: A nuanced view of introspective abilities in llms, 2026. URL https://arxiv.org/abs/2512.12411.
+
Muhammad Umair Haider, Hammad Rizwan, Hassan Sajjad, and A.B. Siddique. Multi-granular node pruning for causal circuit discovery. In Third Conference on Language Modeling, 2026. URL https://openreview.net/forum?id=Ol96ETTo6k.
Tal Haklay, Hadas Orgad, David Bau, Aaron Mueller, and Yonatan Belinkov. Position-aware automatic circuit discovery. In Wanxiang Che, Joyce Nabende, Ekaterina Shutova, and Mohammad Taher Pilehvar (eds.), Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 2792–2817, Vienna, Austria, July 2025. Association for Computational Linguistics. ISBN 979-8-89176-251-0. . URL https://aclanthology.org/2025.acl-long.141/.
+
Tal Haklay, Nikhil Prakash, Sana Pandey, Antonio Torralba, Aaron Mueller, Jacob Andreas, Tamar Rott Shaham, and Yonatan Belinkov. Pitfalls in evaluating interpretability agents, 2026. URL https://arxiv.org/abs/2603.20101.
+
Jiaojiao Han, Wujiang Xu, Mingyu Jin, and Mengnan Du. Sage: An agentic explainer framework for interpreting sae features in language models. In Proceedings of the 19th Conference of the European Chapter of the Association for Computational Linguistics (Volume 5: Industry Track), pp. 483–495, 2026.
Michael Hanna, Ollie Liu, and Alexandre Variengien. How does GPT-2 compute greater-than?: Interpreting mathematical abilities in a pre-trained language model. In Thirty-seventh Conference on Neural Information Processing Systems, 2023. URL https://openreview.net/forum?id=p4PckNQR8k.
Michael Hanna, Sandro Pezzelle, and Yonatan Belinkov. Have faith in faithfulness: Going beyond circuit overlap when finding model mechanisms. In First Conference on Language Modeling, 2024. URL https://openreview.net/forum?id=TZ0CCGDcuT.
-
Abir Harrasse, Florent Draye, Zhijing Jin, and Bernhard Schölkopf. Tracing multilingual representations in llms with cross-layer transcoders. arXiv preprint arXiv:2511.10840, 2025a.
-
Abir Harrasse, Philip Quirke, Clement Neo, Dhruv Nathawani, Luke Marks, and Amir Abdullah. TinySQL: A progressive text-to-SQL dataset for mechanistic interpretability research. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing, pp. 29256–29284, Suzhou, China, November 2025b. Association for Computational Linguistics. ISBN 979-8-89176-332-6. . URL https://aclanthology.org/2025.emnlp-main.1489/.
+
Bear Häon, Kaylene Caswell Stocking, Ian Chuang, and Claire Tomlin. Mechanistic interpretability for steering vision-language-action models. In 9th Annual Conference on Robot Learning, 2025. URL https://openreview.net/forum?id=YvsUD8C9QS.
+
Abir Harrasse, Philip Quirke, Clement Neo, Dhruv Nathawani, Luke Marks, and Amir Abdullah. TinySQL: A progressive text-to-SQL dataset for mechanistic interpretability research. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing, pp. 29256–29284, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-332-6. . URL https://aclanthology.org/2025.emnlp-main.1489/.
+
Abir Harrasse, Florent Draye, Punya Syon Pandey, Zhijing Jin, and Bernhard Schölkopf. Tracing multilingual representations in llms with cross-layer transcoders, 2026. URL https://arxiv.org/abs/2511.10840.
+
Peter Hase, Mohit Bansal, Been Kim, and Asma Ghandeharioun. Does localization inform editing? surprising differences in causality-based localization vs. knowledge editing in language models. In Proceedings of the 37th International Conference on Neural Information Processing Systems, NIPS ’23, Red Hook, NY, USA, 2023. Curran Associates Inc.
Pengfei He, Yuping Lin, Shen Dong, Han Xu, Yue Xing, and Hui Liu. Red-teaming LLM multi-agent systems via communication attacks. In Wanxiang Che, Joyce Nabende, Ekaterina Shutova, and Mohammad Taher Pilehvar (eds.), Findings of the Association for Computational Linguistics: ACL 2025, pp. 6726–6747, Vienna, Austria, July 2025. Association for Computational Linguistics. ISBN 979-8-89176-256-5. . URL https://aclanthology.org/2025.findings-acl.349/.
-
Stefan Heimersheim and Neel Nanda. How to use and interpret activation patching. arXiv preprint arXiv:2404.15255, 2024.
-
Dan Hendrycks, Kevin Zhao, Steven Basart, Jacob Steinhardt, and Dawn Song. Natural adversarial examples. In Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR), pp. 15262–15271, June 2021.
-
Joel Hestness, Sharan Narang, Newsha Ardalani, Gregory Diamos, Heewoo Jun, Hassan Kianinejad, Md. Mostofa Ali Patwary, Yang Yang, and Yanqi Zhou. Deep learning scaling is predictable, empirically, 2017. URL https://arxiv.org/abs/1712.00409.
-
John Hewitt and Percy Liang. Designing and interpreting probes with control tasks. In Kentaro Inui, Jing Jiang, Vincent Ng, and Xiaojun Wan (eds.), Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pp. 2733–2743, Hong Kong, China, November 2019. Association for Computational Linguistics. . URL https://aclanthology.org/D19-1275/.
+
Zhengfu He, Wentao Shu, Xuyang Ge, Lingjie Chen, Junxuan Wang, Yunhua Zhou, Frances Liu, Qipeng Guo, Xuanjing Huang, Zuxuan Wu, Yu-Gang Jiang, and Xipeng Qiu. Llama scope: Extracting millions of features from llama-3.1-8b with sparse autoencoders, 2024. URL https://arxiv.org/abs/2410.20526.
+
Thomas Heap, Tim Lawson, Lucy Farnik, and Laurence Aitchison. Automated interpretability metrics do not distinguish trained and random transformers. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=USyGD0eUod.
+
Lovis Heindrich, Philip Torr, Fazl Barez, and Veronika Thost. Do sparse autoencoders generalize? a case study of answerability. In ICML 2025 Workshop on Reliable and Responsible Foundation Models, 2025. URL https://openreview.net/forum?id=rs3alQ5BV8.
+
Chadi Helwe, Oana Balalau, and Davide Ceolin. Navigating the political compass: Evaluating multilingual LLMs across languages and nationalities. In Wanxiang Che, Joyce Nabende, Ekaterina Shutova, and Mohammad Taher Pilehvar (eds.), Findings of the Association for Computational Linguistics: ACL 2025, pp. 17179–17204, Vienna, Austria, July 2025. Association for Computational Linguistics. ISBN 979-8-89176-256-5. . URL https://aclanthology.org/2025.findings-acl.883/.
+
Dan Hendrycks, Collin Burns, Steven Basart, Andy Zou, Mantas Mazeika, Dawn Song, and Jacob Steinhardt. Measuring massive multitask language understanding. In International Conference on Learning Representations, 2021. URL https://openreview.net/forum?id=d7KBjmI3GmQ.
+
Evan Hernandez, Sarah Schwettmann, David Bau, Teona Bagashvili, Antonio Torralba, and Jacob Andreas. Natural language descriptions of deep features. In International Conference on Learning Representations, 2022. URL https://openreview.net/forum?id=NudBMY-tzDr.
+
Joel Hestness, Sharan Narang, Newsha Ardalani, Gregory Diamos, Heewoo Jun, Hassan Kianinejad, Md. Mostofa Ali Patwary, Yang Yang, and Yanqi Zhou. Deep learning scaling is predictable, empirically, 2017. URL https://arxiv.org/abs/1712.00409.
+
John Hewitt and Percy Liang. Designing and interpreting probes with control tasks. In Kentaro Inui, Jing Jiang, Vincent Ng, and Xiaojun Wan (eds.), Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pp. 2733–2743, Hong Kong, China, November 2019. Association for Computational Linguistics. . URL https://aclanthology.org/D19-1275/.
+
John Hewitt and Christopher D Manning. A structural probe for finding syntax in word representations. In Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 4129–4138, 2019.
John Hewitt, Robert Geirhos, and Been Kim. Position: We can’t understand AI using our existing vocabulary. In Forty-second International Conference on Machine Learning Position Paper Track, 2025. URL https://openreview.net/forum?id=asQJx56NqB.
+
Geoffrey E. Hinton. Learning distributed representations of concepts. In Proceedings of the Eighth Annual Conference of the Cognitive Science Society, pp. 1–12, Amherst, MA, 1986.
+
Geoffrey E. Hinton, James L. McClelland, and David E. Rumelhart. Distributed representations. In David E. Rumelhart, James L. McClelland, and PDP Research Group (eds.), Parallel Distributed Processing: Explorations in the Microstructure of Cognition, Volume 1: Foundations, pp. 77–109. MIT Press, Cambridge, MA, 1986. ISBN 026268053X. .
+
Jason Hoelscher-Obermaier, Julia Persson, Esben Kran, Ioannis Konstas, and Fazl Barez. Detecting edit failures in large language models: An improved specificity benchmark. In Anna Rogers, Jordan Boyd-Graber, and Naoaki Okazaki (eds.), Findings of the Association for Computational Linguistics: ACL 2023, pp. 11548–11559, Toronto, Canada, July 2023. Association for Computational Linguistics. . URL https://aclanthology.org/2023.findings-acl.733/.
+
Jordan Hoffmann, Sebastian Borgeaud, Arthur Mensch, Elena Buchatskaya, Trevor Cai, Eliza Rutherford, Diego de Las Casas, Lisa Anne Hendricks, Johannes Welbl, Aidan Clark, Tom Hennigan, Eric Noland, Katie Millican, George van den Driessche, Bogdan Damoc, Aurelia Guy, Simon Osindero, Karen Simonyan, Erich Elsen, Oriol Vinyals, Jack W. Rae, and Laurent Sifre. Training compute-optimal large language models. In Proceedings of the 36th International Conference on Neural Information Processing Systems, NIPS ’22, Red Hook, NY, USA, 2022. Curran Associates Inc. ISBN 9781713871088.
+
Jiseung Hong, Grace Byun, Seungone Kim, and Kai Shu. Measuring sycophancy of language models in multi-turn dialogues. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Findings of the Association for Computational Linguistics: EMNLP 2025, pp. 2239–2259, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-335-7. . URL https://aclanthology.org/2025.findings-emnlp.121/.
Jesse Hoogland, Alexander Gietelink Oldenziel, Daniel Murfet, and Stan van Wingerden. Towards developmental interpretability. https://www.lesswrong.com/posts/TjaeCWvLZtEDAS5Ex/towards-developmental-interpretability, July 12 2023.
-
Jesse Hoogland, George Wang, Matthew Farrugia-Roberts, Liam Carroll, Susan Wei, and Daniel Murfet. Loss landscape degeneracy and stagewise development in transformers, 2025. URL https://arxiv.org/abs/2402.02364.
+
Jesse Hoogland, George Wang, Matthew Farrugia-Roberts, Liam Carroll, Susan Wei, and Daniel Murfet. Loss landscape degeneracy and stagewise development in transformers. Transactions on Machine Learning Research, 2025. ISSN 2835-8856. URL https://openreview.net/forum?id=45qJyBG8Oj.
+
Harold Hotelling. Analysis of a complex of statistical variables into principal components. Journal of Educational Psychology, 24 (6): 417–441, 1933. .
Edward J Hu, yelong shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, and Weizhu Chen. LoRA: Low-rank adaptation of large language models. In International Conference on Learning Representations, 2022. URL https://openreview.net/forum?id=nZeVKeeFYf9.
+
Michael Y. Hu, Angelica Chen, Naomi Saphra, and Kyunghyun Cho. Latent state models of training dynamics. Transactions on Machine Learning Research, 2023.
Nathan Hu, Jake Ward, Thomas Icard, and Christopher Potts. Transcoder adapters for reasoning-model diffing, 2026. URL https://arxiv.org/abs/2602.20904.
Jing Huang, Zhengxuan Wu, Christopher Potts, Mor Geva, and Atticus Geiger. RAVEL: Evaluating interpretability methods on disentangling language model representations. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 8669–8687, Bangkok, Thailand, August 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.470/.
-
Vincent Huang, Dami Choi, Daniel D Johnson, Sarah Schwettmann, and Jacob Steinhardt. Predictive concept decoders: Training scalable end-to-end interpretability assistants. arXiv preprint arXiv:2512.15712, 2025.
-
Evan Hubinger, Chris van Merwijk, Vladimir Mikulik, Joar Skalse, and Scott Garrabrant. Risks from learned optimization in advanced machine learning systems, 2021. URL https://arxiv.org/abs/1906.01820.
+
Vincent Huang, Dami Choi, Daniel D. Johnson, Sarah Schwettmann, and Jacob Steinhardt. Predictive concept decoders: Training scalable end-to-end interpretability assistants. In Mechanistic Interpretability Workshop at ICML 2026, 2026. URL https://openreview.net/forum?id=s9IaugZ0En.
Evan Hubinger, Carson Denison, Jesse Mu, Mike Lambert, Meg Tong, Monte MacDiarmid, Tamera Lanham, Daniel M. Ziegler, Tim Maxwell, Newton Cheng, Adam Jermyn, Amanda Askell, Ansh Radhakrishnan, Cem Anil, David Duvenaud, Deep Ganguli, Fazl Barez, Jack Clark, Kamal Ndousse, Kshitij Sachan, Michael Sellitto, Mrinank Sharma, Nova DasSarma, Roger Grosse, Shauna Kravec, Yuntao Bai, Zachary Witten, Marina Favaro, Jan Brauner, Holden Karnofsky, Paul Christiano, Samuel R. Bowman, Logan Graham, Jared Kaplan, Sören Mindermann, Ryan Greenblatt, Buck Shlegeris, Nicholas Schiefer, and Ethan Perez. Sleeper agents: Training deceptive llms that persist through safety training, 2024. URL https://arxiv.org/abs/2401.05566.
Minyoung Huh, Brian Cheung, Tongzhou Wang, and Phillip Isola. Position: The platonic representation hypothesis. In Ruslan Salakhutdinov, Zico Kolter, Katherine Heller, Adrian Weller, Nuria Oliver, Jonathan Scarlett, and Felix Berkenkamp (eds.), Proceedings of the 41st International Conference on Machine Learning, volume 235 of Proceedings of Machine Learning Research, pp. 20617–20642. PMLR, 21–27 Jul 2024. URL https://proceedings.mlr.press/v235/huh24a.html.
-
Aapo Hyvärinen and Erkki Oja. Independent component analysis: algorithms and applications. Neural networks, 13 (4-5): 411–430, 2000.
-
Aapo Hyvärinen, Ilyes Khemakhem, and Hiroshi Morioka. Nonlinear independent component analysis for principled disentanglement in unsupervised deep learning. Patterns, 4 (10), 2023.
-
Andrew Ilyas, Sung Min Park, Logan Engstrom, Guillaume Leclerc, and Aleksander Madry. Datamodels: Understanding predictions with data and data with predictions. In Kamalika Chaudhuri, Stefanie Jegelka, Le Song, Csaba Szepesvari, Gang Niu, and Sivan Sabato (eds.), Proceedings of the 39th International Conference on Machine Learning, volume 162 of Proceedings of Machine Learning Research, pp. 9525–9587. PMLR, 17–23 Jul 2022. URL https://proceedings.mlr.press/v162/ilyas22a.html.
-
Alon Jacovi and Yoav Goldberg. Towards faithfully interpretable nlp systems: How should we define and evaluate faithfulness?, 2020. URL https://arxiv.org/abs/2004.03685.
+
Aapo Hyvärinen and Erkki Oja. Independent component analysis: algorithms and applications. Neural networks, 13 (4-5): 411–430, 2000.
+
Zarif Ikram, Arad Firouzkouhi, Stephen Tu, Mahdi Soltanolkotabi, and Paria Rashidinejad. Crispedit: Low-curvature projections for scalable non-destructive LLM editing. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=gTHlB3WK0l.
+
Andrew Ilyas, Sung Min Park, Logan Engstrom, Guillaume Leclerc, and Aleksander Madry. Datamodels: Understanding predictions with data and data with predictions. In Kamalika Chaudhuri, Stefanie Jegelka, Le Song, Csaba Szepesvari, Gang Niu, and Sivan Sabato (eds.), Proceedings of the 39th International Conference on Machine Learning, volume 162 of Proceedings of Machine Learning Research, pp. 9525–9587. PMLR, 17–23 Jul 2022. URL https://proceedings.mlr.press/v162/ilyas22a.html.
+
Tatsuro Inaba, Go Kamoda, Kentaro Inui, Masaru Isonuma, Yusuke Miyao, Yohei Oseki, Yu Takagi, and Benjamin Heinzerling. How a bilingual LM becomes bilingual: Tracing internal representations with sparse autoencoders. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Findings of the Association for Computational Linguistics: EMNLP 2025, pp. 13458–13470, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-335-7. . URL https://aclanthology.org/2025.findings-emnlp.725/.
+
Hakan Inan, Kartikeya Upasani, Jianfeng Chi, Rashi Rungta, Krithika Iyer, Yuning Mao, Michael Tontchev, Qing Hu, Brian Fuller, Davide Testuggine, and Madian Khabsa. Llama guard: Llm-based input-output safeguard for human-ai conversations, 2023. URL https://arxiv.org/abs/2312.06674.
+
Alon Jacovi and Yoav Goldberg. Towards faithfully interpretable NLP systems: How should we define and evaluate faithfulness? In Dan Jurafsky, Joyce Chai, Natalie Schluter, and Joel Tetreault (eds.), Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 4198–4205, Online, July 2020. Association for Computational Linguistics. . URL https://aclanthology.org/2020.acl-main.386/.
+
Neel Jain, Avi Schwarzschild, Yuxin Wen, Gowthami Somepalli, John Kirchenbauer, Ping yeh Chiang, Micah Goldblum, Aniruddha Saha, Jonas Geiping, and Tom Goldstein. Baseline defenses for adversarial attacks against aligned language models, 2023. URL https://arxiv.org/abs/2309.00614.
+
Samyak Jain, Robert Kirk, Ekdeep Singh Lubana, Robert P. Dick, Hidenori Tanaka, Tim Rocktäschel, Edward Grefenstette, and David Krueger. Mechanistically analyzing the effects of fine-tuning on procedurally defined tasks. In The Twelfth International Conference on Learning Representations, 2024a. URL https://openreview.net/forum?id=A0HKeKl4Nl.
+
Samyak Jain, Ekdeep Singh Lubana, Kemal Oksuz, Tom Joy, Philip Torr, Amartya Sanyal, and Puneet K. Dokania. What makes and breaks safety fine-tuning? a mechanistic study. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024b. URL https://openreview.net/forum?id=JEflV4nRlH.
Sarthak Jain and Byron C. Wallace. Attention is not Explanation. In Jill Burstein, Christy Doran, and Thamar Solorio (eds.), Proceedings of the 2019 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers), pp. 3543–3556, Minneapolis, Minnesota, June 2019. Association for Computational Linguistics. . URL https://aclanthology.org/N19-1357/.
-
Thomas Jiralerspong and Trenton Bricken. Cross-Architecture Model Diffing with Crosscoders: Unsupervised Discovery of Differences Between LLMs. In Workshop on Mechanistic Interpretability, September 2025. URL https://openreview.net/forum?id=ZB84SvrZB8.
-
Shruti Joshi, Aaron Mueller, David Klindt, Wieland Brendel, Patrik Reizinger, and Dhanya Sridhar. Causality is key for interpretability claims to generalise, 2026. URL https://arxiv.org/abs/2602.16698.
-
Jaap Jumelet, Lisa Bylinina, Willem Zuidema, and Jakub Szymanik. Black big boxes: Do language models hide a theory of adjective order? arXiv preprint arXiv:2407.02136, 2024.
+
Arya Jakkli, Senthooran Rajamanoharan, and Neel Nanda. Current activation oracles are hard to use on safety-relevant tasks. In Mechanistic Interpretability Workshop at ICML 2026, 2026. URL https://openreview.net/forum?id=7nRmqgz3Wv.
+
Albert Q. Jiang, Alexandre Sablayrolles, Antoine Roux, Arthur Mensch, Blanche Savary, Chris Bamford, Devendra Singh Chaplot, Diego de las Casas, Emma Bou Hanna, Florian Bressand, Gianna Lengyel, Guillaume Bour, Guillaume Lample, Lélio Renard Lavaud, Lucile Saulnier, Marie-Anne Lachaux, Pierre Stock, Sandeep Subramanian, Sophia Yang, Szymon Antoniak, Teven Le Scao, Théophile Gervet, Thibaut Lavril, Thomas Wang, Timothée Lacroix, and William El Sayed. Mixtral of experts, 2024. URL https://arxiv.org/abs/2401.04088.
+
Carlos E Jimenez, John Yang, Alexander Wettig, Shunyu Yao, Kexin Pei, Ofir Press, and Karthik R Narasimhan. SWE-bench: Can language models resolve real-world github issues? In The Twelfth International Conference on Learning Representations, 2024. URL https://openreview.net/forum?id=VTF8yNQM66.
+
Thomas Jiralerspong and Trenton Bricken. Cross-Architecture Model Diffing with Crosscoders: Unsupervised Discovery of Differences Between LLMs. In Workshop on Mechanistic Interpretability, September 2025. URL https://openreview.net/forum?id=ZB84SvrZB8.
+
Manas Joglekar, Jeremy Chen, Gabriel Wu, Jason Yosinski, Jasmine Wang, Boaz Barak, and Amelia Glaese. Training llms for honesty via confessions, 2025. URL https://arxiv.org/abs/2512.08093.
+
Erik Jones, Meg Tong, Jesse Mu, Mohammed Mahfoud, Jan Leike, Roger Grosse, Jared Kaplan, William Fithian, Ethan Perez, and Mrinank Sharma. Forecasting rare language model behaviors, 2025. URL https://arxiv.org/abs/2502.16797.
+
Jaap Jumelet, Lisa Bylinina, Willem Zuidema, and Jakub Szymanik. Black big boxes: Tracing adjective order preferences in large language models, 2026. URL https://arxiv.org/abs/2407.02136.
+
Jeevesh Juneja, Rachit Bansal, Kyunghyun Cho, João Sedoc, and Naomi Saphra. Linear connectivity reveals generalization strategies. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=hY6M0JHl3uL.
Harish Kamath, Emmanuel Ameisen, Isaac Kauvar, Rodrigo Luger, Wes Gurnee, Adam Pearce, Sam Zimmerman, Joshua Batson, Thomas Conerly, Chris Olah, and Jack Lindsey. Tracing attention computation through feature interactions. Transformer Circuits Thread, 2025. URL https://transformer-circuits.pub/2025/attention-qk/index.html.
-
Sara Kangaslahti, Elan Rosenfeld, and Naomi Saphra. Hidden breakthroughs in language model training, 2026. URL https://arxiv.org/abs/2506.15872.
-
Jared Kaplan, Sam McCandlish, Tom Henighan, Tom B. Brown, Benjamin Chess, Rewon Child, Scott Gray, Alec Radford, Jeffrey Wu, and Dario Amodei. Scaling laws for neural language models. arXiv preprint arXiv:2001.08361, 2020. URL https://arxiv.org/abs/2001.08361.
-
Andrej Karpathy, Justin Johnson, and Li Fei-Fei. Visualizing and understanding recurrent networks. arXiv preprint arXiv:1506.02078, 2015.
-
Adam Karvonen, James Chua, Clément Dumas, Kit Fraser-Taliente, Subhash Kantamneni, Julian Minder, Euan Ong, Arnab Sen Sharma, Daniel Wen, Owain Evans, et al. Activation oracles: Training and evaluating llms as general-purpose activation explainers. arXiv preprint arXiv:2512.15674, 2025.
-
Elias Kempf, Simon Schrodi, Bartosz Cywiński, Thomas Brox, Neel Nanda, and Arthur Conmy. Simple llm baselines are competitive for model diffing, 2026. URL https://arxiv.org/abs/2602.10371.
+
Sara Kangaslahti, Elan Rosenfeld, and Naomi Saphra. Hidden breakthroughs in language model training. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=eiEIKGuqaf.
+
Jared Kaplan, Sam McCandlish, Tom Henighan, Tom B. Brown, Benjamin Chess, Rewon Child, Scott Gray, Alec Radford, Jeffrey Wu, and Dario Amodei. Scaling laws for neural language models. arXiv preprint arXiv:2001.08361, 2020. URL https://arxiv.org/abs/2001.08361.
+
Andrej Karpathy, Justin Johnson, and Li Fei-Fei. Visualizing and understanding recurrent networks, 2015. URL https://arxiv.org/abs/1506.02078.
+
Adam Karvonen, Can Rager, Johnny Lin, Curt Tigges, Joseph Isaac Bloom, David Chanin, Yeu-Tong Lau, Eoin Farrell, Callum Stuart McDougall, Kola Ayonrinde, Demian Till, Matthew Wearden, Arthur Conmy, Samuel Marks, and Neel Nanda. SAEBench: A comprehensive benchmark for sparse autoencoders in language model interpretability. In Forty-second International Conference on Machine Learning, 2025.
+
Adam Karvonen, James Chua, Clément Dumas, Kit Fraser-Taliente, Subhash Kantamneni, Julian Minder, Euan Ong, Arnab Sen Sharma, Daniel Wen, Owain Evans, and Samuel Marks. Activation oracles: Training and evaluating LLMs as general-purpose activation explainers. In Forty-third International Conference on Machine Learning, 2026a. URL https://openreview.net/forum?id=DbZjxkZrZm.
+
Adam Karvonen, Euan Ong, Subhash Kantamneni, and Samuel Marks. Would this change your answer? evaluating explanations of llm behavior in the wild with counterfactual experiments, 2026b. URL https://arxiv.org/abs/2608.16747.
+
Maximilian Kempermann, Sai Sarathi Macharla Vasu, Mathusan Raveenthiran, Tracie Farrell, and Ingmar Weber. Safe for whom? Rethinking how we evaluate the safety of LLMs for real users. Proceedings of the IASEAI Conference, 2 (1): 305–317, 2026. URL https://ojs.aaai.org/index.php/IASEAI/article/view/43033.
+
Elias Kempf, Simon Schrodi, Bartosz Cywiński, Thomas Brox, Neel Nanda, and Arthur Conmy. Simple llm baselines are competitive for model diffing, 2026. URL https://arxiv.org/abs/2602.10371.
+
Ayan Antik Khan, Harsh Kohli, Yuekun Yao, Huan Sun, and Ziyu Yao. Can language model agents be helpful circuit explainers in mechanistic interpretability?, 2026. URL https://arxiv.org/abs/2606.24026.
+
Dmitrii Kharlapenko, Stepan Shabalin, Arthur Conmy, and Neel Nanda. Scaling sparse feature circuits for studying in-context learning. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=Mp8Og8Rpop.
Been Kim, Martin Wattenberg, Justin Gilmer, Carrie Cai, James Wexler, Fernanda Viegas, and Rory sayres. Interpretability beyond feature attribution: Quantitative testing with concept activation vectors (TCAV). In Jennifer Dy and Andreas Krause (eds.), Proceedings of the 35th International Conference on Machine Learning, volume 80 of Proceedings of Machine Learning Research, pp. 2668–2677. PMLR, 10–15 Jul 2018. URL https://proceedings.mlr.press/v80/kim18d.html.
-
Been Kim, John Hewitt, Neel Nanda, Noah Fiedel, and Oyvind Tafjord. Because we have llms, we can and should pursue agentic interpretability, 2025. URL https://arxiv.org/abs/2506.12152.
-
Eunji Kim, Dahuin Jung, Sangha Park, Siwon Kim, and Sungroh Yoon. Probabilistic concept bottleneck models. In Proceedings of the 40th International Conference on Machine Learning, ICML’23. JMLR.org, 2023.
+
Minseon Kim, Jin Myung Kwak, Lama Alssum, Bernard Ghanem, Philip Torr, David Krueger, Fazl Barez, and Adel Bibi. Rethinking safety in LLM fine-tuning: An optimization perspective. In Second Conference on Language Modeling, 2025. URL https://openreview.net/forum?id=ZnOoEA2nDn.
+
Pieter-Jan Kindermans, Sara Hooker, Julius Adebayo, Maximilian Alber, Kristof T. Schütt, Sven Dähne, Dumitru Erhan, and Been Kim. The (Un)reliability of Saliency Methods, pp. 267–280. Springer-Verlag, Berlin, Heidelberg, 2022. ISBN 978-3-030-28953-9. URL https://doi.org/10.1007/978-3-030-28954-6_14.
Megan Kinniment, Seraphina Nix, Thomas Broadley, and Neev Parikh. Early work on monitorability evaluations. https://metr.org/blog/2026-01-19-early-work-on-monitorability-evaluations/, January 2026.
Connor Kissane, Robert Krzyzanowski, Joseph Isaac Bloom, Arthur Conmy, and Neel Nanda. Interpreting attention layer outputs with sparse autoencoders, 2024. URL https://arxiv.org/abs/2406.17759.
David Klindt, Charles O’Neill, Patrik Reizinger, Harald Maurer, and Nina Miolane. From superposition to sparse codes: interpretable representations in neural networks, 2025. URL https://arxiv.org/abs/2503.01824.
-
Pang Wei Koh and Percy Liang. Understanding black-box predictions via influence functions. In Proceedings of the 34th International Conference on Machine Learning - Volume 70, ICML’17, pp. 1885–1894. JMLR.org, 2017.
+
A. Sophia Koepke, Daniil Zverev, Shiry Ginosar, and Alexei A. Efros. Back into plato’s cave: Examining cross-modal representational convergence at scale, 2026. URL https://arxiv.org/abs/2604.18572.
+
Pang Wei Koh and Percy Liang. Understanding black-box predictions via influence functions. In Proceedings of the 34th International Conference on Machine Learning - Volume 70, ICML’17, pp. 1885–1894. JMLR.org, 2017.
Pang Wei Koh, Thao Nguyen, Yew Siang Tang, Stephen Mussmann, Emma Pierson, Been Kim, and Percy Liang. Concept bottleneck models. In Hal Daumé III and Aarti Singh (eds.), Proceedings of the 37th International Conference on Machine Learning, volume 119 of Proceedings of Machine Learning Research, pp. 5338–5348. PMLR, 13–18 Jul 2020. URL https://proceedings.mlr.press/v119/koh20a.html.
Arne Köhn. What’s in an embedding? analyzing word embeddings through multilingual evaluation. In Lluís Màrquez, Chris Callison-Burch, and Jian Su (eds.), Proceedings of the 2015 Conference on Empirical Methods in Natural Language Processing, pp. 2067–2073, Lisbon, Portugal, September 2015. Association for Computational Linguistics. . URL https://aclanthology.org/D15-1246/.
+
Tomek Korbak, Mikita Balesni, Elizabeth Barnes, Yoshua Bengio, Joe Benton, Joseph Bloom, Mark Chen, Alan Cooney, Allan Dafoe, Anca Dragan, Scott Emmons, Owain Evans, David Farhi, Ryan Greenblatt, Dan Hendrycks, Marius Hobbhahn, Evan Hubinger, Geoffrey Irving, Erik Jenner, Daniel Kokotajlo, Victoria Krakovna, Shane Legg, David Lindner, David Luan, Aleksander Mądry, Julian Michael, Neel Nanda, Dave Orr, Jakub Pachocki, Ethan Perez, Mary Phuong, Fabien Roger, Joshua Saxe, Buck Shlegeris, Martín Soto, Eric Steinberger, Jasmine Wang, Wojciech Zaremba, Bowen Baker, Rohin Shah, and Vlad Mikulik. Chain of thought monitorability: A new and fragile opportunity for ai safety, 2025. URL https://arxiv.org/abs/2507.11473.
Simon Kornblith, Mohammad Norouzi, Honglak Lee, and Geoffrey Hinton. Similarity of neural network representations revisited. In Kamalika Chaudhuri and Ruslan Salakhutdinov (eds.), Proceedings of the 36th International Conference on Machine Learning, volume 97 of Proceedings of Machine Learning Research, pp. 3519–3529. PMLR, 09–15 Jun 2019. URL https://proceedings.mlr.press/v97/kornblith19a.html.
Anton Korznikov, Andrey Galichin, Alexey Dontsov, Oleg Rogov, Elena Tutubalina, and Ivan Oseledets. Ortsae: Orthogonal sparse autoencoders uncover atomic features, 2025. URL https://arxiv.org/abs/2509.22033.
-
Matthew Kowal, Goncalo Paulo, Louis Jaburi, Tom Tseng, Lev E McKinney, Stefan Heimersheim, Aaron David Tucker, Adam Gleave, and Kellin Pelrine. Concept influence: Leveraging interpretability to improve performance and efficiency in training data attribution. arXiv preprint arXiv:2602.14869, 2026.
-
Satyapriya Krishna, Tessa Han, Alex Gu, Steven Wu, Shahin Jabbari, and Himabindu Lakkaraju. The disagreement problem in explainable machine learning: A practitioner’s perspective. Transactions on Machine Learning Research, 2024. ISSN 2835-8856. URL https://openreview.net/forum?id=jESY2WTZCe.
-
Alex Krizhevsky, Ilya Sutskever, and Geoffrey Hinton. Imagenet classification with deep convolutional neural networks. In Advances in Neural Information Processing Systems, 2012.
-
Tamera Lanham, Anna Chen, Ansh Radhakrishnan, Benoit Steiner, Carson Denison, Danny Hernandez, Dustin Li, Esin Durmus, Evan Hubinger, Jackson Kernion, Kamilė Lukošiūtė, Karina Nguyen, Newton Cheng, Nicholas Joseph, Nicholas Schiefer, Oliver Rausch, Robin Larson, Sam McCandlish, Sandipan Kundu, Saurav Kadavath, Shannon Yang, Thomas Henighan, Timothy Maxwell, Timothy Telleen-Lawton, Tristan Hume, Zac Hatfield-Dodds, Jared Kaplan, Jan Brauner, Samuel R. Bowman, and Ethan Perez. Measuring faithfulness in chain-of-thought reasoning, 2023. URL https://arxiv.org/abs/2307.13702.
-
Edmund Lau, Zach Furman, George Wang, Daniel Murfet, and Susan Wei. The local learning coefficient: A singularity-aware complexity measure. arXiv preprint arXiv:2308.12108, 2023.
-
Quoc V Le. Building high-level features using large scale unsupervised learning. In 2013 IEEE international conference on acoustics, speech and signal processing, pp. 8595–8598. IEEE, 2013.
-
Patrick Leask, Bart Bussmann, Michael Pearce, Joseph Bloom, Curt Tigges, Noura Al Moubayed, Lee Sharkey, and Neel Nanda. Sparse autoencoders do not find canonical units of analysis. arXiv preprint arXiv:2502.04878, 2025.
+
Matthew Kowal, Goncalo Paulo, Louis Jaburi, Tom Tseng, Lev E McKinney, Stefan Heimersheim, Aaron David Tucker, Adam Gleave, and Kellin Pelrine. Concept influence: Leveraging interpretability to improve performance and efficiency in training data attribution, 2026. URL https://arxiv.org/abs/2602.14869.
+
Satyapriya Krishna, Tessa Han, Alex Gu, Steven Wu, Shahin Jabbari, and Himabindu Lakkaraju. The disagreement problem in explainable machine learning: A practitioner’s perspective. Transactions on Machine Learning Research, 2024. ISSN 2835-8856. URL https://openreview.net/forum?id=jESY2WTZCe.
+
Alex Krizhevsky, Ilya Sutskever, and Geoffrey E Hinton. Imagenet classification with deep convolutional neural networks. In F. Pereira, C.J. Burges, L. Bottou, and K. Weinberger (eds.), Advances in Neural Information Processing Systems, volume 25. Curran Associates, Inc., 2012. URL https://proceedings.neurips.cc/paper_files/paper/2012/file/c399862d3b9d6b76c8436e924a68c45b-Paper.pdf.
+
Philippe Laban, Hiroaki Hayashi, Yingbo Zhou, and Jennifer Neville. LLMs get lost in multi-turn conversation. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=VKGTGGcwl6.
+
Andrew K. Lampinen and Surya Ganguli. An analytic theory of generalization dynamics and transfer learning in deep linear networks. In International Conference on Learning Representations, 2019. URL https://openreview.net/forum?id=ryfMLoCqtQ.
+
Michael Lan, Philip Torr, and Fazl Barez. Towards interpretable sequence continuation: Analyzing shared circuits in large language models. In Yaser Al-Onaizan, Mohit Bansal, and Yun-Nung Chen (eds.), Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, pp. 12576–12601, Miami, Florida, USA, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.emnlp-main.699/.
+
Michael Lan, Philip Torr, Austin Meek, Ashkan Khakzar, David Krueger, and Fazl Barez. Quantifying feature space universality across large language models via sparse autoencoders, 2025. URL https://arxiv.org/abs/2410.06981.
+
Michael Lan, Narmeen Fatimah Oozeer, Chaithanya Bandi, Philip Quirke, Austin Meek, Fazl Barez, and Amir Abdullah. Make mechanistic interpretability auditable: A call to develop guidelines via continuous collaborative reviewing. In Maria Liakata, Viviane P. Moreira, Jiajun Zhang, and David Jurgens (eds.), Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 3516–3540, San Diego, California, United States, July 2026. Association for Computational Linguistics. ISBN 979-8-89176-390-6. . URL https://aclanthology.org/2026.acl-long.159/.
+
Edmund Lau, Zach Furman, George Wang, Daniel Murfet, and Susan Wei. The local learning coefficient: A singularity-aware complexity measure. In The 28th International Conference on Artificial Intelligence and Statistics, 2025. URL https://openreview.net/forum?id=1av51ZlsuL.
+
Patrick Leask, Bart Bussmann, Michael T Pearce, Joseph Isaac Bloom, Curt Tigges, Noura Al Moubayed, Lee Sharkey, and Neel Nanda. Sparse autoencoders do not find canonical units of analysis. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=9ca9eHNrdH.
+
Victor Lecomte, Kushal Thaman, Rylan Schaeffer, Naomi Bashkansky, Trevor Chow, and Sanmi Koyejo. What causes polysemanticity? an alternative origin story of mixed selectivity from incidental causes. In ICLR 2024 Workshop on Representational Alignment, 2024. URL https://openreview.net/forum?id=AHfE6WeJLQ.
+
Daniel D. Lee and H. Sebastian Seung. Learning the parts of objects by non-negative matrix factorization. Nature, 401 (6755): 788–791, 1999. .
Honglak Lee, Alexis Battle, Rajat Raina, and Andrew Ng. Efficient sparse coding algorithms. In B. Schölkopf, J. Platt, and T. Hoffman (eds.), Advances in Neural Information Processing Systems, volume 19. MIT Press, 2006. URL https://proceedings.neurips.cc/paper_files/paper/2006/file/2d71b2ae158c7c5912cc0bbde2bb9d95-Paper.pdf.
-
Isabelle Lee, Emmy Liu, Cathy Jiao, Brihi Joshi, Dani Yogatama, Fazl Barez, and Michael Saxon. Rigorous interpretation is a form of evaluation, 2026. URL https://arxiv.org/abs/2605.05508.
-
Jin Hwa Lee, Matthew Smith, Maxwell Adam, and Jesse Hoogland. Influence dynamics and stagewise data attribution. arXiv preprint arXiv:2510.12071, 2025.
-
Anson Lei, Bernhard Schölkopf, and Ingmar Posner. Spartan: A sparse transformer world model attending to what matters. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025.
+
Isabelle Lee, Emmy Liu, Cathy Jiao, Brihi Joshi, Dani Yogatama, Fazl Barez, and Michael Saxon. Rigorous interpretation is a form of evaluation, 2026a. URL https://arxiv.org/abs/2605.05508.
+
Isabelle Lee, Emmy Liu, Kaiser Sun, Stella Biderman, and David Alvarez-Melis. Workshop on the science of training: Emergence, prediction, and steering (steps). https://science-of-training.github.io/assets/proposal.pdf, 2026b. Workshop proposal, NeurIPS 2026.
+
Anson Lei, Bernhard Schölkopf, and Ingmar Posner. Spartan: A sparse transformer world model attending to what matters. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025.
+
Dmitry Lepikhin, HyoukJoong Lee, Yuanzhong Xu, Dehao Chen, Orhan Firat, Yanping Huang, Maxim Krikun, Noam Shazeer, and Zhifeng Chen. {GS}hard: Scaling giant models with conditional computation and automatic sharding. In International Conference on Learning Representations, 2021. URL https://openreview.net/forum?id=qrwe7XHTmYb.
+
Simon Lermen, Mateusz Dziemian, and Natalia Pérez-Campanero Antolín. Deceptive automated interpretability: Language models coordinating to fool oversight systems, 2025. URL https://arxiv.org/abs/2504.07831.
Michael S. Lewicki and Terrence J. Sejnowski. Learning overcomplete representations. Neural Comput., 12 (2): 337–365, February 2000. ISSN 0899-7667. . URL https://doi.org/10.1162/089976600300015826.
+
Belinda Z. Li. Introspective interpretability: a definition, motivation, and open problems. https://belindal.github.io/introspection, February 2026. Blog post.
Belinda Z. Li, Zifan Carl Guo, Vincent Huang, Jacob Steinhardt, and Jacob Andreas. Training language models to explain their own computations, 2025a. URL https://arxiv.org/abs/2511.08579.
-
Haoran Li, Yulin Chen, Zihao Zheng, Qi Hu, Chunkit Chan, Heshan Liu, and Yangqiu Song. Simulate and eliminate: revoke backdoors for generative large language models. In Proceedings of the Thirty-Ninth AAAI Conference on Artificial Intelligence and Thirty-Seventh Conference on Innovative Applications of Artificial Intelligence and Fifteenth Symposium on Educational Advances in Artificial Intelligence, AAAI’25/IAAI’25/EAAI’25. AAAI Press, 2025b. ISBN 978-1-57735-897-8. . URL https://doi.org/10.1609/aaai.v39i1.32018.
+
Changyi Li, Pengfei Lu, Xudong Pan, Fazl Barez, and Min Yang. Autocontrol arena: Synthesizing executable test environments for frontier AI risk evaluation. In Forty-third International Conference on Machine Learning, 2026a. URL https://openreview.net/forum?id=XkDpZusDTK.
+
Chloe Li, Mary Phuong, and Daniel Tan. Spilling the beans: Teaching LLMs to self-report their hidden objectives. In The Fourteenth International Conference on Learning Representations, 2026b. URL https://openreview.net/forum?id=sWs0cCuM8I.
Jiwei Li, Xinlei Chen, Eduard Hovy, and Dan Jurafsky. Visualizing and understanding neural models in NLP. In Kevin Knight, Ani Nenkova, and Owen Rambow (eds.), Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 681–691, San Diego, California, June 2016. Association for Computational Linguistics. . URL https://aclanthology.org/N16-1082/.
+
Kenneth Li, Aspen K Hopkins, David Bau, Fernanda Viégas, Hanspeter Pfister, and Martin Wattenberg. Emergent world representations: Exploring a sequence model trained on a synthetic task. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=DeG07_TcZvT.
+
Millicent Li, Alberto Mario Ceballos Arroyo, Giordano Rogers, Naomi Saphra, and Byron C Wallace. Do activation verbalization methods convey privileged information? In Forty-third International Conference on Machine Learning, 2026c. URL https://openreview.net/forum?id=IFGTKwHwK6.
Nathaniel Li, Alexander Pan, Anjali Gopal, Summer Yue, Daniel Berrios, Alice Gatti, Justin D. Li, Ann-Kathrin Dombrowski, Shashwat Goel, Gabriel Mukobi, Nathan Helm-Burger, Rassin Lababidi, Lennart Justen, Andrew Bo Liu, Michael Chen, Isabelle Barrass, Oliver Zhang, Xiaoyuan Zhu, Rishub Tamirisa, Bhrugu Bharathi, Ariel Herbert-Voss, Cort B Breuer, Andy Zou, Mantas Mazeika, Zifan Wang, Palash Oswal, Weiran Lin, Adam Alfred Hunt, Justin Tienken-Harder, Kevin Y. Shih, Kemper Talley, John Guan, Ian Steneker, David Campbell, Brad Jokubaitis, Steven Basart, Stephen Fitz, Ponnurangam Kumaraguru, Kallol Krishna Karmakar, Uday Tupakula, Vijay Varadharajan, Yan Shoshitaishvili, Jimmy Ba, Kevin M. Esvelt, Alexandr Wang, and Dan Hendrycks. The WMDP benchmark: Measuring and reducing malicious use with unlearning. In Forty-first International Conference on Machine Learning, 2024. URL https://openreview.net/forum?id=xlr6AUDuJz.
-
Yixuan Li, Jason Yosinski, Jeff Clune, Hod Lipson, and John Hopcroft. Convergent learning: Do different neural networks learn the same representations? In Dmitry Storcheus, Afshin Rostamizadeh, and Sanjiv Kumar (eds.), Proceedings of the 1st International Workshop on Feature Extraction: Modern Questions and Challenges at NIPS 2015, volume 44 of Proceedings of Machine Learning Research, pp. 196–212, Montreal, Canada, 11 Dec 2015. PMLR. URL https://proceedings.mlr.press/v44/li15convergent.html.
-
Yuxiao Li, Eric J Michaud, David D Baek, Joshua Engels, Xiaoqing Sun, and Max Tegmark. The geometry of concepts: Sparse autoencoder feature structure. Entropy, 27 (4): 344, 2025c.
+
Yixuan Li, Jason Yosinski, Jeff Clune, Hod Lipson, and John Hopcroft. Convergent learning: Do different neural networks learn the same representations? In Dmitry Storcheus, Afshin Rostamizadeh, and Sanjiv Kumar (eds.), Proceedings of the 1st International Workshop on Feature Extraction: Modern Questions and Challenges at NIPS 2015, volume 44 of Proceedings of Machine Learning Research, pp. 196–212, Montreal, Canada, 11 Dec 2015. PMLR. URL https://proceedings.mlr.press/v44/li15convergent.html.
+
Yuxiao Li, Eric J Michaud, David D Baek, Joshua Engels, Xiaoqing Sun, and Max Tegmark. The geometry of concepts: Sparse autoencoder feature structure. Entropy, 27 (4): 344, 2025b.
+
Tom Lieberum, Senthooran Rajamanoharan, Arthur Conmy, Lewis Smith, Nicolas Sonnerat, Vikrant Varma, Janos Kramar, Anca Dragan, Rohin Shah, and Neel Nanda. Gemma scope: Open sparse autoencoders everywhere all at once on gemma 2. In Yonatan Belinkov, Najoung Kim, Jaap Jumelet, Hosein Mohebbi, Aaron Mueller, and Hanjie Chen (eds.), Proceedings of the 7th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP, pp. 278–300, Miami, Florida, US, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.blackboxnlp-1.19/.
+
Johnny Lin. Neuronpedia: Interactive reference and tooling for analyzing neural networks, 2023. URL https://www.neuronpedia.org. Software available from neuronpedia.org.
+
Jack Lindsey. Emergent introspective awareness in large language models, 2026. URL https://arxiv.org/abs/2601.01828.
Jack Lindsey, Adly Templeton, Jonathan Marcus, Thomas Conerly, Joshua Batson, and Christopher Olah. Sparse crosscoders for cross‑layer features and model diffing. Anthropic Transformer Circuits Thread, 2024. URL https://transformer-circuits.pub/2024/crosscoders/index.html.
Jack Lindsey, Emmanuel Ameisen, Neel Nanda, Stepan Shabalin, Mateusz Piotrowski, Tom McGrath, Michael Hanna, Owen Lewis, Curt Tigges, Jack Merullo, Connor Watts, Gonçalo Paulo, Joshua Batson, Liv Gorton, Elana Simon, Max Loeffler, Callum McDougall, and Johnny Lin. The circuits research landscape: Results and perspectives. Neuronpedia, 2025a. URL https://neuronpedia.org/graph/info.
Jack Lindsey, Brian Chen, Adam Pearce, Sasha Hydrie, Thomas Conerly, and Jeff Wu. Sparse mixtures of linear transforms. https://transformer-circuits.pub/2025/bulk-update/index.html, 2025b.
-
Jack Lindsey, Wes Gurnee, Emmanuel Ameisen, Brian Chen, Adam Pearce, Nicholas L. Turner, Craig Citro, David Abrahams, Shan Carter, Basil Hosmer, Jonathan Marcus, Michael Sklar, Adly Templeton, Trenton Bricken, Callum McDougall, Hoagy Cunningham, Thomas Henighan, Adam Jermyn, Andy Jones, Andrew Persic, Zhenyi Qi, T. Ben Thompson, Sam Zimmerman, Kelley Rivoire, Thomas Conerly, Chris Olah, and Joshua Batson. On the biology of a large language model. Transformer Circuits Thread, 2025c. URL https://transformer-circuits.pub/2025/attribution-graphs/biology.html.
Tal Linzen, Emmanuel Dupoux, and Yoav Goldberg. Assessing the ability of LSTMs to learn syntax-sensitive dependencies. Transactions of the Association for Computational Linguistics, 4: 521–535, 2016. . URL https://aclanthology.org/Q16-1037/.
-
Jiacheng Liu, Sewon Min, Luke Zettlemoyer, Yejin Choi, and Hannaneh Hajishirzi. Infini-gram: Scaling unbounded n-gram language models to a trillion tokens. In First Conference on Language Modeling, 2024. URL https://openreview.net/forum?id=u2vAyMeLMm.
+
Jiacheng Liu, Sewon Min, Luke Zettlemoyer, Yejin Choi, and Hannaneh Hajishirzi. Infini-gram: Scaling unbounded n-gram language models to a trillion tokens. In First Conference on Language Modeling, 2024a. URL https://openreview.net/forum?id=u2vAyMeLMm.
Jiacheng Liu, Taylor Blanton, Yanai Elazar, Sewon Min, Yen-Sung Chen, Arnavi Chheda-Kothary, Huy Tran, Byron Bischoff, Eric Marsh, Michael Schmitz, et al. Olmotrace: Tracing language model outputs back to trillions of training tokens. In Proceedings of the 63rd Annual Meeting of the Association for Computational Linguistics (Volume 3: System Demonstrations), pp. 178–188, 2025a.
-
Ziming Liu, Yixuan Wang, Sachin Vaidya, Fabian Ruehle, James Halverson, Marin Soljacic, Thomas Y. Hou, and Max Tegmark. KAN: Kolmogorovarnold networks. In The Thirteenth International Conference on Learning Representations, 2025b. URL https://openreview.net/forum?id=Ozo7qJ5vZi.
+
Leo Z. Liu, Yizhong Wang, Jungo Kasai, Hannaneh Hajishirzi, and Noah A. Smith. Probing across time: What does RoBERTa know and when? In Findings of the Association for Computational Linguistics: EMNLP 2021, pp. 820–842, 2021.
+
Sida Liu and Feijiang Han. Ica lens: Interpreting language models without training another dictionary, 2026. URL https://arxiv.org/abs/2606.11722.
+
Tianyang Liu, Canwen Xu, and Julian McAuley. Repobench: Benchmarking repository-level code auto-completion systems. In The Twelfth International Conference on Learning Representations, 2024b. URL https://openreview.net/forum?id=pPjZIOuQuF.
+
Ziming Liu, Eric J Michaud, and Max Tegmark. Omnigrok: Grokking beyond algorithmic data. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=zDiHoIWa0q1.
+
Ziming Liu, Yixuan Wang, Sachin Vaidya, Fabian Ruehle, James Halverson, Marin Soljacic, Thomas Y. Hou, and Max Tegmark. KAN: Kolmogorovarnold networks. In The Thirteenth International Conference on Learning Representations, 2025b. URL https://openreview.net/forum?id=Ozo7qJ5vZi.
+
Michelle Lo, Fazl Barez, and Shay B. Cohen. Large language models relearn removed concepts. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Findings of the Association for Computational Linguistics: ACL 2024, pp. 8306–8323, Bangkok, Thailand, August 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.findings-acl.492/.
+
Sobhan Lotfi, Ava Iranmanesh, Lachin Naghashyar, Ali Shirali, Fateme Nateghi Haredasht, Sanmi Koyejo, Philip Torr, Yong Suk Lee, Fazl Barez, Joel Lehman, Peter Norvig, and Arvind Narayanan. Position: There are futures that benchmark-driven AI cannot see. In Forty-third International Conference on Machine Learning Position Paper Track, 2026. URL https://openreview.net/forum?id=RzWlUbmRJi.
+
Ekdeep Singh Lubana, Eric J. Bigelow, Robert P. Dick, David Krueger, and Hidenori Tanaka. Mechanistic mode connectivity. In International Conference on Machine Learning, pp. 22965–23004, 2023.
Scott M. Lundberg and Su-In Lee. A unified approach to interpreting model predictions. In Proceedings of the 31st International Conference on Neural Information Processing Systems, NIPS’17, pp. 4768–4777, Red Hook, NY, USA, 2017. Curran Associates Inc. ISBN 9781510860964.
+
Ziming Luo, Atoosa Kasirzadeh, and Nihar B. Shah. The more you automate, the less you see: Hidden pitfalls of autonomous ai scientists. Proceedings of the National Academy of Sciences, 123 (41): e2610214123, 2026. . URL https://www.pnas.org/doi/abs/10.1073/pnas.2610214123.
Monte MacDiarmid, Timothy Maxwell, Nicholas Schiefer, Jesse Mu, Jared Kaplan, David Duvenaud, Sam Bowman, Alex Tamkin, Ethan Perez, Mrinank Sharma, Carson Denison, and Evan Hubinger. Simple probes can catch sleeper agents. https://www.anthropic.com/news/probes-catch-sleeper-agents, 2024.
-
Monte MacDiarmid, Benjamin Wright, Jonathan Uesato, Joe Benton, Jon Kutasov, Sara Price, Naia Bouscal, Sam Bowman, Trenton Bricken, Alex Cloud, Carson Denison, Johannes Gasteiger, Ryan Greenblatt, Jan Leike, Jack Lindsey, Vlad Mikulik, Ethan Perez, Alex Rodrigues, Drake Thomas, Albert Webson, Daniel Ziegler, and Evan Hubinger. Natural Emergent Misalignment from Reward Hacking in Production RL, November 2025. URL http://arxiv.org/abs/2511.18397. arXiv:2511.18397 [cs].
+
Monte MacDiarmid, Benjamin Wright, Jonathan Uesato, Joe Benton, Jon Kutasov, Sara Price, Naia Bouscal, Sam Bowman, Trenton Bricken, Alex Cloud, Carson Denison, Johannes Gasteiger, Ryan Greenblatt, Jan Leike, Jack Lindsey, Vlad Mikulik, Ethan Perez, Alex Rodrigues, Drake Thomas, Albert Webson, Daniel Ziegler, and Evan Hubinger. Natural emergent misalignment from reward hacking in production rl, 2025. URL https://arxiv.org/abs/2511.18397.
+
Aleksandar Makelov, Georg Lange, Atticus Geiger, and Neel Nanda. Is this the subspace you are looking for? an interpretability illusion for subspace activation patching. In The Twelfth International Conference on Learning Representations, 2024. URL https://openreview.net/forum?id=Ebt7JgMHv1.
+
S.G. Mallat and Zhifeng Zhang. Matching pursuits with time-frequency dictionaries. IEEE Transactions on Signal Processing, 41 (12): 3397–3415, 1993. .
+
Arnau Marin-Llobet and Javier Ferrando. Automated interpretability and feature discovery in language models with agents, 2026. URL https://arxiv.org/abs/2605.01555.
+
Luke Marks, Amir Abdullah, Clement Neo, Rauno Arike, David Krueger, Philip Torr, and Fazl Barez. Interpreting learned feedback patterns in large language models. In Proceedings of the 38th International Conference on Neural Information Processing Systems, NIPS ’24, Red Hook, NY, USA, 2024a. Curran Associates Inc. ISBN 9798331314385.
+
Luke Marks, Alasdair Paren, David Krueger, and Fazl Barez. Enhancing neural network interpretability with feature-aligned sparse autoencoders, 2024b. URL https://arxiv.org/abs/2411.01220.
+
Samuel Marks and Max Tegmark. The geometry of truth: Emergent linear structure in large language model representations of true/false datasets. In First Conference on Language Modeling, 2024. URL https://openreview.net/forum?id=aajyHYjjsk.
Samuel Marks, Can Rager, Eric J Michaud, Yonatan Belinkov, David Bau, and Aaron Mueller. Sparse feature circuits: Discovering and editing interpretable causal graphs in language models. In The Thirteenth International Conference on Learning Representations, 2025a. URL https://openreview.net/forum?id=I4e82CIDxv.
-
Samuel Marks, Johannes Treutlein, Trenton Bricken, Jack Lindsey, Jonathan Marcus, Siddharth Mishra-Sharma, Daniel Ziegler, Emmanuel Ameisen, Joshua Batson, Tim Belonax, Samuel R. Bowman, Shan Carter, Brian Chen, Hoagy Cunningham, Carson Denison, Florian Dietz, Satvik Golechha, Akbir Khan, Jan Kirchner, Jan Leike, Austin Meek, Kei Nishimura-Gasparian, Euan Ong, Christopher Olah, Adam Pearce, Fabien Roger, Jeanne Salle, Andy Shih, Meg Tong, Drake Thomas, Kelley Rivoire, Adam Jermyn, Monte MacDiarmid, Tom Henighan, and Evan Hubinger. Auditing language models for hidden objectives, 2025b. URL https://arxiv.org/abs/2503.10965.
+
Samuel Marks, Johannes Treutlein, Trenton Bricken, Jack Lindsey, Jonathan Marcus, Siddharth Mishra-Sharma, Daniel Ziegler, Emmanuel Ameisen, Joshua Batson, Tim Belonax, Samuel R. Bowman, Shan Carter, Brian Chen, Hoagy Cunningham, Carson Denison, Florian Dietz, Satvik Golechha, Akbir Khan, Jan Kirchner, Jan Leike, Austin Meek, Kei Nishimura-Gasparian, Euan Ong, Christopher Olah, Adam Pearce, Fabien Roger, Jeanne Salle, Andy Shih, Meg Tong, Drake Thomas, Kelley Rivoire, Adam Jermyn, Monte MacDiarmid, Tom Henighan, and Evan Hubinger. Auditing language models for hidden objectives, 2025b. URL https://arxiv.org/abs/2503.10965.
+
James Martens and Roger Grosse. Optimizing neural networks with kronecker-factored approximate curvature. In Francis Bach and David Blei (eds.), Proceedings of the 32nd International Conference on Machine Learning, volume 37 of Proceedings of Machine Learning Research, pp. 2408–2417, Lille, France, 07–09 Jul 2015. PMLR. URL https://proceedings.mlr.press/v37/martens15.html.
Rebecca Marvin and Tal Linzen. Targeted syntactic evaluation of language models. In Ellen Riloff, David Chiang, Julia Hockenmaier, and Jun’ichi Tsujii (eds.), Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing, pp. 1192–1202, Brussels, Belgium, October-November 2018. Association for Computational Linguistics. . URL https://aclanthology.org/D18-1151/.
R. Thomas McCoy, Ellie Pavlick, and Tal Linzen. Right for the wrong reasons: Diagnosing syntactic heuristics in natural language inference. In Anna Korhonen, David Traum, and Lluís Màrquez (eds.), Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pp. 3428–3448, Florence, Italy, July 2019. Association for Computational Linguistics. . URL https://aclanthology.org/P19-1334/.
+
Callum McDougall, Arthur Conmy, János Kramár, Tom Lieberum, Senthooran Rajamanoharan, and Neel Nanda. Gemma scope 2: Technical paper. Google DeepMind technical report, 2025. URL https://storage.googleapis.com/deepmind-media/DeepMind.com/Blog/gemma-scope-2-helping-the-ai-safety-community-deepen-understanding-of-complex-language-model-behavior/Gemma_Scope_2_Technical_Paper.pdf.
Thomas McGrath, Matthew Rahtz, Janos Kramar, Vladimir Mikulik, and Shane Legg. The hydra effect: Emergent self-repair in language model computations, 2023. URL https://arxiv.org/abs/2307.15771.
+
Ian R. McKenzie, Alexander Lyzhov, Michael Pieler, Alicia Parrish, Aaron Mueller, Ameya Prabhu, Euan McLean, Aaron Kirtland, Alexis Ross, Alisa Liu, Andrew Gritsevskiy, Daniel Wurgaft, Derik Kauffman, Gabriel Recchia, Jiacheng Liu, Joe Cavanagh, Max Weiss, Sicong Huang, The Floating Droid, Tom Tseng, Tomasz Korbak, Xudong Shen, Yuhui Zhang, Zhengping Zhou, Najoung Kim, Samuel R. Bowman, and Ethan Perez. Inverse scaling: When bigger isn’t better. Transactions on Machine Learning Research, 2023. URL https://arxiv.org/abs/2306.09479.
+
Maxime Méloux, Silviu Maniu, François Portet, and Maxime Peyrard. Everything, everywhere, all at once: Is mechanistic interpretability identifiable? In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=5IWJBStfU7.
Kevin Meng, David Bau, Alex Andonian, and Yonatan Belinkov. Locating and editing factual associations in gpt. Advances in neural information processing systems, 35: 17359–17372, 2022.
Kevin Meng, Arnab Sen Sharma, Alex J Andonian, Yonatan Belinkov, and David Bau. Mass-editing memory in a transformer. In The Eleventh International Conference on Learning Representations, 2023.
Jack Merullo, Connor Watts, Max Loeffler, Liv Gorton, Elana Simon, Tom McGrath, and Owen Lewis. Replicating circuit tracing for a simple known mechanism. https://www.goodfire.ai/research/replicating-circuit-tracing-for-a-simple-mechanism, 2025.
+
Eric J. Michaud, Ziming Liu, Uzay Girit, and Max Tegmark. The quantization model of neural scaling. In Advances in Neural Information Processing Systems, volume 36, 2023. URL https://openreview.net/forum?id=3tbTw2ga8K.
+
Joseph Miller, Bilal Chughtai, and William Saunders. Transformer circuit evaluation metrics are not robust. In First Conference on Language Modeling, 2024. URL https://openreview.net/forum?id=zSf8PJyQb2.
Moritz Miller, Florent Draye, and Bernhard Schölkopf. Identifying intervenable and interpretable features via orthogonality regularization, 2026. URL https://arxiv.org/abs/2602.04718.
-
Julian Minder, Clément Dumas, Caden Juang, Bilal Chugtai, and Neel Nanda. Overcoming Sparsity Artifacts in Crosscoders to Interpret Chat-Tuning, October 2025. URL http://arxiv.org/abs/2504.02922. arXiv:2504.02922 [cs].
-
Bruno Kacper Mlodozeniec, Isaac Reid, Samuel Power, David Krueger, Murat A Erdogdu, Richard E. Turner, and Roger Baker Grosse. Distributional training data attribution: What do influence functions sample? In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025. URL https://openreview.net/forum?id=UBRFn7YKMe.
-
Ari S. Morcos, Maithra Raghu, and Samy Bengio. Insights on representational similarity in neural networks with canonical correlation, 2018a. URL https://arxiv.org/abs/1806.05759.
-
Ari S. Morcos, Maithra Raghu, and Samy Bengio. Insights on representational similarity in neural networks with canonical correlation. In Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS’18, pp. 5732–5741, Red Hook, NY, USA, 2018b. Curran Associates Inc. URL https://dl.acm.org/doi/10.5555/3327345.3327475.
-
Luca Moschella, Valentino Maiorca, Marco Fumero, Antonio Norelli, Francesco Locatello, and Emanuele Rodolà. Relative representations enable zero-shot latent space communication, 2023. URL https://arxiv.org/abs/2209.15430.
-
Aaron Mueller, Atticus Geiger, Sarah Wiegreffe, Dana Arad, Iván Arcuschin, Adam Belfki, Yik Siu Chan, Jaden Fried Fiotto-Kaufman, Tal Haklay, Michael Hanna, Jing Huang, Rohan Gupta, Yaniv Nikankin, Hadas Orgad, Nikhil Prakash, Anja Reusch, Aruna Sankaranarayanan, Shun Shao, Alessandro Stolfo, Martin Tutek, Amir Zur, David Bau, and Yonatan Belinkov. MIB: A mechanistic interpretability benchmark. In Forty-second International Conference on Machine Learning, 2025a. URL https://openreview.net/forum?id=sSrOwve6vb.
-
Aaron Mueller, Andrew Lee, Shruti Joshi, Ekdeep Singh Lubana, Dhanya Sridhar, and Patrik Reizinger. From isolation to entanglement: When do interpretability methods identify and disentangle known concepts?, 2025b. URL https://arxiv.org/abs/2512.15134.
+
Marvin L. Minsky and Seymour A. Papert. Perceptrons: An Introduction to Computational Geometry. MIT Press, Cambridge, MA, 1969.
+
Bruno Kacper Mlodozeniec, Isaac Reid, Samuel Power, David Krueger, Murat A Erdogdu, Richard E. Turner, and Roger Baker Grosse. Distributional training data attribution: What do influence functions sample? In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025. URL https://openreview.net/forum?id=UBRFn7YKMe.
+
Ali Modarressi, Hanieh Deilamsalehy, Franck Dernoncourt, Trung Bui, Ryan A. Rossi, Seunghyun Yoon, and Hinrich Schuetze. Nolima: Long-context evaluation beyond literal matching. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=0OshX1hiSa.
+
Ari S. Morcos, Maithra Raghu, and Samy Bengio. Insights on representational similarity in neural networks with canonical correlation. In Proceedings of the 32nd International Conference on Neural Information Processing Systems, NIPS’18, pp. 5732–5741, Red Hook, NY, USA, 2018. Curran Associates Inc.
+
Luca Moschella, Valentino Maiorca, Marco Fumero, Antonio Norelli, Francesco Locatello, and Emanuele Rodolà. Relative representations enable zero-shot latent space communication. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=SrC-nwieGJ.
+
Aaron Mueller, Atticus Geiger, Sarah Wiegreffe, Dana Arad, Iván Arcuschin, Adam Belfki, Yik Siu Chan, Jaden Fried Fiotto-Kaufman, Tal Haklay, Michael Hanna, Jing Huang, Rohan Gupta, Yaniv Nikankin, Hadas Orgad, Nikhil Prakash, Anja Reusch, Aruna Sankaranarayanan, Shun Shao, Alessandro Stolfo, Martin Tutek, Amir Zur, David Bau, and Yonatan Belinkov. MIB: A mechanistic interpretability benchmark. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=sSrOwve6vb.
+
Vinod Nair and Geoffrey E. Hinton. Rectified linear units improve restricted boltzmann machines. In Proceedings of the 27th International Conference on International Conference on Machine Learning, ICML’10, pp. 807–814, Madison, WI, USA, 2010. Omnipress. ISBN 9781605589077.
+
Preetum Nakkiran, Gal Kaplun, Dimitris Kalimeris, Tristan Yang, Benjamin L. Edelman, Fred Zhang, and Boaz Barak. SGD on neural networks learns functions of increasing complexity. Curran Associates Inc., Red Hook, NY, USA, 2019.
Neel Nanda, Lawrence Chan, Tom Lieberum, Jess Smith, and Jacob Steinhardt. Progress measures for grokking via mechanistic interpretability. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=9XFSbDPmdW.
Neel Nanda, Josh Engel, Arthur Conmy, Senthooran Rajamanoharan, bilalchughtai, CallumMcDougall, János Kramár, and lewis smith. A pragmatic vision for interpretability. Alignment Forum, 2025. URL https://www.alignmentforum.org/posts/StENzDcD3kpfGJssR/a-pragmatic-vision-for-interpretability.
-
Joe Needham, Giles Edkins, Govind Pimpale, Henning Bartsch, and Marius Hobbhahn. Large language models often know when they are being evaluated, 2025. URL https://arxiv.org/abs/2505.23836.
+
Joe Needham, Giles Edkins, Govind Pimpale, Henning Bartsch, and Marius Hobbhahn. Large language models often know when they are being evaluated, 2025. URL https://arxiv.org/abs/2505.23836.
+
Clement Neo, Shay B. Cohen, and Fazl Barez. Interpreting context look-ups in transformers: Investigating attention-MLP interactions. In Yaser Al-Onaizan, Mohit Bansal, and Yun-Nung Chen (eds.), Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, pp. 16681–16697, Miami, Florida, USA, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.emnlp-main.930/.
+
Clement Neo, Luke Ong, Philip Torr, Mor Geva, David Krueger, and Fazl Barez. Towards interpreting visual information processing in vision-language models. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=chanJGoa7f.
Andrew Ng. Sparse autoencoder. Stanford CS294A Lecture Notes, 2011.
+
Yaniv Nikankin, Anja Reusch, Aaron Mueller, and Yonatan Belinkov. Arithmetic without algorithms: Language models solve math with a bag of heuristics. In Y. Yue, A. Garg, N. Peng, F. Sha, and R. Yu (eds.), International Conference on Learning Representations, volume 2025, pp. 55939–55965, 2025. URL https://proceedings.iclr.cc/paper_files/paper/2025/file/8c5f30296296d2ae402ebbd09aaa9c12-Paper-Conference.pdf.
nostalgebraist. Interpreting gpt: The logit lens. LessWrong post, 2020. URL https://www.lesswrong.com/posts/AcKRB8wDpdaN6v6ru/interpreting-gpt-the-logit-lens.
+
Kyle O’Brien, Stephen Casper, Quentin Gregory Anthony, Tomek Korbak, Robert Kirk, Xander Davies, Ishan Mishra, Geoffrey Irving, Yarin Gal, and Stella Biderman. Deep ignorance: Filtering pretraining data builds tamper-resistant safeguards into open-weight LLMs. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=xcf0QcTcGS.
+
Tuomas Oikarinen and Tsui-Wei Weng. CLIP-dissect: Automatic description of neuron representations in deep vision networks. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=iPWiwWHc1V.
Tuomas Oikarinen, Subhro Das, Lam M. Nguyen, and Tsui-Wei Weng. Label-free concept bottleneck models. In The Eleventh International Conference on Learning Representations, 2023.
+
Chris Olah, Alexander Mordvintsev, and Ludwig Schubert. Feature visualization. Distill, 2017. . https://distill.pub/2017/feature-visualization.
Chris Olah, Nick Cammarata, Ludwig Schubert, Gabriel Goh, Michael Petrov, and Shan Carter. Zoom in: An introduction to circuits. Distill, 2020. . https://distill.pub/2020/circuits/zoom-in.
James Oldfield, Markos Georgopoulos, Grigorios Chrysos, Christos Tzelepis, Yannis Panagakis, Mihalis Nicolaou, Jiankang Deng, and Ioannis Patras. Multilinear mixture of experts: Scalable expert specialization through factorization. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024.
James Oldfield, Shawn Im, Sharon Li, Mihalis Nicolaou, Ioannis Patras, and Grigorios Chrysos. Towards interpretability without sacrifice: Faithful dense layer decomposition with mixture of decoders. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025. URL https://openreview.net/forum?id=jcvX8XFNqX.
-
Bruno A. Olshausen and David J. Field. Emergence of simple-cell receptive field properties by learning a sparse code for natural images. Nature, 381: 607–609, 1996. .
-
Catherine Olsson, Nelson Elhage, Neel Nanda, Nicholas Joseph, Nova DasSarma, Tom Henighan, Ben Mann, Amanda Askell, Yuntao Bai, Anna Chen, Tom Conerly, Dawn Drain, Deep Ganguli, Zac Hatfield-Dodds, Danny Hernandez, Scott Johnston, Andy Jones, Jackson Kernion, Liane Lovitt, Kamal Ndousse, Dario Amodei, Tom Brown, Jack Clark, Jared Kaplan, Sam McCandlish, and Chris Olah. In-context learning and induction heads. Transformer Circuits Thread, 2022a. https://transformer-circuits.pub/2022/in-context-learning-and-induction-heads/index.html.
-
Catherine Olsson et al. In-context learning and induction heads. arXiv preprint arXiv:2209.11895, 2022b.
-
Narmeen Fatimah Oozeer, Luke Marks, Fazl Barez, and Amir Abdullah. Beyond linear steering: Unified multi-attribute control for language models. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Findings of the Association for Computational Linguistics: EMNLP 2025, pp. 23513–23557, Suzhou, China, November 2025a. Association for Computational Linguistics. ISBN 979-8-89176-335-7. . URL https://aclanthology.org/2025.findings-emnlp.1278/.
-
Narmeen Fatimah Oozeer, Dhruv Nathawani, Nirmalendu Prakash, Michael Lan, Abir HARRASSE, and Amir Abdullah. Activation space interventions can be transferred between large language models. In Forty-second International Conference on Machine Learning, 2025b. URL https://openreview.net/forum?id=HXOicJsmMQ.
+
James Oldfield, Philip Torr, Ioannis Patras, Adel Bibi, and Fazl Barez. Beyond linear probes: Dynamic safety monitoring for language models. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=AGWa8whf92.
+
Bruno A. Olshausen and David J. Field. Emergence of simple-cell receptive field properties by learning a sparse code for natural images. Nature, 381 (6583): 607–609, June 1996. .
+
Bruno A. Olshausen and David J. Field. Sparse coding with an overcomplete basis set: A strategy employed by V1? Vision Research, 37 (23): 3311–3325, December 1997. .
+
Catherine Olsson, Nelson Elhage, Neel Nanda, Nicholas Joseph, Nova DasSarma, Tom Henighan, Ben Mann, Amanda Askell, Yuntao Bai, Anna Chen, Tom Conerly, Dawn Drain, Deep Ganguli, Zac Hatfield-Dodds, Danny Hernandez, Scott Johnston, Andy Jones, Jackson Kernion, Liane Lovitt, Kamal Ndousse, Dario Amodei, Tom Brown, Jack Clark, Jared Kaplan, Sam McCandlish, and Chris Olah. In-context learning and induction heads. Transformer Circuits Thread, 2022. https://transformer-circuits.pub/2022/in-context-learning-and-induction-heads/index.html.
+
Christian W. Omlin and C.Lee Giles. Extraction of rules from discrete-time recurrent neural networks. Neural Networks, 9 (1): 41–52, 1996. ISSN 0893-6080. . URL https://www.sciencedirect.com/science/article/pii/0893608095000860.
+
Charles O’Neill, Alim Gumran, and David Klindt. Compute optimal inference and provable amortisation gap in sparse autoencoders. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=8forr1FkvC.
+
Narmeen Fatimah Oozeer, Luke Marks, Fazl Barez, and Amir Abdullah. Beyond linear steering: Unified multi-attribute control for language models. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Findings of the Association for Computational Linguistics: EMNLP 2025, pp. 23513–23557, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-335-7. . URL https://aclanthology.org/2025.findings-emnlp.1278/.
OpenAI, Josh Achiam, Steven Adler, Sandhini Agarwal, Lama Ahmad, Ilge Akkaya, Florencia Leoni Aleman, Diogo Almeida, Janko Altenschmidt, Sam Altman, Shyamal Anadkat, Red Avila, Igor Babuschkin, Suchir Balaji, Valerie Balcom, Paul Baltescu, Haiming Bao, Mohammad Bavarian, Jeff Belgum, Irwan Bello, Jake Berdine, Gabriel Bernadett-Shapiro, Christopher Berner, Lenny Bogdonoff, Oleg Boiko, Madelaine Boyd, Anna-Luisa Brakman, Greg Brockman, Tim Brooks, Miles Brundage, Kevin Button, Trevor Cai, Rosie Campbell, Andrew Cann, Brittany Carey, Chelsea Carlson, Rory Carmichael, Brooke Chan, Che Chang, Fotis Chantzis, Derek Chen, Sully Chen, Ruby Chen, Jason Chen, Mark Chen, Ben Chess, Chester Cho, Casey Chu, Hyung Won Chung, Dave Cummings, Jeremiah Currier, Yunxing Dai, Cory Decareaux, Thomas Degry, Noah Deutsch, Damien Deville, Arka Dhar, David Dohan, Steve Dowling, Sheila Dunning, Adrien Ecoffet, Atty Eleti, Tyna Eloundou, David Farhi, Liam Fedus, Niko Felix, Simón Posada Fishman, Juston Forte, Isabella Fulford, Leo Gao, Elie Georges, Christian Gibson, Vik Goel, Tarun Gogineni, Gabriel Goh, Rapha Gontijo-Lopes, Jonathan Gordon, Morgan Grafstein, Scott Gray, Ryan Greene, Joshua Gross, Shixiang Shane Gu, Yufei Guo, Chris Hallacy, Jesse Han, Jeff Harris, Yuchen He, Mike Heaton, Johannes Heidecke, Chris Hesse, Alan Hickey, Wade Hickey, Peter Hoeschele, Brandon Houghton, Kenny Hsu, Shengli Hu, Xin Hu, Joost Huizinga, Shantanu Jain, Shawn Jain, Joanne Jang, Angela Jiang, Roger Jiang, Haozhun Jin, Denny Jin, Shino Jomoto, Billie Jonn, Heewoo Jun, Tomer Kaftan, Łukasz Kaiser, Ali Kamali, Ingmar Kanitscheider, Nitish Shirish Keskar, Tabarak Khan, Logan Kilpatrick, Jong Wook Kim, Christina Kim, Yongjik Kim, Jan Hendrik Kirchner, Jamie Kiros, Matt Knight, Daniel Kokotajlo, Łukasz Kondraciuk, Andrew Kondrich, Aris Konstantinidis, Kyle Kosic, Gretchen Krueger, Vishal Kuo, Michael Lampe, Ikai Lan, Teddy Lee, Jan Leike, Jade Leung, Daniel Levy, Chak Ming Li, Rachel Lim, Molly Lin, Stephanie Lin, Mateusz Litwin, Theresa Lopez, Ryan Lowe, Patricia Lue, Anna Makanju, Kim Malfacini, Sam Manning, Todor Markov, Yaniv Markovski, Bianca Martin, Katie Mayer, Andrew Mayne, Bob McGrew, Scott Mayer McKinney, Christine McLeavey, Paul McMillan, Jake McNeil, David Medina, Aalok Mehta, Jacob Menick, Luke Metz, Andrey Mishchenko, Pamela Mishkin, Vinnie Monaco, Evan Morikawa, Daniel Mossing, Tong Mu, Mira Murati, Oleg Murk, David Mély, Ashvin Nair, Reiichiro Nakano, Rajeev Nayak, Arvind Neelakantan, Richard Ngo, Hyeonwoo Noh, Long Ouyang, Cullen O’Keefe, Jakub Pachocki, Alex Paino, Joe Palermo, Ashley Pantuliano, Giambattista Parascandolo, Joel Parish, Emy Parparita, Alex Passos, Mikhail Pavlov, Andrew Peng, Adam Perelman, Filipe de Avila Belbute Peres, Michael Petrov, Henrique Ponde de Oliveira Pinto, Michael, Pokorny, Michelle Pokrass, Vitchyr H. Pong, Tolly Powell, Alethea Power, Boris Power, Elizabeth Proehl, Raul Puri, Alec Radford, Jack Rae, Aditya Ramesh, Cameron Raymond, Francis Real, Kendra Rimbach, Carl Ross, Bob Rotsted, Henri Roussez, Nick Ryder, Mario Saltarelli, Ted Sanders, Shibani Santurkar, Girish Sastry, Heather Schmidt, David Schnurr, John Schulman, Daniel Selsam, Kyla Sheppard, Toki Sherbakov, Jessica Shieh, Sarah Shoker, Pranav Shyam, Szymon Sidor, Eric Sigler, Maddie Simens, Jordan Sitkin, Katarina Slama, Ian Sohl, Benjamin Sokolowsky, Yang Song, Natalie Staudacher, Felipe Petroski Such, Natalie Summers, Ilya Sutskever, Jie Tang, Nikolas Tezak, Madeleine B. Thompson, Phil Tillet, Amin Tootoonchian, Elizabeth Tseng, Preston Tuggle, Nick Turley, Jerry Tworek, Juan Felipe Cerón Uribe, Andrea Vallone, Arun Vijayvergiya, Chelsea Voss, Carroll Wainwright, Justin Jay Wang, Alvin Wang, Ben Wang, Jonathan Ward, Jason Wei, CJ Weinmann, Akila Welihinda, Peter Welinder, Jiayi Weng, Lilian Weng, Matt Wiethoff, Dave Willner, Clemens Winter, Samuel Wolrich, Hannah Wong, Lauren Workman, Sherwin Wu, Jeff Wu, Michael Wu, Kai Xiao, Tao Xu, Sarah Yoo, Kevin Yu, Qiming Yuan, Wojciech Zaremba, Rowan Zellers, Chong Zhang, Marvin Zhang, Shengjia Zhao, Tianhao Zheng, Juntang Zhuang, William Zhuk, and Barret Zoph. Gpt-4 technical report, 2024. URL https://arxiv.org/abs/2303.08774.
-
Hadas Orgad, Fazl Barez, Tal Haklay, Isabelle Lee, Marius Mosbach, Anja Reusch, Naomi Saphra, Byron Wallace, Sarah Wiegreffe, Eric Wong, Ian Tenney, and Mor Geva. Interpretability can be actionable, 2026. URL https://arxiv.org/abs/2605.11161.
+
OpenAI, :, Sandhini Agarwal, Lama Ahmad, Jason Ai, Sam Altman, Andy Applebaum, Edwin Arbus, Rahul K. Arora, Yu Bai, Bowen Baker, Haiming Bao, Boaz Barak, Ally Bennett, Tyler Bertao, Nivedita Brett, Eugene Brevdo, Greg Brockman, Sebastien Bubeck, Che Chang, Kai Chen, Mark Chen, Enoch Cheung, Aidan Clark, Dan Cook, Marat Dukhan, Casey Dvorak, Kevin Fives, Vlad Fomenko, Timur Garipov, Kristian Georgiev, Mia Glaese, Tarun Gogineni, Adam Goucher, Lukas Gross, Katia Gil Guzman, John Hallman, Jackie Hehir, Johannes Heidecke, Alec Helyar, Haitang Hu, Romain Huet, Jacob Huh, Saachi Jain, Zach Johnson, Chris Koch, Irina Kofman, Dominik Kundel, Jason Kwon, Volodymyr Kyrylov, Elaine Ya Le, Guillaume Leclerc, James Park Lennon, Scott Lessans, Mario Lezcano-Casado, Yuanzhi Li, Zhuohan Li, Ji Lin, Jordan Liss, Lily, Liu, Jiancheng Liu, Kevin Lu, Chris Lu, Zoran Martinovic, Lindsay McCallum, Josh McGrath, Scott McKinney, Aidan McLaughlin, Song Mei, Steve Mostovoy, Tong Mu, Gideon Myles, Alexander Neitz, Alex Nichol, Jakub Pachocki, Alex Paino, Dana Palmie, Ashley Pantuliano, Giambattista Parascandolo, Jongsoo Park, Leher Pathak, Carolina Paz, Ludovic Peran, Dmitry Pimenov, Michelle Pokrass, Elizabeth Proehl, Huida Qiu, Gaby Raila, Filippo Raso, Hongyu Ren, Kimmy Richardson, David Robinson, Bob Rotsted, Hadi Salman, Suvansh Sanjeev, Max Schwarzer, D. Sculley, Harshit Sikchi, Kendal Simon, Karan Singhal, Yang Song, Dane Stuckey, Zhiqing Sun, Philippe Tillet, Sam Toizer, Foivos Tsimpourlas, Nikhil Vyas, Eric Wallace, Xin Wang, Miles Wang, Olivia Watkins, Kevin Weil, Amy Wendling, Kevin Whinnery, Cedric Whitney, Hannah Wong, Lin Yang, Yu Yang, Michihiro Yasunaga, Kristen Ying, Wojciech Zaremba, Wenting Zhan, Cyril Zhang, Brian Zhang, Eddie Zhang, and Shengjia Zhao. gpt-oss-120b & gpt-oss-20b model card, 2025. URL https://arxiv.org/abs/2508.10925.
+
Hadas Orgad, Fazl Barez, Tal Haklay, Isabelle Lee, Marius Mosbach, Anja Reusch, Naomi Saphra, Byron C Wallace, Sarah Wiegreffe, Eric Wong, Ian Tenney, and Mor Geva. Position: Interpretability can be actionable. In Forty-third International Conference on Machine Learning Position Paper Track, 2026. URL https://openreview.net/forum?id=yRUy3w5y56.
+
Keiron O’Shea and Ryan Nash. An introduction to convolutional neural networks, 2015. URL https://arxiv.org/abs/1511.08458.
Simon Ott, Adriano Barbosa-Silva, Kathrin Blagec, Jan Brauner, and Matthias Samwald. Mapping global dynamics of benchmark creation and saturation in artificial intelligence. Nature Communications, 13 (1): 6793, 2022. ISSN 2041-1723. . URL https://www.nature.com/articles/s41467-022-34591-0.
-
Long Ouyang, Jeffrey Wu, Xu Jiang, Diogo Almeida, Carroll Wainwright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, John Schulman, Jacob Hilton, Fraser Kelton, Luke Miller, Maddie Simens, Amanda Askell, Peter Welinder, Paul F Christiano, Jan Leike, and Ryan Lowe. Training language models to follow instructions with human feedback. In S. Koyejo, S. Mohamed, A. Agarwal, D. Belgrave, K. Cho, and A. Oh (eds.), Advances in Neural Information Processing Systems, volume 35, pp. 27730–27744. Curran Associates, Inc., 2022. URL https://proceedings.neurips.cc/paper_files/paper/2022/file/b1efde53be364a73914f58805a001731-Paper-Conference.pdf.
-
Alexander Pan, Lijie Chen, and Jacob Steinhardt. Latentqa: Teaching llms to decode activations into natural language, 2024. URL https://arxiv.org/abs/2412.08686.
+
Long Ouyang, Jeffrey Wu, Xu Jiang, Diogo Almeida, Carroll Wainwright, Pamela Mishkin, Chong Zhang, Sandhini Agarwal, Katarina Slama, Alex Ray, John Schulman, Jacob Hilton, Fraser Kelton, Luke Miller, Maddie Simens, Amanda Askell, Peter Welinder, Paul F Christiano, Jan Leike, and Ryan Lowe. Training language models to follow instructions with human feedback. In S. Koyejo, S. Mohamed, A. Agarwal, D. Belgrave, K. Cho, and A. Oh (eds.), Advances in Neural Information Processing Systems, volume 35, pp. 27730–27744. Curran Associates, Inc., 2022. URL https://proceedings.neurips.cc/paper_files/paper/2022/file/b1efde53be364a73914f58805a001731-Paper-Conference.pdf.
+
Alexander Pan, Lijie Chen, and Jacob Steinhardt. LatentQA: Teaching LLMs to decode activations into natural language. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=niUroX9EOd.
Jungwoo Park, Ahn Young Jin, Kee-Eung Kim, and Jaewoo Kang. Monet: Mixture of monosemantic experts for transformers. In The Thirteenth International Conference on Learning Representations, 2025.
+
Kiho Park, Yo Joong Choe, and Victor Veitch. The linear representation hypothesis and the geometry of large language models. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.
Sung Min Park, Kristian Georgiev, Andrew Ilyas, Guillaume Leclerc, and Aleksander Madry. TRAK: Attributing model behavior at scale. In Andreas Krause, Emma Brunskill, Kyunghyun Cho, Barbara Engelhardt, Sivan Sabato, and Jonathan Scarlett (eds.), Proceedings of the 40th International Conference on Machine Learning, volume 202 of Proceedings of Machine Learning Research, pp. 27074–27113. PMLR, 23–29 Jul 2023. URL https://proceedings.mlr.press/v202/park23c.html.
-
Gonçalo Paulo and Nora Belrose. Sparse autoencoders trained on the same data learn different features. arXiv preprint arXiv:2501.16615, 2025.
-
Gonçalo Santos Paulo, Alex Troy Mallen, Caden Juang, and Nora Belrose. Automatically interpreting millions of features in large language models. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=EemtbhJOXc.
+
Gonçalo Paulo and Nora Belrose. Sparse autoencoders trained on the same data learn different features. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=EjInprGpk9.
+
Gonçalo Paulo, Thomas Marshall, and Nora Belrose. Do transformer interpretability methods transfer to rnns? In Proceedings of the Thirty-Ninth AAAI Conference on Artificial Intelligence and Thirty-Seventh Conference on Innovative Applications of Artificial Intelligence and Fifteenth Symposium on Educational Advances in Artificial Intelligence, AAAI’25/IAAI’25/EAAI’25. AAAI Press, 2025a. ISBN 978-1-57735-897-8. . URL https://doi.org/10.1609/aaai.v39i26.34969.
+
Gonçalo Santos Paulo, Alex Troy Mallen, Caden Juang, and Nora Belrose. Automatically interpreting millions of features in large language models. In Forty-second International Conference on Machine Learning, 2025b. URL https://openreview.net/forum?id=EemtbhJOXc.
+
Gonçalo Paulo, Stepan Shabalin, and Nora Belrose. Transcoders beat sparse autoencoders for interpretability, 2025c. URL https://arxiv.org/abs/2501.18823.
Michael Pearce, Thomas Dooms, Alice Rigg, Jose Oramas, and Lee Sharkey. Bilinear mlps enable weight-based mechanistic interpretability. In Y. Yue, A. Garg, N. Peng, F. Sha, and R. Yu (eds.), International Conference on Learning Representations, volume 2025, pp. 47283–47310, 2025. URL https://proceedings.iclr.cc/paper_files/paper/2025/file/7504142a20a3e1fe9dd7de42f475828c-Paper-Conference.pdf.
-
Ethan Perez, Saffron Huang, Francis Song, Trevor Cai, Roman Ring, John Aslanides, Amelia Glaese, Nat McAleese, and Geoffrey Irving. Red teaming language models with language models. In Yoav Goldberg, Zornitsa Kozareva, and Yue Zhang (eds.), Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, pp. 3419–3448, Abu Dhabi, United Arab Emirates, December 2022a. Association for Computational Linguistics. . URL https://aclanthology.org/2022.emnlp-main.225/.
-
Ethan Perez, Saffron Huang, H. Francis Song, Trevor Cai, Roman Ring, John Aslanides, Amelia Glaese, Nat McAleese, and Geoffrey Irving. Red teaming language models with language models. In Yoav Goldberg, Zornitsa Kozareva, and Yue Zhang (eds.), Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, EMNLP 2022, Abu Dhabi, United Arab Emirates, December 7-11, 2022, pp. 3419–3448. Association for Computational Linguistics, 2022b. . URL https://doi.org/10.18653/v1/2022.emnlp-main.225.
+
Karl Pearson. On lines and planes of closest fit to systems of points in space. The London, Edinburgh, and Dublin Philosophical Magazine and Journal of Science, 2 (11): 559–572, 1901. .
+
Bo Peng, Eric Alcaide, Quentin Anthony, Alon Albalak, Samuel Arcadinho, Stella Biderman, Huanqi Cao, Xin Cheng, Michael Chung, Leon Derczynski, Xingjian Du, Matteo Grella, Kranthi Gv, Xuzheng He, Haowen Hou, Przemyslaw Kazienko, Jan Kocon, Jiaming Kong, Bartłomiej Koptyra, Hayden Lau, Jiaju Lin, Krishna Sri Ipsit Mantri, Ferdinand Mom, Atsushi Saito, Guangyu Song, Xiangru Tang, Johan Wind, Stanisław Woźniak, Zhenyuan Zhang, Qinghua Zhou, Jian Zhu, and Rui-Jie Zhu. RWKV: Reinventing RNNs for the transformer era. In Houda Bouamor, Juan Pino, and Kalika Bali (eds.), Findings of the Association for Computational Linguistics: EMNLP 2023, pp. 14048–14077, Singapore, December 2023. Association for Computational Linguistics. . URL https://aclanthology.org/2023.findings-emnlp.936/.
+
Bo Peng, Ruichong Zhang, Daniel Goldstein, Eric Alcaide, Xingjian Du, Haowen Hou, Jiaju Lin, Jiaxing Liu, Janna Lu, William Merrill, Guangyu Song, Kaifeng Tan, Saiteja Utpala, Nathan Wilce, Johan S. Wind, Tianyi Wu, Daniel Wuttke, and Christian Zhou-Zheng. RWKV-7 ”goose” with expressive dynamic state evolution. In Second Conference on Language Modeling, 2025. URL https://openreview.net/forum?id=ayB1PACN5j.
+
Ethan Perez, Saffron Huang, Francis Song, Trevor Cai, Roman Ring, John Aslanides, Amelia Glaese, Nat McAleese, and Geoffrey Irving. Red teaming language models with language models. In Yoav Goldberg, Zornitsa Kozareva, and Yue Zhang (eds.), Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing, pp. 3419–3448, Abu Dhabi, United Arab Emirates, December 2022. Association for Computational Linguistics. . URL https://aclanthology.org/2022.emnlp-main.225/.
+
Ivo Petrov, Jasper Dekoninck, and Martin Vechev. Brokenmath: A benchmark for sycophancy in theorem proving with LLMs. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=Y5bEI4cuMd.
Long Phan, Alice Gatti, Ziwen Han, Nathaniel Li, et al. A benchmark of expert-level academic questions to assess ai capabilities. Nature, 649 (8099): 1139–1146, 2026. ISSN 1476-4687. . URL https://www.nature.com/articles/s41586-025-09962-4.
-
Aleksandra Piktus, Christopher Akiki, Paulo Villegas, Hugo Laurençon, Gérard Dupont, Sasha Luccioni, Yacine Jernite, and Anna Rogers. The ROOTS search tool: Data transparency for LLMs. In Danushka Bollegala, Ruihong Huang, and Alan Ritter (eds.), Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 3: System Demonstrations), pp. 304–314, Toronto, Canada, July 2023a. Association for Computational Linguistics. . URL https://aclanthology.org/2023.acl-demo.29/.
-
Aleksandra Piktus, Odunayo Ogundepo, Christopher Akiki, Akintunde Oladipo, Xinyu Zhang, Hailey Schoelkopf, Stella Biderman, Martin Potthast, and Jimmy Lin. Gaia search: Hugging face and pyserini interoperability for nlp training data exploration. In Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 3: System Demonstrations), pp. 588–598, 2023b.
-
Alethea Power, Yuri Burda, Harri Edwards, Igor Babuschkin, and Vedant Misra. Grokking: Generalization beyond overfitting on small algorithmic datasets, 2022. URL https://arxiv.org/abs/2201.02177.
-
Garima Pruthi, Frederick Liu, Mukund Sundararajan, and Satyen Kale. Estimating training data influence by tracing gradient descent, 2020. URL https://arxiv.org/abs/2002.08484.
+
Aleksandra Piktus, Christopher Akiki, Paulo Villegas, Hugo Laurençon, Gérard Dupont, Sasha Luccioni, Yacine Jernite, and Anna Rogers. The ROOTS search tool: Data transparency for LLMs. In Danushka Bollegala, Ruihong Huang, and Alan Ritter (eds.), Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 3: System Demonstrations), pp. 304–314, Toronto, Canada, July 2023a. Association for Computational Linguistics. . URL https://aclanthology.org/2023.acl-demo.29/.
+
Aleksandra Piktus, Odunayo Ogundepo, Christopher Akiki, Akintunde Oladipo, Xinyu Zhang, Hailey Schoelkopf, Stella Biderman, Martin Potthast, and Jimmy Lin. Gaia search: Hugging face and pyserini interoperability for nlp training data exploration. In Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 3: System Demonstrations), pp. 588–598, 2023b.
+
Dillon Plunkett, Adam Morris, Keerthi Reddy, and Jorge Morales. Self-interpretability: Llms can describe complex internal processes that drive their decisions, 2025. URL https://arxiv.org/abs/2505.17120.
+
Alethea Power, Yuri Burda, Harri Edwards, Igor Babuschkin, and Vedant Misra. Grokking: Generalization beyond overfitting on small algorithmic datasets, 2022. URL https://arxiv.org/abs/2201.02177.
Philip Quirke and Fazl Barez. Understanding addition in transformers. In The Twelfth International Conference on Learning Representations, 2024. URL https://openreview.net/forum?id=rIx1YXVWZb.
+
Philip Quirke, Clement Neo, and Fazl Barez. Understanding addition and subtraction in transformers, 2026. URL https://arxiv.org/abs/2402.02619.
QwenAI. Qwen3 technical report, 2025. URL https://arxiv.org/abs/2505.09388.
-
Maithra Raghu, Justin Gilmer, Jason Yosinski, and Jascha Sohl-Dickstein. SVCCA: Singular Vector Canonical Correlation Analysis for Deep Learning Dynamics and Interpretability. In Advances in Neural Information Processing Systems, volume 30. Curran Associates, Inc., 2017. URL https://proceedings.neurips.cc/paper_files/paper/2017/hash/dc6a7e655d7e5840e66733e9ee67cc69-Abstract.html.
-
Daking Rai, Yilun Zhou, Shi Feng, Abulhair Saparov, and Ziyu Yao. A practical review of mechanistic interpretability for transformer-based language models. arXiv preprint arXiv:2407.02646, 2024.
-
Senthooran Rajamanoharan, Arthur Conmy, Lewis Smith, Tom Lieberum, Vikrant Varma, János Kramár, Rohin Shah, and Neel Nanda. Improving dictionary learning with gated sparse autoencoders, 2024a. URL https://arxiv.org/abs/2404.16014.
-
Senthooran Rajamanoharan, Tom Lieberum, Nicolas Sonnerat, Arthur Conmy, Vikrant Varma, János Kramár, and Neel Nanda. Jumping ahead: Improving reconstruction fidelity with jumprelu sparse autoencoders, 2024b. URL https://arxiv.org/abs/2407.14435.
-
Nazneen Fatema Rajani, Ben Krause, Wengpeng Yin, Tong Niu, Richard Socher, and Caiming Xiong. Explaining and improving model behavior with k nearest neighbor representations, 2020. URL https://arxiv.org/abs/2010.09030.
-
Tilman Raukur, An Chang Ho, Stephen Casper, and Dylan Hadfield-Menell. Toward transparent ai: A survey on interpreting the inner structures of deep neural networks. 2023 IEEE Conference on Secure and Trustworthy Machine Learning (SaTML), pp. 464–483, 2022.
-
Shauli Ravfogel, Yanai Elazar, Hila Gonen, Michael Twiton, and Yoav Goldberg. Null it out: Guarding protected attributes by iterative nullspace projection. In Dan Jurafsky, Joyce Chai, Natalie Schluter, and Joel Tetreault (eds.), Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 7237–7256, Online, July 2020. Association for Computational Linguistics. . URL https://aclanthology.org/2020.acl-main.647/.
+
Maithra Raghu, Justin Gilmer, Jason Yosinski, and Jascha Sohl-Dickstein. SVCCA: Singular Vector Canonical Correlation Analysis for Deep Learning Dynamics and Interpretability. In Advances in Neural Information Processing Systems, volume 30. Curran Associates, Inc., 2017. URL https://proceedings.neurips.cc/paper_files/paper/2017/hash/dc6a7e655d7e5840e66733e9ee67cc69-Abstract.html.
+
Nasim Rahaman, Aristide Baratin, Devansh Arpit, Felix Draxler, Min Lin, Fred Hamprecht, Yoshua Bengio, and Aaron Courville. On the spectral bias of neural networks. In Kamalika Chaudhuri and Ruslan Salakhutdinov (eds.), Proceedings of the 36th International Conference on Machine Learning, volume 97 of Proceedings of Machine Learning Research, pp. 5301–5310. PMLR, 09–15 Jun 2019. URL https://proceedings.mlr.press/v97/rahaman19a.html.
+
Daking Rai, Yilun Zhou, Shi Feng, Abulhair Saparov, and Ziyu Yao. A practical review of mechanistic interpretability for transformer-based language models, 2025. URL https://arxiv.org/abs/2407.02646.
+
Senthooran Rajamanoharan, Arthur Conmy, Lewis Smith, Tom Lieberum, Vikrant Varma, Janos Kramar, Rohin Shah, and Neel Nanda. Improving sparse decomposition of language model activations with gated sparse autoencoders. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024a. URL https://openreview.net/forum?id=zLBlin2zvW.
+
Senthooran Rajamanoharan, Tom Lieberum, Nicolas Sonnerat, Arthur Conmy, Vikrant Varma, János Kramár, and Neel Nanda. Jumping ahead: Improving reconstruction fidelity with jumprelu sparse autoencoders, 2024b. URL https://arxiv.org/abs/2407.14435.
+
Neil Rathi and Alec Radford. Shaping capabilities with token-level data filtering, 2026. URL https://arxiv.org/abs/2601.21571.
+
Tilman Rauker, Anson Ho, Stephen Casper, and Dylan Hadfield-Menell. Toward Transparent AI: A Survey on Interpreting the Inner Structures of Deep Neural Networks . In 2023 IEEE Conference on Secure and Trustworthy Machine Learning (SaTML), pp. 464–483, Los Alamitos, CA, USA, February 2023. IEEE Computer Society. . URL https://doi.ieeecomputersociety.org/10.1109/SaTML54575.2023.00039.
+
Shivam Raval, Hae Jin Song, Linlin Wu, Abir Harrasse, Jeff M. Phillips, Fazl Barez, and Amirali Abdullah. Curveball steering: The right direction to steer isn’t always linear, 2026. URL https://arxiv.org/abs/2603.09313.
+
Shauli Ravfogel, Yanai Elazar, Hila Gonen, Michael Twiton, and Yoav Goldberg. Null it out: Guarding protected attributes by iterative nullspace projection. In Dan Jurafsky, Joyce Chai, Natalie Schluter, and Joel Tetreault (eds.), Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics, pp. 7237–7256, Online, July 2020. Association for Computational Linguistics. . URL https://aclanthology.org/2020.acl-main.647/.
+
Yasaman Razeghi, Robert L Logan IV, Matt Gardner, and Sameer Singh. Impact of pretraining term frequencies on few-shot numerical reasoning. In Findings of the Association for Computational Linguistics: EMNLP 2022, pp. 840–854, 2022.
+
Maria Refinetti, Alessandro Ingrosso, and Sebastian Goldt. Neural networks trained with sgd learn distributions of increasing complexity. In International Conference on Machine Learning, 2023.
Marco Ribeiro, Sameer Singh, and Carlos Guestrin. “why should I trust you?”: Explaining the predictions of any classifier. In John DeNero, Mark Finlayson, and Sravana Reddy (eds.), Proceedings of the 2016 Conference of the North American Chapter of the Association for Computational Linguistics: Demonstrations, pp. 97–101, San Diego, California, June 2016. Association for Computational Linguistics. . URL https://aclanthology.org/N16-3020/.
Nina Rimsky, Nick Gabrieli, Julian Schulz, Meg Tong, Evan Hubinger, and Alexander Turner. Steering llama 2 via contrastive activation addition. In Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 15504–15522, Bangkok, Thailand, August 2024. Association for Computational Linguistics.
+
Herbert Robbins and Sutton Monro. A Stochastic Approximation Method. The Annals of Mathematical Statistics, 22 (3): 400 – 407, 1951. . URL https://doi.org/10.1214/aoms/1177729586.
Anna Rogers, Olga Kovaleva, and Anna Rumshisky. A primer in BERTology: What we know about how BERT works. Transactions of the Association for Computational Linguistics, 8: 842–866, 2020. . URL https://aclanthology.org/2020.tacl-1.54/.
-
Jonathan S. Rosenfeld, Amir Rosenfeld, Yonatan Belinkov, and Nir Shavit. A constructive prediction of the generalization error across scales, 2019. URL https://arxiv.org/abs/1909.12673.
+
Jonathan S. Rosenfeld, Amir Rosenfeld, Yonatan Belinkov, and Nir Shavit. A constructive prediction of the generalization error across scales. In International Conference on Learning Representations, 2020. URL https://openreview.net/forum?id=ryenvpEKDr.
Tamar Rott Shaham, Sarah Schwettmann, Franklin Wang, Achyuta Rajaram, Evan Hernandez, Jacob Andreas, and Antonio Torralba. A multimodal automated interpretability agent. In Forty-first International Conference on Machine Learning, 2024.
Hassan Sajjad, Nadir Durrani, and Fahim Dalvi. Neuron-level interpretation of deep NLP models: A survey. Transactions of the Association for Computational Linguistics, 10: 1285–1303, 2022. . URL https://aclanthology.org/2022.tacl-1.74/.
-
Naomi Saphra. Interpretability creationism. The Gradient, 2023. URL https://thegradient.pub/interpretability-creationism/.
-
Naomi Saphra and Sarah Wiegreffe. Mechanistic? arXiv preprint arXiv:2404.02698, 2024.
-
Andrew M Saxe, James L McClelland, and Surya Ganguli. A mathematical theory of semantic development in deep neural networks. Proceedings of the National Academy of Sciences, 116 (23): 11537–11546, 2019.
-
Rylan Schaeffer, Joshua Kazdan, Baber Abbasi, Ken Ziyu Liu, Brando Miranda, Ahmed Ahmed, Abhay Puri, Niloofar Mireshghallah, and Sanmi Koyejo. Quantifying the effect of test set contamination on generative evaluations, 2026. URL https://arxiv.org/abs/2601.04301.
-
Karin Schnass and Pierre Vandergheynst. Dictionary preconditioning for greedy algorithms. IEEE Transactions on Signal Processing, 56 (5): 1994–2002, 2008.
-
Junghoon Seo, Jeongyeol Choe, Jamyoung Koo, Seunghyeon Jeon, Beomsu Kim, and Taegyun Jeon. Noise-adding methods of saliency map as series of higher order partial derivative. arXiv preprint arXiv:1806.03000, 2018.
-
Or Shafran, Atticus Geiger, and Mor Geva. Decomposing mlp activations into interpretable features via semi-nonnegative matrix factorization. arXiv preprint arXiv:2506.10920, 2025.
+
Naomi Saphra. Interpretability creationism. The Gradient, 2023. URL https://thegradient.pub/interpretability-creationism/.
+
Naomi Saphra and Sarah Wiegreffe. Mechanistic?, 2024. URL https://arxiv.org/abs/2410.09087.
+
Shashata Sawmya, Micah Adler, and Nir Shavit. The birth of knowledge: Emergent features across time, space, and scale in large language models, 2025. URL https://arxiv.org/abs/2505.19440.
+
Andrew M Saxe, James L McClelland, and Surya Ganguli. Exact solutions to the nonlinear dynamics of learning in deep linear neural networks. In The Twelfth International Conference on Learning Representations, 2014.
+
Andrew M Saxe, James L McClelland, and Surya Ganguli. A mathematical theory of semantic development in deep neural networks. Proceedings of the National Academy of Sciences, 116 (23): 11537–11546, 2019.
+
Rylan Schaeffer, Brando Miranda, and Sanmi Koyejo. Are emergent abilities of large language models a mirage? In Advances in Neural Information Processing Systems, volume 36, 2023. URL https://arxiv.org/abs/2304.15004.
+
Rylan Schaeffer, Hailey Schoelkopf, Brando Miranda, Gabriel Mukobi, Varun Madan, Adam Ibrahim, Herbie Bradley, Stella Biderman, and Sanmi Koyejo. Why has predicting downstream capabilities of frontier AI models with scale remained elusive? In Proceedings of the 42nd International Conference on Machine Learning, 2025. URL https://arxiv.org/abs/2406.04391.
+
Rylan Schaeffer, Joshua Kazdan, Baber Abbasi, Ken Ziyu Liu, Brando Miranda, Ahmed Ahmed, Abhay Puri, Niloofar Mireshghallah, and Sanmi Koyejo. Quantifying the effect of test set contamination on generative evaluations, 2026. URL https://arxiv.org/abs/2601.04301.
+
Karin Schnass and Pierre Vandergheynst. Dictionary preconditioning for greedy algorithms. IEEE Transactions on Signal Processing, 56 (5): 1994–2002, 2008.
+
Simon Schrodi, Elias Kempf, Fazl Barez, and Thomas Brox. Towards understanding subliminal learning: When and how hidden biases transfer. In The Fourteenth International Conference on Learning Representations, 2026. URL https://openreview.net/forum?id=IelhmYSjPt.
+
Lisa Schut, Nenad Tomašev, Thomas McGrath, Demis Hassabis, Ulrich Paquet, and Been Kim. Bridging the human–ai knowledge gap through concept discovery and transfer in alphazero. Proceedings of the National Academy of Sciences, 122 (13): e2406675122, 2025. . URL https://www.pnas.org/doi/abs/10.1073/pnas.2406675122.
+
Sarah Schwettmann, Tamar Shaham, Joanna Materzynska, Neil Chowdhury, Shuang Li, Jacob Andreas, David Bau, and Antonio Torralba. Find: A function description benchmark for evaluating interpretability methods. In A. Oh, T. Naumann, A. Globerson, K. Saenko, M. Hardt, and S. Levine (eds.), Advances in Neural Information Processing Systems, volume 36, pp. 75688–75715. Curran Associates, Inc., 2023. . URL https://proceedings.neurips.cc/paper_files/paper/2023/file/ef0164c1112f56246224af540857348f-Paper-Datasets_and_Benchmarks.pdf.
+
Thibault Sellam, Steve Yadlowsky, Ian Tenney, Jason Wei, Naomi Saphra, Alexander D’Amour, Tal Linzen, Jasmijn Bastings, Iulia Raluca Turc, Jacob Eisenstein, Dipanjan Das, and Ellie Pavlick. The multiBERTs: BERT reproductions for robustness analysis. In International Conference on Learning Representations, 2022. URL https://openreview.net/forum?id=K0E_F0gFDgA.
+
Ramprasaath R Selvaraju, Michael Cogswell, Abhishek Das, Ramakrishna Vedantam, Devi Parikh, and Dhruv Batra. Grad-cam: Visual explanations from deep networks via gradient-based localization. In Proceedings of the IEEE International Conference on Computer Vision, pp. 618–626, 2017.
+
Or David Shafran, Atticus Geiger, and Mor Geva. Constructing interpretable features from compositional neuron groups. In Maria Liakata, Viviane P. Moreira, Jiajun Zhang, and David Jurgens (eds.), Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 42326–42348, San Diego, California, United States, July 2026. Association for Computational Linguistics. ISBN 979-8-89176-390-6. . URL https://aclanthology.org/2026.acl-long.1959/.
Lee Sharkey, Bilal Chughtai, Joshua Batson, Jack Lindsey, Jeffrey Wu, Lucius Bushnaq, Nicholas Goldowsky-Dill, Stefan Heimersheim, Alejandro Ortega, Joseph Isaac Bloom, Stella Biderman, Adrià Garriga-Alonso, Arthur Conmy, Neel Nanda, Jessica Mary Rumbelow, Martin Wattenberg, Nandi Schoots, Joseph Miller, William Saunders, Eric J Michaud, Stephen Casper, Max Tegmark, David Bau, Eric Todd, Atticus Geiger, Mor Geva, Jesse Hoogland, Daniel Murfet, and Thomas McGrath. Open problems in mechanistic interpretability. Transactions on Machine Learning Research, 2025. ISSN 2835-8856. URL https://openreview.net/forum?id=91H76m9Z94.
+
Arnab Sen Sharma, David Atkinson, and David Bau. Locating and editing factual associations in mamba. In First Conference on Language Modeling, 2024a. URL https://openreview.net/forum?id=yoVRyrEgix.
Mrinank Sharma, Meg Tong, Tomasz Korbak, David Duvenaud, Amanda Askell, Samuel R. Bowman, Esin DURMUS, Zac Hatfield-Dodds, Scott R Johnston, Shauna M Kravec, Timothy Maxwell, Sam McCandlish, Kamal Ndousse, Oliver Rausch, Nicholas Schiefer, Da Yan, Miranda Zhang, and Ethan Perez. Towards understanding sycophancy in language models. In The Twelfth International Conference on Learning Representations, 2024b. URL https://openreview.net/forum?id=tvhaxkMKAn.
+
Mrinank Sharma, Meg Tong, Jesse Mu, Jerry Wei, Jorrit Kruthoff, Scott Goodfriend, Euan Ong, Alwin Peng, Raj Agarwal, Cem Anil, Amanda Askell, Nathan Bailey, Joe Benton, Emma Bluemke, Samuel R. Bowman, Eric Christiansen, Hoagy Cunningham, Andy Dau, Anjali Gopal, Rob Gilson, Logan Graham, Logan Howard, Nimit Kalra, Taesung Lee, Kevin Lin, Peter Lofgren, Francesco Mosconi, Clare O’Hara, Catherine Olsson, Linda Petrini, Samir Rajani, Nikhil Saxena, Alex Silverstein, Tanya Singh, Theodore Sumers, Leonard Tang, Kevin K. Troy, Constantin Weisser, Ruiqi Zhong, Giulio Zhou, Jan Leike, Jared Kaplan, and Ethan Perez. Constitutional classifiers: Defending against universal jailbreaks across thousands of hours of red teaming, 2025. URL https://arxiv.org/abs/2501.18837.
+
Pratyusha Sharma, Jordan T. Ash, and Dipendra Misra. The truth is in there: Improving reasoning in language models with layer-selective rank reduction. In The Twelfth International Conference on Learning Representations, 2024c. URL https://openreview.net/forum?id=ozX92bu8VA.
+
Noam Shazeer. Glu variants improve transformer, 2020. URL https://arxiv.org/abs/2002.05202.
Xing Shi, Inkit Padhi, and Kevin Knight. Does string-based neural MT learn source syntax? In Jian Su, Kevin Duh, and Xavier Carreras (eds.), Proceedings of the 2016 Conference on Empirical Methods in Natural Language Processing, pp. 1526–1534, Austin, Texas, November 2016. Association for Computational Linguistics. . URL https://aclanthology.org/D16-1159/.
+
Igor Shilov, Alex Cloud, Aryo Pradipta Gema, Jacob Goldman-Wetzler, Nina Panickssery, Henry Sleight, Erik Jones, and Cem Anil. Beyond data filtering: Knowledge localization for capability removal in llms, 2025. URL https://arxiv.org/abs/2512.05648.
+
Edward H. Shortliffe. MYCIN: A knowledge-based computer program applied to infectious diseases. In Proceedings of the Annual Symposium on Computer Application in Medical Care, pp. 66–69, 1977. PMCID: PMC2464549.
+
Avanti Shrikumar, Peyton Greenside, and Anshul Kundaje. Learning important features through propagating activation differences. In Proceedings of the 34th International Conference on Machine Learning - Volume 70, ICML’17, pp. 3145–3153. JMLR.org, 2017.
Dong Shu, Xuansheng Wu, Haiyan Zhao, Daking Rai, Ziyu Yao, Ninghao Liu, and Mengnan Du. A survey on sparse autoencoders: Interpreting the internal mechanisms of large language models. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Findings of the Association for Computational Linguistics: EMNLP 2025, pp. 1690–1712, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-335-7. . URL https://aclanthology.org/2025.findings-emnlp.89/.
-
Karen Simonyan, Andrea Vedaldi, and Andrew Zisserman. Deep inside convolutional networks: Visualising image classification models and saliency maps. In Yoshua Bengio and Yann LeCun (eds.), 2nd International Conference on Learning Representations, ICLR 2014, Banff, AB, Canada, April 14-16, 2014, Workshop Track Proceedings, 2014.
+
Adi Simhi, Itay Itzhak, Fazl Barez, Gabriel Stanovsky, and Yonatan Belinkov. Trust me, I’m wrong: LLMs hallucinate with certainty despite knowing the answer. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Findings of the Association for Computational Linguistics: EMNLP 2025, pp. 14665–14688, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-335-7. . URL https://aclanthology.org/2025.findings-emnlp.792/.
+
Adi Simhi, Fazl Barez, Martin Tutek, Yonatan Belinkov, and Shay B Cohen. Old habits die hard: How conversational history geometrically traps LLMs. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=jNhEZteJfq.
+
Jamie Simon, Daniel Kunin, Alexander Atanasov, Enric Boix-Adserà, Blake Bordelon, Jeremy Cohen, Nikhil Ghosh, Florentin Guth, Arthur Jacot, Mason Kamb, Dhruva Karkada, Eric J. Michaud, Berkan Ottlik, and Joseph Turnbull. There will be a scientific theory of deep learning, 2026. URL https://arxiv.org/abs/2604.21691.
+
Karen Simonyan, Andrea Vedaldi, and Andrew Zisserman. Deep inside convolutional networks: Visualising image classification models and saliency maps. In Yoshua Bengio and Yann LeCun (eds.), 2nd International Conference on Learning Representations, ICLR 2014, Banff, AB, Canada, April 14-16, 2014, Workshop Track Proceedings, 2014.
+
Aaditya Singh, Stephanie C. Y. Chan, Ted Moskovitz, Erin Grant, Andrew Saxe, and Felix Hill. The transient nature of emergent in-context learning in transformers. In Advances in Neural Information Processing Systems, volume 36, 2023.
Aaditya K. Singh, Ted Moskovitz, Felix Hill, Stephanie C. Y. Chan, and Andrew M. Saxe. What needs to go right for an induction head? a mechanistic study of in-context learning circuits and their formation. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.
-
Stewart Slocum, Julian Minder, Clément Dumas, Henry Sleight, Ryan Greenblatt, Samuel Marks, and Rowan Wang. Believe it or not: How deeply do llms believe implanted facts? arXiv preprint arXiv:2510.17941, 2025.
+
Daniel Smilkov, Nikhil Thorat, Been Kim, Fernanda Viégas, and Martin Wattenberg. Smoothgrad: removing noise by adding noise, 2017. URL https://arxiv.org/abs/1706.03825.
Lewis Smith. The ’strong’ feature hypothesis could be wrong. LessWrong, August 2024. URL https://www.lesswrong.com/posts/tojtPCCRpKLSHBdpn/the-strong-feature-hypothesis-could-be-wrong.
-
Mukund Sundararajan, Ankur Taly, and Qiqi Yan. Axiomatic attribution for deep networks. In Proceedings of the 34th International Conference on Machine Learning, pp. 3319–3328. JMLR.org, 2017.
+
Charlie Snell, Eric Wallace, Dan Klein, and Sergey Levine. Predicting emergent capabilities by finetuning, 2024. URL https://arxiv.org/abs/2411.16035.
+
Jascha Sohl-Dickstein, Eric Weiss, Niru Maheswaranathan, and Surya Ganguli. Deep unsupervised learning using nonequilibrium thermodynamics. In Francis Bach and David Blei (eds.), Proceedings of the 32nd International Conference on Machine Learning, volume 37 of Proceedings of Machine Learning Research, pp. 2256–2265, Lille, France, 07–09 Jul 2015. PMLR. URL https://proceedings.mlr.press/v37/sohl-dickstein15.html.
+
Siyuan Song, Harvey Lederman, Jennifer Hu, and Kyle Mahowald. Privileged self-access matters for introspection in AI. In ICML 2026 Workshop: Philosophy Meets Machine Learning, 2026. URL https://openreview.net/forum?id=ZcqCJHOWAA.
+
Aarohi Srivastava, Abhinav Rastogi, Abhishek Rao, Abu Awal Md Shoeb, Abubakar Abid, Adam Fisch, Adam R. Brown, Adam Santoro, Aditya Gupta, Adrià Garriga-Alonso, Agnieszka Kluska, Aitor Lewkowycz, Akshat Agarwal, Alethea Power, Alex Ray, Alex Warstadt, Alexander W. Kocurek, Ali Safaya, Ali Tazarv, Alice Xiang, Alicia Parrish, Allen Nie, Aman Hussain, Amanda Askell, Amanda Dsouza, Ambrose Slone, Ameet Rahane, Anantharaman S. Iyer, Anders Johan Andreassen, Andrea Madotto, Andrea Santilli, Andreas Stuhlmüller, Andrew M. Dai, Andrew La, Andrew Kyle Lampinen, Andy Zou, Angela Jiang, Angelica Chen, Anh Vuong, Animesh Gupta, Anna Gottardi, Antonio Norelli, Anu Venkatesh, Arash Gholamidavoodi, Arfa Tabassum, Arul Menezes, Arun Kirubarajan, Asher Mullokandov, Ashish Sabharwal, Austin Herrick, Avia Efrat, Aykut Erdem, Ayla Karakaş, B. Ryan Roberts, Bao Sheng Loe, Barret Zoph, Bartłomiej Bojanowski, Batuhan Özyurt, Behnam Hedayatnia, Behnam Neyshabur, Benjamin Inden, Benno Stein, Berk Ekmekci, Bill Yuchen Lin, Blake Howald, Bryan Orinion, Cameron Diao, Cameron Dour, Catherine Stinson, Cedrick Argueta, Cesar Ferri, Chandan Singh, Charles Rathkopf, Chenlin Meng, Chitta Baral, Chiyu Wu, Chris Callison-Burch, Christopher Waites, Christian Voigt, Christopher D Manning, Christopher Potts, Cindy Ramirez, Clara E. Rivera, Clemencia Siro, Colin Raffel, Courtney Ashcraft, Cristina Garbacea, Damien Sileo, Dan Garrette, Dan Hendrycks, Dan Kilman, Dan Roth, C. Daniel Freeman, Daniel Khashabi, Daniel Levy, Daniel Moseguı́ González, Danielle Perszyk, Danny Hernandez, Danqi Chen, Daphne Ippolito, Dar Gilboa, David Dohan, David Drakard, David Jurgens, Debajyoti Datta, Deep Ganguli, Denis Emelin, Denis Kleyko, Deniz Yuret, Derek Chen, Derek Tam, Dieuwke Hupkes, Diganta Misra, Dilyar Buzan, Dimitri Coelho Mollo, Diyi Yang, Dong-Ho Lee, Dylan Schrader, Ekaterina Shutova, Ekin Dogus Cubuk, Elad Segal, Eleanor Hagerman, Elizabeth Barnes, Elizabeth Donoway, Ellie Pavlick, Emanuele Rodolà, Emma Lam, Eric Chu, Eric Tang, Erkut Erdem, Ernie Chang, Ethan A Chi, Ethan Dyer, Ethan Jerzak, Ethan Kim, Eunice Engefu Manyasi, Evgenii Zheltonozhskii, Fanyue Xia, Fatemeh Siar, Fernando Martı́nez-Plumed, Francesca Happé, Francois Chollet, Frieda Rong, Gaurav Mishra, Genta Indra Winata, Gerard de Melo, Germàn Kruszewski, Giambattista Parascandolo, Giorgio Mariani, Gloria Xinyue Wang, Gonzalo Jaimovitch-Lopez, Gregor Betz, Guy Gur-Ari, Hana Galijasevic, Hannah Kim, Hannah Rashkin, Hannaneh Hajishirzi, Harsh Mehta, Hayden Bogar, Henry Francis Anthony Shevlin, Hinrich Schuetze, Hiromu Yakura, Hongming Zhang, Hugh Mee Wong, Ian Ng, Isaac Noble, Jaap Jumelet, Jack Geissinger, Jackson Kernion, Jacob Hilton, Jaehoon Lee, Jaime Fernández Fisac, James B Simon, James Koppel, James Zheng, James Zou, Jan Kocon, Jana Thompson, Janelle Wingfield, Jared Kaplan, Jarema Radom, Jascha Sohl-Dickstein, Jason Phang, Jason Wei, Jason Yosinski, Jekaterina Novikova, Jelle Bosscher, Jennifer Marsh, Jeremy Kim, Jeroen Taal, Jesse Engel, Jesujoba Alabi, Jiacheng Xu, Jiaming Song, Jillian Tang, Joan Waweru, John Burden, John Miller, John U. Balis, Jonathan Batchelder, Jonathan Berant, Jörg Frohberg, Jos Rozen, Jose Hernandez-Orallo, Joseph Boudeman, Joseph Guerr, Joseph Jones, Joshua B. Tenenbaum, Joshua S. Rule, Joyce Chua, Kamil Kanclerz, Karen Livescu, Karl Krauth, Karthik Gopalakrishnan, Katerina Ignatyeva, Katja Markert, Kaustubh Dhole, Kevin Gimpel, Kevin Omondi, Kory Wallace Mathewson, Kristen Chiafullo, Ksenia Shkaruta, Kumar Shridhar, Kyle McDonell, Kyle Richardson, Laria Reynolds, Leo Gao, Li Zhang, Liam Dugan, Lianhui Qin, Lidia Contreras-Ochando, Louis-Philippe Morency, Luca Moschella, Lucas Lam, Lucy Noble, Ludwig Schmidt, Luheng He, Luis Oliveros-Colón, Luke Metz, Lütfi Kerem Senel, Maarten Bosma, Maarten Sap, Maartje Ter Hoeve, Maheen Farooqi, Manaal Faruqui, Mantas Mazeika, Marco Baturan, Marco Marelli, Marco Maru, Maria Jose Ramirez-Quintana, Marie Tolkiehn, Mario Giulianelli, Martha Lewis, Martin Potthast, Matthew L Leavitt, Matthias Hagen, Mátyás Schubert, Medina Orduna Baitemirova, Melody Arnaud, Melvin McElrath, Michael Andrew Yee, Michael Cohen, Michael Gu, Michael Ivanitskiy, Michael Starritt, Michael Strube, Michał Swędrowski, Michele Bevilacqua, Michihiro Yasunaga, Mihir Kale, Mike Cain, Mimee Xu, Mirac Suzgun, Mitch Walker, Mo Tiwari, Mohit Bansal, Moin Aminnaseri, Mor Geva, Mozhdeh Gheini, Mukund Varma T, Nanyun Peng, Nathan Andrew Chi, Nayeon Lee, Neta Gur-Ari Krakover, Nicholas Cameron, Nicholas Roberts, Nick Doiron, Nicole Martinez, Nikita Nangia, Niklas Deckers, Niklas Muennighoff, Nitish Shirish Keskar, Niveditha S. Iyer, Noah Constant, Noah Fiedel, Nuan Wen, Oliver Zhang, Omar Agha, Omar Elbaghdadi, Omer Levy, Owain Evans, Pablo Antonio Moreno Casares, Parth Doshi, Pascale Fung, Paul Pu Liang, Paul Vicol, Pegah Alipoormolabashi, Peiyuan Liao, Percy Liang, Peter W Chang, Peter Eckersley, Phu Mon Htut, Pinyu Hwang, Piotr Miłkowski, Piyush Patil, Pouya Pezeshkpour, Priti Oli, Qiaozhu Mei, Qing Lyu, Qinlang Chen, Rabin Banjade, Rachel Etta Rudolph, Raefer Gabriel, Rahel Habacker, Ramon Risco, Raphaël Millière, Rhythm Garg, Richard Barnes, Rif A. Saurous, Riku Arakawa, Robbe Raymaekers, Robert Frank, Rohan Sikand, Roman Novak, Roman Sitelew, Ronan Le Bras, Rosanne Liu, Rowan Jacobs, Rui Zhang, Russ Salakhutdinov, Ryan Andrew Chi, Seungjae Ryan Lee, Ryan Stovall, Ryan Teehan, Rylan Yang, Sahib Singh, Saif M. Mohammad, Sajant Anand, Sam Dillavou, Sam Shleifer, Sam Wiseman, Samuel Gruetter, Samuel R. Bowman, Samuel Stern Schoenholz, Sanghyun Han, Sanjeev Kwatra, Sarah A. Rous, Sarik Ghazarian, Sayan Ghosh, Sean Casey, Sebastian Bischoff, Sebastian Gehrmann, Sebastian Schuster, Sepideh Sadeghi, Shadi Hamdan, Sharon Zhou, Shashank Srivastava, Sherry Shi, Shikhar Singh, Shima Asaadi, Shixiang Shane Gu, Shubh Pachchigar, Shubham Toshniwal, Shyam Upadhyay, Shyamolima Shammie Debnath, Siamak Shakeri, Simon Thormeyer, Simone Melzi, Siva Reddy, Sneha Priscilla Makini, Soo-Hwan Lee, Spencer Torene, Sriharsha Hatwar, Stanislas Dehaene, Stefan Divic, Stefano Ermon, Stella Biderman, Stephanie Lin, Stephen Prasad, Steven Piantadosi, Stuart Shieber, Summer Misherghi, Svetlana Kiritchenko, Swaroop Mishra, Tal Linzen, Tal Schuster, Tao Li, Tao Yu, Tariq Ali, Tatsunori Hashimoto, Te-Lin Wu, Théo Desbordes, Theodore Rothschild, Thomas Phan, Tianle Wang, Tiberius Nkinyili, Timo Schick, Timofei Kornev, Titus Tunduny, Tobias Gerstenberg, Trenton Chang, Trishala Neeraj, Tushar Khot, Tyler Shultz, Uri Shaham, Vedant Misra, Vera Demberg, Victoria Nyamai, Vikas Raunak, Vinay Venkatesh Ramasesh, vinay uday prabhu, Vishakh Padmakumar, Vivek Srikumar, William Fedus, William Saunders, William Zhang, Wout Vossen, Xiang Ren, Xiaoyu Tong, Xinran Zhao, Xinyi Wu, Xudong Shen, Yadollah Yaghoobzadeh, Yair Lakretz, Yangqiu Song, Yasaman Bahri, Yejin Choi, Yichi Yang, Sophie Hao, Yifu Chen, Yonatan Belinkov, Yu Hou, Yufang Hou, Yuntao Bai, Zachary Seid, Zhuoye Zhao, Zijian Wang, Zijie J. Wang, Zirui Wang, and Ziyi Wu. Beyond the imitation game: Quantifying and extrapolating the capabilities of language models. Transactions on Machine Learning Research, 2023. ISSN 2835-8856. URL https://openreview.net/forum?id=uyTL5Bvosj. Featured Certification.
+
Dashiell Stander, Qinan Yu, Honglu Fan, and Stella Biderman. Grokking group multiplication with cosets. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.
+
Mukund Sundararajan, Ankur Taly, and Qiqi Yan. Axiomatic attribution for deep networks. In Proceedings of the 34th International Conference on Machine Learning - Volume 70, ICML’17, pp. 3319–3328. JMLR.org, 2017.
+
Denis Sutter, Julian Minder, Thomas Hofmann, and Tiago Pimentel. The non-linear representation dilemma: Is causal abstraction enough for mechanistic interpretability? In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025. URL https://openreview.net/forum?id=ZYXTLo7kCi.
+
Aaquib Syed, Can Rager, and Arthur Conmy. Attribution patching outperforms automated circuit discovery. In Yonatan Belinkov, Najoung Kim, Jaap Jumelet, Hosein Mohebbi, Aaron Mueller, and Hanjie Chen (eds.), Proceedings of the 7th BlackboxNLP Workshop: Analyzing and Interpreting Neural Networks for NLP, pp. 407–416, Miami, Florida, US, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.blackboxnlp-1.25/.
Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed, Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, and Andrew Rabinovich. Going deeper with convolutions. In Proceedings of the IEEE conference on computer vision and pattern recognition, pp. 1–9, 2015.
-
Daniel Tan, David Chanin, Aengus Lynch, Brooks Paige, Dimitrios Kanoulas, Adrià Garriga-Alonso, and Robert Kirk. Analysing the generalisation and reliability of steering vectors. In Proceedings of the 38th International Conference on Neural Information Processing Systems, NIPS ’24, Red Hook, NY, USA, 2024a. Curran Associates Inc. ISBN 9798331314385.
-
Daniel Chee Hian Tan, David Chanin, Aengus Lynch, Brooks Paige, Dimitrios Kanoulas, Adrià Garriga-Alonso, and Robert Kirk. Analysing the generalisation and reliability of steering vectors. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024b.
+
Daniel Chee Hian Tan, David Chanin, Aengus Lynch, Brooks Paige, Dimitrios Kanoulas, Adrià Garriga-Alonso, and Robert Kirk. Analysing the generalisation and reliability of steering vectors. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024.
Ruixiang Tang, Dehan Kong, Longtao Huang, and Hui Xue. Large language models can be lazy learners: Analyze shortcuts in in-context learning. In Anna Rogers, Jordan Boyd-Graber, and Naoaki Okazaki (eds.), Findings of the Association for Computational Linguistics: ACL 2023, pp. 4645–4657, Toronto, Canada, July 2023. Association for Computational Linguistics. . URL https://aclanthology.org/2023.findings-acl.284/.
-
Gemma Team, Thomas Mesnard, Cassidy Hardin, Robert Dadashi, Surya Bhupatiraju, Shreya Pathak, Laurent Sifre, Morgane Rivière, Mihir Sanjay Kale, Juliette Love, Pouya Tafti, Léonard Hussenot, Pier Giuseppe Sessa, Aakanksha Chowdhery, Adam Roberts, Aditya Barua, Alex Botev, Alex Castro-Ros, Ambrose Slone, Amélie Héliou, Andrea Tacchetti, Anna Bulanova, Antonia Paterson, Beth Tsai, Bobak Shahriari, Charline Le Lan, Christopher A. Choquette-Choo, Clément Crepy, Daniel Cer, Daphne Ippolito, David Reid, Elena Buchatskaya, Eric Ni, Eric Noland, Geng Yan, George Tucker, George-Christian Muraru, Grigory Rozhdestvenskiy, Henryk Michalewski, Ian Tenney, Ivan Grishchenko, Jacob Austin, James Keeling, Jane Labanowski, Jean-Baptiste Lespiau, Jeff Stanway, Jenny Brennan, Jeremy Chen, Johan Ferret, Justin Chiu, Justin Mao-Jones, Katherine Lee, Kathy Yu, Katie Millican, Lars Lowe Sjoesund, Lisa Lee, Lucas Dixon, Machel Reid, Maciej Mikuła, Mateo Wirth, Michael Sharman, Nikolai Chinaev, Nithum Thain, Olivier Bachem, Oscar Chang, Oscar Wahltinez, Paige Bailey, Paul Michel, Petko Yotov, Rahma Chaabouni, Ramona Comanescu, Reena Jana, Rohan Anil, Ross McIlroy, Ruibo Liu, Ryan Mullins, Samuel L Smith, Sebastian Borgeaud, Sertan Girgin, Sholto Douglas, Shree Pandya, Siamak Shakeri, Soham De, Ted Klimenko, Tom Hennigan, Vlad Feinberg, Wojciech Stokowiec, Yu hui Chen, Zafarali Ahmed, Zhitao Gong, Tris Warkentin, Ludovic Peran, Minh Giang, Clément Farabet, Oriol Vinyals, Jeff Dean, Koray Kavukcuoglu, Demis Hassabis, Zoubin Ghahramani, Douglas Eck, Joelle Barral, Fernando Pereira, Eli Collins, Armand Joulin, Noah Fiedel, Evan Senter, Alek Andreev, and Kathleen Kenealy. Gemma: Open models based on gemini research and technology, 2024. URL https://arxiv.org/abs/2403.08295.
+
Mia Taylor, James Chua, Jan Betley, Johannes Treutlein, and Owain Evans. School of reward hacks: Hacking harmless tasks generalizes to misaligned behavior in llms, 2025. URL https://arxiv.org/abs/2508.17511.
+
Gemma Team, Thomas Mesnard, Cassidy Hardin, Robert Dadashi, Surya Bhupatiraju, Shreya Pathak, Laurent Sifre, Morgane Rivière, Mihir Sanjay Kale, Juliette Love, Pouya Tafti, Léonard Hussenot, Pier Giuseppe Sessa, Aakanksha Chowdhery, Adam Roberts, Aditya Barua, Alex Botev, Alex Castro-Ros, Ambrose Slone, Amélie Héliou, Andrea Tacchetti, Anna Bulanova, Antonia Paterson, Beth Tsai, Bobak Shahriari, Charline Le Lan, Christopher A. Choquette-Choo, Clément Crepy, Daniel Cer, Daphne Ippolito, David Reid, Elena Buchatskaya, Eric Ni, Eric Noland, Geng Yan, George Tucker, George-Christian Muraru, Grigory Rozhdestvenskiy, Henryk Michalewski, Ian Tenney, Ivan Grishchenko, Jacob Austin, James Keeling, Jane Labanowski, Jean-Baptiste Lespiau, Jeff Stanway, Jenny Brennan, Jeremy Chen, Johan Ferret, Justin Chiu, Justin Mao-Jones, Katherine Lee, Kathy Yu, Katie Millican, Lars Lowe Sjoesund, Lisa Lee, Lucas Dixon, Machel Reid, Maciej Mikuła, Mateo Wirth, Michael Sharman, Nikolai Chinaev, Nithum Thain, Olivier Bachem, Oscar Chang, Oscar Wahltinez, Paige Bailey, Paul Michel, Petko Yotov, Rahma Chaabouni, Ramona Comanescu, Reena Jana, Rohan Anil, Ross McIlroy, Ruibo Liu, Ryan Mullins, Samuel L Smith, Sebastian Borgeaud, Sertan Girgin, Sholto Douglas, Shree Pandya, Siamak Shakeri, Soham De, Ted Klimenko, Tom Hennigan, Vlad Feinberg, Wojciech Stokowiec, Yu hui Chen, Zafarali Ahmed, Zhitao Gong, Tris Warkentin, Ludovic Peran, Minh Giang, Clément Farabet, Oriol Vinyals, Jeff Dean, Koray Kavukcuoglu, Demis Hassabis, Zoubin Ghahramani, Douglas Eck, Joelle Barral, Fernando Pereira, Eli Collins, Armand Joulin, Noah Fiedel, Evan Senter, Alek Andreev, and Kathleen Kenealy. Gemma: Open models based on gemini research and technology, 2024a. URL https://arxiv.org/abs/2403.08295.
+
Gemma Team, Morgane Riviere, Shreya Pathak, Pier Giuseppe Sessa, Cassidy Hardin, Surya Bhupatiraju, Léonard Hussenot, Thomas Mesnard, Bobak Shahriari, Alexandre Ramé, Johan Ferret, Peter Liu, Pouya Tafti, Abe Friesen, Michelle Casbon, Sabela Ramos, Ravin Kumar, Charline Le Lan, Sammy Jerome, Anton Tsitsulin, Nino Vieillard, Piotr Stanczyk, Sertan Girgin, Nikola Momchev, Matt Hoffman, Shantanu Thakoor, Jean-Bastien Grill, Behnam Neyshabur, Olivier Bachem, Alanna Walton, Aliaksei Severyn, Alicia Parrish, Aliya Ahmad, Allen Hutchison, Alvin Abdagic, Amanda Carl, Amy Shen, Andy Brock, Andy Coenen, Anthony Laforge, Antonia Paterson, Ben Bastian, Bilal Piot, Bo Wu, Brandon Royal, Charlie Chen, Chintu Kumar, Chris Perry, Chris Welty, Christopher A. Choquette-Choo, Danila Sinopalnikov, David Weinberger, Dimple Vijaykumar, Dominika Rogozińska, Dustin Herbison, Elisa Bandy, Emma Wang, Eric Noland, Erica Moreira, Evan Senter, Evgenii Eltyshev, Francesco Visin, Gabriel Rasskin, Gary Wei, Glenn Cameron, Gus Martins, Hadi Hashemi, Hanna Klimczak-Plucińska, Harleen Batra, Harsh Dhand, Ivan Nardini, Jacinda Mein, Jack Zhou, James Svensson, Jeff Stanway, Jetha Chan, Jin Peng Zhou, Joana Carrasqueira, Joana Iljazi, Jocelyn Becker, Joe Fernandez, Joost van Amersfoort, Josh Gordon, Josh Lipschultz, Josh Newlan, Ju yeong Ji, Kareem Mohamed, Kartikeya Badola, Kat Black, Katie Millican, Keelin McDonell, Kelvin Nguyen, Kiranbir Sodhia, Kish Greene, Lars Lowe Sjoesund, Lauren Usui, Laurent Sifre, Lena Heuermann, Leticia Lago, Lilly McNealus, Livio Baldini Soares, Logan Kilpatrick, Lucas Dixon, Luciano Martins, Machel Reid, Manvinder Singh, Mark Iverson, Martin Görner, Mat Velloso, Mateo Wirth, Matt Davidow, Matt Miller, Matthew Rahtz, Matthew Watson, Meg Risdal, Mehran Kazemi, Michael Moynihan, Ming Zhang, Minsuk Kahng, Minwoo Park, Mofi Rahman, Mohit Khatwani, Natalie Dao, Nenshad Bardoliwalla, Nesh Devanathan, Neta Dumai, Nilay Chauhan, Oscar Wahltinez, Pankil Botarda, Parker Barnes, Paul Barham, Paul Michel, Pengchong Jin, Petko Georgiev, Phil Culliton, Pradeep Kuppala, Ramona Comanescu, Ramona Merhej, Reena Jana, Reza Ardeshir Rokni, Rishabh Agarwal, Ryan Mullins, Samaneh Saadat, Sara Mc Carthy, Sarah Cogan, Sarah Perrin, Sébastien M. R. Arnold, Sebastian Krause, Shengyang Dai, Shruti Garg, Shruti Sheth, Sue Ronstrom, Susan Chan, Timothy Jordan, Ting Yu, Tom Eccles, Tom Hennigan, Tomas Kocisky, Tulsee Doshi, Vihan Jain, Vikas Yadav, Vilobh Meshram, Vishal Dharmadhikari, Warren Barkley, Wei Wei, Wenming Ye, Woohyun Han, Woosuk Kwon, Xiang Xu, Zhe Shen, Zhitao Gong, Zichuan Wei, Victor Cotruta, Phoebe Kirk, Anand Rao, Minh Giang, Ludovic Peran, Tris Warkentin, Eli Collins, Joelle Barral, Zoubin Ghahramani, Raia Hadsell, D. Sculley, Jeanine Banks, Anca Dragan, Slav Petrov, Oriol Vinyals, Jeff Dean, Demis Hassabis, Koray Kavukcuoglu, Clement Farabet, Elena Buchatskaya, Sebastian Borgeaud, Noah Fiedel, Armand Joulin, Kathleen Kenealy, Robert Dadashi, and Alek Andreev. Gemma 2: Improving open language models at a practical size, 2024b. URL https://arxiv.org/abs/2408.00118.
+
Kimi Team, Yifan Bai, Yiping Bao, Y. Charles, Cheng Chen, Guanduo Chen, Haiting Chen, Huarong Chen, Jiahao Chen, Ningxin Chen, Ruijue Chen, Yanru Chen, Yuankun Chen, Yutian Chen, Zhuofu Chen, Jialei Cui, Hao Ding, Mengnan Dong, Angang Du, Chenzhuang Du, Dikang Du, Yulun Du, Yu Fan, Yichen Feng, Kelin Fu, Bofei Gao, Chenxiao Gao, Hongcheng Gao, Peizhong Gao, Tong Gao, Yuyao Ge, Shangyi Geng, Qizheng Gu, Xinran Gu, Longyu Guan, Haiqing Guo, Jianhang Guo, Xiaoru Hao, Tianhong He, Weiran He, Wenyang He, Yunjia He, Chao Hong, Hao Hu, Yangyang Hu, Zhenxing Hu, Weixiao Huang, Zhiqi Huang, Zihao Huang, Tao Jiang, Zhejun Jiang, Xinyi Jin, Yongsheng Kang, Guokun Lai, Cheng Li, Fang Li, Haoyang Li, Ming Li, Wentao Li, Yang Li, Yanhao Li, Yiwei Li, Zhaowei Li, Zheming Li, Hongzhan Lin, Xiaohan Lin, Zongyu Lin, Chengyin Liu, Chenyu Liu, Hongzhang Liu, Jingyuan Liu, Junqi Liu, Liang Liu, Shaowei Liu, T. Y. Liu, Tianwei Liu, Weizhou Liu, Yangyang Liu, Yibo Liu, Yiping Liu, Yue Liu, Zhengying Liu, Enzhe Lu, Haoyu Lu, Lijun Lu, Yashuo Luo, Shengling Ma, Xinyu Ma, Yingwei Ma, Shaoguang Mao, Jie Mei, Xin Men, Yibo Miao, Siyuan Pan, Yebo Peng, Ruoyu Qin, Zeyu Qin, Bowen Qu, Zeyu Shang, Lidong Shi, Shengyuan Shi, Feifan Song, Jianlin Su, Zhengyuan Su, Lin Sui, Xinjie Sun, Flood Sung, Yunpeng Tai, Heyi Tang, Jiawen Tao, Qifeng Teng, Chaoran Tian, Chensi Wang, Dinglu Wang, Feng Wang, Hailong Wang, Haiming Wang, Jianzhou Wang, Jiaxing Wang, Jinhong Wang, Shengjie Wang, Shuyi Wang, Si Wang, Xinyuan Wang, Yao Wang, Yejie Wang, Yiqin Wang, Yuxin Wang, Yuzhi Wang, Zhaoji Wang, Zhengtao Wang, Zhengtao Wang, Zhexu Wang, Chu Wei, Qianqian Wei, Haoning Wu, Wenhao Wu, Xingzhe Wu, Yuxin Wu, Chenjun Xiao, Jin Xie, Xiaotong Xie, Weimin Xiong, Boyu Xu, Jinjing Xu, L. H. Xu, Lin Xu, Suting Xu, Weixin Xu, Xinran Xu, Yangchuan Xu, Ziyao Xu, Jing Xu, Jing Xu, Junjie Yan, Yuzi Yan, Hao Yang, Xiaofei Yang, Yi Yang, Ying Yang, Zhen Yang, Zhilin Yang, Zonghan Yang, Haotian Yao, Xingcheng Yao, Wenjie Ye, Zhuorui Ye, Bohong Yin, Longhui Yu, Enming Yuan, Hongbang Yuan, Mengjie Yuan, Siyu Yuan, Haobing Zhan, Dehao Zhang, Hao Zhang, Wanlu Zhang, Xiaobin Zhang, Yadong Zhang, Yangkun Zhang, Yichi Zhang, Yizhi Zhang, Yongting Zhang, Yu Zhang, Yutao Zhang, Yutong Zhang, Zheng Zhang, Haotian Zhao, Yikai Zhao, Zijia Zhao, Huabin Zheng, Shaojie Zheng, Longguang Zhong, Jianren Zhou, Xinyu Zhou, Zaida Zhou, Jinguo Zhu, Zhen Zhu, Weiyu Zhuang, and Xinxing Zu. Kimi k2: Open agentic intelligence, 2026. URL https://arxiv.org/abs/2507.20534.
Adly Templeton, Tom Conerly, Jonathan Marcus, Jack Lindsey, Trenton Bricken, Brian Chen, Adam Pearce, Craig Citro, Emmanuel Ameisen, Andy Jones, Hoagy Cunningham, Nicholas L Turner, Callum McDougall, Monte MacDiarmid, C. Daniel Freeman, Theodore R. Sumers, Edward Rees, Joshua Batson, Adam Jermyn, Shan Carter, Chris Olah, and Tom Henighan. Scaling monosemanticity: Extracting interpretable features from claude 3 sonnet. Transformer Circuits Thread, 2024. URL https://transformer-circuits.pub/2024/scaling-monosemanticity/index.html.
-
Ian Tenney, Dipanjan Das, and Ellie Pavlick. Bert rediscovers the classical nlp pipeline, 2019. URL https://arxiv.org/abs/1905.05950.
-
Harrish Thasarathan, Julian Forsyth, Thomas Fel, Matthew Kowal, and Konstantinos G. Derpanis. Universal sparse autoencoders: Interpretable cross-model concept alignment. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=UoaxRN88oR.
-
Federico Tiblias, Irina Bigoulaeva, Jingcheng Niu, Simone Balloccu, and Iryna Gurevych. Shape happens: Automatic feature manifold discovery in llms via supervised multi-dimensional scaling. arXiv preprint arXiv:2510.01025, 2025.
-
Curt Tigges, Michael Hanna, Qinan Yu, and Stella Biderman. Llm circuit analyses are consistent across training and scale. Advances in Neural Information Processing Systems, 37: 40699–40731, 2024.
+
Ian Tenney, Dipanjan Das, and Ellie Pavlick. BERT rediscovers the classical NLP pipeline. In Anna Korhonen, David Traum, and Lluís Màrquez (eds.), Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pp. 4593–4601, Florence, Italy, July 2019. Association for Computational Linguistics. . URL https://aclanthology.org/P19-1452/.
+
Harrish Thasarathan, Julian Forsyth, Thomas Fel, Matthew Kowal, and Konstantinos G. Derpanis. Universal sparse autoencoders: Interpretable cross-model concept alignment. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=UoaxRN88oR.
+
Federico Tiblias, Irina Bigoulaeva, Jingcheng Niu, Simone Balloccu, and Iryna Gurevych. Hypothesis-driven feature manifold analysis in LLMs via supervised multi-dimensional scaling. Transactions on Machine Learning Research, 2026. ISSN 2835-8856. URL https://openreview.net/forum?id=vCKZ40YYPr.
+
Curt Tigges, Michael Hanna, Qinan Yu, and Stella Biderman. Llm circuit analyses are consistent across training and scale. Advances in Neural Information Processing Systems, 37: 40699–40731, 2024.
+
Berk Tinaz, Zalan Fabian, and Mahdi Soltanolkotabi. Emergence and evolution of interpretable concepts in diffusion models. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025. URL https://openreview.net/forum?id=C7LxkebvVW.
Hugo Touvron, Thibaut Lavril, Gautier Izacard, Xavier Martinet, Marie-Anne Lachaux, Timothée Lacroix, Baptiste Rozière, Naman Goyal, Eric Hambro, Faisal Azhar, Aurelien Rodriguez, Armand Joulin, Edouard Grave, and Guillaume Lample. Llama: Open and efficient foundation language models, 2023. URL https://arxiv.org/abs/2302.13971.
Alexander Matt Turner, Lisa Thiergart, Gavin Leech, David Udell, Juan J. Vazquez, Ulisse Mini, and Monte MacDiarmid. Steering language models with activation engineering, 2024. URL https://arxiv.org/abs/2308.10248.
-
Miles Turpin, Julian Michael, Ethan Perez, and Samuel R. Bowman. Language models don’t always say what they think: unfaithful explanations in chain-of-thought prompting. In Proceedings of the 37th International Conference on Neural Information Processing Systems, NIPS ’23, Red Hook, NY, USA, 2023. Curran Associates Inc.
-
Moritz Vandenhirtz, Sonia Laguna, Ričards Marcinkevičs, and Julia E. Vogt. Stochastic concept bottleneck models. In A. Globerson, L. Mackey, D. Belgrave, A. Fan, U. Paquet, J. Tomczak, and C. Zhang (eds.), Advances in Neural Information Processing Systems, volume 37, pp. 51787–51810. Curran Associates, Inc., 2024. . URL https://proceedings.neurips.cc/paper_files/paper/2024/file/5c7894ac8788555f1cecf536f1e0fd35-Paper-Conference.pdf.
-
Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Ł ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In I. Guyon, U. Von Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (eds.), Advances in Neural Information Processing Systems, volume 30. Curran Associates, Inc., 2017a. URL https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf.
-
Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Ł ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In I. Guyon, U. Von Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (eds.), Advances in Neural Information Processing Systems, volume 30. Curran Associates, Inc., 2017b. URL https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf.
+
Miles Turpin, Julian Michael, Ethan Perez, and Samuel R. Bowman. Language models don’t always say what they think: unfaithful explanations in chain-of-thought prompting. In Proceedings of the 37th International Conference on Neural Information Processing Systems, NIPS ’23, Red Hook, NY, USA, 2023. Curran Associates Inc.
+
Oskar van der Wal, Pietro Lesci, Max Müller-Eberstein, Naomi Saphra, Hailey Schoelkopf, Willem Zuidema, and Stella Biderman. PolyPythias: Stability and outliers across fifty language model pre-training runs. In The Thirteenth International Conference on Learning Representations, 2025.
+
Moritz Vandenhirtz, Sonia Laguna, Ričards Marcinkevičs, and Julia E Vogt. Stochastic concept bottleneck models. In The Thirty-eighth Annual Conference on Neural Information Processing Systems, 2024. URL https://openreview.net/forum?id=iSjqTQ5S1f.
+
Vikrant Varma, Rohin Shah, Zachary Kenton, János Kramár, and Ramana Kumar. Explaining grokking through circuit efficiency, 2023. URL https://arxiv.org/abs/2309.02390.
+
Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N Gomez, Ł ukasz Kaiser, and Illia Polosukhin. Attention is all you need. In I. Guyon, U. Von Luxburg, S. Bengio, H. Wallach, R. Fergus, S. Vishwanathan, and R. Garnett (eds.), Advances in Neural Information Processing Systems, volume 30. Curran Associates, Inc., 2017. URL https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf.
Jesse Vig. A multiscale visualization of attention in the transformer model. In Marta R. Costa-jussà and Enrique Alfonseca (eds.), Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics: System Demonstrations, pp. 37–42, Florence, Italy, July 2019. Association for Computational Linguistics. . URL https://aclanthology.org/P19-3007/.
-
Jesse Vig, Sebastian Gehrmann, Yonatan Belinkov, Sharon Qian, Daniel Nevo, Yaron Singer, and Stuart Shieber. Investigating gender bias in language models using causal mediation analysis. In H. Larochelle, M. Ranzato, R. Hadsell, M.F. Balcan, and H. Lin (eds.), Advances in Neural Information Processing Systems, volume 33, pp. 12388–12401. Curran Associates, Inc., 2020. URL https://proceedings.neurips.cc/paper_files/paper/2020/file/92650b2e92217715fe312e6fa7b90d82-Paper.pdf.
-
Elena Voita and Ivan Titov. Information-theoretic probing with minimum description length. In Bonnie Webber, Trevor Cohn, Yulan He, and Yang Liu (eds.), Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 183–196, Online, November 2020. Association for Computational Linguistics. . URL https://aclanthology.org/2020.emnlp-main.14/.
-
Elena Voita, David Talbot, Fedor Moiseev, Rico Sennrich, and Ivan Titov. Analyzing multi-head self-attention: Specialized heads do the heavy lifting, the rest can be pruned. In Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, 2019.
-
Alex Wang, Yada Pruksachatkun, Nikita Nangia, Amanpreet Singh, Julian Michael, Felix Hill, Omer Levy, and Samuel Bowman. Superglue: A stickier benchmark for general-purpose language understanding systems. In H. Wallach, H. Larochelle, A. Beygelzimer, F. d' Alché-Buc, E. Fox, and R. Garnett (eds.), Advances in Neural Information Processing Systems, volume 32. Curran Associates, Inc., 2019. URL https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf.
-
Andrew Wang, Elisa Nguyen, Runshi Yang, Juhan Bae, Sheila A. McIlraith, and Roger Baker Grosse. Better training data attribution via better inverse hessian-vector products. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025a. URL https://openreview.net/forum?id=7LTTzYXyJ1.
+
Jesse Vig and Yonatan Belinkov. Analyzing the structure of attention in a transformer language model. In Tal Linzen, Grzegorz Chrupała, Yonatan Belinkov, and Dieuwke Hupkes (eds.), Proceedings of the 2019 ACL Workshop BlackboxNLP: Analyzing and Interpreting Neural Networks for NLP, pp. 63–76, Florence, Italy, August 2019. Association for Computational Linguistics. . URL https://aclanthology.org/W19-4808/.
+
Elena Voita and Ivan Titov. Information-theoretic probing with minimum description length. In Bonnie Webber, Trevor Cohn, Yulan He, and Yang Liu (eds.), Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP), pp. 183–196, Online, November 2020. Association for Computational Linguistics. . URL https://aclanthology.org/2020.emnlp-main.14/.
+
Elena Voita, David Talbot, Fedor Moiseev, Rico Sennrich, and Ivan Titov. Analyzing multi-head self-attention: Specialized heads do the heavy lifting, the rest can be pruned. In Anna Korhonen, David Traum, and Lluís Màrquez (eds.), Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics, pp. 5797–5808, Florence, Italy, July 2019. Association for Computational Linguistics. . URL https://aclanthology.org/P19-1580/.
+
Eric Wallace, Kai Xiao, Reimar Leike, Lilian Weng, Johannes Heidecke, and Alex Beutel. The instruction hierarchy: Training llms to prioritize privileged instructions, 2024. URL https://arxiv.org/abs/2404.13208.
+
Alex Wang, Yada Pruksachatkun, Nikita Nangia, Amanpreet Singh, Julian Michael, Felix Hill, Omer Levy, and Samuel Bowman. Superglue: A stickier benchmark for general-purpose language understanding systems. In H. Wallach, H. Larochelle, A. Beygelzimer, F. d'Alché-Buc, E. Fox, and R. Garnett (eds.), Advances in Neural Information Processing Systems, volume 32. Curran Associates, Inc., 2019. URL https://proceedings.neurips.cc/paper_files/paper/2019/file/4496bf24afe7fab6f046bf4923da8de6-Paper.pdf.
+
Andrew Wang, Elisa Nguyen, Runshi Yang, Juhan Bae, Sheila A. McIlraith, and Roger Baker Grosse. Better training data attribution via better inverse hessian-vector products. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025a. URL https://openreview.net/forum?id=7LTTzYXyJ1.
+
George Wang, Jesse Hoogland, Stan van Wingerden, Zach Furman, and Daniel Murfet. Differentiation and specialization of attention heads via the refined local learning coefficient. In International Conference on Learning Representations, 2025b.
Kevin Ro Wang, Alexandre Variengien, Arthur Conmy, Buck Shlegeris, and Jacob Steinhardt. Interpretability in the wild: a circuit for indirect object identification in GPT-2 small. In The Eleventh International Conference on Learning Representations, 2023. URL https://openreview.net/forum?id=NpsVSN6o4ul.
-
Liwei Wang, Lunjia Hu, Jiayuan Gu, Zhiqiang Hu, Yue Wu, Kun He, and John Hopcroft. Towards Understanding Learning Representations: To What Extent Do Different Neural Networks Learn the Same Representation. In Advances in Neural Information Processing Systems, volume 31. Curran Associates, Inc., 2018. URL https://proceedings.neurips.cc/paper_files/paper/2018/hash/5fc34ed307aac159a30d81181c99847e-Abstract.html.
-
Miles Wang, Tom Dupré la Tour, Olivia Watkins, Alex Makelov, Ryan A. Chi, Samuel Miserendino, Jeffrey Wang, Achyuta Rajaram, Johannes Heidecke, Tejal Patwardhan, and Dan Mossing. Persona Features Control Emergent Misalignment, October 2025b. URL http://arxiv.org/abs/2506.19823. arXiv:2506.19823 [cs].
+
Liwei Wang, Lunjia Hu, Jiayuan Gu, Zhiqiang Hu, Yue Wu, Kun He, and John Hopcroft. Towards Understanding Learning Representations: To What Extent Do Different Neural Networks Learn the Same Representation. In Advances in Neural Information Processing Systems, volume 31. Curran Associates, Inc., 2018. URL https://proceedings.neurips.cc/paper_files/paper/2018/hash/5fc34ed307aac159a30d81181c99847e-Abstract.html.
+
Miles Wang, Tom Dupré la Tour, Olivia Watkins, Aleksandar Makelov, Ryan Chi, Samuel Miserendino, Jeffrey Wang, Achyuta Rajaram, Johannes Heidecke, Tejal Patwardhan, and Daniel Mossing. Persona features control emergent misalignment. In The Fourteenth International Conference on Learning Representations, 2026a.
+
Wenxuan Wang, Zirui Liu, Haoxuan Kou, Xuefeng Liu, and Jiaxing Shen. The chatbot knows it’s you: Dialogue attribution in unauthenticated human–llm sessions. In Proceedings of the ACM Web Conference 2026, WWW ’26, pp. 8797–8807, New York, NY, USA, 2026b. Association for Computing Machinery. ISBN 9798400723070. . URL https://doi.org/10.1145/3774904.3793048.
+
Xu Wang, Bingqing Jiang, Yu Wan, Baosong Yang, Lingpeng Kong, and Difan Zou. Dlm-scope: Mechanistic interpretability of diffusion language models via sparse autoencoders, 2026c. URL https://arxiv.org/abs/2602.05859.
Yubo Wang, Xueguang Ma, Ge Zhang, Yuansheng Ni, Abhranil Chandra, Shiguang Guo, Weiming Ren, Aaran Arulraj, Xuan He, Ziyan Jiang, Tianle Li, Max Ku, Kai Wang, Alex Zhuang, Rongqi Fan, Xiang Yue, and Wenhu Chen. MMLU-pro: A more robust and challenging multi-task language understanding benchmark. In The Thirty-eight Conference on Neural Information Processing Systems Datasets and Benchmarks Track, 2024. URL https://openreview.net/forum?id=y10DM6R2r3.
-
Sumio Watanabe. Algebraic Geometry and Statistical Learning Theory. Cambridge University Press, 2009. ISBN 9780521864671.
+
Farooq Ahmad Wani, Alessandro Suglia, Rohit Saxena, Aryo Pradipta Gema, Wai-Chung Kwan, Fazl Barez, Maria Sofia Bucarelli, Fabrizio Silvestri, and Pasquale Minervini. Same answer, different representations: Hidden instability in VLMs. In Hung-Yi Lee, Sunipa Dev, Junyi Jessy Li, and Desmond Elliott (eds.), Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing, pp. 41059–41087, Budapest, Hungary, October 2026. Association for Computational Linguistics. ISBN 979-8-89176-459-0. . URL https://aclanthology.org/2026.emnlp-main.1803/.
+
Sumio Watanabe. Algebraic Geometry and Statistical Learning Theory. Cambridge University Press, 2009. ISBN 9780521864671.
+
Jason Wei, Yi Tay, Rishi Bommasani, Colin Raffel, Barret Zoph, Sebastian Borgeaud, Dani Yogatama, Maarten Bosma, Denny Zhou, Donald Metzler, Ed H. Chi, Tatsunori Hashimoto, Oriol Vinyals, Percy Liang, Jeff Dean, and William Fedus. Emergent abilities of large language models. Transactions on Machine Learning Research, 2022. URL https://openreview.net/forum?id=yzkSU5zdwD.
+
Zeming Wei, Yifei Wang, Ang Li, Yichuan Mo, and Yisen Wang. Jailbreak and guard aligned language models with only few in-context demonstrations. IEEE Transactions on Pattern Analysis and Machine Intelligence, 48 (6): 6835–6846, 2026. .
Sarah Wiegreffe and Yuval Pinter. Attention is not not explanation. In Kentaro Inui, Jing Jiang, Vincent Ng, and Xiaojun Wan (eds.), Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP), pp. 11–20, Hong Kong, China, November 2019. Association for Computational Linguistics. . URL https://aclanthology.org/D19-1002/.
-
Tung-Yu Wu and Fazl Barez. Query circuits: Explaining how language models answer user prompts. arXiv preprint arXiv:2509.24808, 2025.
-
Zhengxuan Wu, Aryaman Arora, Atticus Geiger, Zheng Wang, Jing Huang, Dan Jurafsky, Christopher D Manning, and Christopher Potts. Axbench: Steering LLMs? even simple baselines outperform sparse autoencoders. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=K2CckZjNy0.
-
Hao Xu, Jiacheng Liu, Yejin Choi, Noah A. Smith, and Hannaneh Hajishirzi. Infini-gram mini: Exact n-gram search at the Internet scale with FM-index. In Christos Christodoulopoulos, Tanmoy Chakraborty, Carolyn Rose, and Violet Peng (eds.), Proceedings of the 2025 Conference on Empirical Methods in Natural Language Processing, pp. 24944–24969, Suzhou, China, November 2025. Association for Computational Linguistics. ISBN 979-8-89176-332-6. . URL https://aclanthology.org/2025.emnlp-main.1268/.
+
Workshop Organizers. Workshop on actionable interpretability. ICML 2025, 2025. URL https://actionable-interpretability.github.io/.
+
Tung-Yu Wu and Fazl Barez. Query circuits: Explaining how language models answer user prompts. In Forty-third International Conference on Machine Learning, 2026. URL https://openreview.net/forum?id=7F0sragazb.
+
Tung-Yu Wu and Pei-Yu Lo. U-shaped and inverted-u scaling behind emergent abilities of large language models. In International Conference on Learning Representations, 2025. URL https://arxiv.org/abs/2410.01692.
+
Tung-Yu Wu, Yu-Xiang Lin, and Tsui-Wei Weng. And: audio network dissection for interpreting deep acoustic models. In Proceedings of the 41st International Conference on Machine Learning, ICML’24. JMLR.org, 2024.
+
Wilson Wu, Louis Jaburi, jacob drori, and Jason Gross. Towards a unified and verified understanding of group-operation networks. In The Thirteenth International Conference on Learning Representations, 2025a. URL https://openreview.net/forum?id=8xxEBAtD7y.
+
Zhengxuan Wu, Atticus Geiger, Joshua Rozner, Elisa Kreiss, Hanson Lu, Thomas Icard, Christopher Potts, and Noah Goodman. Causal distillation for language models. In Marine Carpuat, Marie-Catherine de Marneffe, and Ivan Vladimir Meza Ruiz (eds.), Proceedings of the 2022 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, pp. 4288–4295, Seattle, United States, July 2022. Association for Computational Linguistics. . URL https://aclanthology.org/2022.naacl-main.318/.
+
Zhengxuan Wu, Aryaman Arora, Atticus Geiger, Zheng Wang, Jing Huang, Dan Jurafsky, Christopher D Manning, and Christopher Potts. Axbench: Steering LLMs? even simple baselines outperform sparse autoencoders. In Forty-second International Conference on Machine Learning, 2025b. URL https://openreview.net/forum?id=K2CckZjNy0.
+
Daniel Wurgaft, Can Rager, Matthew Kowal, Vasudev Shyam, Sheridan Feucht, Usha Bhalla, Tal Haklay, Eric Bigelow, Raphael Sarfati, Thomas McGrath, Owen Lewis, Jack Merullo, Noah Goodman, Thomas Fel, Atticus Geiger, and Ekdeep Singh Lubana. Manifold steering reveals the shared geometry of neural network representation and behavior, 2026. URL https://arxiv.org/abs/2605.05115.
+
Yueqi Xie, Jingwei Yi, Jiawei Shao, Justin Curl, Lingjuan Lyu, Qifeng Chen, Xing Xie, and Fangzhao Wu. Defending ChatGPT against jailbreak attack via self-reminders. Nature Machine Intelligence, 5 (12): 1486–1496, 2023. .
+
Yang Xu, Yi Wang, Hengguan Huang, and Hao Wang. Tracking the feature dynamics in llm training: A mechanistic study, 2025b. URL https://arxiv.org/abs/2412.17626.
Xingyi Yang, Constantin Venhoff, Ashkan Khakzar, Christian Schroeder de Witt, Puneet K. Dokania, Adel Bibi, and Philip Torr. Mixture of experts made intrinsically interpretable. In Forty-second International Conference on Machine Learning, 2025. URL https://openreview.net/forum?id=6QERrXMLP2.
Yue Yang, Artemis Panagopoulou, Shenghao Zhou, Daniel Jin, Chris Callison-Burch, and Mark Yatskar. Language in a bottle: Language model guided concept bottlenecks for interpretable image classification. In Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition, pp. 19187–19197, 2023.
Wenqian Ye, Luyang Jiang, Eric Xie, Guangtao Zheng, Yunsheng Ma, Xu Cao, Dongliang Guo, Daiqing Qi, Zeyu He, Yijun Tian, Christopher W. Porter, Megan Coffee, Zhe Zeng, Sheng Li, Ziran Wang, Ting-Hao Kenneth Huang, James Matthew Rehg, Henry Kautz, and Aidong Zhang. The clever hans mirage: A comprehensive survey on spurious correlations in machine learning. Transactions on Machine Learning Research, 2026. ISSN 2835-8856. URL https://openreview.net/forum?id=kIuqPmS1b1.
+
Kayo Yin and Jacob Steinhardt. Which attention heads matter for in-context learning? In Aarti Singh, Maryam Fazel, Daniel Hsu, Simon Lacoste-Julien, Felix Berkenkamp, Tegan Maharaj, Kiri Wagstaff, and Jerry Zhu (eds.), Proceedings of the 42nd International Conference on Machine Learning, volume 267 of Proceedings of Machine Learning Research, pp. 72428–72461. PMLR, 13–19 Jul 2025. URL https://proceedings.mlr.press/v267/yin25e.html.
+
Cheng Yu and Orestis Papakyriakopoulos. Safety devolution in AI agents. In ICLR 2025 Workshop on Human-AI Coevolution, 2025. URL https://openreview.net/forum?id=7nJmuFFkWd.
Xuemin Yu, Fahim Dalvi, Nadir Durrani, Marzia Nouri, and Hassan Sajjad. Latent concept-based explanation of NLP models. In Yaser Al-Onaizan, Mohit Bansal, and Yun-Nung Chen (eds.), Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, pp. 12435–12459, Miami, Florida, USA, November 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.emnlp-main.692/.
-
Zeyu Yun, Yubei Chen, Bruno Olshausen, and Yann LeCun. Transformer visualization via dictionary learning: contextualized embedding as a linear superposition of transformer factors. In Eneko Agirre, Marianna Apidianaki, and Ivan Vulić (eds.), Proceedings of Deep Learning Inside Out (DeeLIO): The 2nd Workshop on Knowledge Extraction and Integration for Deep Learning Architectures, pp. 1–10, Online, June 2021. Association for Computational Linguistics. . URL https://aclanthology.org/2021.deelio-1.1/.
-
Shahin Zanbaghi, Ryan Rostampour, Farhan Abid, and Salim Al Jarmakani. Detecting sleeper agents in large language models via semantic drift analysis, 2025. URL https://arxiv.org/abs/2511.15992.
+
Xuemin Yu, Ankur Garg, Samira Ebrahimi Kahou, and Hassan Sajjad. Vector quantized latent concepts: A scalable alternative to clustering-based concept discovery, 2026. URL https://arxiv.org/abs/2602.02726.
+
Zeyu Yun, Yubei Chen, Bruno Olshausen, and Yann LeCun. Transformer visualization via dictionary learning: contextualized embedding as a linear superposition of transformer factors. In Eneko Agirre, Marianna Apidianaki, and Ivan Vulić (eds.), Proceedings of Deep Learning Inside Out (DeeLIO): The 2nd Workshop on Knowledge Extraction and Integration for Deep Learning Architectures, pp. 1–10, Online, June 2021. Association for Computational Linguistics. . URL https://aclanthology.org/2021.deelio-1.1/.
Mateo Espinosa Zarlenga, Pietro Barbiero, Gabriele Ciravegna, Giuseppe Marra, Francesco Giannini, Michelangelo Diligenti, Zohreh Shams, Frederic Precioso, Stefano Melacci, Adrian Weller, Pietro Lio, and Mateja Jamnik. Concept embedding models: beyond the accuracy-explainability trade-off. In Proceedings of the 36th International Conference on Neural Information Processing Systems, NIPS ’22, Red Hook, NY, USA, 2022. Curran Associates Inc. ISBN 9781713871088.
Matthew Zeiler and Rob Fergus. Visualizing and understanding convolutional networks. European Conference on Computer Vision, 2014.
-
Andy K Zhang, Neil Perry, Riya Dulepet, Joey Ji, Celeste Menders, Justin W Lin, Eliot Jones, Gashon Hussein, Samantha Liu, Donovan Julian Jasper, Pura Peetathawatchai, Ari Glenn, Vikram Sivashankar, Daniel Zamoshchin, Leo Glikbarg, Derek Askaryar, Haoxiang Yang, Aolin Zhang, Rishi Alluri, Nathan Tran, Rinnara Sangpisit, Kenny O Oseleononmen, Dan Boneh, Daniel E. Ho, and Percy Liang. Cybench: A framework for evaluating cybersecurity capabilities and risks of language models. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=tc90LV0yRL.
-
Fred Zhang and Neel Nanda. Towards best practices of activation patching in language models: Metrics and methods. In The Twelfth International Conference on Learning Representations, 2024. URL https://openreview.net/forum?id=Hf17y6u9BC.
-
Hengyuan Zhang, Zhihao Zhang, Mingyang Wang, Zunhai Su, Yiwei Wang, Qianli Wang, Shuzhou Yuan, Ercong Nie, Xufeng Duan, Qibo Xue, Zeping Yu, Chenming Shang, Xiao Liang, Jing Xiong, Hui Shen, Chaofan Tao, Zhengwu Liu, Senjie Jin, Zhiheng Xi, Dongdong Zhang, Sophia Ananiadou, Tao Gui, Ruobing Xie, Hayden Kwok-Hay So, Hinrich Schütze, Xuanjing Huang, Qi Zhang, and Ngai Wong. Locate, steer, and improve: A practical survey of actionable mechanistic interpretability in large language models, 2026. URL https://arxiv.org/abs/2601.14004.
+
Andy K Zhang, Neil Perry, Riya Dulepet, Joey Ji, Celeste Menders, Justin W Lin, Eliot Jones, Gashon Hussein, Samantha Liu, Donovan Julian Jasper, Pura Peetathawatchai, Ari Glenn, Vikram Sivashankar, Daniel Zamoshchin, Leo Glikbarg, Derek Askaryar, Haoxiang Yang, Aolin Zhang, Rishi Alluri, Nathan Tran, Rinnara Sangpisit, Kenny O Oseleononmen, Dan Boneh, Daniel E. Ho, and Percy Liang. Cybench: A framework for evaluating cybersecurity capabilities and risks of language models. In The Thirteenth International Conference on Learning Representations, 2025a. URL https://openreview.net/forum?id=tc90LV0yRL.
+
Fred Zhang and Neel Nanda. Towards best practices of activation patching in language models: Metrics and methods. In The Twelfth International Conference on Learning Representations, 2024. URL https://openreview.net/forum?id=Hf17y6u9BC.
+
Hengyuan Zhang, Zhihao Zhang, Ercong Nie, Mingyang Wang, Zunhai Su, Yiwei Wang, Qianli Wang, Shuzhou Yuan, Xufeng Duan, Qibo Xue, Zeping Yu, Chenming Shang, Xiao Liang, Jing Xiong, Hui Shen, Chaofan Tao, Zhengwu Liu, Senjie Jin, Zhiheng Xi, Dongdong Zhang, Sophia Ananiadou, Tao Gui, Ruobing Xie, Hayden Kwok-Hay So, Hinrich Schuetze, Xuanjing Huang, Qi Zhang, and Ngai Wong. Locate, steer, and improve: A practical survey of actionable mechanistic interpretability in large language models. In Maria Liakata, Viviane P. Moreira, Jiajun Zhang, and David Jurgens (eds.), Findings of the Association for Computational Linguistics: ACL 2026, pp. 10317–10362, San Diego, California, United States, July 2026a. Association for Computational Linguistics. ISBN 979-8-89176-395-1. . URL https://aclanthology.org/2026.findings-acl.502/.
+
Lin Zhang, Wenshuo Dong, Zhuoran Zhang, Shu Yang, Lijie Hu, Ninghao Liu, Pan Zhou, and Di Wang. EAP-GP: Mitigating saturation effect in gradient-based automated circuit identification. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025b. URL https://openreview.net/forum?id=lGyXq0LOeQ.
+
Xinrong Zhang, Yingfa Chen, Shengding Hu, Zihang Xu, Junhao Chen, Moo Hao, Xu Han, Zhen Thai, Shuo Wang, Zhiyuan Liu, and Maosong Sun. \inftyBench: Extending long context evaluation beyond 100K tokens. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 15262–15277, Bangkok, Thailand, August 2024a. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.814/.
+
Yuchen Zhang, Anietta Weckauff, Diego Garcia-Olano, and Maksym Andriushchenko. What shapes emergent misalignment? insights from training dynamics, model priors, and data, 2026b. URL https://arxiv.org/abs/2606.20814.
+
Zhexin Zhang, Junxiao Yang, Pei Ke, Fei Mi, Hongning Wang, and Minlie Huang. Defending large language models against jailbreaking attacks through goal prioritization. In Lun-Wei Ku, Andre Martins, and Vivek Srikumar (eds.), Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), pp. 8865–8887, Bangkok, Thailand, August 2024b. Association for Computational Linguistics. . URL https://aclanthology.org/2024.acl-long.481/.
Haiyan Zhao, Hanjie Chen, Fan Yang, Ninghao Liu, Huiqi Deng, Hengyi Cai, Shuaiqiang Wang, Dawei Yin, and Mengnan Du. Explainability for large language models: A survey. ACM Trans. Intell. Syst. Technol., 15 (2), February 2024a. ISSN 2157-6904. . URL https://doi.org/10.1145/3639372.
-
Rosie Zhao, Naomi Saphra, and Sham M Kakade. Distributional scaling laws for emergent capabilities. In NeurIPS 2024 Workshop on Scientific Methods for Understanding Deep Learning, 2024b.
+
Jianli Zhao, Tingchen Fu, Rylan Schaeffer, Mrinank Sharma, and Fazl Barez. Chain-of-thought hijacking, 2026a. URL https://arxiv.org/abs/2510.26418.
+
Rosie Zhao, Naomi Saphra, and Sham M Kakade. Distributional scaling laws for emergent capabilities. In NeurIPS 2024 Workshop on Scientific Methods for Understanding Deep Learning, 2024b.
Ruochen Zhao, Tan Wang, Yongjie Wang, and Shafiq Joty. Explaining language model predictions with high-impact concepts. In Yvette Graham and Matthew Purver (eds.), Findings of the Association for Computational Linguistics: EACL 2024, pp. 995–1012, St. Julian’s, Malta, March 2024c. Association for Computational Linguistics. . URL https://aclanthology.org/2024.findings-eacl.67/.
Yu Zhao, Alessio Devoto, Giwon Hong, Xiaotang Du, Aryo Pradipta Gema, Hongru Wang, Xuanli He, Kam-Fai Wong, and Pasquale Minervini. Steering knowledge selection behaviours in llms via sae-based representation engineering. In Proceedings of the 2025 Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers), pp. 5117–5136, 2025.
-
Nicolas Zucchet, Jörg Bornschein, Stephanie Chan, Andrew Lampinen, Razvan Pascanu, and Soham De. How do language models learn facts? dynamics, curricula and hallucinations. arXiv preprint arXiv:2503.21676, 2025a.
-
Nicolas Zucchet, Francesco d’Angelo, Andrew K Lampinen, and Stephanie CY Chan. The emergence of sparse attention: impact of data distribution and benefits of repetition. arXiv preprint arXiv:2505.17863, 2025b.
+
Zheng Zhao, Yeskendir Koishekenov, Xianjun Yang, Naila Murray, and Nicola Cancedda. Verifying chain-of-thought reasoning via its computational graph. In The Fourteenth International Conference on Learning Representations, 2026b. URL https://openreview.net/forum?id=CxiNICq0Rr.
+
Wanjun Zhong, Ruixiang Cui, Yiduo Guo, Yaobo Liang, Shuai Lu, Yanlin Wang, Amin Saied, Weizhu Chen, and Nan Duan. AGIEval: A human-centric benchmark for evaluating foundation models. In Kevin Duh, Helena Gomez, and Steven Bethard (eds.), Findings of the Association for Computational Linguistics: NAACL 2024, pp. 2299–2314, Mexico City, Mexico, June 2024. Association for Computational Linguistics. . URL https://aclanthology.org/2024.findings-naacl.149/.
+
Bolei Zhou, Aditya Khosla, Agata Lapedriza, Aude Oliva, and Antonio Torralba. Learning deep features for discriminative localization. In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pp. 2921–2929, 2016.
+
Yuxuan Zhu, Tengjun Jin, Yada Pruksachatkun, Andy K Zhang, Shu Liu, Sasha Cui, Sayash Kapoor, Shayne Longpre, Kevin Meng, Rebecca Weiss, Fazl Barez, Rahul Gupta, Jwala Dhamala, Jacob Merizian, Mario Giulianelli, Harry Coppock, Cozmin Ududec, Antony Kellermann, Jasjeet S Sekhon, Jacob Steinhardt, Sarah Schwettmann, Arvind Narayanan, Matei Zaharia, Ion Stoica, Percy Liang, and Daniel Kang. Establishing best practices in building rigorous agentic benchmarks. In The Thirty-ninth Annual Conference on Neural Information Processing Systems Datasets and Benchmarks Track, 2025. URL https://openreview.net/forum?id=E58HNCqoaA.
+
Itamar Zimerman, Ameen Ali Ali, and Lior Wolf. Explaining modern gated-linear RNNs via a unified implicit attention formulation. In The Thirteenth International Conference on Learning Representations, 2025. URL https://openreview.net/forum?id=wnT8bfJCDx.
+
Nicolas Zucchet, Jörg Bornschein, Stephanie Chan, Andrew Lampinen, Razvan Pascanu, and Soham De. How do language models learn facts? dynamics, curricula and hallucinations, 2025a. URL https://arxiv.org/abs/2503.21676.
+
Nicolas Zucchet, Francesco D’Angelo, Andrew Kyle Lampinen, and Stephanie C.Y. Chan. The emergence of sparse attention: impact of data distribution and benefits of repetition. In The Thirty-ninth Annual Conference on Neural Information Processing Systems, 2025b. URL https://openreview.net/forum?id=jMhRbV47pS.
\ No newline at end of file diff --git a/src/generated/withheld-fingerprints.json b/src/generated/withheld-fingerprints.json index 374b290b2f..e05ab90f5e 100644 --- a/src/generated/withheld-fingerprints.json +++ b/src/generated/withheld-fingerprints.json @@ -1,1731 +1,4 @@ { "about": "SHA-256 prefixes of normalised sentences, 10-word sequences, headings and ids of withheld draft chapters. tests/site.test.ts checks that none appear anywhere in the built site.", - "chapters": { - "learning-dynamics": { - "sentences": [ - "0017b07dfe1e99c9748285d0", - "004507333b6068adf7f44b00", - "004c46db9fd8229962ddc463", - "006471f30e02e5f7ac82d0e1", - "006a194c30086641ce527992", - "006f73ffa45bffe039fcb3c3", - "009ea2ae04dff2d921e1565e", - "00a1aee88607fcc350be2838", - "01c641ad76512cc9c27cfa0e", - "020201fa4562fc7c4a30c38e", - "021d0bdfd86f08830f4a8906", - "0246b9cecf0781cf7f5823fe", - "0257b0f39271bc60d15908e4", - "02790d81456fab98d8de89aa", - "0279a0b4d254bc2ffcb99e14", - "027a57c1edbadf1c3ba2e0a6", - "0299bd444cb4abd8377d5911", - "02e2ffccbbf6a11e6295fb0c", - "031e0b6912d3f1df576ad1ac", - "0365c6bf8ffe81a24964e00f", - "0378a82db770b3c061e9885b", - "03814111154460744fd6edbb", - "03dabc3604ea4279df792e5a", - "03e71cfc23221d020b432bbd", - "040722cf0aaa7b28123974c4", - "0456aafb04965b97d2984b61", - "04869fd77ed8bd7883283d99", - "049217829456b1366ad13cee", - "0494d1cf65f955cce8b28923", - "04b00c60e49feff2822a5165", - "04d2141dd2d1ac76229a7582", - "04e669c8f98376fdd9b3c5a7", - "04f69a79ac138dbb49e8200b", - "0525068b111ed4977bb42a18", - "0527677b3441a74bec05c767", - "056b3c76b77bba139d5436ba", - "0570cf0ae8edd88c0ac0385e", - "064521c0f2ce146716ff6bee", - "0656ec4bd88eb606b6961330", - "065fe2b2f800d9b91e3ef02b", - "06739d6fb9d9fff7fdae024e", - "06a88cc8c353a186f465f406", - "06e9d748c4721d0a6df1fcf7", - "06f0b9fba0f09838afe5c69b", - "06f861686f713bf8c396008c", - "070043acbc347571bc849587", - "0705bec32a8bc4c82289b91e", - "073c78cbdd17fe069582a535", - "074621deefee964e0ed005a5", - "07495fd56bcb1e4c1e65c382", - "077684a3d6496aab01c26477", - "0782d7715413ef8e3744ec23", - "07e4ea78440df13e95cac1bb", - "07f5b4639de4b0c24d36bdb1", - "07fa6a61c46d371a5f4625c8", - "0802d30072138a26981ee0cc", - "08ff7ef8814ad6a856655f43", - "091233d1d125ebe932e19a67", - "091b8a0e653a9997d334ea73", - "093d1e1c6109dcd927799d42", - "094445cefbaa6c6a90d10814", - "099b70693bbd19c08130f094", - "09b39a85b4fb534637bedb1d", - "09c9191f9317c4cce88efc23", - "0a754282329a20b753615315", - "0a87b27cc77249261c9b443a", - "0a87df7ad2849b72d8573a58", - "0ac1895af34e749c6d8559b8", - "0af3d5c66a0438d509b33068", - "0b0eea35c3b08774cc75657a", - "0b723a94896af995c5735a58", - "0b83e9f61820d4b3aeefffe2", - "0b9a742351f94558cf6ca8a2", - "0bcc0d2bbc3d4b934d85665f", - "0bd02ebe71ae6695ac0b658b", - "0be5e1e8dd1d4e0d6a9adf14", - "0bf6d8178eb49c746943667d", - "0bff496a485e59587e6bd581", - "0c012de6d1da0738cd9191c2", - "0c5bf17bac824f3463cbe682", - "0c5dfc7df57442087e3d9096", - "0c8cc820697dbcde26e23e6c", - "0d06c490aba03ebe9386ca2f", - "0d7616456699809a18d490e4", - "0d7adcbfa8872f335c57a5f2", - "0d9da0bda7bd5352abf4c58c", - "0ddb7e13d543fcea3055eec4", - "0e1203a710d1d1b9939571b5", - "0e1363b16f428ad4b3073c37", - "0e2dd7c5f20fb467bd8de1bb", - "0e5cee67a13d97021eccfa10", - "0e653994040c88fce0c21b8b", - "0e6f6cff3f582463f1d495e4", - "0e991b005b5bdd842c86199e", - "0eb101ce30fc1c65cfef4332", - "0eeb968cd1052981babb65af", - "0f1f0a0a7c53a39fbff7ba28", - "0f4984b0bb476a01b9ddc92a", - "0f50208b0bc84d956a6df11d", - "0fc22d8a11abe61881ef5477", - "102e07d30c3ab3a7b65f6c26", - "104c1ddc56f7176d0f3f705e", - "1056ebd88f7f067725a365a4", - "1070b581689ac16b48b0ad5c", - "10a0b3f6629634929a588955", - "10c042fca3c7995e71be6c4c", - "10fe8f98400e77bad74d33e0", - "1163379c3326f9f6a3b7ec11", - "11d1795452548042ad5c0007", - "12d6205d8896b835e30c8d91", - "12db428d9192b2dbbce0484f", - "12f15ca1a5b9b684f4c15aa9", - "133406a30706546e745f0ac7", - "133420f8485ea8481b81603c", - "1335a15150d55c98cc9c87ff", - "134385fb6bee1b16ce4861e4", - "1365ea10d4cd65d3baf1c7fd", - "136e657577c22d758f36ff63", - "139f720300af2c02900f2668", - "13cd99263819826b21adb071", - "13f37fe81271c25039bbeae1", - "13f9fac04027a5e4b25bcdb2", - "1445ed7db5e1bcc083660c5b", - "1473bb7362a67f9daf8489a7", - "150257053d48df0a02c457ff", - "15167b64b2c2ccd3e991b4aa", - "1551ad805f22ee7593c5b24f", - "1555345b1025f8868c0377e1", - "155ffce3f4084f264d36b1a2", - "1595d154d29fe86106c368ff", - "15a936d3bb6e1107f589bd09", - "15ae554af867266c63197695", - "15dc4c874027b604792dfff8", - "15e88ffa50583a4eb054ef89", - "15faf66539c700554b2a0f9c", - "160c51be75f6683575cfbeff", - "16566435e917e3093ac4e0ed", - "165b1879856a041a8e5c013e", - "166ebaf9ba080a674c3e7f28", - "168d71c908780ec1fc9555f2", - "1693f2096a056ba076117f7c", - "16ad03c909e80423151a38d9", - "17331c8a7b16d4feef42722c", - "17510b064d2cf4efd6b06fc1", - "17652ea3cfc9b7efd62f992f", - "1786152ab4585db7beecedb0", - "178f7a24d22e0df8ccacb048", - "17c21f196351190928e43d86", - "17d322f1aecbbf8095e4d540", - "17d7630b878c71f4f980f23f", - "185bcf24b6ed8ff8662bb107", - "18c17b09881b5777796caa7e", - "190d091e72b9e23ce5e2a836", - "19146660ace769302fa46339", - "1952ae5d3ec800cc35e7ad8f", - "195ee57e909f7c12be5d9153", - "19ad537b52b4e2b6c9cd9c61", - "19af24d975dd2790cc0f81a8", - "19ec85a8992d1c7ba98b59a7", - "1a3497f4b487ff54dba870d2", - "1a6e67f2f51a08d361492c31", - "1ac6ea0d53de3c8901a62f5b", - "1b0504ce503f83f43e614f7e", - "1b1fdfcafb910181e1df0635", - "1b241a30c6309e30d9011a7d", - "1b4d8b749d877318ecc6e68e", - "1b4e6340fa0d45674985f32c", - "1b4f2f9674d195a840e9ddf6", - "1b52719879b9efd54d67504a", - "1b566fb021098b439ebad098", - "1b8096eb61838a1577ca72fb", - "1bc716bd4965e068cc2c3608", - "1bddd594b552645ea48cdc59", - "1bdf1825c36338e4ae2d89ca", - "1bf71902dc90627069fc1cc7", - "1c7d8fdcbf9291fd833e4ebe", - "1c94ef4d2c2026554ce78fa9", - "1c97e1e45646c484b268bdca", - "1cb9f09a695dccab4c04939e", - "1ce0ccf469cefe164507a601", - "1d2d07052a419d710e333938", - "1d5ee32b14b122b0944ade47", - "1d8026d05d66b515748578f9", - "1d9254ecb45bd499de78b56d", - "1df02171bbda430b6a7693f9", - "1dfb7cfb80f231cd762f2409", - "1e024a86a7b31202d5e3f56c", - "1e0848bcdb31bc99e192b96a", - "1e0f384157e5f75679a06fa1", - "1e3da19962cfb3fd73f6ed80", - "1e77c7d73774fe1d2f93be45", - "1eac90a151586f7d307d1011", - "1f0706d80f105ceb6a0d673b", - "1f1c5921d5fa8f2c1981726a", - "1f2fb1dcd5f301da324ed809", - "1f480ad2bdf70867e4b8e5db", - "1f6fcb7e80f6dbe717b66388", - "1f735fe676ec57566be4e03a", - "1fd63e5cf4b6d63a3016c2ed", - "1fd6e987d04e56c574e4702a", - "2012c10f4dff592591345b4a", - "202d3213219e1432e197eeb7", - "202d3700d244006d5b486e0c", - "203ede5a4a25b1bef4c6d5f6", - "2051804f74e577778aa1a99a", - "20552c80cdd178dc6981b855", - "2056aa82f640d88661abc19c", - "205eb2c4c38f7005a78f50e9", - "20984de2ef9965ee31b9f35d", - "20c5df3adb5b113e35149cb0", - "210ff267b15544fab4c216f6", - "21172d371b8ac9192642e390", - "2134f416a78276a37d992395", - "216aa705cfdf73dc332141ad", - "217ad82c67fd3edf6003e525", - "2182c3f93d7fa9957beea7f3", - "2198e87630456c56da1b49c3", - "21b3e3597a2a77baddf47b48", - "21c204e9e0096798d2ea642e", - "21c4193e7c91555112cda37d", - "21c629ec49665242bd56c11d", - "21fb30b7cd41c1705ffccc5c", - "226b279819fe8bc340bea7f2", - "227365c59ade845c8847079c", - "2273882da95a1b39fd6a594f", - "228425245d5e4bf0929359ed", - "22cdf658908dde6cc0f730d7", - "22e19e6ae1c44131b760a21d", - "22e6bb3eba50b6795856e1b0", - "230163f552c3691562d95b5d", - "23512d95f47c69896d23fb86", - "23554917e19eba2d909f6c59", - "23c756c98259c4b7bcd5b52f", - "23e55200962d523e45178206", - "244cfe408e89c1c0c01b912c", - "24a9fe3531fe328759e102cd", - "24b0729247739cb8421c6c87", - "24ed657244a93394ee8c2c8d", - "2558d0d0701f0810182a566c", - "2564b2bdfafec49331e264f4", - "257760fff031ee964d32dd41", - "25c69e85d7c710978972c270", - "25ce9895b7a1a6f46192f754", - "2623a174ce2acf540165bbd6", - "262722c9d3235f311a97d842", - "26367b6d840720b52dd53279", - "26451538b68fbbb9b7e2f314", - "26956a6803ce92e9483b07fd", - "26cbb452bdfcedc87af9342b", - "26cfe2c3e27037fc20f2f37d", - "271d0d02a079d182b0e0925d", - "272a1834a33b39b41fe2f15d", - "275f9bde9e4456ee27c5f3fc", - "276da39203b7deb7bb6cd09b", - "276ea85efd43df61a8efe0be", - "27789a06f211f89e3b9c06d3", - "277a7e54c61514f7990d5c56", - "27ab997000a8267c35474846", - "27b80b22188c18f29deb0611", - "27f334e2f8bff17fc0b9c5dc", - "27fde2896ea6d81ca43ce01d", - "2828da22ac2ca3bc15a7d2a7", - "28835e1a024019fcb83d8bf4", - "28bad692f4b3fc9645608b6b", - "28d2051683a78b32df64f35e", - "290b4b67f31a49d4079ad0ca", - "2939e26895937d34a315a997", - "29717415f4f840737f36dc97", - "29cce9ab0b03f6d91e931d88", - "29f63aa186b2a4729e9fdc6c", - "2a4c1afd8d369d06aa6536a6", - "2a5ecd812fed98fd5a8f4022", - "2aa82335877b8b467bc42ea5", - "2ad80ca77651d25857dc0aaf", - "2b19628b5f708e4dc14b6923", - "2bacfe2f6b5f777669f4f8c0", - "2bbf69c8be6122447ae1344a", - "2bcfc91a60035d009ad57960", - "2bde8370da40b358b60bc1bd", - "2bfa09ab2f547909b548c4a9", - "2c1f215b2dd73081fb71124a", - "2c46abf10a5bdaaabf4693af", - "2c8eadfe164ea5aa7aabab09", - "2c8ee1626bcaf5be4960f3b8", - "2cd6b5ae6901d2e2ea753ad2", - "2cdfc1b5bbc2782bd78e0706", - "2ce98b0ff6cf235b8eecb9d2", - "2cf5b1f237b6e63a4eb2a697", - "2d0ae2fc1a72fed8a462b957", - "2d3d07d7009857bea734cad8", - "2d625819ec19d6f2eaadcd5d", - "2d65e62cae134a787ac06432", - "2d784a8dcaeecaebf43cbb82", - "2db9fbce86071d7019d97b7b", - "2df22992e2c65275f03920d0", - "2e40c82afb1d45aa89535d15", - "2e4db339c1ea6b0c5c1d87ce", - "2e4f1d521807a90df8550e97", - "2e6dc4d7abf9bfdb2971b326", - "2e792200d804b8ff0aece2c7", - "2e94684d2df1e4438a47aa4e", - "2ec8b605410649df471fb656", - "2efc4746f52b8d71258d7bae", - "2f1df0f91cd4d6654e1aec8a", - "2f254e852c792f5d1ff7824f", - "2f361cb0d9ad48c6889c6739", - "2f77fa4b08275f24e7c9ab75", - "2f784c6d11c0dc2930ce1c60", - "2f84a2a38d451c1857107482", - "2fd3a2f44d3dea4f8677ce1f", - "2ff0a2ac40308e4a3ffa2f3d", - "300051c118c41b3933d659bf", - "3013799061b74ad01a94da82", - "304a177d7e22de27eb6fe1ec", - "3057cc2cb49c7f71a934d16a", - "30db8b1fe5a526785c1f7a2a", - "30e532591659d6a0b0407d9b", - "30f4906e25e152c02a61f05f", - "310612efda9d20b336bd8ffe", - "313b4d83f74b5682d8772a1b", - "3144604b4b5e50bf8bb45d43", - "31b7a325e4a1766db7790568", - "31c45d4a10fd3718fe850bee", - "31deabd432a56f7d37bf22ec", - "31f5475c33cb2f6fe7bbca85", - "32547e2d39ed98cd46a59c64", - "325a0c01a6528590fb55261e", - "325a3461a0f559c037e7170f", - "327b87e269e47902415cee80", - "32ca3407054b933cf22cf724", - "33487ea00896b62949acda33", - "3369bed52ff44790a6cace23", - "336ead1ace6f6bc4709b4568", - "33707af9498355766fe51a71", - "33848c8628a9319ed87daa98", - "338a5958cc13c36cf0688043", - "33a7e53187273a2542e568f5", - "33d136d04786b37f521ee4b5", - "33ff8ee7ecd7270bc25dc141", - "340922939c7d2d90d418da95", - "340c87ad83cad01eb7726662", - "346b4c7eafbdfd602f854cdb", - "348b5a1c5ec0a73dcb92515c", - "3499cbf24d54585a2c680717", - "350d795b4ed9f2682262a134", - "350f717b9d5329569098bf38", - "357e57e69660774baa10b6ab", - "359d928635710d32891744ac", - "35bceb60f03e7a4098aadc8c", - "3614d3cfd864a1043c4dc872", - "36535617db68b53064d8984d", - "36a30f26ae6cdf09f6d0d1bc", - "36afbe5182e93c7952831833", - "36b6c2430d66b231563e5a3a", - "36c70b1cede096ed1723b837", - "36c8fff5546b17d6aa5e41f0", - "36d3d0d9da61f7c248b633b8", - "36e4889350951a1dd7646017", - "36f0523fa2b04bb276ba5afe", - "36f6f2a022e8ffdca5f94147", - "36fe358f2844e64e474e5ae5", - "379e72a6814674d7756bfa12", - "37ea5f15314df476f3f6714d", - "37f893b7cea225bed3260cd9", - "383a8b8250343606f731d34f", - "389395517907b60724b0c134", - "38ff2e426d0e8f29b6fe54cf", - "391b0b0950bbb32d232cc664", - "392a4ceb75288cb5be929a2f", - "3969daf4ee2a19bbc8b078f3", - "398db040cfa052bcf6b500d5", - "3a2085546dcc3642a76121b3", - "3a209b32738b7d5252f1089c", - "3a576ff1801f1fa169d93a1a", - "3a632046d99e888b54cae25c", - "3a94d92237fb2a47626c61f6", - "3aa75a31d5b3aef32529a364", - "3aba68682030630281f3c76e", - "3b59bc22e521ddf39c68de14", - "3b5d4189b7b712c4c8042f0d", - "3b76165410d3a204954fb989", - "3b96cae0b7a3db8eb41df898", - "3bc0fd0d9645b30a65b431a3", - "3bdd55ef0c11f5bc79b5e8dd", - "3bf405e3dd0827d54d00fdf5", - "3bfb4033e60d209d7519b630", - "3ca8f22a6e3bf1cee0f1f84a", - "3ccc64c23df7ad3d639f7c7c", - "3d515046556cf008f210d986", - "3d530a3f0d775f5c600f48b2", - "3d65878df9735e98f63ab972", - "3dc3725b9cf4cbb778cda45b", - "3dfb7843b0d25c00158af815", - "3e0b0002d41b010be60fb544", - "3e3a82ba18eddcfac70bdf7f", - "3e3ab0478f4c03fd7e0d4d7e", - "3e634982b7989a191ce95feb", - "3e69085f3ecc15a116f2ac44", - "3eaae58a26a8d36aabf5a639", - "3ec26a7b1b05259d2899ad79", - "3ed29e58da055788452c2b2a", - "3eea26d4e422e067913fd7df", - "3f26f0aa55fb745b1f081f3a", - "3f622b3fc6d54a5a5248fe7c", - "3f98b74660b1220715d624fc", - "4024e8b6e1f5ccd063c996b8", - "4041f01f1ee98f673e7bd697", - "404b93094de680387029f11c", - "405a3df3a9ab8624c92b744b", - "405bedc6e0113f1d247beff8", - "40cab9daaffa56c171fd226e", - "40ee40fcbd9578d17d70fc89", - "413874edca4bea311228fa1b", - "4139c250895a8424d6bb2ba7", - "41679ca2dae4a0a69d628d5e", - "41ab0597993fc47ca814fd70", - "41d35f85e5766db09b7be215", - "41e137fa01e6fdb444a90120", - "41f3e73ef3da78324a1b2617", - "41f5a2adcd13bc9f07bb1552", - "41fa34beb17f5c8eaaae63d3", - "4278786ce13e351d4339898b", - "4292b36fb480bb540a557f3e", - "42b4e36fbd680f28bfaa750b", - "42d8426838c7aff4feb0824d", - "42eeb71eed9dbab9a7bcb4b8", - "4319817ade7764cc0147e62e", - "4322ea19402c0ea7a2a9072b", - "43499859cad826f9c0ee0ce2", - "436f3cd5ac0ead8c934504bf", - "43d510fa5e58b31fec0d66dc", - "4400cbd0805d39cb350b136c", - "4420915f8e7fa81bdf5505b3", - "449566c63f958bd4e48d5726", - "4502ae50dea5a926b56e83e8", - "45366d49128f8841a194dbc2", - "458f4a8f10da66d73d471365", - "45e804cc7c7bc7ac700b63f0", - "45f04a2ac8487f6ab78029ef", - "462e6ccca396c1ce03f796e6", - "46390380254f9a4f9d5cdcc7", - "46543f2b46af2c36c725d1d3", - "466151e150a8e8baf27058cd", - "46e7337f75a502ca5e3a0886", - "46f2f8c29518b0a76e56908f", - "46fc6773b73165472d32f8e5", - "46ff3eec4ccfd2f795590412", - "472012e4c034a45810b5b593", - "47526bc454e1b38ed6787ccc", - "475b7ad0d40b9d6ca1be7775", - "47655666dca3177b943dd025", - "476701d1cf11c66f78eb04aa", - "476a2c557304edff410c2f5c", - "4789ef3f13649a5aa496fc43", - "484610e1355f8642ea314784", - "48ce93df82f5f14f6eb49bd1", - "48f317e9475352f84aca6370", - "492490890b075b2ed5e6c293", - "4925fb8a69cf295f5860c33a", - "4940bea12d645bfd6c888517", - "4a014b41b16b4cd4509d4bb4", - "4a174f5a6a952d6601d280c7", - "4a1b8cd0c4311c7204d3b227", - "4a65ddf84f0fa38c1f80cbd9", - "4a912b813b3da91f3cfc3fd7", - "4a9e8deb43bc55cfc7d33aa9", - "4ac64ae2686a470b209f24c7", - "4adfba4c4f957733c6e7529a", - "4b1f8f9ca6456337d3b4a94f", - "4b890e39ba9487353431276a", - "4b996454010bc7bad3729b13", - "4bbc945e80a950aac0829bf9", - "4be57ef3502cf44c25fe3a78", - "4c024518e59eb6e6dcadd136", - "4c66298a3185efdaa5160da1", - "4c7d5c1036a888b7e7aed03e", - "4c7d6966df389fb3bbccc2b5", - "4c7f13352802b9c32eb0415f", - "4c9d8aa83f33fc1d1d4fc78e", - "4ca145f63d825af0b9593c4d", - "4cb3a7be1b65e46ca8458136", - "4cb4cd95cc219dc289cee9da", - "4d2d330ec70ef6f9fd2f38b6", - "4d4d471062ccfebb45bb70bb", - "4d58b442cadcedd3722aa1af", - "4df394b91ad3e69068f55a46", - "4dfb1e42ebd89ae20653781e", - "4e755958a2701bc81805c790", - "4e84648d1767033c517bd843", - "4eec80e560aece1e815d953d", - "4eeca05c15c3cfbaf619dd8a", - "4ef07f7595eb2e6f2f3c1caf", - "4f3b87964460b6788beeb36e", - "4f477a27eb485ebb5a4f2265", - "4f5e519c1dd1a3d9c8135f11", - "4fe0109bd300ea53c9c247b0", - "502cd0cd8dc7534f9956bdb0", - "502d733000b14e4864787e7a", - "5030468b557f449f4fccf990", - "504c0a84ddf5dbfcdbcd7129", - "50d161852d1db68111ec7009", - "515691fee2535ff5e511c11f", - "5158ef347dcfd1338ec06320", - "51b0035a67fcb43ea48fbf51", - "51cfdba9c2292aca98c4f4a9", - "51f3752f34c76e6c76e09a8f", - "5218c20681438ba93e8b80af", - "523074006ef21af1eeaab1fe", - "5235f60aa9a6baf9f90c6032", - "52412275f60e3e98e2dd98f0", - "527d97af9af51a479e7b89c0", - "5294a9f2fdc95e64fea4b180", - "52cf75297cad129f2b343ae2", - "52dd61f69fe2e44aaef20946", - "53333df30028ca3296cf62c0", - "53381be1de4328a152e122f9", - "535f6ffc31d3397eaac51921", - "5364a493ca944fae42ed8042", - "536857c6decd7ce46ce2d33b", - "5409d437a50018006fb79990", - "541e337f5ef0997f1db6655a", - "5438f1fe94e955720dbef961", - "546e7a5ae7991471a4a6bc5b", - "549369e7b5bdba18b30d2afe", - "54ba06cba113c6c19dafaa74", - "54c31f5730c4914cbe7bf655", - "54d162f2ad7e9565348789f2", - "54fe96d8c7925471ca54bb91", - "55108467723557b081c3969c", - "5516683bfa92420ae300605d", - "5545c9a7b4a3b9eaf19b03d2", - "559561a624015343fafa7a98", - "559c1102fda7267c4a4c42a7", - "55cf51a85ed97c8300ba9029", - "5601d56ad77e078a0c5d8d87", - "561e57b1dfc77b5575b382cb", - "561fb4e23b4a862792a981d9", - "5629c1d0349558b6e7965fe2", - "563ad574e0265e47640c70d8", - "56cc9bf0b36d2365f99428f1", - "56e2e12b82dea0c5dd1b2be6", - "571999cad79f744dc231197b", - "571e1587906e1070a4125b38", - "576ae78413ab4faf71a6a23c", - "57a1a3d72eda73ed8282ac8e", - "57a2923d7e99cab1b5414509", - "582eef0514043d390e618683", - "58462ebb5b2c984848284248", - "585b1347015ddbc60b0f5fd6", - "585d2865fb0f02a18be4fef1", - "5888d10e8b10a1365a389487", - "58d1b9aa12af49f0ca80c67c", - "58e4058de5c7ccdfdd730a43", - "5918ac04b964c410ac3371a1", - "59622d92755c4eb6b46c5c20", - "5966ab1cccff06a3d4aeee63", - "598451e35e27cb3d65307a9a", - "599f0d0f379bdb41f9e201f1", - "5a0bfdb14314b84532a1ca67", - "5a7bbe6d2ce5ddc4e9009fb0", - "5a7fcf7c364c3ebe6409e338", - "5a87b3d05231dfc055b3bcd3", - "5afcd2b8fdfc95e348e66996", - "5b6d7c677c065c2df0463fce", - "5b6f29eeda6c2e8ee781fb50", - "5be5f5964abbbd73bc8d6c4c", - "5bf2b4a641b11e486f3e8823", - "5bfeb06dbbb6ffc6dc2e9ba7", - "5c03dc4fae880ca080873bfe", - "5c06ec267d48d9d432ed8ce7", - "5c521850ff1265dc273ba41a", - "5c539646590287a2eeb12fde", - "5cc31e1e4527c1e88f95db8d", - "5cf4c58582103b66cf997730", - "5d5010367c13a212180cc57b", - "5da6ab30a70d8916b68fdab4", - "5e009c32ac96230f8d98ac16", - "5e41656d626ebb5871da7f72", - "5e5418cdc93fd5d5a4f547c8", - "5e5538bf452352d10bf4f1ed", - "5e95a40bd312ba89a6ea1ab1", - "5ea444230b548c2a199307ac", - "5ed6d6c80db4e581a40fc77a", - "5f151482bb69f01edf59bb89", - "5f19783ee138cdc9604f5727", - "5f9af07cefe610f9f3c7df39", - "6045ed508899eea9ef30d7ec", - "6059656a74f00ac47fed78fa", - "607076ccfcd7d6a36530bdb3", - "60914ce21aba8802a85bde84", - "60a0a6d831812a73e5e6ce6b", - "60ad1bda7bb3dfa70feeebf3", - "60b78d8f6e62633eaa438b1c", - "60ec71cee9a6a0642dc8f1ca", - "60f1e6ce3b42e97650c8799a", - "6101d4c510d291aa2e390480", - "614c81048a30370cc1f99fd2", - "614f3049eda21207162b02ea", - "61554361a66327e58ebe5f35", - "6186dd38d868b5930673a54e", - "61c17a6c8be799877f9f6604", - "61c6da12340a7210be9aede3", - "61d7f8f988357234eddcf6fd", - "62383f3fb946714908117a42", - "626aee7a76a05c18e9ac36d5", - "626fe640e78705d106c070e8", - "62944aed20fdb12c6e4f37ea", - "62a2d717eeb5f2125861c2bb", - "62ce11c90560f091e74942ba", - "62d0b6dc3ff9726383a4c760", - "62d14aac6559598435ab3043", - "6300e3003e8f42b7affd9bcc", - "631620faaa70ef250ec06805", - "6334c5763596e1b2c6550f6e", - "6398a5174473a9c5563de884", - "63b55ff6a362b5d69772626c", - "63bd6438202f52fbf0235b04", - "63cc83fc75db0ddfed28f76c", - "63ea62527d4bf4443d7af742", - "63f049978840c2d6f9b23cf8", - "63ff89651bd7514d54aac6b5", - "6401403f1a054c87ef641db8", - "644d9521c52be31f9ece48e7", - "6474145f7c49c2ee9bd5fa36", - "6480cddb25e6e09bbd47a4a9", - "648d67e279e513f85f3d72a2", - "64b2a3b424a20dc4e124315a", - "64b9a8cf3025b3ce940d0be8", - "64bf13a4b9afca68e87a16ac", - "64c031f9f42d399431b68d12", - "64d24b8d80bb16a497d3776c", - "64f25eb48ace841e8ba7e63c", - "65059f82b5e40d2dbcc9b5c2", - "6548bed3bbcb212859804e4c", - "6581c44d5d25cf9d642af048", - "658ab7506b6bdd3ce96e5ea0", - "659685310926cbd4dc48e2cb", - "65a3b66f57eeade01f6a2b47", - "65c2180995d1aa0ac7927097", - "66008e90c638ac5f252d0ad9", - "6605b3cc4c145f47c6d06dbb", - "6630fd4d1b491217d74257f6", - "66bad1133f4ca7120b86aecd", - "66c65874082c99cc4fd0ae85", - "673a177ce0e94b6ea2488384", - "673f753c28a8535c381dc6e6", - "67cc836eb54a4748f40a8353", - "67d7d4a152b6d6ba892c4b56", - "6883e80268079b1fcf79bd25", - "68c45b6e3fd3362bd271062f", - "68ced2de8e4d4d9b4684d84e", - "6910a682f6af8f72eafdb0c6", - "691ab81715ebc6de62674565", - "691bd41bb7c9d255e01b0c0c", - "695c465e3ac41d9f44a01683", - "69ac126c821fea946514581b", - "69b45ba807ef1ef796b9519e", - "69b5564b1f389e0666e444b8", - "69b89bf9b9b67c09d92a4c6e", - "6a2efbe236ebb8c0d1e2880b", - "6a7aa8a330b80086d81a221f", - "6a909079294bc16aeecf70c3", - "6ab212f9711c8cfbb4edae32", - "6ab906261a18f878c7773fd0", - "6ad9b00f98674aef3454e316", - "6b56898bce2fcf339806330e", - "6b88a0aec920d9302d660de6", - "6bc96a9b8cf844bdcc005ebc", - "6c0750585c6b4ffc82230d93", - "6c47e996a975ed5dddcc311c", - "6c4ae71bacbc487d0e8156ce", - "6ccfefedf55fd2aa069aec2c", - "6d1ca8cf613730c9a79c6e58", - "6d38eafb9106e8bf44358713", - "6d4593fb8266555f94af111f", - "6d45ea70bb1697e6e6cbe8ca", - "6d6a58d8e32b59a578bdaa7a", - "6d89fbabbd0e4194be630efb", - "6de78c67637e7d9169ad0263", - "6e127d6903d92ca1e9e7c768", - "6e15b521f275b83ee0be1178", - "6e1c25c10cadb66d49fc07b9", - "6e1d668aaecf00d7f3e4a058", - "6e43efb77914ceb22b92bea0", - "6ec67458865104ae9030eb2a", - "6ecf679d5b019917e948f2c6", - "6ee94f3ea3fa392a04242b5f", - "6f0262240c041a416d0349b4", - "6ff8e489b6979c207540d5c9", - "701e0dafc73e579f3654375a", - "70665aad66111dc8cf25d98b", - "709e4dc74309357b6803c1e5", - "70a3c15974f6073850927f90", - "70dc0737cd4b3f7b7bc71a30", - "71030bcfcddadbca19ed14da", - "71044f0ec9a0358d8a4df8d6", - "71220675a1ecdf1ea71ea955", - "719c30b4752b0edf19cb3c85", - "71c9443b3212ad7ff7d7f1b5", - "71dac0fed104ffa3e4e54add", - "72609b65d45c4d3432846f7d", - "726779fa791a38e2fa65820f", - "729a80d6127f6d488800d5b6", - "72a6f614fae38884ccfc9b6c", - "72e4d3085cdac864b312f72a", - "72ea1767f63448605428a683", - "72f62ea8b251f384dc107c63", - "730dd22c5d0f9308a1ee2ac1", - "7343f30b634f3db0c3c0b92e", - "73b47a6095cfffe023362a1c", - "740bb36cc90927773da1c66e", - "745fb5b231327527e7666269", - "74667e521a4ebc95196b7974", - "74d796ffbc04c3dfdf816b3b", - "74dbe0831c09576ef9eee841", - "751136256293b20add40ec3f", - "7519202c131bb616851a1b64", - "756c994b8ab745e38b774a4c", - "7584e256257e82b8f5656f43", - "758a5df90bbe0477f9e19183", - "75c992e1bdd185fb85549c7b", - "75ec48e816d99919f06bcca8", - "7683de847c96d3f076c0d60e", - "76a6543cb76ff0381b060213", - "76f5264be2697d00e5e126c0", - "774c0ac27cd3dbba70b46241", - "77836ece88a103e1bc9e358a", - "77a28b727fae18f27c4ae108", - "77bd1063abe17086770af15c", - "77bf58bf82022e2f1678b531", - "77e740277257c05337c695df", - "77ea32945a037ec7b731936d", - "780b32944a3c2996dbacf3bc", - "784c02f20aaeef5dbb3b85cc", - "785962ea7bc5d11b539961c5", - "78873dbb086e10aef87b3b70", - "788db3f9393fa8540edcd20b", - "78f9bf850966fcf7718edd9b", - "79300f475c27ab3e079184de", - "795e1533a0f8b633baf0785c", - "796310dfd3f07ae5c8103263", - "796e626677e92f6fceb5412b", - "79730196f9a7f1ae820e5efd", - "79ab05f35b43ece704bf86c0", - "79b04fe15af27d5dc296af47", - "7a9ee266eef87d39fb7352e1", - "7a9f1eefc33add27d7a6e720", - "7aaa83b8a68a63e79d291eec", - "7acea856a1f56dbd14dd78c9", - "7ad56f898230a4d8d19a4b23", - "7ae256c22789c01eda5ce7a9", - "7ae485a62d3797ba9e6666c4", - "7afbb72666924bde40476aaa", - "7b2dba46f54d23663370d39c", - "7b58878f0a95db3d8a748897", - "7b632c72de0ea6800b2c5b78", - "7b8cdddc0a6bfca799f6a814", - "7b9932749870e4e1be8c56bb", - "7bebc05252f61ff1477c3167", - "7c0d94d4fd5b01a1a4b1dfd0", - "7c27c807b4023597dd8e0453", - "7c32b38e93efc9c035e7427e", - "7c3ee02ec76ae46cd33ad644", - "7c460dd459ce89a4dff306b8", - "7c46ac358c8d7aa6fcb1c3ec", - "7c50c015a0f97eb7b5d2f49a", - "7c52ee37a3933853ab32a654", - "7c608f033dccd6cfd478dee4", - "7c74cf9d3a316f24f8164224", - "7c7580d6d9e817475d5d54fe", - "7c96ad73cbda39ebe2574eb3", - "7c9b71b1c0fe0d5db8875a53", - "7c9e359fd265c2cfa03b206d", - "7cd27b74ee38e1cdf0afa293", - "7cfb6a5b810a8a1c11e575e4", - "7d175a4b766bf6463987cd6d", - "7d2ed901b85c3c9dd6a5cd6b", - "7d47ab28b9f9ec57320aa812", - "7d5599df1c152460c8edf197", - "7d5a45ca81ab4d3e1bc1dd55", - "7d7ae6471a0071c7abd84052", - "7d869065dcaa7a2115da95fe", - "7d8d700183019d1ece28e0cf", - "7da10de81955cfd8171eb511", - "7da43cc94740225519b557f6", - "7da8441a19696585b496a56e", - "7e1a6b4ff7dc35b7b4c1fa1e", - "7e2a980ef20a7a4950b87f24", - "7e3fe6e1a4cadf604a78c579", - "7e672c6c4718d3634bf329d1", - "7e94e2c4d6090d17f4146bb7", - "7ebf2874fbc8a9914177a752", - "7eee8c2daec666a53e69d748", - "7f1448e29297fa92567ea1e7", - "7f7b2aa3690df08ecc78099c", - "7f97b53f3296523a66ca1430", - "7f9f92e0aba77220025252d9", - "7fb50c83e323368b78e22c7c", - "7fbcc7709de212680fe5f930", - "8083007115ddc37f9a02aee9", - "80c175e6fb5e6f51a13022ed", - "80d9e959633e5cbf01da77dd", - "80da3a9c058c3c31b5aa20a9", - "81777585554f3b065e958775", - "817c365d1f6d72b2968517bc", - "8191b8e8ff2ad21508ad074e", - "81cb823c33703a5da56d2c92", - "81e3d98fd069838122b045fa", - "822017b0760f4f50e919d1cc", - "823eaeedab810acd19f3f1e3", - "825fa9e2161561d89853346e", - "827312330f158abc7c4415f3", - "8281af3004fe025a121203d5", - "8284180d2bf80dd127ab4d90", - "82c70bd64718e93e9b034f05", - "831af6b7d66f8f44de837106", - "833bdbab5925ec37eb54c8b1", - "838c5b0a99abc07cb3b6ce34", - "83a72c18c9cc4a04b117c34c", - "83da6516a392f07d4c98bba2", - "83db15d83ecaa5e02ca159a2", - "83f26ed31584b32631122f73", - "842845a899f4604c2a776571", - "842b97fdb93972f16738c67b", - "842e114c03b74899e1f31c16", - "844643506c075c34327f6ac2", - "8452e6ad99bb3ce274fe664e", - "847f3323b33c0c4ae484a50b", - "848c838c99cffcd26206d946", - "84c00d9ce14986a388365780", - "84edc197f9a17a05f5e69e6a", - "84efddd2f7ddd3e7337df474", - "8501fb4822adcddf3f94959b", - "851c544587f165e109cdd194", - "852c66174202a7456fb607ad", - "85408647b13bf36fdca1bac4", - "8543d47f3216ec71b04b2c9a", - "85639b8903488d5d376aed86", - "8589b95a718a8bfc2f7df7d4", - "85a308d94667bf4df88bcac6", - "85a5c60f1820d43ab1f1080a", - "85b9416d56e0f398d779d70e", - "85d8a2c123f373a999dd154a", - "85e4207c197dfcd4a4543087", - "8648429b43947c1716a23235", - "867181460fb9b11eeccf7b6c", - "86e17057016e25475199967b", - "87089f109fd0e66f5eab2d78", - "871e5f292ea74b75ccd95c2a", - "8753db6abbdfbe2ff99047e0", - "8761403ab0844434c32f06c2", - "87815da6155c5b6230f99358", - "87a011a8b88fe0be7563e736", - "87ea43d496283c260110d433", - "882cc9f0de9d4b58b3b17a21", - "883df15d75bd3ed41eea3f36", - "884a29ecdecfd52c7f3b408b", - "886864fe28803b8d7ed339c4", - "887b357e5b833abd81dd5239", - "88a205ec0627ea05482b952d", - "88a4f2e043e81b5c7327a4e4", - "88d7824c18652a46cccd6711", - "892fa59be80e4a51987b2aa7", - "8978d9a1c938022ca8d8cd4d", - "8981fd04366166831330ec63", - "8984a0f376b53993513e9749", - "898787cef62c0c039ec6e8d1", - "899ba88ad5da53dbcf95d2da", - "899ee0049abb0af622db07ce", - "89a4f6286424816ca2309cd6", - "89b6da46040a46ee76ab3830", - "89d661bbc2030c511473af48", - "8a1cbdf26f651ae582b9dea4", - "8a67a4ef29e087a79c2914bc", - "8ada7eaed949cef5a3d01a47", - "8ae3a64c84ab5f330c03adad", - "8aea039d75e6debd3741c157", - "8b37d4e69e003b405b220994", - "8b3a2d39dc23f18b5a365c43", - "8b44053fd21aa2f977f64d1c", - "8b7ca7bf0b0663fc1c37a014", - "8bacb33011b613ba808277b5", - "8bc92057f5bbc9fc9d4c3b57", - "8be0b3d6ed9cbf902c67f506", - "8be3f5d408a0895f92847cd6", - "8c1693caa00a2ee3d9f91080", - "8c22ec14c10e3abce3819bc4", - "8c9c902b71596a030e6361b0", - "8cc7681b115eb86770bd99b6", - "8ce01466fdaa23e73dea8ee9", - "8d04533eb7812ce6f102d7fb", - "8d31c21b80092ac688f8ae7f", - "8d392353da2158499fab5b4b", - "8da7344de8921a3fe6ca1d2e", - "8db33c1fe8d5d6eda65af472", - "8db718d99c0d9c884894c7a8", - "8e0bb135db1e130434299be5", - "8e0fa1108e42179acb33f8f5", - "8e8f4f80e3492f9cd74ff36d", - "8e91bad7833e8c5daade1a4e", - "8eedf7eb387f3fdb4cd9c192", - "8efee0c8df2134c905cfa393", - "8f06bea2d2cf23fbc113456d", - "8f3197ccceb4fb90b15f7c6a", - "8f37db896f00129222a31980", - "8f641e7b46d47e2acf0250bd", - "8f796ed3036da296f5ffd183", - "8f89e25b2003e82706affd28", - "8fa1dc0283da47409eced1fc", - "8fa24f739a6ebfaf2f8ab7b2", - "8fb38b05e188c47290f02fdd", - "8fc44c1141b1864ba73fbba6", - "8fea9f601dda14b985256cc4", - "8ffbc52f2562ce5e8e140086", - "90275f7e5635ff4a17b6a63d", - "902d9bd5610c6d521e75c561", - "9035934ebe04bf45447cded2", - "9057509779be9a026cf8315d", - "9096df77b2d45193e8ba704f", - "90cfbdaad487985b8f7d4ca3", - "90d3f4a603408a2f6c1d45de", - "90e9252a0030e9115ff8b398", - "90f0f441bc1ab2f484707df5", - "90f6d2b3c953451d0f1eb2fd", - "9128030e8d4f4fc9e32279e0", - "9128fc05c1a213600f6d2ff9", - "91586b964124cf2453f7116a", - "91671a9cc700d9dd51cb991b", - "91882677b395f60145db712d", - "91c3466f5f7b7d6f3919e87c", - "92037ef8c20c84201fe95960", - "92a1edc9b0ebff6657e03e3e", - "92c1e82cd88d644e7e38cef0", - "93149995c21da5be89c9a708", - "933838f5ec8dbb4c5b4f5819", - "934c748b47eb72130521f125", - "938c1a74e3d8e4be0c25b823", - "93c1239f59376556b3bcbbaa", - "94aba88e35368729e7ad8ae7", - "94be82c7c0604f307cd64b5e", - "94d681718fb8ae99a9c31fe9", - "94fa3a8d8ef6d45f5eda20b0", - "94fd48f29664c03b9a5f46eb", - "95486e46e5ad68c7a5dc4ae2", - "95813d3ff4ba0f2a9bce56bd", - "9585ba16cfb960c9acfe2d1c", - "9593192860db288b53561df3", - "959f28d5b3e73e224d5dc761", - "9606f5354feeb51b2edbba45", - "960da586578bfa9442a6f0b9", - "962726582989ba85d0ed2b30", - "963706821e37540869cb35e4", - "9669efba4ed028bd618f7a99", - "96989ad3175336723c92db8e", - "96b9ff859da079799af95cc7", - "96ea7769b38404f0992056fc", - "96ecb46cea608ea5d49dbed4", - "96f864123c031de66e5e61a2", - "97267a1656f894d1441ad239", - "972c8580559ecd6213871bfd", - "972cfa1c5c418e09f7684c26", - "9730a49db763c1949363492e", - "9762835acd9b0d81222b8b8d", - "9776d1597459abc00cf61490", - "97e53e2f28a076477ad7a708", - "97e87ff23c837ec5bdf5e89b", - "982d07572d534f73bc758508", - "9852865b6241664de72d70b5", - "9918391d40375fcb145975d0", - "99220857f2de8b9d5ce2aba7", - "992793ba9100eae2251a020a", - "994990381194397932cacf10", - "994aebe0abbc8c182403a38f", - "99a82df1ff58fd9f2261c2f8", - "99dd4fa8886a43315931015c", - "99eb9bab36c78132494b2fb6", - "9a035e8ec7121af68a5236aa", - "9a0b27d05eebbca6781fe8a6", - "9a1670b4d50944c98c64fed5", - "9a3f8f1cabc6b170a3b68f6a", - "9a88dec165773688c92cf0e6", - "9abf6b37d6cf9a9c1508a5c1", - "9ac009e7b051fecbbd583187", - "9ad559af9b9f0ecb4ce49387", - "9aedfadaeafc3ce4a5d78ea2", - "9b4b787c4bf7df9decb54d5d", - "9bec98a32e6c6c3e8492a078", - "9bee8f1fbf4ca744e4988640", - "9bf88effdb006e5c59c9dfa5", - "9c00b5461cd2c5567f4aae49", - "9c1aefd6f72b02ff8f31ee83", - "9c1de47cc8b1d4495a1884b6", - "9c3b66dd8e115ac571a5b702", - "9c57bb4b82757f80a3706b82", - "9c81957688a08cf0a7084f32", - "9cb076c648859a830a12b940", - "9cd536578cadc6c8d9584289", - "9ce457300f835ae170479841", - "9d125a2358ca21d9ae6e742a", - "9d1767f10df5a5d83358f810", - "9d576dbbf4042045bf5c7aea", - "9d6f1e0ad131ec99b06a5277", - "9da27a06a574f94deb3d7f32", - "9dba7a5921a688634986340b", - "9de800ea34838647ce607340", - "9e08a82fa268aedd8c46b7eb", - "9e0c09202af4e7167a8bdcac", - "9e63b1dd3274a87af4b5ed80", - "9e679df7ce6e51904763bbd9", - "9eacfb7fd88291d5c8249fbc", - "9f2aaebd58362471588733e3", - "9f50ac3b6b3694c561c20687", - "9f6b0aa31179344f83021dfe", - "9f7543b7d2d6f84b7e674244", - "9f7b856ec2622883fb7c0387", - "9f90d7f459a57e8b1fcb8abe", - "9fcb80c026a9c0cfe7892a5d", - "9fe6dc2169fbf2992f583629", - "a09dcefb4a99d8c3e02a13b7", - "a0a73835178c4056652a49ac", - "a0cb9e82888d5abbe6de5db2", - "a0d87b6740f5a4031e47c31e", - "a0dabc4236a39e499d966df9", - "a103c5b805062e260e3bbb78", - "a1156d61e7d6ebd44460e09e", - "a12b8fca77c6e287aead45d4", - "a190440bd86dad947b38a172", - "a1973d8e05ff3cf5e902f740", - "a1f9e97aec011844760036a0", - "a207260105c9ef272d7c6746", - "a214dacdf4dc1bc9742c0180", - "a218237e8187b8d6ffb84a02", - "a22e94fafcc97c88594f1e32", - "a23997727ac775b220281f25", - "a24a61162c3b820259cb7e55", - "a256d08c42e59098e9c8afce", - "a258df7b76ad3200d0201f2c", - "a2719ca0bc4bf01ced11cd65", - "a2a9dfc3b80b30a85f58571e", - "a2c083b825ffdfd48d7d0c3b", - "a2c4b57b3d4b751dc3f2c98e", - "a2c68f95842ffcd7b11bc93b", - "a2f35c00b134f2314149df22", - "a31dba5b20ef94fdf1fed3a7", - "a33abef5caf04cc244290ace", - "a36bcc4c925664f567d3a057", - "a3bc00277708b0b13acccad0", - "a41b7f9d0ff1a7c107f8ff8a", - "a4219ede2ea3c7422ad06a91", - "a4696058b2248aa9bcf7dcef", - "a46cd01fb22c9918da9a86ee", - "a48baac0b6a63df67b558596", - "a492ef9916121559d41d0c0a", - "a49e1c226e4efe8fc995aee6", - "a4d0cd81b758b3626163c889", - "a4e6732d48e2c05046cf4f93", - "a535b253f558ba045c31ae5b", - "a53d766cd7376d73550bc039", - "a570fe5faed8f4d386ecea23", - "a58bd106e7fae7f7cb8f6d83", - "a5a621c546308c95fdd6195f", - "a5b6a88fc9d694e13000aa44", - "a5e1f93af40b411d8042d6f5", - "a639328d73d3cd5a50f5bf85", - "a64c55fa0c89eeb94368c181", - "a68a5bd436ce7a56a3de4dca", - "a72275ea23ca106d70b8063c", - "a73c23eb560ca6c69b682754", - "a75282b6cc233dcfef933fc5", - "a768dd7e2df585ea9bd8f1b5", - "a779f581ae38811cbd37ddbd", - "a786c739bafd1c80541cedac", - "a7c0fac3287313716248c813", - "a810106055ba6ca44e93f347", - "a821a5495e70fb00c6d67f48", - "a82b05c037e3b85fb60cf97c", - "a8470dea01ee2bb2226de3c4", - "a84cace3699175e09d1a7831", - "a857dd07cb03e301f113e0f3", - "a884350baf1937add4a1403f", - "a8e1cb9019ed524b61e68b0c", - "a8e28c09e75b88a393238af5", - "a8fda042779dafe1b3bdb842", - "a924c0390f1eb14535846533", - "a9375788b5497fc9a820831a", - "a97275afd5c63c1e9469cee9", - "a99c1d37c26a973458ac8013", - "aa02d480fbf8e30f4298874b", - "aa1a61b21e78833f9a48a0e8", - "aa3d5a38180cb3f4457c2ac4", - "aa5ef238fcbe6ad924a37068", - "aa924ce04ce11ce5d35a7d63", - "aadcff4d05aaf695a527e88e", - "ab02abc2a3a5b5d312b29719", - "ab2d974c62a0234c5583e96f", - "ab464f1aa23976f120d558fa", - "ab68e5d5af4460b8c471af93", - "ab852474bd51f2e01c90ff7b", - "aba9ecdec8645aaa9c93ba91", - "abe894c60d12ee9ffb4361d0", - "abf0f82d753efd328eecd626", - "abfcb7da56dfb5e59a74269b", - "ac4028b432ee6aa7208a0e1b", - "ac5ad0b972a5b640c00967d0", - "ac6757a934caf188b6c81596", - "acb2ac5f0bc83bef4df87949", - "ad09d3a0742fe6ac4f22d71c", - "ad0d96b586f14043a3e95e56", - "ad1d31443ba4ec0dde27010a", - "ad1f6fc57c26636026c97672", - "ad26bb497d7d3446d2369828", - "ad2f6d7bd719998fd5f8c3e5", - "ad357ee685baa0eb8b86ee5d", - "ad5ac1af454f9820b7127698", - "adb05f4a9932e4515d59c577", - "adef27b06b8af84a849e975a", - "adfa6396371b9fb2bdec77bc", - "ae06acbd04fc6373493b3474", - "ae225d168886ea2a4f77673a", - "ae2cf1e57d1799e5f63d7159", - "ae543d9dde7313d25f013875", - "ae610127db6cd629e8f6e1f3", - "aea75903bb3d977d00a3128f", - "aeb241cc0d384ebe0c4e779a", - "aec9aa7c9a043850f52a907f", - "aedcddaa6952114c745018f3", - "aef64b439980e45a00d7c0de", - "af19d1decfb5ed657fb00e68", - "af54d0dae3ef38207850f4db", - "affd3675f8dbf605ed632979", - "b09f244b6a72da56fe62b215", - "b0afcf740e88b2e039751b30", - "b0d2dbfae94557b2fcf5b1f7", - "b0d3f91f8cfc2228d90d316e", - "b0e9b074173777db296b4498", - "b109d45dd7a2aa1e767b1cdc", - "b126c58d6c1cc896e855d2a6", - "b136bfddea053ca483151b50", - "b140974318afab50e9fd349a", - "b14d35a794e47cef8ba8c989", - "b14eebae256f175e7631fdf9", - "b15167db866dc37871e305af", - "b15bbf492797b2f9579ab928", - "b19747b2a9c185160afcaf3d", - "b1c4bb26d40e868887a1eb73", - "b20ccc35d2fb950e7bbb04e9", - "b30331aaffec4124d6b92c74", - "b3049ff5e5682fc87ba44785", - "b31eb134ce5545f37de841e1", - "b337399ee0945c3fc874fff0", - "b3cb70b1073f8afc469a4556", - "b3fa0438cedb4a479ace4675", - "b3fe080ef990d2e299874b18", - "b4032f4a28d3b0cb537f1ae3", - "b42f787305c49b43a2736eda", - "b4307871b261896ebfbc5bb1", - "b4585ef9fbc1a258187d65d9", - "b4621a7d21f35524b6eb0ac7", - "b4ba08890be3600152745b27", - "b4c1d9a9d68327269338ab72", - "b4c64a56b7b6bc79fae225a8", - "b4c9a3b7a625fad2183a606d", - "b4f4b510fda65263de6782b9", - "b4f703ec956c3db7507f2b2d", - "b56c6206a7afc171ddb0af8e", - "b5ac94bc6d6a02f3209b061f", - "b5e76da73c5847a3c039f146", - "b6685df74fd99049f14b0f26", - "b66e12b58b5cb980a8e035b8", - "b673ef8168026525685de225", - "b678f64c6567c6cd56d1a261", - "b69752e3b57f8fa6f259230f", - "b69adf3bf8b5daae2fcb6808", - "b6c818f647428e1daba91c03", - "b6f19645f1aac77f12d94538", - "b70b999a0e0cf1a511c4e8ba", - "b71e2983658438f7140b9fbb", - "b725e302374787f6c0d172ce", - "b756244bf7455b493eb35d15", - "b76ab397a79635778abf8cea", - "b7b00f1a08d32f2a771a7dea", - "b7b9242425ee8f4dc85bdd79", - "b7e1d8a04de7d86d61c43429", - "b800b19a9a949d2a2cf36949", - "b82c00e777fd79ff467615eb", - "b8689f4a629c5b05346d07a3", - "b892d33c4f5b6346ce2e59a4", - "b8cc63b77da0910f51ecc701", - "b9377ad7084d3539694d3647", - "b9473e228c1263a25687e69a", - "b9692d2c12dc283170b403bf", - "b97fb2af1d48f4b73e88f2a5", - "b9a3f14dcf6fcd209bfb6082", - "b9b1fbdcdfb743c44118c9aa", - "ba19b6657ad47d6efa2b8ed2", - "ba1c7fb356a458bb44ead8a8", - "ba4094f45db9cc29e763f236", - "ba7b914f67c93159636e4c8e", - "ba89efcf75ae5c1bf3e595cc", - "baa63ffbc611c25294b44a51", - "baeeea44752ea022c1d8d3d0", - "bb145ff910a06c03e473fc1b", - "bb1ef6439e0548d7872c76cf", - "bb5c75271b29028a35719cb9", - "bb8cd69a3b01a8a14c00ef5d", - "bc0583da5f1c6a1ea4bbc887", - "bc0cdc9358389c5cd290e477", - "bc1f2ab1cc9376cae1bc88f4", - "bc2f256af0b915f45a2705b2", - "bc32be7761f312b338a05981", - "bc7351d354fe6cce5310ac7f", - "bd1a65c8c7b191b96647d3a8", - "bd2c599bf5b48e1a3c6c21cd", - "bd49230e7b692c8d8becb87b", - "bd5c0ce238177e0a29ea4bef", - "bd615554a8af6ae0f22fc51f", - "bd7e309dabeb268d12b5a52a", - "bdce4c36620314d2d2c2f38b", - "be2010a8f4103d400b61d888", - "be67a34513e630f9e7235716", - "beab896af6eed7d08e6739c3", - "beda128492779de554e3debf", - "beeb98373f0f64412691f637", - "bef453ce197c7eadf0bfa7cd", - "bf447b7e80f2f017eca81b63", - "bf523d90c4ff842aee8a197d", - "bf7f644ca1edeeb3c70f97b0", - "bfae88be6003ba47178fc43a", - "bfeace643e355fe2419ec8fb", - "c026578949631bb36e0ffc77", - "c047ccc1b6051b3b1708b6d4", - "c063d9b8712870e639e3833a", - "c098e02b63aef7cce7ea0aa1", - "c0adc1ff094b46982f28bebc", - "c0cbc18e6e7ba7a26c72afc6", - "c11c2d3fd17ccd9098769237", - "c14fe640a9e45507173a7d22", - "c152adbed72647d9991852bd", - "c15b2945c95b7d9c60441797", - "c1696b859bf23a8f84a682a1", - "c192c9db97995695193388f2", - "c1e6ffda6830b2a8b9d846fc", - "c1f71a219f63fa38c1fb1c69", - "c27b073ffa60f55aa1b9b97e", - "c2a60dc6f373d6b7818ab7b7", - "c2e96675eaedbaea986c993d", - "c345c08d27ee1a182bd31fbc", - "c3731b4ab37d77a293fb4c3c", - "c395e629bcc4b57c0256dc5b", - "c3b9c8adb5f253b8553cfee1", - "c3c07462f8667ac1485cf15e", - "c3c19dd7e9b6ffbb33e1e834", - "c4042e7b5d5d00abd1cf2f0a", - "c437e7a1fdd28830e5c00ef7", - "c46b82f38881e7474a2713df", - "c46edd3a9ffc75fa8f8c1182", - "c488ce308dc4935b92fe3cd6", - "c4e68a7ab49b32a1adaddbaf", - "c5421f6b06e05e216a10a592", - "c54ac74469de2e81e86a90c6", - "c55676aa3771094193d11e3e", - "c569685aefc1ead482afbdba", - "c56ecaae40e8b0d1a44ce878", - "c57f322c9e34ff30dd5cb5e2", - "c581cc85c59bb9c7aeccf4a6", - "c5be7ac4e18acd4360d669c0", - "c602ec938437b0d50b8682e2", - "c61ea5ec9b760cb73154a87b", - "c623ae9e5e5b9c4b1d4bdcbf", - "c65118f000599ae643efeaa6", - "c65af7e0722ef78071c23e28", - "c66b3bc494654492c1eb42f1", - "c69fbe7b87cc199d9afd08e2", - "c6ad69dca86ad03f9aa950bb", - "c6c638a8973c641eaf90114a", - "c6e870b9f3b27ab7b8220054", - "c71107f91690c342feafc2eb", - "c7270f5cca7c61ceb2e71b85", - "c74de8025ec132e23568d99f", - "c7555f301a9780f8f2136e07", - "c75d3060822cc6cad737b61f", - "c7625969b232d843478d7331", - "c7700e2eba2a1b93244a6a4d", - "c77dd26988b6edc25a0191db", - "c789132c9253da7a14540db3", - "c7bcaccce63355c1b92046cc", - "c7bee0c8df3750f25f0e328a", - "c7d96840a2a1717a542f6078", - "c7dd4355c60dcf163fe92fe0", - "c7f46e9224ab00f5b897776d", - "c7fb5ce600e426af4d4c4df8", - "c811c5e45772ecf11e24736e", - "c8265ad41ca1f4834ae4c05e", - "c841d4c1a102ae83335c137e", - "c85e21167d2f56c446857868", - "c889c1f2f7cf702746313161", - "c8be66c53843cbc89b6b8296", - "c8c01d810ed32192c794479e", - "c8dd1e6520a8e801ba2c84cc", - "c933a1368e41b2d2e9cebccf", - "c93c94302b8d572411bd6715", - "c9439e02fed6ca48602e721f", - "c950dd8c9fa43f39963c6642", - "c9661bfef4fd44fd33067a30", - "ca17493ea8620f9075f8ad82", - "ca1ad70a68c5e13ec0a18f89", - "ca46f62f691f2d4d9b0af4a9", - "ca4c7d9b438d47ea76a31a57", - "caa6863729a5159b05d6282e", - "cad250840e00d3c5988a6b5b", - "cb021a1e72cc1baeb8be455e", - "cb369407ca814f5508a7c3b2", - "cb46709ccccded9a3dc2864f", - "cb7866a624f17e25259ef6f1", - "cba063cd5b7101a4f6f4169e", - "cbd852d99bbeb319fe47a546", - "cbd95e66183ed8ece981b464", - "cc3483d0963809fd9a2806bd", - "cc69ce79c41582abd277253d", - "ccc3eaa02fb6096219ca788b", - "ccccddbb36e18b883f81237d", - "cceebdf0b1cd3692231d42ff", - "ccfa50e2cac018f54075be06", - "ccfd73ddfb674b354fc23465", - "cd03468350d361c4fa871536", - "cd21624b9dba3a1843c92bdf", - "cd519b485c751e678cc51997", - "cd5dbd9647b16572b3a36767", - "cd69a60fe4f8dd3f71ddcac8", - "cda74d47a0abee95fcc37049", - "cdf903ddc3250844e9c13487", - "ce11dd66b5fb8b99066af622", - "ce147c50d8693f2597434b9b", - "ce147e60d1b7b21c0b3bc2ee", - "ce2b586c13699422029913db", - "ce3d772a1a834811079165ed", - "ce8c88c023a136aeb256a627", - "ceb32dd9b2d35ca144c7b5ef", - "cec86f43daf570e3c7b7cdce", - "cee897368427aeab946a3d10", - "ceeee97c108b36ae1060770e", - "cf0c04d397bb0ee7142ef82c", - "cf28164b34847a5e861509d4", - "cf28759f11e075b8939f772d", - "cf3411a4b5cdf594edf1b598", - "cf40b39bbb923e82105d6581", - "cf62486cac06f343aeed59ff", - "cf6b9163b7e3b1f82e3c2b54", - "cfab164fa297d5e8c49bc619", - "cfc113eaccedbac65a874530", - "cfe27574024f230fbb91c8c6", - "cffedc67a668e2411fb95a06", - "d038d91ec2350a7a9d2d35a6", - "d08021064694e05336843c00", - "d08c2a27d9598479cee9fbe7", - "d0ba4b2311f122a26afef6d2", - "d105322f855bb87c94518202", - "d13189ed78073560fd9fac22", - "d13413eb2cc968c6f09b26ae", - "d174d76b15eb082ed027ca9c", - "d18254556fa9602e1220fc1b", - "d183b5c77504e35c9c2ac548", - "d1d1c7f8c7ca139c45d55a24", - "d1db452cf664667a24ebf208", - "d2294dc8f81b9d866e53e8ed", - "d25925e5146d2faeffe94f02", - "d286bea8c4d096049441518d", - "d30229df8f6e5c880a7df9d4", - "d30a8969cae905a8acfb9f35", - "d31a41645e4874694b8cf299", - "d33c9434a3d631165eecbb9f", - "d3b79b0815237a8a3cf8db05", - "d3c0281e83af62fb1917fa28", - "d3ccc581e7d9fdc6df07e61e", - "d3d5a9f1d0c62989109b76bf", - "d3f6d16ba9597afeb152d203", - "d43d0a77167f9e542525eeaa", - "d457a14222b6cf6c34b006a8", - "d45dc10239c4a19338023156", - "d47e99abe714f79dac6f3cd8", - "d49ec7b4e7d21ab1b4476762", - "d4aa9789068499b1caec193a", - "d4c1e7edc57eed8a738b1673", - "d522bd60c867dc27368c7350", - "d532b6efec5c483d147408f7", - "d54693e1175b1341499fd1b5", - "d5737b6a96ab54b37fa232e6", - "d59ce88d5f8c799a94e0eb83", - "d5d9a55cd7e85ba124f690dd", - "d5dc8f04431eae5a67d05da1", - "d61ae7503fefa9f49e6bbcf4", - "d626574884161cd5ebe0eb8e", - "d677d66182dfa18790524fff", - "d68e78733d81016cbca6fd8e", - "d6d805c23384c6eca02e599d", - "d7096419fb4e5d6acc33cd7b", - "d71cde725ec6a192b47e2529", - "d77d75320a2472e928b8bee7", - "d799f8d7742ad93944a69b86", - "d7a4e5698dff87c4eee9c03c", - "d7b7f3456f0d54bc34f58d69", - "d80c44718edd67c3938e3824", - "d8656543fe06172eb2fe920a", - "d8d4a1c42d7952049e0ff7b9", - "d8dc7b9c9bc46405758bdab7", - "d911dfc3e933053bdc63640b", - "d91ce222173f2f1afd9ceb0a", - "d927032816b6bead0f45cd98", - "d94207348cb58516cbe21372", - "d9918f229da51ea9c53e71f3", - "d9b64c80d10d2ce1ef1588f8", - "d9eff9a5332b59b5b7cac9a0", - "da34adccd0e2433723559691", - "da5237a75404b03c5e1136ad", - "da77362e803193f9646f23ca", - "da9f76688772e560315d5008", - "dac80db72b6dde9516344e2c", - "dacb6918a3d593ce1dcfef71", - "daee70676e659d7a000d78f5", - "db6588108b8c3aaebfa77ac7", - "dbccfb13787ef879a7b7cccf", - "dbcdf91a37ef83888ca04170", - "dc0a134cd8afae108a88cba9", - "dc1a81de3f30d195790d166d", - "dc3adc026c022d7fa117950f", - "dc5714c97da6bcab949bbabd", - "dc75017b3cb894ad99b93cc6", - "dca6fe0b105bc6482f52c2a7", - "dcbb2a4d0943c42c93740cd3", - "dcbe30158f7ce4593625849e", - "dcdd5c46c64d09d783b2336f", - "dceabe11c47b4b279c7cfddf", - "dd0677d05336b6311eb2eae2", - "dd0dd64228d3cf83c747601c", - "dd4cefd24f4e7e50b62ebb10", - "dd547127382bcb2af643c99c", - "dd7fe66596c62e0e63ad6dd8", - "dd8d03eaf33330987c807ed2", - "ddc412d7d818075f8ead7965", - "ddf6c0d258dacde3c771d908", - "de02527685c998b0639ab8c2", - "de2829543733b3d4be27143b", - "de3afdd58b7dd578ed5d4d4c", - "de4908c64bc269948694839b", - "de5c55c5f861f389dbd52468", - "de6c70125d0197646cb241f1", - "de6c755ff909bca53d29fe17", - "de7794afdcc64aa29a70c57a", - "de87b5a02c3521e401fc08dd", - "de96eadac6ef834a84c43ee0", - "dea11ec13d5feee576d0281d", - "decb730443817e45449511fe", - "df0226f647224f50cf8dedd2", - "df16a56591cbbd1ffcf9d358", - "df43eb5c9b2ee26623149c77", - "df59fb36214866a309e0ba30", - "df7932c4a858cb0b9edea842", - "df99e3868adec5de46de1adc", - "dfa63b64f445a228cef92bf7", - "dfa7bb94b3bf0d6e62ed2493", - "dfc4f330c324769975792eed", - "e01994c725c8cfbf6c4e679e", - "e04c2ee033a5dfc014d71072", - "e06ac66fb04b470803c6d840", - "e072a8c1f44287ee570457ac", - "e07b967ccf5249c98f8f20d5", - "e09c0dce174ed8dc530fe501", - "e0dbdef8ee4af28409b3462b", - "e0f0e4acd0bc1b0ec6747119", - "e13367571cfbe4b34db09216", - "e17ff2bf493777f5b64f903f", - "e190119a908eda268cf997b1", - "e192955b110656aa21172851", - "e1937e7166787cb4231e9c77", - "e1f72052128658ff1b9b7e6e", - "e1ff5eeb71bc3ba5d8280222", - "e1fff8c3c017ec114f2ef372", - "e248e9d84f0aeee7253ceebc", - "e277cfd963d40bbff15a8fef", - "e2d8b440d09fd83ca2658eef", - "e2daf08736f39ab6e89be7f9", - "e31901ded552b5ec2310668c", - "e31de847d2c578d62080b099", - "e340aa22c1ab15b269f3032a", - "e36310c0738d31ae90a11fb9", - "e370d8fe95f84b8245dbec4f", - "e3b199dd543100fc10f74e86", - "e3c66a2d8664bd678b05b505", - "e4097c4ab62f497a37d84c5c", - "e471f2b967226ae5e96b28e4", - "e47d6b8b83a7b472f2272058", - "e4b31adb708ccb177481c64b", - "e4ccb6e8a9847b1310dbaf6f", - "e50a50b736579c0cfea12603", - "e53636c436d1decd42c0d5c8", - "e578f38798c0bace6fba78c2", - "e57a331751bcbbee8571ca7f", - "e581baa0867f160848e69979", - "e587830613d18d734906fe3d", - "e58920ebbc39cfd762fb84b3", - "e5a2a21148ef834fc6d2ed3e", - "e5c45b133eb7362d5d80b2c7", - "e6080fc167645cfd36b13e1e", - "e60a4571ebc08494eec3e657", - "e6342a4cf45e1404b22243a7", - "e655c2119ddf616550f3fc85", - "e68ee491b2283b911a4fa358", - "e694f7706cbd8a2275a23900", - "e7137fcba0de8fb9a3ca4f2e", - "e746865fde88dbeb1809d26a", - "e7794973c7723a4835f09655", - "e78023316592ee1aa3320ae2", - "e7995869957227a83cf8283d", - "e81256c839e22e081c4134b1", - "e82a35e99a3ed547ed67f67c", - "e84587ef80b0a1b42b8473ba", - "e85354a7bee5565d7854d82f", - "e857773ace59b27b937a92f2", - "e87750e3c2f0b9a2408024dd", - "e892e40856a180cfcb94daa3", - "e8b5e46890ceddfb5237a621", - "e8e28f7c5ee54dbf2193869b", - "e9326796ec8d756eb62078db", - "e94a6c5dd1a380d4e2ef00e4", - "e97fa02030444dd48d122ce6", - "e9813b6e57023138f2180a83", - "e9941831c962d890bcdb6d7b", - "e9bc6d61412f5306f7779190", - "e9d6c357d4f933eb58617395", - "e9fb3aa72b2a5d5cd8077864", - "ea08f8db74c691cb8c005242", - "ea0b6af51fd3606aa97e4a7c", - "ea5b8818142bb9ba799c84e7", - "ea5ca51c12a5b0885764e918", - "ea77f5b44339eccdfd697fa9", - "eaa667f3c9a8f07bb1f8a0b2", - "eabe261d6fff54d7c34e2419", - "ead931303e6cc86d36d43091", - "ead963a5a0d803174a31b407", - "eaee1360196a0c94af867ebe", - "eaf6e27102a9ac45c6c58138", - "eb2b19b01bd9d61812ee631a", - "eb636c8c3cf7da3cc9fe2952", - "eb78eefb8ca8356d00b7900c", - "eb7a32ec010006ef37754183", - "ebf1cd76caf56c86d490195a", - "ec89e06b7a6e0ff112301a2e", - "eca906a273c3d3170a5cad4b", - "ecbedb89d13879814fab406b", - "ed38769f730a66a24aa1c6e7", - "ed7f75c976d7dc2dc0ab5c7a", - "edbfd4ba49a31000b7837d12", - "ee02f902d4e65021043f4906", - "ee4924ce532f1922462d4b94", - "ee6b829482c0c6c7c60516cf", - "ee76dc3952215b0c0298a9e0", - "eeac81ad9c864c82793f35d5", - "eedde10bbed3307affdfbb2a", - "eeeebc7395d71715b26ac658", - "eef10954a69a04b449480711", - "ef18c811ddd7651d407648b4", - "ef27585fe67356b4507bf7c9", - "ef772d0debea52302dbaca22", - "efdc28bc6794a4b4a9e78563", - "f015a4c3fb5468174c3ee62b", - "f02073bb24df178381d4c34b", - "f029ab5fa8f552051d757918", - "f04c3d71614505b7002c8cdb", - "f0b76059940b0c9dff240d11", - "f0f36fe272825fa04413565b", - "f11a46615be244a8595c9f54", - "f1368a799dfa9e4930a4027f", - "f14a923cf18398921aa81bfa", - "f15521d32f65855507fe37da", - "f16a272c2e2383ad1df685a3", - "f17c2453819f1c5f44b6b9a7", - "f19d8f1a1f7709ede4143558", - "f1ccca9ef4cde4c876dd0f92", - "f2058e7da78393d320dcd826", - "f21674a817ffa843e36c0ea4", - "f25725ce5d0cc65acdb4f465", - "f2980e9525c53f4e99d0293d", - "f29a0958529a4c702bf27fb1", - "f2b8e0e0f1b00ee1a6c292b4", - "f2e583d2413dbec63b652af8", - "f381e97ae0944666e7c4445a", - "f3c1788d183638783fb544e7", - "f3ea65288e5f89a2ccde4ddc", - "f410cef705fa4ee104c0bce4", - "f419e32c2059b73ebe38c6ab", - "f424866ce0b2810221248364", - "f473a28eced6063484de5227", - "f4b2e202d9831a545d8f31a7", - "f4be3053169fa1f89cd26d4b", - "f4d21b0404a4b5f0a0e584da", - "f4ddccd0d5e51753b0eba7bf", - "f4edb1ffcac7358e171879d8", - "f50c2ce896a777a3604035c5", - "f534e5e30d75fa55a9c4e4ee", - "f55f3df681998cca1d75d1ba", - "f575690e21d1b1eeb6dcbff5", - "f579c5e3c7d003a68331ba9c", - "f5a6b0057f590ab3717ca740", - "f5c00dff841f564a1f5686d1", - "f5e63d4292bf225748b5ac5f", - "f60e5b609abd2b59ae02ec1b", - "f62340bb5d3959505f8f5986", - "f642677e7bb7a2dc0ba3ba42", - "f647a7d1f8042a0208027629", - "f6bbf3944db7fdcc06bf5079", - "f6d10a6987d38df963f9dd32", - "f6dcefa60da3faff6ca4a877", - "f6f5d422bdcfdaf3f41304e7", - "f70035f8e1939263fb44a754", - "f72f8106d8da715c329ae71a", - "f742933dc7ba7cd7d0dca63c", - "f74ecb0a5340e601b9b906b5", - "f75c145f32652fa43cef0a41", - "f7bd0f7fb8966d5c587f2d9a", - "f7dd6d290358ab99a9c88fbe", - "f84cf7c17fd4e956a5583db9", - "f8b677965da863b6c43423e4", - "f8be6ce150dfd5d7c49087ec", - "f8c90c73fd91a9729e6979ac", - "f8ca2f842a7d94cad4564e59", - "f8d56a2dabfa3821c6a22452", - "f8f81e56cefad830f192ec45", - "f92dd96740de409918e81198", - "f95e3de64851f58ebd9d2f17", - "f978841858e59a3f0ccd2dfd", - "f9a9a577464b9b008290159a", - "f9be1e24710cf4830081a897", - "f9ca2ad6cad4bf0f84ab0dab", - "f9da74c024ca60fa6167cdb8", - "f9eef381c2a2543d36f9ee20", - "f9f03d7165cc27c84b5b613e", - "fa8085e1d1fa37b500717aa2", - "fa87f241022fb92e6a2ba86d", - "faa82500b3107f258a06b475", - "fabaa73c2b99d6b3408183e7", - "fad5a508a0601dbc76c32e7d", - "fb22ac08cc53e7344dca1162", - "fb33ea85698649b6e1b9d496", - "fb47fa2125e242bfd2b40a8c", - "fb580f7311fd4bb47e3a4c19", - "fba4aeee5ccbbfdc354a6e92", - "fbd79e78d45b37b47f938d39", - "fc32637245aeba9de15ef93d", - "fcb50a861db050b289f659b6", - "fcd842fab71f2069d4f36900", - "fce8f87ee6ee6418151c1b61", - "fcea40dc989bf2c8f4b32e46", - "fd0a4b21a7884831dda0a01a", - "fd2693ffffcceec0d3af3581", - "fd53e3b06ee3d047b5026bba", - "fd67845025b8460bf877df66", - "fd9ceeddd047701230288df8", - "fdc4f3700e74cd000bf0c7a5", - "fe0df1929735cbb373110d5c", - "fe424def774af8212d88c5ed", - "fe7dae3c7bf55b604e588e75", - "fe84628fef72b96b5b3ec4bc", - "fed841dac3206c63de30e73e", - "fee7aa0c539d7471f3120666", - "ff0a3bdb44864f29ca3f0100", - "ff3cfeb7b6395cc8b848e343", - "ff4a0be36d0a1a0d0b56e898", - "ff580e381da9d5b1522c75e3", - "ff5e06b3cc8122f183f9e17e", - "ff6c5bea2c57227d0544c047", - "ff70ab6d32dcd8b2897df28d", - "ff87dcc5d328155abd6e5483", - "ffe1da44600c808f463068df", - "ffe3ac62a0fa61809ec24625" - ], - "headings": [ - "1544f59729126e94ce473333", - "25f8e5dcd0c55af93b51bdc0", - "3039410d9447864aad0f7192", - "38ea5495d2d5ff3b7528ee13", - "470c313f0118afc5a5f327e3", - "4752b3186dcd1afbd8aed5c7", - "56aa3e3c27b98b1371799b2f", - "627fe72a26a0c4003c9e2066", - "73b67c02138c61bb5f97418b", - "7a3ac0daaded45bef155c385", - "7d48667eaa21d4fcbdc3c879", - "99f787c4577ecb99dd828224", - "b26574da9227c23c8ca6c777", - "b4f3632844a10d3add59e23a", - "bc62e88de51eee77fdeeb42f", - "be52cea7d48ae35bd3a74c9f", - "c7da2f12c8fe54143779b1e6", - "e582d7c8006a09b57fb72590", - "ec0e524d657131d6efdd0739", - "edb8ea204b96700292b181c5", - "fd7615d451c5f345d9f64bf0" - ], - "ids": [ - "017496de8c73a620510f04f8", - "04570549e6a54a598fa2612c", - "1544f59729126e94ce473333", - "2e1b534c284b28a143d39527", - "43815d2cee8b34a1d69729d1", - "4ffd3c06beff6d3df01d9cfb", - "5f195910401ad7245f983dad", - "5f8d3e78c13184d990427e74", - "6a4b3447ae51eceb7c6411c8", - "759f2ca71563b81d0671c541", - "7dabfa3cf09b15ee995e588f", - "7eed6b8d8f5126765d8d842b", - "7f571119ca4e0ed772e951a3", - "84f570202c477d2177cf03df", - "89a5c7985df4efcb390dc79c", - "a6622655ae40c8331c4dcf47", - "b16462ad53da295e1ca6983e", - "c355e8937c69b4c56390153f", - "c4a7abdc795d19c55b0c2670", - "c9e21a20c89743f231a8d00c", - "cff6364373b80c0b12d46290", - "edb8ea204b96700292b181c5", - "fb1c7e6b2afad3c5e24ae00b" - ] - } - } + "chapters": {} } \ No newline at end of file diff --git a/src/lib/data.ts b/src/lib/data.ts index e923270cef..1edb0581aa 100644 --- a/src/lib/data.ts +++ b/src/lib/data.ts @@ -44,7 +44,10 @@ export function demoLinks(id: string) { return (links.demos as any)[id] ?? {}; } /** Key papers = works cited in the method's primary sections (computed from the manuscript). */ export function keyPapers(m: any, limit = 12) { const keys = citationsInSections(m.sections?.slice(0, 1) ?? []); - return keys.slice(0, limit).map((k) => referenceByKey.get(k)).filter(Boolean) as { key: string; text: string; tier: string | null }[]; + // Reference titles keep a few TeX symbols from the .bbl (e.g. "$\\infty$Bench"); show them as characters. + const TEX: Record = { '\\infty': '∞', '\\alpha': 'α', '\\beta': 'β', '\\lambda': 'λ', '\\ell': 'ℓ' }; + return keys.slice(0, limit).map((k) => referenceByKey.get(k)).filter(Boolean) + .map((r: any) => ({ ...r, text: r.text.replace(/\\(infty|alpha|beta|lambda|ell)/g, (m: string) => TEX[m] ?? m) })) as { key: string; text: string; tier: string | null }[]; } /** Find a method whose name appears in a free-text method mention from the framework data. */ diff --git a/src/lib/manuscript.ts b/src/lib/manuscript.ts index 198a3be003..3e1490e0d2 100644 --- a/src/lib/manuscript.ts +++ b/src/lib/manuscript.ts @@ -2,7 +2,7 @@ // and renders maths with KaTeX at build time, so pages ship no maths JavaScript. import katex from 'katex'; import manifestJson from '../generated/manifest.json'; -import { base } from './site'; +import { base, SITE } from './site'; import withheldJson from '../data/withheld.json'; export interface TocSection { id: string; num: string | null; title: string; level: number; page: string | null } @@ -104,11 +104,13 @@ export function referencesHtml(): string { } export function pdfHref(page: string | null | undefined): string | null { - if (!manifest.provenance.pdf_included || !page) return null; + if (!page) return null; // Roman-numbered front matter pages are not addressable by number; skip them if (!/^\d+$/.test(page)) return null; // PDF viewers use physical page indices; the offset accounts for front matter. - return `${base}survey.pdf#page=${Number(page) + pdfOffset}`; + // Without a local copy, link the published arXiv PDF (the ingested source is that build). + const pdf = manifest.provenance.pdf_included ? `${base}survey.pdf` : manifest.provenance.status === 'final' ? SITE.paperPdf : null; + return pdf ? `${pdf}#page=${Number(page) + pdfOffset}` : null; } // Physical index of printed page 1. Set by the ingest step from the PDF if known. diff --git a/src/pages/demos/index.astro b/src/pages/demos/index.astro index b020e3bf3f..8b8c6f9303 100644 --- a/src/pages/demos/index.astro +++ b/src/pages/demos/index.astro @@ -10,11 +10,11 @@ const demos = [ text: 'Find where GPT-2 small resolves indirect object identification by patching clean activations into a corrupted run, layer by layer and head by head.' }, ]; --- - +

Hands-on

Interactive demos

-

Three core ideas from the survey that you can run yourself. Nothing here is a drawing: the first two train real models in your browser, and the third shows activation-patching measurements on GPT-2 small from a script in this repository.

+

Three ideas from the survey to try yourself. The first two train small models in your browser; the third shows activation patching on GPT-2 small.

{demos.map((d) => {d.label} diff --git a/src/pages/demos/patching.astro b/src/pages/demos/patching.astro index f79254a64e..fd96a5e8b8 100644 --- a/src/pages/demos/patching.astro +++ b/src/pages/demos/patching.astro @@ -12,7 +12,7 @@ import data from '../../data/experiments/patching_gpt2_ioi.json';

How these numbers were produced

Model: {data.model}. Task: indirect object identification over {data.n_prompts} prompt pairs ({data.templates.length} templates × {data.name_pairs.length} name pairs × 2 orders). In the corrupted prompt the second-clause subject is swapped, which flips the answer. Metric: {data.metric}. A value of 0 equals the corrupted run and 1 equals the clean run. The clean run is correct on {(data.clean_accuracy * 100).toFixed(0)}% of prompts. Activation patching runs the model once per component. Attribution patching approximates the same quantity with a single backward pass, using a first-order Taylor expansion. Across all 144 heads the two agree with Pearson r = {data.pearson_activation_vs_attribution.toFixed(3)}.

Reproduce: python3 {data.script} (torch {data.versions.torch}, transformer-lens {data.versions.transformer_lens}; about {Math.round(data.runtime_seconds)} s on a laptop CPU, deterministic). Generated {data.generated}. The setup follows the companion notebook on circuit discovery, scaled up to more prompts.

-

A patching effect shows that a component is causally relevant for this metric on these prompts. It does not by itself show what the component computes. See the survey’s discussion of causal validity and the evaluation framework.

+

A patching effect shows that a component matters for this metric on these prompts, not what it computes.

diff --git a/src/pages/demos/sae.astro b/src/pages/demos/sae.astro index beddd0ddd0..800780936c 100644 --- a/src/pages/demos/sae.astro +++ b/src/pages/demos/sae.astro @@ -11,7 +11,7 @@ import SaeDemo from '../../components/demos/SaeDemo.tsx';

What is being computed

Data: x = Σi ai gi, where the gi are 16 random unit vectors in ℝ⁸ and each ai is 0 with probability 0.9, otherwise Uniform[0, 1]. SAE: f = ReLU(We(x − bd) + be), x̂ = Wdf + bd, with 32 latents and unit-norm decoder columns. The loss is ‖x − x̂‖² + λ‖f‖₁, or the TopK variant, which keeps the k largest activations. Training is Adam (learning rate 0.003, batch 256) for 2,000 steps. MMCS (mean max cosine similarity) averages, over planted features, the best cosine with any decoder column. A feature counts as recovered when that cosine exceeds 0.9.

Checked results (seed 1, tested in tests/sim.test.ts): with λ = 0 the SAE reconstructs almost perfectly but recovers 0 of 16 features (MMCS ≈ 0.69). With λ = 0.3 it recovers all 16 (MMCS ≈ 0.97) at about 13% variance unexplained. With λ = 1 reconstruction degrades to about 60% variance unexplained. Sparsity is what makes the dictionary line up with the true features, and too much sparsity costs reconstruction.

-

This is a deliberately easy, idealised setting. Real activations have no guarantee of a sparse linear ground truth. That limitation is discussed in the survey’s sections on feature extraction and evaluation.

+

An idealised setting: real activations need not have a sparse linear ground truth.

diff --git a/src/pages/evaluation.astro b/src/pages/evaluation.astro index e7ed0531b9..f1c99c4b02 100644 --- a/src/pages/evaluation.astro +++ b/src/pages/evaluation.astro @@ -21,7 +21,7 @@ if (__hide) return Astro.rewrite(`${import.meta.env.BASE_URL.replace(/\/?$/, '/'

Technical methods · Evaluation

A unified evaluation framework

-

How do you tell whether an interpretability result is right? The survey proposes judging every method on the same five dimensions.

+

Five dimensions for evaluating interpretability claims.

{e.framework_intro.map((t: string) => )} @@ -34,26 +34,24 @@ if (__hide) return Astro.rewrite(`${import.meta.env.BASE_URL.replace(/\/?$/, '/' diff --git a/src/pages/handbook/[slug].astro b/src/pages/handbook/[slug].astro index c7a69486d8..2aca731e40 100644 --- a/src/pages/handbook/[slug].astro +++ b/src/pages/handbook/[slug].astro @@ -41,7 +41,7 @@ const description = `${label}: ${ch.title} — from the survey "The State of Mec

Contents

{chapters.map((c) => (
- {`${c.num}.`} {c.title}{withheld.has(c.slug) && (with the final manuscript)} + {c.num ? `${c.num}. ` : ''}{c.title}{withheld.has(c.slug) && (with the final manuscript)} {!withheld.has(c.slug) && c.sections.filter((s) => s.level === 2).length > 0 && (
    {c.sections.filter((s) => s.level === 2).map((s) => ( @@ -94,24 +94,31 @@ const description = `${label}: ${ch.title} — from the survey "The State of Mec document.querySelectorAll('.prose section[id]').forEach((s) => map.has(s.id) && obs.observe(s)); // citation previews (hover / focus), full entry on the references page - let pop; - const label = { established: 'Peer-reviewed', emerging: 'Preprint / emerging', flagged: 'Flagged in manuscript' }; + let pop, timer; + const label = { established: 'Peer-reviewed', emerging: 'Preprint', flagged: 'Preprint' }; function show(a) { const key = a.dataset.cite; const text = refs[key]; if (!text) return; - hide(); + clearTimeout(timer); pop?.remove(); pop = document.createElement('div'); pop.className = 'cite-pop'; pop.setAttribute('role', 'tooltip'); + const body = document.createElement('div'); body.textContent = text.replace(/\s*URL\s+https?:\/\/\S+\s*$/, ''); pop.append(body); + const foot = document.createElement('div'); foot.className = 'small'; foot.style.marginTop = '.45rem'; const tier = tiers[key]; - pop.textContent = text; - if (tier) { const s = document.createElement('div'); s.className = 'muted small'; s.style.marginTop = '.35rem'; s.textContent = label[tier]; pop.append(s); } + if (tier) { const t = document.createElement('span'); t.className = 'muted'; t.textContent = label[tier] + ' · '; foot.append(t); } + const url = (text.match(/https?:\/\/[^\s]+[^\s.,)]/) || [])[0]; + if (url) { const l = document.createElement('a'); l.href = url; l.target = '_blank'; l.rel = 'noopener'; l.textContent = 'Open paper'; foot.append(l, document.createTextNode(' · ')); } + const r2 = document.createElement('a'); r2.href = a.href; r2.textContent = 'In the reference list'; foot.append(r2); + pop.append(foot); + pop.addEventListener('mouseenter', () => clearTimeout(timer)); pop.addEventListener('mouseleave', hideSoon); document.body.append(pop); const r = a.getBoundingClientRect(); const left = Math.min(window.scrollX + r.left, window.scrollX + document.documentElement.clientWidth - pop.offsetWidth - 12); pop.style.left = Math.max(8, left) + 'px'; pop.style.top = window.scrollY + r.bottom + 6 + 'px'; } - function hide() { pop?.remove(); pop = null; } + function hide() { clearTimeout(timer); pop?.remove(); pop = null; } + function hideSoon() { clearTimeout(timer); timer = setTimeout(hide, 250); } document.querySelectorAll('a.cite').forEach((a) => { - a.addEventListener('mouseenter', () => show(a)); a.addEventListener('mouseleave', hide); - a.addEventListener('focus', () => show(a)); a.addEventListener('blur', hide); + a.addEventListener('mouseenter', () => show(a)); a.addEventListener('mouseleave', hideSoon); + a.addEventListener('focus', () => show(a)); a.addEventListener('blur', hideSoon); }); document.addEventListener('keydown', (e) => e.key === 'Escape' && hide()); diff --git a/src/pages/handbook/index.astro b/src/pages/handbook/index.astro index 3dd55777e8..f15a132d0c 100644 --- a/src/pages/handbook/index.astro +++ b/src/pages/handbook/index.astro @@ -10,25 +10,20 @@ const words = shown.reduce((a, c) => a + c.words, 0); import { draftHidden as __hide } from '../../lib/manuscript'; if (__hide) return Astro.rewrite(`${import.meta.env.BASE_URL.replace(/\/?$/, '/')}held/`); --- - +

    The handbook

    Read the survey

    -

    {isDraft ? 'A preview of the working draft' : 'The survey'} as an online book: {main.filter((c) => !withheld.has(c.slug)).length} chapters, {apps.length} appendices and {manifest.references.length} references, about {Math.round(words / 1000)}k words. - {withheld.size > 0 && <>{withheld.size === 1 ? 'One chapter is' : `${withheld.size} chapters are`} held back until the final manuscript. } - Numbering matches the {isDraft ? 'draft ' : ''}PDF, and every section, figure and table has a link you can share.

    -

    The paper: arXiv:2611.08438 (PDF, alphaXiv) · BibTeX

    -

    Citation colours follow the manuscript: - ■ peer-reviewed (plain), - ■ preprints and other non-peer-reviewed work (dotted underline; tagged “Preprint” in the reference list).

    +

    arXiv:2611.08438 · PDF · alphaXiv · BibTeX

    +

    Citations: ■ peer-reviewed · ■ preprint

      {main.map((c) => (
    1. {c.num}{c.title} - · p. {c.page} · {Math.max(1, Math.round(c.words / 230))} min read + {c.page && · p. {c.page}} {withheld.has(c.slug) ? · coming with the final manuscript : c.sections.filter((s) => s.level === 2).length > 0 && (
      {c.sections.filter((s) => s.level === 2).length} sections diff --git a/src/pages/index.astro b/src/pages/index.astro index cdafa34eff..90530984ba 100644 --- a/src/pages/index.astro +++ b/src/pages/index.astro @@ -48,13 +48,13 @@ const roman = ['I', 'II', 'III', 'IV']; // the arrangement the toy model in demos/superposition settles into when inputs are sparse). const feats = [0, 1, 2, 3, 4].map((i) => { const a = Math.PI / 2 + (i * 2 * Math.PI) / 5; return [Math.cos(a), Math.sin(a)]; }); const explore = [ - ['Read', 'handbook/', 'The handbook', `${isDraft ? 'A preview of the working draft' : 'The survey'} online: ${chapters.filter((c) => !c.appendix && !withheld.has(c.slug)).length} chapters${withheld.size ? ` (${withheld.size} held back until the final manuscript)` : ''} and ${manifest.references.length} references, numbered as in the ${isDraft ? 'draft ' : ''}PDF.`], - ['Compare', 'methods/', 'Methods explorer', 'Every method the survey covers, side by side, quoted from the text.'], - ['Judge', 'evaluation/', 'Evaluation framework', 'Five dimensions for deciding whether an interpretability claim holds.'], - ['Orient', 'orientations/', 'Research orientations', 'The agendas behind the methods: what each assumes and what it trades away.'], - ['Apply', 'safety/', 'Safety and open problems', 'Where interpretability already supports safety work, and what is unsolved.'], - ['Run', 'demos/', 'Experiments', 'Superposition, sparse autoencoders and activation patching, computed for real.'], - ['Code', 'notebooks/', 'Notebooks', `${notebooks.notebooks.length} companion notebooks with compute requirements and Colab links.`], + ['Read', 'handbook/', 'The handbook', `The full survey, by chapter.`], + ['Compare', 'methods/', 'Methods explorer', 'The methods in the survey, side by side.'], + ['Judge', 'evaluation/', 'Evaluation framework', 'Five dimensions for evaluating interpretability claims.'], + ['Orient', 'orientations/', 'Research orientations', 'Six research agendas and their trade-offs.'], + ['Apply', 'safety/', 'Safety and open problems', 'Safety applications and open problems.'], + ['Run', 'demos/', 'Experiments', 'Superposition, sparse autoencoders and activation patching.'], + ['Code', 'notebooks/', 'Notebooks', `${notebooks.notebooks.length} companion notebooks.`], ]; const exploreShown = draftHidden ? explore.filter(([, href]) => href === 'demos/' || href === 'notebooks/') : explore; // BibTeX exactly as arXiv issues it for 2611.08438. @@ -96,7 +96,7 @@ const jsonld = { {feats.map(([x, y], i) => f{i + 1})} -
      Five features, two dimensions. When features are rarely active together, a network can store more of them than it has dimensions, at the cost of interference. Train it yourself.
      +
      Superposition. Five features in two dimensions: when features are rarely active together, a network can store more features than it has dimensions, at the cost of interference. Train it yourself.
@@ -145,7 +145,7 @@ const jsonld = {
-

CompanionUsing this site

+

ExploreOn this site

{exploreShown.map(([verb, href, title, text]) => (
{verb}
@@ -154,7 +154,7 @@ const jsonld = { ))}
-
+
Taxonomy
@@ -192,7 +192,7 @@ const jsonld = { @media (max-width: 860px) { .front-grid { grid-template-columns: minmax(0, 1fr); padding-top: 2.5rem; } } .front-text { background: var(--paper); padding: 2.2rem 2.4rem 2rem; outline: 1px solid var(--hair); } @media (max-width: 600px) { .front-text { padding: 1.4rem 1.2rem; } } - .front h1 { font-size: clamp(2.3rem, 5.4vw, 4rem); line-height: 1.04; margin: 0 0 .8rem; } + .front h1 { font-size: clamp(1.9rem, 4.2vw, 3.2rem); line-height: 1.08; margin: 0 0 .8rem; overflow-wrap: normal; word-break: normal; hyphens: manual; } .sub { font-family: var(--serif); font-style: italic; font-weight: 300; font-size: clamp(1.1rem, 2vw, 1.4rem); color: var(--ink-2); margin: 0 0 2rem; } .question { font-family: var(--serif); font-size: clamp(1.25rem, 2.3vw, 1.6rem); line-height: 1.35; margin: 1.4rem 0 2rem; padding-top: 1.2rem; border-top: 1px solid var(--hair); max-width: 30ch; color: var(--accent); } .front-links { display: flex; flex-wrap: wrap; gap: .6rem 1.6rem; font-size: .92rem; margin: 0; } diff --git a/src/pages/methods/[id].astro b/src/pages/methods/[id].astro index d353572973..5ce76360d6 100644 --- a/src/pages/methods/[id].astro +++ b/src/pages/methods/[id].astro @@ -33,18 +33,17 @@ const nbAnchor = (p: string) => p.split('/').pop()!.replace('.ipynb', '');

- {m.definition && } + {m.definition && } - {groups.map(([h, list]) => ( + {groups.filter(([, list]) => list?.length).map(([h, list]) => (

{h}

- {list?.length ? list.map((t) => ) :

The survey doesn’t state this explicitly for {m.name}.

} + {list.map((t) => )}
))} {(m.related?.length > 0 || relatedBack.length > 0) && (

Related methods

-

These are methods the survey explicitly relates to this one or contrasts with it.

x.id)].slice(0, 3).join(',')}`)}>Compare side by side →

@@ -66,7 +65,7 @@ const nbAnchor = (p: string) => p.split('/').pop()!.replace('.ipynb', ''); {papers.length > 0 && (

Key papers

-

Works cited in {primary?.num ? `§${primary.num}` : 'the primary section'}, taken automatically from the survey’s citations.

+ {primary?.num &&

Cited in §{primary.num}.

}
    {papers.map((p) =>
  • ■ {p.tier === 'emerging' && Preprint}{p.tier === 'flagged' && Flagged}{p.text}
  • )}
)} diff --git a/src/pages/methods/index.astro b/src/pages/methods/index.astro index 898ebc4724..d0465baa68 100644 --- a/src/pages/methods/index.astro +++ b/src/pages/methods/index.astro @@ -13,7 +13,8 @@ const rows = methods.map((m) => { section: s?.num ?? null, sectionHref: s?.href ?? null, definition: m.definition ?? '', assumptions: m.assumptions ?? [], evaluation: m.evaluation ?? [], applications: m.applications ?? [], limitations: m.limitations ?? [], html: Object.fromEntries(['definition', 'assumptions', 'evaluation', 'applications', 'limitations'].map((k) => { - const v = m[k]; return [k, Array.isArray(v) ? v.map(excerptHtml) : v ? [excerptHtml(v)] : []]; + const v = m[k]; const cont = (t: string) => (k === 'definition' && !/[.!?”)\]]$/.test(t.trim()) ? `${t} …` : t); + return [k, Array.isArray(v) ? v.map(excerptHtml) : v ? [excerptHtml(cont(v))] : []]; })), related: (m.related ?? []).map((id: string) => ({ id, name: methodById.get(id)?.name ?? id })), notebooks: notebooksFor('methods', m.id).length, demos: demosFor('methods', m.id).length, href: url(`methods/${m.id}/`), @@ -27,22 +28,9 @@ if (__hide) return Astro.rewrite(`${import.meta.env.BASE_URL.replace(/\/?$/, '/'

Technical methods

Methods explorer

-

Every method the survey covers in its own section, grouped the way the survey groups them. All text here is quoted from the survey. - We don’t add scores, rankings or numbers the survey doesn’t report.

+

The methods covered in the survey, grouped as in the paper.

-

Core hypotheses

-

These are the assumptions many of the methods rely on. They are not methods themselves.

-
- {hypotheses.map((h) => { - const s = sections(h.sections)[0]; - return

{h.name}

-

“{h.definition}”

- {s?.href && §{s.num} {s.title}} - {demosFor('hypotheses', h.id).map((d) => <> · Demo: {d.title})} -
; - })} -
diff --git a/src/pages/notebooks.astro b/src/pages/notebooks.astro index 7646cb221b..7008df49e1 100644 --- a/src/pages/notebooks.astro +++ b/src/pages/notebooks.astro @@ -55,11 +55,7 @@ const stats = { total: notebooks.length, cpu: notebooks.filter((n) => n.compute.

Hands-on

Companion notebooks

-

{stats.total} notebooks from TSGLAB/Interpretability_Survey_Safety, catalogued automatically from the notebooks themselves. {stats.cpu} of them run on a CPU.

-

Catalogued from commit {notebookMeta.commit}. - {notebookMeta.link_check_run ? ' Every source link was checked at build time and resolves.' : ' Links were not checked in this build.'} - Compute needs are estimated from the largest model each notebook loads. They are not benchmarks. - “Open in Colab” opens the notebook. It has not been run end to end in Colab unless marked.

+

{stats.total} notebooks from TSGLAB/Interpretability_Survey_Safety, {stats.cpu} run on a CPU.

@@ -69,7 +65,7 @@ const stats = { total: notebooks.length, cpu: notebooks.filter((n) => n.compute. {groups.map((g) => (

{FOLDER[g.key]}

-

Prerequisites (from the repository README): {PREREQ[g.key]}

+

Prerequisites: {PREREQ[g.key]}

{g.items.map((n) => { const r = resolve(n.survey_reference); @@ -89,11 +85,11 @@ const stats = { total: notebooks.length, cpu: notebooks.filter((n) => n.compute.
{r ? (<> {draftHidden ? {n.survey_reference.replace(/\(\[.*$/, '').replace(/\]\(.*$/, '').trim()} : <> {r.match ? §{r.match.num} {r.match.title} : {n.survey_reference.replace(/\(\[.*$/, '').trim()}}} - ) : Not stated in notebook}
+ ) : —}
Learning outcomes
{n.learning_objectives.length ?
    {n.learning_objectives.slice(0, 5).map((o: string) =>
  • {o.replace(/\*\*/g, '')}
  • )}
- : n.key_methods.length ? <>Not listed as outcomes. Key methods covered:
    {n.key_methods.map((o: string) =>
  • {o.replace(/\*\*/g, '')}
  • )}
- : Not stated in notebook}
+ : n.key_methods.length ? <>Covers:
    {n.key_methods.map((o: string) =>
  • {o.replace(/\*\*/g, '')}
  • )}
+ : —}
Models
{n.models.length ? n.models.join(', ') : 'None (toy or synthetic data)'}{n.gated_models.length > 0 && gated: {n.gated_models.join(', ')}}
Dependencies
{n.imports.join(', ')} @@ -113,7 +109,7 @@ const stats = { total: notebooks.length, cpu: notebooks.filter((n) => n.compute. {n.preview_image && (
{`First -
Saved output from the notebook (first plot)
+
)} diff --git a/src/pages/orientations.astro b/src/pages/orientations.astro index edca14d24a..6c40e47f32 100644 --- a/src/pages/orientations.astro +++ b/src/pages/orientations.astro @@ -19,8 +19,7 @@ if (__hide) return Astro.rewrite(`${import.meta.env.BASE_URL.replace(/\/?$/, '/'

Research directions

Research orientations

-

An orientation is a research agenda, defined by its goals, assumptions and trade-offs. It is not a method: several orientations can use the same tool for different ends.

-

Quoted from §{src.chapter} {src.chapter_title} of the paper, arXiv:{src.id}{src.version}.

+

From §{src.chapter} of the paper.

{data.intro.map((t: string) => )}
diff --git a/src/pages/safety.astro b/src/pages/safety.astro index 0467853d12..2c6f1ab14a 100644 --- a/src/pages/safety.astro +++ b/src/pages/safety.astro @@ -10,9 +10,11 @@ import { url } from '../lib/site'; const s = framework.safety_applications; const apps: any[] = s.applications; const problems: any[] = framework.open_problems.problems; -const orient: any[] = framework.orientations.orientations; +import orientationsArxiv from '../data/orientations-arxiv.json'; const sec = (id: string) => sections([id])[0]; -const orientLink = (name: string) => { const o = orient.find((x) => name.toLowerCase().includes(x.name.toLowerCase().split(' ')[0])); return o ? url(`orientations/#${o.id}`) : null; }; +// The six orientations of the paper (§12), keyed by the word that tells them apart. +const ORIENT_KEYS: [string, string][] = orientationsArxiv.orientations.map((o: any) => [o.name.replace(/ Interpretability$/, '').replace(/^Interpretability /, '').toLowerCase(), o.id]); +const orientLink = (name: string) => { const k = ORIENT_KEYS.find(([w]) => name.toLowerCase().includes(w)); return k ? url(`orientations/#${k[1]}`) : null; }; const methodLinks = (names: string[]) => names.map((n) => ({ n, m: resolveMethod(n) })); const search = (o: any) => [o.name ?? o.title, o.definition ?? o.statement, ...(o.details ?? []), ...(o.evidence ?? []), ...(o.limitations ?? []), ...(o.methods ?? o.related_methods ?? [])].join(' ').toLowerCase(); import { draftHidden as __hide } from '../lib/manuscript'; @@ -22,7 +24,7 @@ if (__hide) return Astro.rewrite(`${import.meta.env.BASE_URL.replace(/\/?$/, '/'

Safety & open problems

Safety applications and open problems

-

Where interpretability supports concrete safety work, what evidence the survey cites for it, and what remains unsolved. Each entry links to the methods it relies on and the orientations it belongs to.

+

Where mechanistic interpretability supports safety work, and the open problems that remain.

@@ -31,7 +33,7 @@ if (__hide) return Astro.rewrite(`${import.meta.env.BASE_URL.replace(/\/?$/, '/' - +

@@ -47,21 +49,18 @@ if (__hide) return Astro.rewrite(`${import.meta.env.BASE_URL.replace(/\/?$/, '/' {a.methods?.length > 0 &&

Methods: {methodLinks(a.methods).map(({ n, m }, j) => <>{j > 0 && ', '}{m ? {n} : n})}

} {a.orientations?.length > 0 &&

Orientations: {a.orientations.map((n: string, j: number) => <>{j > 0 && ', '}{orientLink(n) ? {n} : n})}

}
- Evidence ({a.evidence?.length ?? 0}) and limitations ({a.limitations?.length ?? 0}) -

Evidence cited in the survey

- {a.evidence?.length ? a.evidence.map((t: string) => ) :

The survey reports no empirical evidence for this application.

} + Evidence and limitations + {a.evidence?.length > 0 && <>

Evidence

{a.evidence.map((t: string) => )}} {a.worked_example?.length > 0 && <>

Worked example Illustrative

-

This is a walkthrough from the survey. Its “results” illustrate the steps and are not measured outcomes.

{a.worked_example.map((t: string) => )}} -

Limitations

- {a.limitations?.length ? a.limitations.map((t: string) => ) :

None stated specifically.

} + {a.limitations?.length > 0 && <>

Limitations

{a.limitations.map((t: string) => )}}
{notebooksFor('applications', a.id).map((n) =>

Open “{n.title}” in Colab

)} ))}
-

Risks and verification

+

Risks

Risk taxonomy

@@ -71,18 +70,10 @@ if (__hide) return Astro.rewrite(`${import.meta.env.BASE_URL.replace(/\/?$/, '/'
-

Verification checklist

-
- {s.verification_checklist.groups.map((g: any) =>
{g.heading}{g.items.map((t: string) => )}
)} -
-

The ticks are for your own use. They aren’t saved.

+

Mitigation strategies

+ {s.verification_checklist.groups.map((g: any) => <>

{g.heading}

{g.items.map((t: string) => )})}
-
-

Limitations of mechanistic safety interventions

- {s.chapter_limitations.excerpts.map((t: string) => )} - -

Open problems

@@ -90,12 +81,12 @@ if (__hide) return Astro.rewrite(`${import.meta.env.BASE_URL.replace(/\/?$/, '/'
{problems.map((p) => (
-

{p.title}

Open problem{p.source_chapter !== 15 && from chapter {p.source_chapter}}
+

{p.title}

Open problem{p.source_chapter !== 14 && from chapter {p.source_chapter}}
{p.related_methods?.length > 0 &&

Methods mentioned: {methodLinks(p.related_methods).map(({ n, m }, j) => <>{j > 0 && ', '}{m ? {n} : n})}

} {p.related_orientations?.length > 0 &&

Orientations: {p.related_orientations.map((n: string, j: number) => <>{j > 0 && ', '}{orientLink(n) ? {n} : n})}

} - {p.details?.length > 0 &&
More from the survey ({p.details.length}){p.details.map((t: string) => )}
} + {p.details?.length > 0 &&
More ({p.details.length}){p.details.map((t: string) => )}
}
))}
diff --git a/src/pages/search.astro b/src/pages/search.astro index 484a1a5526..cd47ffb86a 100644 --- a/src/pages/search.astro +++ b/src/pages/search.astro @@ -7,7 +7,7 @@ import { url } from '../lib/site';

Search

Search

-

Searches every page on this site. The index is built statically, so it works offline once loaded.

+

Search the survey, methods, notebooks and demos.

diff --git a/src/styles/global.css b/src/styles/global.css index 15ba66cf54..e09b075748 100644 --- a/src/styles/global.css +++ b/src/styles/global.css @@ -157,7 +157,7 @@ table.data th[scope="row"] { font-weight: 500; } .label.manuscript { color: var(--ink-3); } .label.editorial { color: var(--accent); } -blockquote.excerpt { margin: .7rem 0 1rem; padding: 0 0 0 1.1rem; border-left: 1px solid var(--hair-strong); font-family: var(--serif); font-size: 1.08rem; line-height: 1.55; color: var(--ink); } +blockquote.excerpt { margin: .5rem 0 .9rem; padding: 0; border: 0; font-family: var(--serif); font-size: 1.08rem; line-height: 1.55; color: var(--ink); } blockquote.excerpt footer { font-family: var(--sans); font-size: .78rem; color: var(--ink-3); margin-top: .3rem; } blockquote.excerpt footer a { color: var(--ink-3); } @@ -214,13 +214,12 @@ blockquote.excerpt footer a { color: var(--ink-3); } .prose a.cite-established { color: var(--tier-established); } .prose a.cite-emerging { color: var(--tier-emerging); } .prose a.cite-missing, .prose a.cite-flagged { color: var(--tier-flagged); } -/* evidence tier without relying on colour: preprints dotted, missing/flagged wavy */ -.prose a.cite-emerging, .cite-emerging-sample { text-decoration: underline dotted 1px; text-underline-offset: .2em; } +/* evidence tier: colour (peer-reviewed vs preprint); missing/flagged also wavy */ .prose a.cite-flagged, .prose a.cite-missing { text-decoration: underline wavy 1px; text-underline-offset: .2em; } .tier-tag { display: inline-block; font: 500 .66rem/1.3 var(--sans); letter-spacing: .05em; text-transform: uppercase; border: 1px solid currentColor; padding: 0 .35em; margin-right: .45em; vertical-align: .12em; color: var(--ink-2); } .prose a.cite:hover { text-decoration: underline; } -.prose :is(.definitionbox, .examplebox, .warningbox, .theorybox) { margin: 1.8rem 0; padding: .2rem 0 .2rem 1.2rem; border-left: 1px solid var(--hair-strong); font-size: 1.05rem; } +.prose :is(.definitionbox, .examplebox, .warningbox, .theorybox, .insightbox, .practicebox) { margin: 1.8rem 0; padding: .2rem 0 .2rem 1.2rem; border-left: 1px solid var(--hair-strong); font-size: 1.05rem; } .prose .warningbox { border-left-color: var(--accent); } .prose .unrendered { display: block; padding: 1rem 0; border-top: 1px solid var(--hair); border-bottom: 1px solid var(--hair); font-family: var(--sans); font-size: .9rem; color: var(--ink-2); } .prose .math-error { color: var(--tier-flagged); }