diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index f8c54f79..608a1139 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -221,6 +221,10 @@ impl Tokenizer { self.run(); } + pub fn set_plaintext_state(&mut self) { + self.state = states::Plaintext; + } + fn process_token(&mut self, token: Token) { if self.opts.profile { let (_, dt) = time!(self.sink.process_token(token));