diff --git a/docs/changelog.md b/docs/changelog.md index 76350a27..e0ce5ae5 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -16,6 +16,8 @@ See the [Contributing Guide](contributing.md) for details. ### Fixed +* Fix quadratic-time parsing of many consecutive reference definitions, and of + many consecutive footnote definitions in the `footnotes` extension. * Support automatic links with URL schemes other than HTTP and FTP, such as `` (#1244). * Fix quadratic-time id generation in the `toc` extension when many headers diff --git a/markdown/blockprocessors.py b/markdown/blockprocessors.py index 728bf8f9..d3574045 100644 --- a/markdown/blockprocessors.py +++ b/markdown/blockprocessors.py @@ -588,16 +588,25 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool: block = blocks.pop(0) m = self.RE.search(block) if m: - id = m.group(1).strip().lower() - link = m.group(2).lstrip('<').rstrip('>') - title = m.group(5) or m.group(6) - self.parser.md.references[id] = (link, title) - if block[m.end():].strip(): + start = end = m.start() + # Record the definitions on the following lines in this pass rather + # than one per pass through the block processors. A definition after + # the first which spans lines is left for the next pass, as another + # processor may claim part of it. + for m in self.RE.finditer(block, start): + if m.start() > start and (block[end:m.start()].strip('\n') or '\n' in m.group()): + break + id = m.group(1).strip().lower() + link = m.group(2).lstrip('<').rstrip('>') + title = m.group(5) or m.group(6) + self.parser.md.references[id] = (link, title) + end = m.end() + if block[end:].strip(): # Add any content after match back to blocks as separate block - blocks.insert(0, block[m.end():].lstrip('\n')) - if block[:m.start()].strip(): + blocks.insert(0, block[end:].lstrip('\n')) + if block[:start].strip(): # Add any content before match back to blocks as separate block - blocks.insert(0, block[:m.start()].rstrip('\n')) + blocks.insert(0, block[:start].rstrip('\n')) return True # No match. Restore block. blocks.insert(0, block) diff --git a/markdown/extensions/footnotes.py b/markdown/extensions/footnotes.py index 46bb93c3..cc425216 100644 --- a/markdown/extensions/footnotes.py +++ b/markdown/extensions/footnotes.py @@ -261,6 +261,16 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool: m = self.RE.search(block) if m: + preceding = block[:m.start()] + # Record the definitions which are directly followed by another + # definition in this pass rather than one per pass through the block + # processors. + for m2 in self.RE.finditer(block, m.end()): + if block[m.end():m2.start()].strip('\n'): + break + self.footnotes.setFootnote(m.group(1), m.group(2).rstrip()) + m = m2 + id = m.group(1) fn_blocks = [m.group(2)] @@ -284,9 +294,9 @@ def run(self, parent: etree.Element, blocks: list[str]) -> bool: footnote = "\n\n".join(fn_blocks) self.footnotes.setFootnote(id, footnote.rstrip()) - if block[:m.start()].strip(): + if preceding.strip(): # Add any content before match back to blocks as separate block - blocks.insert(0, block[:m.start()].rstrip('\n')) + blocks.insert(0, preceding.rstrip('\n')) return True # No match. Restore block. blocks.insert(0, block) diff --git a/tests/test_syntax/blocks/test_reference_links.py b/tests/test_syntax/blocks/test_reference_links.py index 8bd67521..5c9884b0 100644 --- a/tests/test_syntax/blocks/test_reference_links.py +++ b/tests/test_syntax/blocks/test_reference_links.py @@ -60,3 +60,154 @@ def test_malformed_reference_with_long_run_of_spaces(self): self.assertMarkdownRenders( text, f'

{text}

', expected_attrs={'references': {}} ) + + def test_reference_definitions_before_setext_underline(self): + """ + A setext underline after consecutive reference definitions is a paragraph, + and does not turn the last definition into a header. + """ + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\n===', + '

===

', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_reference_definitions_before_split_definition(self): + """ + A definition split across lines after other definitions is left for the + next pass, so a horizontal rule on its second line is not taken as the link. + """ + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]:\n***', + '

[b]:

\n
', + expected_attrs={'references': {'a': ('http://example.com/a', None)}} + ) + + def test_reference_definitions_before_definition_split_across_lines(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]:\nhttp://example.com/b', + '', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_reference_definitions_title_on_own_line(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\n"Title"', + '', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', 'Title')} + } + ) + + def test_reference_definitions_before_paragraph(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\nSome text', + '

Some text

', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_paragraph_before_reference_definitions(self): + self.assertMarkdownRenders( + 'Some text\n[a]: http://example.com/a\n[b]: http://example.com/b', + '

Some text

', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_reference_definitions_separated_by_paragraph(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\nSome text\n[b]: http://example.com/b', + '

Some text

', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_indented_reference_definitions(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n [b]: http://example.com/b', + '', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_indented_reference_definitions_short_tab_length(self): + """ + A definition indented by `tab_length` spaces directly after another + definition is a definition, not an indented code block. + """ + self.assertMarkdownRenders( + '[a]: http://example.com/a\n [b]: http://example.com/b', + '', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + }, + tab_length=2 + ) + + def test_duplicate_reference_definitions(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[a]: http://example.com/b', + '', + expected_attrs={'references': {'a': ('http://example.com/b', None)}} + ) + + def test_reference_definitions_before_hash_header(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\n# Header', + '

Header

', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_reference_definitions_before_hr(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\n***', + '
', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_reference_definitions_before_blockquote(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\n> quote', + '
\n

quote

\n
', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_reference_definitions_before_code_block(self): + self.assertMarkdownRenders( + '[a]: http://example.com/a\n[b]: http://example.com/b\n code', + '
code\n
', + expected_attrs={ + 'references': {'a': ('http://example.com/a', None), 'b': ('http://example.com/b', None)} + } + ) + + def test_many_reference_definitions(self): + """ + Consecutive reference definitions should be consumed together rather + than returning the remainder of the block to be parsed once per + definition. + """ + count = 10000 + text = ''.join(f'[id{i}]: http://example.com/{i}\n' for i in range(count)) + self.assertMarkdownRenders( + text, + '', + expected_attrs={ + 'references': {f'id{i}': (f'http://example.com/{i}', None) for i in range(count)} + } + ) diff --git a/tests/test_syntax/extensions/test_footnotes.py b/tests/test_syntax/extensions/test_footnotes.py index 896e1fcd..5cd1037d 100644 --- a/tests/test_syntax/extensions/test_footnotes.py +++ b/tests/test_syntax/extensions/test_footnotes.py @@ -265,6 +265,157 @@ def test_footnote_multple_complex_no_blank_line_between(self): '' ) + def test_consecutive_footnotes_last_multi_line(self): + self.assertMarkdownRenders( + self.dedent( + """ + [^1]: Footnote 1 + [^2]: Footnote 2 + line 2 + """ + ), + '
\n' + '
\n' + '
    \n' + '
  1. \n' + '

    Footnote 1 ↩

    \n' + '
  2. \n' + '
  3. \n' + '

    Footnote 2\nline 2 ↩

    \n' + '
  4. \n' + '
\n' + '
' + ) + + def test_consecutive_footnotes_last_multi_line_lazy_indent(self): + self.assertMarkdownRenders( + self.dedent( + """ + [^1]: Footnote 1 + [^2]: Footnote 2 + line 2 + """ + ), + '
\n' + '
\n' + '
    \n' + '
  1. \n' + '

    Footnote 1 ↩

    \n' + '
  2. \n' + '
  3. \n' + '

    Footnote 2\nline 2 ↩

    \n' + '
  4. \n' + '
\n' + '
' + ) + + def test_consecutive_footnotes_last_multi_paragraph(self): + self.assertMarkdownRenders( + self.dedent( + """ + [^1]: Footnote 1 + [^2]: Footnote 2 + + paragraph 2 + """ + ), + '
\n' + '
\n' + '
    \n' + '
  1. \n' + '

    Footnote 1 ↩

    \n' + '
  2. \n' + '
  3. \n' + '

    Footnote 2

    \n' + '

    paragraph 2 ↩

    \n' + '
  4. \n' + '
\n' + '
' + ) + + def test_paragraph_before_consecutive_footnotes(self): + self.assertMarkdownRenders( + self.dedent( + """ + paragraph[^1] + [^1]: Footnote 1 + [^2]: Footnote 2 + """ + ), + '

paragraph1

\n' + '
\n' + '
\n' + '
    \n' + '
  1. \n' + '

    Footnote 1 ↩

    \n' + '
  2. \n' + '
  3. \n' + '

    Footnote 2 ↩

    \n' + '
  4. \n' + '
\n' + '
' + ) + + def test_consecutive_footnotes_before_setext_underline(self): + """ + A setext underline after consecutive footnotes is a lazily indented line + of the last footnote, and does not turn that definition into a header. + """ + self.assertMarkdownRenders( + self.dedent( + """ + [^1]: Footnote 1 + [^2]: Footnote 2 + === + """ + ), + '
\n' + '
\n' + '
    \n' + '
  1. \n' + '

    Footnote 1 ↩

    \n' + '
  2. \n' + '
  3. \n' + '

    Footnote 2

    \n' + '

    ↩

    \n' + '
  4. \n' + '
\n' + '
' + ) + + def test_many_footnote_definitions(self): + """ + Consecutive footnote definitions should be consumed together rather + than returning the remainder of the block to be parsed once per + definition. + """ + count = 5000 + self.assertMarkdownRenders( + ''.join(f'[^{i}]: Footnote {i}\n' for i in range(count)), + '
\n' + '
\n' + '
    \n' + + ''.join( + f'
  1. \n' + f'

    Footnote {i} ↩

    \n' + '
  2. \n' + for i in range(count) + ) + + '
\n' + '
' + ) + def test_backlink_text(self): """Test back-link configuration."""