diff --git a/README.md b/README.md index 94f9385c..10211e58 100644 --- a/README.md +++ b/README.md @@ -80,8 +80,8 @@ If a file is not properly formatted, the exit code will be non-zero. foo@bar:~$ mdformat --help usage: mdformat [-h] [--check] [--no-validate] [--version] [--number] [--wrap {keep,no,INTEGER}] [--end-of-line {lf,crlf,keep}] - [--exclude PATTERN] [--extensions EXTENSION] - [--codeformatters LANGUAGE] + [--max-nesting INTEGER] [--exclude PATTERN] + [--extensions EXTENSION] [--codeformatters LANGUAGE] [paths ...] CommonMark compliant Markdown formatter @@ -99,6 +99,11 @@ options: paragraph word wrap mode (default: keep) --end-of-line {lf,crlf,keep} output file line ending mode (default: lf) + --max-nesting INTEGER + maximum allowed nesting depth of blockquotes and lists + (default: 20). Content nested deeper than this is + silently dropped by the parser; raise this for deeply + nested documents --exclude PATTERN exclude files that match the Unix-style glob pattern (multiple allowed) --extensions EXTENSION diff --git a/docs/users/configuration_file.md b/docs/users/configuration_file.md index 508f19e9..8ada81bc 100644 --- a/docs/users/configuration_file.md +++ b/docs/users/configuration_file.md @@ -21,6 +21,7 @@ wrap = "keep" # options: {"keep", "no", INTEGER} number = false # options: {false, true} end_of_line = "lf" # options: {"lf", "crlf", "keep"} validate = true # options: {false, true} +max_nesting = 20 # options: a positive integer # extensions = [ # options: a list of enabled extensions (default: all installed are enabled) # "gfm", # "toc", diff --git a/src/mdformat/_cli.py b/src/mdformat/_cli.py index 682b7896..30329aa1 100644 --- a/src/mdformat/_cli.py +++ b/src/mdformat/_cli.py @@ -13,7 +13,7 @@ import mdformat from mdformat._conf import DEFAULT_OPTS, InvalidConfError, read_toml_opts -from mdformat._util import detect_newline_type, is_md_equal +from mdformat._util import detect_newline_type, is_md_equal, required_nesting_depth import mdformat.plugins @@ -23,6 +23,39 @@ def emit(self, record: logging.LogRecord) -> None: sys.stderr.write(f"Warning: {record.msg}\n") +def _print_max_nesting_error( + path_str: str, + original_str: str, + *, + opts: Mapping, + extensions: Mapping, + codeformatters: Mapping, +) -> bool: + """Print error and return `True` if `max_nesting` caused the mismatch.""" + configured_max_nesting = opts["max_nesting"] + needed_depth = required_nesting_depth( + original_str, + options=opts, + extensions=extensions, + codeformatters=codeformatters, + ) + if needed_depth is None or needed_depth <= configured_max_nesting: + return False + print_error( + f'Could not format "{path_str}".', + paragraphs=[ + f"This document nests blockquotes and/or lists {needed_depth} " + "levels deep, but 'max_nesting' is set to " + f"{configured_max_nesting}. Content nested beyond that " + "limit is silently dropped by the parser, " + "which is why formatting changed the document's meaning. " + f"Raise the limit, e.g. `--max-nesting={needed_depth}` " + "or `max_nesting` in `.mdformat.toml`.", + ], + ) + return True + + def run(cli_args: Sequence[str], cache_toml: bool = True) -> int: # noqa: C901 arg_parser = make_arg_parser( mdformat.plugins._PARSER_EXTENSION_DISTS, @@ -162,6 +195,14 @@ def run(cli_args: Sequence[str], cache_toml: bool = True) -> int: # noqa: C901 codeformatters=enabled_codeformatters, ) ): + if _print_max_nesting_error( + path_str, + original_str, + opts=opts, + extensions=enabled_parserplugins, + codeformatters=enabled_codeformatters, + ): + return 1 print_error( f'Could not format "{path_str}".', paragraphs=[ @@ -195,6 +236,13 @@ def validate_wrap_arg(value: str) -> str | int: return width +def validate_max_nesting_arg(value: str) -> int: + max_nesting = int(value) + if max_nesting < 1: + raise ValueError("max-nesting must be a positive integer") + return max_nesting + + def make_arg_parser( parser_extension_dists: Mapping[str, tuple[str, list[str]]], codeformatter_dists: Mapping[str, tuple[str, list[str]]], @@ -241,6 +289,14 @@ def make_arg_parser( choices=("lf", "crlf", "keep"), help="output file line ending mode (default: lf)", ) + parser.add_argument( + "--max-nesting", + type=validate_max_nesting_arg, + metavar="INTEGER", + help="maximum allowed nesting depth of blockquotes and lists " + "(default: 20). Content nested deeper than this is silently " + "dropped by the parser; raise this for deeply nested documents", + ) if sys.version_info >= (3, 13): # pragma: >=3.13 cover parser.add_argument( "--exclude", @@ -327,6 +383,7 @@ class InvalidPath(Exception): """Exception raised when a path does not exist.""" def __init__(self, path: Path): + super().__init__(path) self.path = path diff --git a/src/mdformat/_conf.py b/src/mdformat/_conf.py index d50a798d..d29216f7 100644 --- a/src/mdformat/_conf.py +++ b/src/mdformat/_conf.py @@ -6,7 +6,7 @@ from types import MappingProxyType from mdformat._compat import tomllib -from mdformat._util import EMPTY_MAP +from mdformat._util import DEFAULT_MAX_NESTING, EMPTY_MAP DEFAULT_OPTS = MappingProxyType( { @@ -18,6 +18,7 @@ "plugin": EMPTY_MAP, "extensions": None, "codeformatters": None, + "max_nesting": DEFAULT_MAX_NESTING, } ) @@ -94,6 +95,14 @@ def _validate_values(opts: Mapping, conf_path: Path) -> None: # noqa: C901 for lang in opts["codeformatters"]: if not isinstance(lang, str): raise InvalidConfError(f"Invalid 'codeformatters' value in {conf_path}") + if "max_nesting" in opts: + max_nesting_value = opts["max_nesting"] + if not ( + isinstance(max_nesting_value, int) + and not isinstance(max_nesting_value, bool) + and max_nesting_value > 0 + ): + raise InvalidConfError(f"Invalid 'max_nesting' value in {conf_path}") def _validate_keys(opts: Mapping, conf_path: Path) -> None: diff --git a/src/mdformat/_util.py b/src/mdformat/_util.py index 23472528..15166334 100644 --- a/src/mdformat/_util.py +++ b/src/mdformat/_util.py @@ -13,6 +13,7 @@ NULL_CTX = nullcontext() EMPTY_MAP: MappingProxyType = MappingProxyType({}) +DEFAULT_MAX_NESTING = 20 RE_NEWLINES = re.compile(r"\r\n|\r|\n") RE_HTML_START_SPACE_PREFIX = re.compile(r" (<[a-zA-Z][-a-zA-Z0-9]*>)") @@ -31,6 +32,7 @@ def build_mdit( mdit = MarkdownIt(renderer_cls=renderer_cls) mdit.options["mdformat"] = mdformat_opts + mdit.options["maxNesting"] = mdformat_opts.get("max_nesting", DEFAULT_MAX_NESTING) # store reference labels in link/image tokens mdit.options["store_labels"] = True @@ -117,6 +119,32 @@ def is_md_equal( return html_texts["md1"] == html_texts["md2"] +def required_nesting_depth( + md: str, + *, + options: Mapping[str, Any] = EMPTY_MAP, + extensions: Iterable[str] = (), + codeformatters: Iterable[str] = (), +) -> int | None: + """Return how many levels of blockquote/list nesting is required.""" + # Lazy import to improve module import time + from markdown_it.renderer import RendererHTML + + nesting_probe_ceiling = 500 # Below default Python recursion limit + probe_opts = {**options, "max_nesting": nesting_probe_ceiling} + try: + mdit = build_mdit( + RendererHTML, + mdformat_opts=probe_opts, + extensions=extensions, + codeformatters=codeformatters, + ) + tokens = mdit.parse(md) + except RecursionError: + return None + return max((token.level for token in tokens), default=-1) + 1 + + def detect_newline_type(md: str, eol_setting: str) -> Literal["\n", "\r\n"]: """Returns the newline-character to be used for output. diff --git a/tests/test_api.py b/tests/test_api.py index 73177d7a..820dd947 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -6,6 +6,7 @@ import mdformat from mdformat._util import is_md_equal from mdformat.renderer import MDRenderer +from tests.utils import nested_list_markdown UNFORMATTED_MARKDOWN = "\n\n# A header\n\n" FORMATTED_MARKDOWN = "# A header\n" @@ -90,6 +91,16 @@ def test_api_options(): assert mdformat.text(non_numbered, options={"number": True}) == numbered +def test_max_nesting__deeply_nested_list(): + text = nested_list_markdown(100) + assert mdformat.text(text, options={"max_nesting": 500}) == text + + +def test_max_nesting__default_truncates_deeply_nested_list(): + text = nested_list_markdown(100) + assert mdformat.text(text) != text + + def test_eol__lf(tmp_path): file_path = tmp_path / "test.md" file_path.write_bytes(b"Oi\r\n") diff --git a/tests/test_cli.py b/tests/test_cli.py index 28c36223..62c6b7a2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -12,6 +12,7 @@ UNFORMATTED_MARKDOWN, ASTChangingPlugin, PrefixPostprocessPlugin, + nested_list_markdown, ) @@ -269,6 +270,38 @@ def test_bad_wrap_width(capsys): assert "error: argument --wrap" in captured.err +def test_max_nesting(tmp_path): + file_path = tmp_path / "test.md" + text = nested_list_markdown(100) + file_path.write_text(text) + + assert run([str(file_path), "--max-nesting=500"]) == 0 + assert file_path.read_text() == text + + +def test_max_nesting__too_low(tmp_path, capsys): + file_path = tmp_path / "test.md" + text = nested_list_markdown(100) + file_path.write_text(text) + + assert run([str(file_path)]) == 1 + assert file_path.read_text() == text + captured = capsys.readouterr() + err = " ".join(captured.err.split()) + assert "nests blockquotes and/or lists" in err + assert "'max_nesting' is set to 20" in err + assert "--max-nesting=" in err + + +def test_bad_max_nesting(capsys): + with pytest.raises(SystemExit) as exc_info: + run(["some-path.md", "--max-nesting=0"]) + + assert exc_info.value.code == 2 + captured = capsys.readouterr() + assert "error: argument --max-nesting" in captured.err + + def test_eol__lf(tmp_path): file_path = tmp_path / "test.md" file_path.write_bytes(b"Oi\r\n") @@ -362,15 +395,12 @@ def test_get_plugin_info_str(): {"mdformat-tables": ("0.1.0", ["tables"])}, {"mdformat-black": ("12.1.0", ["python"])}, ) - assert ( - info - == """\ + assert info == """\ installed codeformatters: mdformat-black: python installed extensions: mdformat-tables: tables""" - ) def test_no_timestamp_modify(tmp_path): diff --git a/tests/test_config_file.py b/tests/test_config_file.py index ec16eea8..895b6e53 100644 --- a/tests/test_config_file.py +++ b/tests/test_config_file.py @@ -4,7 +4,19 @@ import pytest from mdformat._cli import run -from tests.utils import FORMATTED_MARKDOWN, UNFORMATTED_MARKDOWN +from tests.utils import FORMATTED_MARKDOWN, UNFORMATTED_MARKDOWN, nested_list_markdown + + +def test_max_nesting_conf(tmp_path): + config_path = tmp_path / ".mdformat.toml" + config_path.write_text("max_nesting = 500") + + file_path = tmp_path / "test_markdown.md" + text = nested_list_markdown(100) + file_path.write_text(text) + + assert run((str(file_path),)) == 0 + assert file_path.read_text() == text def test_cli_override(tmp_path): @@ -73,6 +85,8 @@ def test_invalid_toml(tmp_path, capsys): ("extensions", "extensions = 'gfm'"), ("codeformatters", "codeformatters = ['python', 1]"), ("extensions", "extensions = ['gfm', 1]"), + ("max_nesting", "max_nesting = 0"), + ("max_nesting", "max_nesting = 'deep'"), ], ) def test_invalid_conf_value(bad_conf, conf_key, tmp_path, capsys): diff --git a/tests/test_util.py b/tests/test_util.py index 62c8dcdb..f15c8d36 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,4 +1,7 @@ -from mdformat._util import is_md_equal +import sys + +from mdformat._util import DEFAULT_MAX_NESTING, is_md_equal, required_nesting_depth +from tests.utils import nested_list_markdown def test_is_md_equal(): @@ -48,3 +51,31 @@ def test_is_md_equal__not(): """ assert not is_md_equal(md1, md2) assert not is_md_equal(md1, md2, codeformatters=("js",)) + + +def test_required_nesting_depth__scales_with_actual_nesting(): + shallow_depth = required_nesting_depth(nested_list_markdown(1)) + deep_depth = required_nesting_depth(nested_list_markdown(30)) + + assert shallow_depth is not None + assert deep_depth is not None + assert shallow_depth < deep_depth + + +def test_required_nesting_depth__exceeds_default_max_nesting_for_deep_lists(): + text = nested_list_markdown(30) + + depth = required_nesting_depth(text) + + assert depth is not None + assert depth > DEFAULT_MAX_NESTING + + +def test_required_nesting_depth__recursion_limit_exceeded(): + original_limit = sys.getrecursionlimit() + sys.setrecursionlimit(50) + + try: + assert required_nesting_depth(nested_list_markdown(50)) is None + finally: + sys.setrecursionlimit(original_limit) diff --git a/tests/utils.py b/tests/utils.py index 209c4bc2..99539bad 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -12,6 +12,11 @@ FORMATTED_MARKDOWN = "# A header\n" +def nested_list_markdown(depth: int) -> str: + """A bulleted list nested `depth` levels deep.""" + return "".join(f"{' ' * i}- item{i}\n" for i in range(depth)) + + class JSONFormatterPlugin: """A code formatter plugin that formats JSON."""