From b8399200dfbd68f60eb655509305a029f031cd73 Mon Sep 17 00:00:00 2001 From: Xianpeng Shen Date: Wed, 12 Aug 2026 12:54:05 +0000 Subject: [PATCH 1/2] Add scaling and serve-loop benchmarks to the CodSpeed suite Extend the benchmark suite merged in #88 with the scenarios needed to track the build-performance work for upstream issue mkdocs/mkdocs#3695: * full builds of the deterministic corpus at several sizes (10/50/200 pages by default, overridable via MKDOCS_BENCH_SIZES for local scaling studies) - a local run at 100/400/1600 pages measured 7.8/9.3/22.1 ms per page, i.e. clearly superlinear growth * a dirty rebuild with no modified files, approximating the 'mkdocs serve --dirty' inner loop * a single page rendered against the largest site's navigation, isolating the O(N^2) sitewide template-rendering term corpus.write_site() gains optional sections/pages_per_section parameters (defaults unchanged, existing benchmarks unaffected). --- benchmarks/corpus.py | 11 +++- benchmarks/test_scaling.py | 118 +++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 3 deletions(-) create mode 100644 benchmarks/test_scaling.py diff --git a/benchmarks/corpus.py b/benchmarks/corpus.py index 2997eb83..dec75e82 100644 --- a/benchmarks/corpus.py +++ b/benchmarks/corpus.py @@ -111,7 +111,12 @@ def make_markdown(seed: int, *, headings: int = 8, links: tuple[str, ...] = ()) return "\n".join(parts) + "\n" -def write_site(root: str) -> str: +def write_site( + root: str, + *, + sections: int = SECTIONS, + pages_per_section: int = PAGES_PER_SECTION, +) -> str: """ Write a full MkDocs project (config + nested docs tree) under `root`. @@ -119,7 +124,7 @@ def write_site(root: str) -> str: """ docs_dir = os.path.join(root, "docs") os.makedirs(docs_dir, exist_ok=True) - section_names = [f"section-{i:02d}" for i in range(SECTIONS)] + section_names = [f"section-{i:02d}" for i in range(sections)] with open(os.path.join(docs_dir, "index.md"), "w", encoding="utf-8") as f: f.write(make_markdown(0, links=tuple(f"{n}/index.md" for n in section_names))) @@ -127,7 +132,7 @@ def write_site(root: str) -> str: for s, name in enumerate(section_names): section_dir = os.path.join(docs_dir, name) os.makedirs(section_dir, exist_ok=True) - page_names = tuple(f"page-{p:02d}.md" for p in range(PAGES_PER_SECTION)) + page_names = tuple(f"page-{p:02d}.md" for p in range(pages_per_section)) links = ("../index.md", *page_names) with open(os.path.join(section_dir, "index.md"), "w", encoding="utf-8") as f: f.write(make_markdown(100 + s, links=links)) diff --git a/benchmarks/test_scaling.py b/benchmarks/test_scaling.py new file mode 100644 index 00000000..5ca5d01f --- /dev/null +++ b/benchmarks/test_scaling.py @@ -0,0 +1,118 @@ +""" +Scaling benchmarks: how build cost grows with site size. + +These complement the fixed-size suite by building the same deterministic +corpus at several sizes, plus two scenarios from the `mkdocs serve` loop. +The default sizes are modest so CI stays fast; override them locally for +larger scaling studies: + + MKDOCS_BENCH_SIZES=100,400,1600 hatch run bench:codspeed benchmarks/test_scaling.py +""" + +from __future__ import annotations + +import logging +import os + +import pytest + +from benchmarks.corpus import PAGES_PER_SECTION, write_site +from mkdocs.commands.build import _build_page, _populate_page, build +from mkdocs.config.base import load_config +from mkdocs.structure.files import get_files +from mkdocs.structure.nav import get_navigation + +SIZES = [int(s) for s in os.environ.get("MKDOCS_BENCH_SIZES", "10,50,200").split(",")] + + +@pytest.fixture(autouse=True, scope="module") +def _quiet_mkdocs_logging(): + """Dirty rebuilds legitimately log a warning; keep benchmark output clean.""" + logger = logging.getLogger("mkdocs") + handler = logging.NullHandler() + logger.addHandler(handler) + old_level = logger.level + logger.setLevel(logging.ERROR) + yield + logger.setLevel(old_level) + logger.removeHandler(handler) + + +def _sections_for(n_pages: int) -> int: + # Each section contributes an index page plus its regular pages, and the + # site has one root index page on top. + return max(1, round((n_pages - 1) / (PAGES_PER_SECTION + 1))) + + +@pytest.fixture(scope="module") +def sized_site_factory(tmp_path_factory): + """Return ``get(n_pages) -> str`` (config file path), one site per size.""" + cache: dict[int, str] = {} + + def get(n_pages: int) -> str: + if n_pages not in cache: + root = tmp_path_factory.mktemp(f"scaling_site_{n_pages}") + cache[n_pages] = write_site(str(root), sections=_sections_for(n_pages)) + return cache[n_pages] + + return get + + +@pytest.mark.parametrize("n_pages", SIZES) +def test_full_build_scaling(benchmark, sized_site_factory, tmp_path, n_pages) -> None: + """Clean full build at increasing site sizes (upstream mkdocs/mkdocs#3695).""" + config_file = sized_site_factory(n_pages) + counter = iter(range(1000)) + + def setup(): + site_dir = str(tmp_path / f"site-{next(counter)}") + return (load_config(config_file, site_dir=site_dir),), {} + + benchmark.pedantic(build, setup=setup, rounds=1, warmup_rounds=0) + + +def test_dirty_rebuild(benchmark, sized_site_factory, tmp_path) -> None: + """ + Proxy for the `mkdocs serve --dirty` inner loop (no file modified). + + Even with nothing changed, every rebuild re-walks the docs directory, + rebuilds the navigation, recreates the Jinja environment and re-runs all + plugin events. + """ + config_file = sized_site_factory(SIZES[len(SIZES) // 2]) + site_dir = str(tmp_path / "site") + build(load_config(config_file, site_dir=site_dir)) + + def setup(): + return (load_config(config_file, site_dir=site_dir),), {"dirty": True} + + benchmark.pedantic(build, setup=setup, rounds=1, warmup_rounds=0) + + +def test_template_render_large_nav(benchmark, sized_site_factory, tmp_path) -> None: + """ + Render a single page against the largest site's navigation. + + Built-in themes iterate the full navigation for every page, so template + rendering costs O(pages) per page — the O(N^2) sitewide term behind + upstream issue mkdocs/mkdocs#3695. + """ + config = load_config( + sized_site_factory(max(SIZES)), site_dir=str(tmp_path / "large-site") + ) + config = config.plugins.on_config(config) + config.plugins.on_pre_build(config=config) + files = get_files(config) + env = config.theme.get_env() + files.add_files_from_theme(env, config) + nav = get_navigation(files, config) + doc_files = files.documentation_pages() + file = doc_files[len(doc_files) // 2] + assert file.page is not None + _populate_page(file.page, config, files) + env = config.plugins.on_env(env, config=config, files=files) + + def render_one() -> None: + _build_page(file.page, config, doc_files, nav, env) + + benchmark(render_one) From b5f10dc5afa0abfb0ac518f0d4f457f3cc9930dd Mon Sep 17 00:00:00 2001 From: Xianpeng Shen Date: Wed, 12 Aug 2026 12:55:07 +0000 Subject: [PATCH 2/2] Update release notes for CodSpeed benchmarking suite (#88, #89) --- docs/about/release-notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/about/release-notes.md b/docs/about/release-notes.md index 680caba5..acb5312a 100644 --- a/docs/about/release-notes.md +++ b/docs/about/release-notes.md @@ -32,6 +32,7 @@ Update documentation to fix broken link anchors. #62 * Built-in themes now bundle highlight.js locally instead of loading it from the cdnjs CDN, so syntax highlighting works in offline and privacy-sensitive environments. #75 * Add a stable public Python API — `mkdocs.build()` and `mkdocs.serve()` — for building and serving documentation programmatically. #76 * The built-in search plugin no longer filters out English stop words, so searching for words like `while`, `if`, `for` or `from` now returns results. A new `stop_words` option restores the previous behavior when set to `true`. #80 +* Add a continuous benchmarking suite (`benchmarks/`, `hatch run bench:run`) tracked by [CodSpeed](https://codspeed.io) in CI, including scaling and serve-loop scenarios, establishing a performance baseline for build-pipeline optimization work. #88 #89 ### Changed