Add CodSpeed continuous benchmarking

.github/workflows/benchmarks.yml

@@ -0,0 +1,36 @@
+name: Benchmarks
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  # Allows CodSpeed to trigger backtest performance analysis.
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  id-token: write  # Required for OpenID Connect authentication with CodSpeed
+
+jobs:
+  benchmarks:
+    name: Run benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Setup Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.12'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e . pytest pytest-codspeed
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@4296e51e7041e24dadb86d1d6e8b9320d223dbe8 # v5.0.3
+        with:
+          mode: simulation
+          run: pytest benchmarks --codspeed

CONTRIBUTING.md

@@ -58,6 +58,14 @@ hatch run integration:test
 hatch run types:check
 ```
 
+Performance-sensitive changes (building, rendering, navigation, search index)
+are covered by the benchmarks in `benchmarks/`. They run on every pull request
+with [CodSpeed], and can be run locally with:
+
+```bash
+hatch run bench:run
+```
+
 If you changed documentation, preview it locally:
 
 ```bash
@@ -86,6 +94,7 @@ consistent with the surrounding style.
 Everyone interacting in MkDocs NG spaces is expected to follow the
 [PyPA Code of Conduct].
 
+[CodSpeed]: https://app.codspeed.io/mkdocs-ng/mkdocs
 [GitHub Discussions]: https://github.com/orgs/mkdocs-ng/discussions
 [GitHub issues]: https://github.com/mkdocs-ng/mkdocs/issues
 [Hatch]: https://hatch.pypa.io/

README.md

@@ -5,6 +5,7 @@
 [![PyPI Version][pypi-v-image]][pypi-v-link]
 [![Build Status][GHAction-image]][GHAction-link]
 [![Coverage Status][codecov-image]][codecov-link]
+[![CodSpeed][codspeed-image]][codspeed-link]
 
 MkDocs is a **fast**, **simple**, and **downright gorgeous** static site
 generator that's geared towards building project documentation. Documentation
@@ -69,6 +70,8 @@ discussion forums are expected to follow the [PyPA Code of Conduct].
 [pypi-v-link]: https://pypi.org/project/mkdocs-ng/
 [GHAction-image]: https://github.com/mkdocs-ng/mkdocs/actions/workflows/ci.yml/badge.svg
 [GHAction-link]: https://github.com/mkdocs-ng/mkdocs/actions/workflows/ci.yml
+[codspeed-image]: https://img.shields.io/endpoint?url=https://codspeed.io/badge.json
+[codspeed-link]: https://app.codspeed.io/mkdocs-ng/mkdocs?utm_source=badge
 <!-- Links -->
 [mkdocs]: https://mkdocs-ng.github.io/mkdocs/
 [mkdocs/mkdocs]: https://github.com/mkdocs/mkdocs

benchmarks/conftest.py

@@ -0,0 +1,51 @@
+"""Shared fixtures for the MkDocs benchmark suite."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from benchmarks.corpus import write_site
+from mkdocs.config.base import load_config
+from mkdocs.structure.files import get_files
+from mkdocs.structure.nav import get_navigation
+
+if TYPE_CHECKING:
+    from mkdocs.config.defaults import MkDocsConfig
+    from mkdocs.structure.files import Files
+    from mkdocs.structure.nav import Navigation
+    from mkdocs.structure.pages import Page
+
+
+@pytest.fixture(scope="session")
+def config_file_path(tmp_path_factory: pytest.TempPathFactory) -> str:
+    """Generate the benchmark site on disk once for the whole session."""
+    return write_site(str(tmp_path_factory.mktemp("mkdocs_bench_site")))
+
+
+@pytest.fixture
+def config(config_file_path: str, tmp_path) -> MkDocsConfig:
+    """A freshly loaded and validated configuration for the generated site."""
+    return load_config(config_file_path, site_dir=str(tmp_path / "site"))
+
+
+@pytest.fixture
+def files(config: MkDocsConfig) -> Files:
+    return get_files(config)
+
+
+@pytest.fixture
+def navigation(config: MkDocsConfig, files: Files) -> Navigation:
+    return get_navigation(files, config)
+
+
+@pytest.fixture
+def rendered_pages(
+    config: MkDocsConfig, files: Files, navigation: Navigation
+) -> list[Page]:
+    """Every documentation page of the generated site, read and rendered."""
+    for page in navigation.pages:
+        page.read_source(config)
+        page.render(config, files)
+    return list(navigation.pages)

benchmarks/corpus.py

@@ -0,0 +1,148 @@
+"""
+Deterministic documentation corpus used by the benchmarks.
+
+Everything here is generated from a fixed pseudo-random seed so that every
+benchmark run measures exactly the same workload and results only move when
+MkDocs itself changes.
+"""
+
+from __future__ import annotations
+
+import os
+import random
+
+WORDS = (
+    "documentation markdown static site generator theme plugin navigation "
+    "configuration template rendering directory anchor heading section link "
+    "reference build server extension content page index deploy source output"
+).split()
+
+LANGUAGES = ("python", "yaml", "bash", "text")
+
+SECTIONS = 5
+PAGES_PER_SECTION = 6
+
+MKDOCS_YML = """\
+site_name: Benchmark Site
+site_url: https://example.com/benchmark/
+site_description: A generated site used to benchmark MkDocs.
+repo_url: https://github.com/mkdocs-ng/mkdocs/
+edit_uri: blob/main/docs/
+
+theme:
+  name: mkdocs
+
+markdown_extensions:
+  - toc:
+      permalink: true
+  - admonition
+  - attr_list
+  - def_list
+  - footnotes
+  - tables
+
+plugins:
+  - search
+"""
+
+
+def _sentence(rng: random.Random) -> str:
+    return " ".join(rng.choices(WORDS, k=rng.randint(6, 14))).capitalize() + "."
+
+
+def _paragraph(rng: random.Random) -> str:
+    return " ".join(_sentence(rng) for _ in range(rng.randint(3, 6)))
+
+
+def _code_block(rng: random.Random) -> str:
+    lines = "\n".join(
+        f"    {rng.choice(WORDS)} = {rng.randint(0, 1000)}"
+        for _ in range(rng.randint(3, 8))
+    )
+    return f"```{rng.choice(LANGUAGES)}\n{lines}\n```"
+
+
+def _table(rng: random.Random) -> str:
+    header = "| Option | Type | Default |\n| --- | --- | --- |"
+    rows = "\n".join(
+        f"| `{rng.choice(WORDS)}` | {rng.choice(WORDS)} | `{rng.randint(0, 99)}` |"
+        for _ in range(rng.randint(3, 6))
+    )
+    return f"{header}\n{rows}"
+
+
+def _bullet_list(rng: random.Random) -> str:
+    return "\n".join(f"- {_sentence(rng)}" for _ in range(rng.randint(3, 6)))
+
+
+def _admonition(rng: random.Random) -> str:
+    return f'!!! note "{rng.choice(WORDS).title()}"\n\n    {_sentence(rng)}'
+
+
+_BUILDERS = (_paragraph, _code_block, _table, _bullet_list, _admonition)
+
+
+def make_markdown(seed: int, *, headings: int = 8, links: tuple[str, ...] = ()) -> str:
+    """Generate a deterministic Markdown document with realistic constructs."""
+    rng = random.Random(seed)
+    parts = [
+        "---",
+        f"title: Page {seed}",
+        "tags:",
+        f"  - {rng.choice(WORDS)}",
+        f"  - {rng.choice(WORDS)}",
+        "---",
+        "",
+        f"# Page {seed} about {rng.choice(WORDS)}",
+        "",
+        _paragraph(rng),
+    ]
+    for i in range(headings):
+        level = "##" if i % 3 else "###"
+        parts += ["", f"{level} {rng.choice(WORDS).title()} {i}", "", _paragraph(rng)]
+        parts += ["", _BUILDERS[i % len(_BUILDERS)](rng)]
+        if links:
+            target = links[i % len(links)]
+            parts += [
+                "",
+                f"See [{rng.choice(WORDS)}]({target}) and "
+                f"[{rng.choice(WORDS)}]({target}#{rng.choice(WORDS)}) for details.",
+            ]
+    return "\n".join(parts) + "\n"
+
+
+def write_site(root: str) -> str:
+    """
+    Write a full MkDocs project (config + nested docs tree) under `root`.
+
+    Returns the path of the generated `mkdocs.yml`.
+    """
+    docs_dir = os.path.join(root, "docs")
+    os.makedirs(docs_dir, exist_ok=True)
+    section_names = [f"section-{i:02d}" for i in range(SECTIONS)]
+
+    with open(os.path.join(docs_dir, "index.md"), "w", encoding="utf-8") as f:
+        f.write(make_markdown(0, links=tuple(f"{n}/index.md" for n in section_names)))
+
+    for s, name in enumerate(section_names):
+        section_dir = os.path.join(docs_dir, name)
+        os.makedirs(section_dir, exist_ok=True)
+        page_names = tuple(f"page-{p:02d}.md" for p in range(PAGES_PER_SECTION))
+        links = ("../index.md", *page_names)
+        with open(os.path.join(section_dir, "index.md"), "w", encoding="utf-8") as f:
+            f.write(make_markdown(100 + s, links=links))
+        for p, page_name in enumerate(page_names):
+            with open(os.path.join(section_dir, page_name), "w", encoding="utf-8") as f:
+                f.write(make_markdown(1000 + s * 100 + p, links=links))
+
+    # A few static assets, so the file collection is not only Markdown.
+    css_dir = os.path.join(docs_dir, "css")
+    os.makedirs(css_dir, exist_ok=True)
+    for i in range(3):
+        with open(os.path.join(css_dir, f"extra-{i}.css"), "w", encoding="utf-8") as f:
+            f.write("body { margin: 0; }\n")
+
+    config_file = os.path.join(root, "mkdocs.yml")
+    with open(config_file, "w", encoding="utf-8") as f:
+        f.write(MKDOCS_YML)
+    return config_file

benchmarks/test_build.py

@@ -0,0 +1,79 @@
+"""End-to-end benchmarks for a full site build."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from mkdocs.commands.build import _build_page, _populate_page, build
+from mkdocs.config.base import load_config
+from mkdocs.structure.files import get_files
+from mkdocs.structure.nav import get_navigation
+
+if TYPE_CHECKING:
+    from mkdocs.config.defaults import MkDocsConfig
+    from mkdocs.structure.files import Files
+    from mkdocs.structure.nav import Navigation
+
+
+def test_build_site(benchmark, config_file_path: str, tmp_path) -> None:
+    """Full `mkdocs build` of the generated site: files, nav, render, templates."""
+    counter = iter(range(1000))
+
+    def setup():
+        site_dir = str(tmp_path / f"site-{next(counter)}")
+        return (load_config(config_file_path, site_dir=site_dir),), {}
+
+    benchmark.pedantic(build, setup=setup, rounds=1, warmup_rounds=0)
+
+
+def test_build_site_no_directory_urls(
+    benchmark, config_file_path: str, tmp_path
+) -> None:
+    """Same build with `use_directory_urls` disabled, which changes URL resolution."""
+    counter = iter(range(1000))
+
+    def setup():
+        site_dir = str(tmp_path / f"flat-site-{next(counter)}")
+        config = load_config(
+            config_file_path, site_dir=site_dir, use_directory_urls=False
+        )
+        return (config,), {}
+
+    benchmark.pedantic(build, setup=setup, rounds=1, warmup_rounds=0)
+
+
+def test_populate_pages(
+    benchmark, config: MkDocsConfig, files: Files, navigation: Navigation
+) -> None:
+    """Read and convert the Markdown of every page, without writing any output."""
+
+    def populate_all() -> int:
+        for file in files.documentation_pages():
+            assert file.page is not None
+            _populate_page(file.page, config, files)
+        return len(navigation.pages)
+
+    assert benchmark(populate_all) > 0
+
+
+def test_build_theme_pages(benchmark, config: MkDocsConfig) -> None:
+    """Render the `mkdocs` theme templates and write the HTML of every page."""
+    config = config.plugins.on_config(config)
+    config.plugins.on_pre_build(config=config)
+
+    files = get_files(config)
+    env = config.theme.get_env()
+    files.add_files_from_theme(env, config)
+    nav = get_navigation(files, config)
+    doc_files = files.documentation_pages()
+    for file in doc_files:
+        assert file.page is not None
+        _populate_page(file.page, config, files)
+    env = config.plugins.on_env(env, config=config, files=files)
+
+    def build_all() -> None:
+        for file in doc_files:
+            assert file.page is not None
+            _build_page(file.page, config, doc_files, nav, env)
+
+    benchmark(build_all)

benchmarks/test_config.py

@@ -0,0 +1,33 @@
+"""Benchmarks for configuration loading and validation."""
+
+from __future__ import annotations
+
+import io
+
+from mkdocs.config.base import load_config
+from mkdocs.utils import yaml as yaml_utils
+from mkdocs.utils.yaml import yaml_load
+
+
+def test_load_config(benchmark, config_file_path: str, tmp_path) -> None:
+    """Read `mkdocs.yml`, apply the schema defaults and validate every option."""
+    site_dir = str(tmp_path / "site")
+
+    config = benchmark(load_config, config_file_path, site_dir=site_dir)
+    assert config.site_name == "Benchmark Site"
+
+
+def test_yaml_load(benchmark, config_file_path: str) -> None:
+    """Parse the YAML configuration with MkDocs' loader (env tags, includes)."""
+    with open(config_file_path, "rb") as f:
+        raw = f.read()
+
+    def load() -> dict:
+        return yaml_load(io.BytesIO(raw))
+
+    assert benchmark(load)["site_name"] == "Benchmark Site"
+
+
+def test_get_yaml_loader(benchmark) -> None:
+    """Build the YAML loader class, done for every config and every page."""
+    assert benchmark(yaml_utils.get_yaml_loader) is not None