From 18f9abe09401087445f708334ec6fd596ad78363 Mon Sep 17 00:00:00 2001 From: CodSpeed Bot Date: Wed, 12 Aug 2026 12:42:03 +0000 Subject: [PATCH] Add CodSpeed continuous benchmarking Add a pytest-codspeed benchmark suite covering the build pipeline, page rendering, file collection, navigation, configuration loading, the search index and the shared utilities, plus a GitHub Actions workflow that runs them on every pull request. --- .github/workflows/benchmarks.yml | 36 ++++++++ CONTRIBUTING.md | 9 ++ README.md | 3 + benchmarks/__init__.py | 0 benchmarks/conftest.py | 51 +++++++++++ benchmarks/corpus.py | 148 +++++++++++++++++++++++++++++++ benchmarks/test_build.py | 79 +++++++++++++++++ benchmarks/test_config.py | 33 +++++++ benchmarks/test_search.py | 55 ++++++++++++ benchmarks/test_structure.py | 82 +++++++++++++++++ benchmarks/test_utils.py | 76 ++++++++++++++++ pyproject.toml | 9 ++ 12 files changed, 581 insertions(+) create mode 100644 .github/workflows/benchmarks.yml create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/conftest.py create mode 100644 benchmarks/corpus.py create mode 100644 benchmarks/test_build.py create mode 100644 benchmarks/test_config.py create mode 100644 benchmarks/test_search.py create mode 100644 benchmarks/test_structure.py create mode 100644 benchmarks/test_utils.py diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..b5c15af7 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,36 @@ +name: Benchmarks +on: + push: + branches: + - main + pull_request: + # Allows CodSpeed to trigger backtest performance analysis. + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + id-token: write # Required for OpenID Connect authentication with CodSpeed + +jobs: + benchmarks: + name: Run benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -e . pytest pytest-codspeed + - name: Run benchmarks + uses: CodSpeedHQ/action@4296e51e7041e24dadb86d1d6e8b9320d223dbe8 # v5.0.3 + with: + mode: simulation + run: pytest benchmarks --codspeed diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a4579b22..fbff74b0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,6 +58,14 @@ hatch run integration:test hatch run types:check ``` +Performance-sensitive changes (building, rendering, navigation, search index) +are covered by the benchmarks in `benchmarks/`. They run on every pull request +with [CodSpeed], and can be run locally with: + +```bash +hatch run bench:run +``` + If you changed documentation, preview it locally: ```bash @@ -86,6 +94,7 @@ consistent with the surrounding style. Everyone interacting in MkDocs NG spaces is expected to follow the [PyPA Code of Conduct]. +[CodSpeed]: https://app.codspeed.io/mkdocs-ng/mkdocs [GitHub Discussions]: https://github.com/orgs/mkdocs-ng/discussions [GitHub issues]: https://github.com/mkdocs-ng/mkdocs/issues [Hatch]: https://hatch.pypa.io/ diff --git a/README.md b/README.md index ec574668..ccdba887 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ [![PyPI Version][pypi-v-image]][pypi-v-link] [![Build Status][GHAction-image]][GHAction-link] [![Coverage Status][codecov-image]][codecov-link] +[![CodSpeed][codspeed-image]][codspeed-link] MkDocs is a **fast**, **simple**, and **downright gorgeous** static site generator that's geared towards building project documentation. Documentation @@ -69,6 +70,8 @@ discussion forums are expected to follow the [PyPA Code of Conduct]. [pypi-v-link]: https://pypi.org/project/mkdocs-ng/ [GHAction-image]: https://github.com/mkdocs-ng/mkdocs/actions/workflows/ci.yml/badge.svg [GHAction-link]: https://github.com/mkdocs-ng/mkdocs/actions/workflows/ci.yml +[codspeed-image]: https://img.shields.io/endpoint?url=https://codspeed.io/badge.json +[codspeed-link]: https://app.codspeed.io/mkdocs-ng/mkdocs?utm_source=badge [mkdocs]: https://mkdocs-ng.github.io/mkdocs/ [mkdocs/mkdocs]: https://github.com/mkdocs/mkdocs diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py new file mode 100644 index 00000000..78986df3 --- /dev/null +++ b/benchmarks/conftest.py @@ -0,0 +1,51 @@ +"""Shared fixtures for the MkDocs benchmark suite.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from benchmarks.corpus import write_site +from mkdocs.config.base import load_config +from mkdocs.structure.files import get_files +from mkdocs.structure.nav import get_navigation + +if TYPE_CHECKING: + from mkdocs.config.defaults import MkDocsConfig + from mkdocs.structure.files import Files + from mkdocs.structure.nav import Navigation + from mkdocs.structure.pages import Page + + +@pytest.fixture(scope="session") +def config_file_path(tmp_path_factory: pytest.TempPathFactory) -> str: + """Generate the benchmark site on disk once for the whole session.""" + return write_site(str(tmp_path_factory.mktemp("mkdocs_bench_site"))) + + +@pytest.fixture +def config(config_file_path: str, tmp_path) -> MkDocsConfig: + """A freshly loaded and validated configuration for the generated site.""" + return load_config(config_file_path, site_dir=str(tmp_path / "site")) + + +@pytest.fixture +def files(config: MkDocsConfig) -> Files: + return get_files(config) + + +@pytest.fixture +def navigation(config: MkDocsConfig, files: Files) -> Navigation: + return get_navigation(files, config) + + +@pytest.fixture +def rendered_pages( + config: MkDocsConfig, files: Files, navigation: Navigation +) -> list[Page]: + """Every documentation page of the generated site, read and rendered.""" + for page in navigation.pages: + page.read_source(config) + page.render(config, files) + return list(navigation.pages) diff --git a/benchmarks/corpus.py b/benchmarks/corpus.py new file mode 100644 index 00000000..2997eb83 --- /dev/null +++ b/benchmarks/corpus.py @@ -0,0 +1,148 @@ +""" +Deterministic documentation corpus used by the benchmarks. + +Everything here is generated from a fixed pseudo-random seed so that every +benchmark run measures exactly the same workload and results only move when +MkDocs itself changes. +""" + +from __future__ import annotations + +import os +import random + +WORDS = ( + "documentation markdown static site generator theme plugin navigation " + "configuration template rendering directory anchor heading section link " + "reference build server extension content page index deploy source output" +).split() + +LANGUAGES = ("python", "yaml", "bash", "text") + +SECTIONS = 5 +PAGES_PER_SECTION = 6 + +MKDOCS_YML = """\ +site_name: Benchmark Site +site_url: https://example.com/benchmark/ +site_description: A generated site used to benchmark MkDocs. +repo_url: https://github.com/mkdocs-ng/mkdocs/ +edit_uri: blob/main/docs/ + +theme: + name: mkdocs + +markdown_extensions: + - toc: + permalink: true + - admonition + - attr_list + - def_list + - footnotes + - tables + +plugins: + - search +""" + + +def _sentence(rng: random.Random) -> str: + return " ".join(rng.choices(WORDS, k=rng.randint(6, 14))).capitalize() + "." + + +def _paragraph(rng: random.Random) -> str: + return " ".join(_sentence(rng) for _ in range(rng.randint(3, 6))) + + +def _code_block(rng: random.Random) -> str: + lines = "\n".join( + f" {rng.choice(WORDS)} = {rng.randint(0, 1000)}" + for _ in range(rng.randint(3, 8)) + ) + return f"```{rng.choice(LANGUAGES)}\n{lines}\n```" + + +def _table(rng: random.Random) -> str: + header = "| Option | Type | Default |\n| --- | --- | --- |" + rows = "\n".join( + f"| `{rng.choice(WORDS)}` | {rng.choice(WORDS)} | `{rng.randint(0, 99)}` |" + for _ in range(rng.randint(3, 6)) + ) + return f"{header}\n{rows}" + + +def _bullet_list(rng: random.Random) -> str: + return "\n".join(f"- {_sentence(rng)}" for _ in range(rng.randint(3, 6))) + + +def _admonition(rng: random.Random) -> str: + return f'!!! note "{rng.choice(WORDS).title()}"\n\n {_sentence(rng)}' + + +_BUILDERS = (_paragraph, _code_block, _table, _bullet_list, _admonition) + + +def make_markdown(seed: int, *, headings: int = 8, links: tuple[str, ...] = ()) -> str: + """Generate a deterministic Markdown document with realistic constructs.""" + rng = random.Random(seed) + parts = [ + "---", + f"title: Page {seed}", + "tags:", + f" - {rng.choice(WORDS)}", + f" - {rng.choice(WORDS)}", + "---", + "", + f"# Page {seed} about {rng.choice(WORDS)}", + "", + _paragraph(rng), + ] + for i in range(headings): + level = "##" if i % 3 else "###" + parts += ["", f"{level} {rng.choice(WORDS).title()} {i}", "", _paragraph(rng)] + parts += ["", _BUILDERS[i % len(_BUILDERS)](rng)] + if links: + target = links[i % len(links)] + parts += [ + "", + f"See [{rng.choice(WORDS)}]({target}) and " + f"[{rng.choice(WORDS)}]({target}#{rng.choice(WORDS)}) for details.", + ] + return "\n".join(parts) + "\n" + + +def write_site(root: str) -> str: + """ + Write a full MkDocs project (config + nested docs tree) under `root`. + + Returns the path of the generated `mkdocs.yml`. + """ + docs_dir = os.path.join(root, "docs") + os.makedirs(docs_dir, exist_ok=True) + section_names = [f"section-{i:02d}" for i in range(SECTIONS)] + + with open(os.path.join(docs_dir, "index.md"), "w", encoding="utf-8") as f: + f.write(make_markdown(0, links=tuple(f"{n}/index.md" for n in section_names))) + + for s, name in enumerate(section_names): + section_dir = os.path.join(docs_dir, name) + os.makedirs(section_dir, exist_ok=True) + page_names = tuple(f"page-{p:02d}.md" for p in range(PAGES_PER_SECTION)) + links = ("../index.md", *page_names) + with open(os.path.join(section_dir, "index.md"), "w", encoding="utf-8") as f: + f.write(make_markdown(100 + s, links=links)) + for p, page_name in enumerate(page_names): + with open(os.path.join(section_dir, page_name), "w", encoding="utf-8") as f: + f.write(make_markdown(1000 + s * 100 + p, links=links)) + + # A few static assets, so the file collection is not only Markdown. + css_dir = os.path.join(docs_dir, "css") + os.makedirs(css_dir, exist_ok=True) + for i in range(3): + with open(os.path.join(css_dir, f"extra-{i}.css"), "w", encoding="utf-8") as f: + f.write("body { margin: 0; }\n") + + config_file = os.path.join(root, "mkdocs.yml") + with open(config_file, "w", encoding="utf-8") as f: + f.write(MKDOCS_YML) + return config_file diff --git a/benchmarks/test_build.py b/benchmarks/test_build.py new file mode 100644 index 00000000..7a61a45d --- /dev/null +++ b/benchmarks/test_build.py @@ -0,0 +1,79 @@ +"""End-to-end benchmarks for a full site build.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from mkdocs.commands.build import _build_page, _populate_page, build +from mkdocs.config.base import load_config +from mkdocs.structure.files import get_files +from mkdocs.structure.nav import get_navigation + +if TYPE_CHECKING: + from mkdocs.config.defaults import MkDocsConfig + from mkdocs.structure.files import Files + from mkdocs.structure.nav import Navigation + + +def test_build_site(benchmark, config_file_path: str, tmp_path) -> None: + """Full `mkdocs build` of the generated site: files, nav, render, templates.""" + counter = iter(range(1000)) + + def setup(): + site_dir = str(tmp_path / f"site-{next(counter)}") + return (load_config(config_file_path, site_dir=site_dir),), {} + + benchmark.pedantic(build, setup=setup, rounds=1, warmup_rounds=0) + + +def test_build_site_no_directory_urls( + benchmark, config_file_path: str, tmp_path +) -> None: + """Same build with `use_directory_urls` disabled, which changes URL resolution.""" + counter = iter(range(1000)) + + def setup(): + site_dir = str(tmp_path / f"flat-site-{next(counter)}") + config = load_config( + config_file_path, site_dir=site_dir, use_directory_urls=False + ) + return (config,), {} + + benchmark.pedantic(build, setup=setup, rounds=1, warmup_rounds=0) + + +def test_populate_pages( + benchmark, config: MkDocsConfig, files: Files, navigation: Navigation +) -> None: + """Read and convert the Markdown of every page, without writing any output.""" + + def populate_all() -> int: + for file in files.documentation_pages(): + assert file.page is not None + _populate_page(file.page, config, files) + return len(navigation.pages) + + assert benchmark(populate_all) > 0 + + +def test_build_theme_pages(benchmark, config: MkDocsConfig) -> None: + """Render the `mkdocs` theme templates and write the HTML of every page.""" + config = config.plugins.on_config(config) + config.plugins.on_pre_build(config=config) + + files = get_files(config) + env = config.theme.get_env() + files.add_files_from_theme(env, config) + nav = get_navigation(files, config) + doc_files = files.documentation_pages() + for file in doc_files: + assert file.page is not None + _populate_page(file.page, config, files) + env = config.plugins.on_env(env, config=config, files=files) + + def build_all() -> None: + for file in doc_files: + assert file.page is not None + _build_page(file.page, config, doc_files, nav, env) + + benchmark(build_all) diff --git a/benchmarks/test_config.py b/benchmarks/test_config.py new file mode 100644 index 00000000..dc080a62 --- /dev/null +++ b/benchmarks/test_config.py @@ -0,0 +1,33 @@ +"""Benchmarks for configuration loading and validation.""" + +from __future__ import annotations + +import io + +from mkdocs.config.base import load_config +from mkdocs.utils import yaml as yaml_utils +from mkdocs.utils.yaml import yaml_load + + +def test_load_config(benchmark, config_file_path: str, tmp_path) -> None: + """Read `mkdocs.yml`, apply the schema defaults and validate every option.""" + site_dir = str(tmp_path / "site") + + config = benchmark(load_config, config_file_path, site_dir=site_dir) + assert config.site_name == "Benchmark Site" + + +def test_yaml_load(benchmark, config_file_path: str) -> None: + """Parse the YAML configuration with MkDocs' loader (env tags, includes).""" + with open(config_file_path, "rb") as f: + raw = f.read() + + def load() -> dict: + return yaml_load(io.BytesIO(raw)) + + assert benchmark(load)["site_name"] == "Benchmark Site" + + +def test_get_yaml_loader(benchmark) -> None: + """Build the YAML loader class, done for every config and every page.""" + assert benchmark(yaml_utils.get_yaml_loader) is not None diff --git a/benchmarks/test_search.py b/benchmarks/test_search.py new file mode 100644 index 00000000..86ee1378 --- /dev/null +++ b/benchmarks/test_search.py @@ -0,0 +1,55 @@ +"""Benchmarks for the built-in search plugin index generation.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from mkdocs.contrib.search.search_index import ContentParser, SearchIndex + +if TYPE_CHECKING: + from mkdocs.structure.pages import Page + +INDEX_CONFIG = dict( + lang=["en"], + separator=r"[\s\-]+", + min_search_length=3, + prebuild_index=False, + indexing="full", +) + + +def test_search_index_add_entries(benchmark, rendered_pages: list[Page]) -> None: + """Parse the HTML of every page and add its sections to the search index.""" + + def index_all() -> SearchIndex: + index = SearchIndex(**INDEX_CONFIG) + for page in rendered_pages: + index.add_entry_from_context(page) + return index + + assert len(benchmark(index_all)._entries) > 0 + + +def test_search_index_generate(benchmark, rendered_pages: list[Page]) -> None: + """Serialize a fully populated search index to JSON.""" + index = SearchIndex(**INDEX_CONFIG) + for page in rendered_pages: + index.add_entry_from_context(page) + + assert len(benchmark(index.generate_search_index)) > 0 + + +def test_content_parser(benchmark, rendered_pages: list[Page]) -> None: + """Run the HTML parser that splits rendered pages into indexable sections.""" + html = [page.content for page in rendered_pages] + + def parse_all() -> int: + sections = 0 + for content in html: + parser = ContentParser() + parser.feed(content) + parser.close() + sections += len(parser.data) + return sections + + assert benchmark(parse_all) > 0 diff --git a/benchmarks/test_structure.py b/benchmarks/test_structure.py new file mode 100644 index 00000000..7400bd22 --- /dev/null +++ b/benchmarks/test_structure.py @@ -0,0 +1,82 @@ +"""Benchmarks for the file collection, navigation and page structures.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import markdown + +from benchmarks.corpus import make_markdown +from mkdocs.structure.files import File, get_files +from mkdocs.structure.nav import get_navigation +from mkdocs.structure.pages import Page +from mkdocs.structure.toc import get_toc + +if TYPE_CHECKING: + from mkdocs.config.defaults import MkDocsConfig + from mkdocs.structure.files import Files + + +def test_get_files(benchmark, config: MkDocsConfig) -> None: + """Walk the docs directory and build the `Files` collection.""" + assert len(benchmark(get_files, config)) > 0 + + +def test_get_navigation(benchmark, config: MkDocsConfig, files: Files) -> None: + """Build the navigation tree out of the file collection.""" + assert len(benchmark(get_navigation, files, config).pages) > 0 + + +def test_file_creation(benchmark, config: MkDocsConfig) -> None: + """Instantiate `File` objects, which computes source/destination URIs.""" + paths = [f"section-{i % 5:02d}/page-{i:03d}.md" for i in range(500)] + docs_dir = config.docs_dir + site_dir = config.site_dir + + def create_files() -> int: + return len([File(path, docs_dir, site_dir, True) for path in paths]) + + assert benchmark(create_files) == len(paths) + + +def test_files_lookup(benchmark, files: Files) -> None: + """Resolve source URIs against the file collection, as link resolution does.""" + uris = [file.src_uri for file in files] + + def lookup_all() -> int: + found = 0 + for uri in uris: + if files.get_file_from_path(uri) is not None: + found += 1 + return found + + assert benchmark(lookup_all) == len(uris) + + +def test_page_render(benchmark, config: MkDocsConfig, files: Files) -> None: + """Convert a single page from Markdown to HTML, including link resolution.""" + file = files.documentation_pages()[1] + page = Page(None, file, config) + page.read_source(config) + + benchmark(page.render, config, files) + assert page.content + + +def test_page_render_large(benchmark, config: MkDocsConfig, files: Files) -> None: + """Convert a much larger page, dominated by the Markdown conversion itself.""" + file = files.documentation_pages()[0] + page = Page(None, file, config) + page.markdown = make_markdown(42, headings=60) + + benchmark(page.render, config, files) + assert page.content + + +def test_get_toc(benchmark) -> None: + """Turn the Markdown `toc` tokens into MkDocs' table of contents objects.""" + md = markdown.Markdown(extensions=["toc"]) + md.convert(make_markdown(7, headings=60)) + toc_tokens = md.toc_tokens + + assert len(list(benchmark(get_toc, toc_tokens))) > 0 diff --git a/benchmarks/test_utils.py b/benchmarks/test_utils.py new file mode 100644 index 00000000..c3fcc07b --- /dev/null +++ b/benchmarks/test_utils.py @@ -0,0 +1,76 @@ +"""Benchmarks for the utility helpers used throughout a build.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from benchmarks.corpus import make_markdown +from mkdocs.utils import ( + get_markdown_title, + get_relative_url, + meta, + nest_paths, + normalize_url, +) +from mkdocs.utils.rendering import _strip_tags + +if TYPE_CHECKING: + from mkdocs.structure.pages import Page + +PATHS = [ + f"section-{s:02d}/{'sub/' * (s % 3)}page-{p:02d}.md" + for s in range(10) + for p in range(20) +] + + +def test_meta_get_data(benchmark) -> None: + """Split the YAML front matter from the Markdown body of every page.""" + documents = [make_markdown(seed) for seed in range(20)] + + def parse_all() -> int: + return sum(len(meta.get_data(doc)[1]) for doc in documents) + + assert benchmark(parse_all) > 0 + + +def test_get_markdown_title(benchmark) -> None: + """Extract the title from a Markdown document without rendering it.""" + document, _ = meta.get_data(make_markdown(3, headings=40)) + + assert benchmark(get_markdown_title, document) + + +def test_get_relative_url(benchmark) -> None: + """Compute relative URLs, done for every link of every page.""" + urls = [(a, b) for a in PATHS[:40] for b in PATHS[:10]] + + def relative_all() -> int: + return sum(len(get_relative_url(url, other)) for url, other in urls) + + assert benchmark(relative_all) > 0 + + +def test_normalize_url(benchmark) -> None: + """Normalize the URLs used by the theme templates.""" + urls = [*PATHS, "https://example.com/", "#anchor", "/absolute/path"] + + def normalize_all() -> int: + return sum(len(normalize_url(url)) for url in urls) + + assert benchmark(normalize_all) > 0 + + +def test_nest_paths(benchmark) -> None: + """Turn a flat list of source paths into the implicit navigation tree.""" + assert len(benchmark(nest_paths, PATHS)) > 0 + + +def test_strip_tags(benchmark, rendered_pages: list[Page]) -> None: + """Strip the HTML out of rendered content, as the search index does.""" + html = [page.content for page in rendered_pages] + + def strip_all() -> int: + return sum(len(_strip_tags(content)) for content in html) + + assert benchmark(strip_all) > 0 diff --git a/pyproject.toml b/pyproject.toml index 846eac19..f6ae2bf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,6 +141,15 @@ matrix.type.features = [ { value = "i18n", if = ["default"] }, ] +[tool.hatch.envs.bench] +dependencies = [ + "pytest", + "pytest-codspeed", +] +[tool.hatch.envs.bench.scripts] +run = "pytest benchmarks {args}" +codspeed = "pytest benchmarks --codspeed {args}" + [tool.hatch.envs.types] dependencies = [ "mypy",