From 77d075287110dc6a23144847a94fa93c5c1ec892 Mon Sep 17 00:00:00 2001 From: Rafael Vuijk Date: Thu, 1 Oct 2026 20:41:39 +0000 Subject: [PATCH] The documentation samples run in CI, the wiki's and the website's DocSamples compiles every code sample in the wiki, and every one on the website marked with an amcheck comment, against the library in the checkout. It runs them and checks each stated output against the one produced. It fails on a sample that does not compile, throws, or prints something other than its page says. A sample that does not finish fails nothing. It clones the wiki beside itself. CI also checks AngouriMathSite out beside the library and passes it with --site. The job is its own, since it builds a generated project per language. A change to the F# wrapper alone now runs the workflow, because the F# samples compile against it. Its first run, against cd3b27b9, found three wiki outputs that master no longer prints. All three were equal values in a new form, and the wiki is corrected (AngouriMath.wiki d23ca13). With that, 89 samples pass: 60 stated outputs verified, 0 failures. Measured.Commit(path) is back in the shared code, because this harness builds the library rather than referencing it. Part of #1256. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012sonx8iAspMiwRwokT1Ura --- .github/workflows/Harnesses.yml | 37 +++ .gitignore | 5 + CLAUDE.md | 10 +- .../Harnesses/DocSamples/DocSamples.csproj | 33 +++ .../Harnesses/DocSamples/FSharpGenerator.cs | 113 +++++++ .../Tests/Harnesses/DocSamples/Generator.cs | 176 +++++++++++ Sources/Tests/Harnesses/DocSamples/Program.cs | 251 ++++++++++++++++ Sources/Tests/Harnesses/DocSamples/README.md | 83 ++++++ Sources/Tests/Harnesses/DocSamples/Report.cs | 134 +++++++++ .../Tests/Harnesses/DocSamples/Snippets.cs | 276 ++++++++++++++++++ Sources/Tests/Harnesses/README.md | 1 + .../Tests/Harnesses/Shared/MeasuredCommit.cs | 6 + 12 files changed, 1120 insertions(+), 5 deletions(-) create mode 100644 Sources/Tests/Harnesses/DocSamples/DocSamples.csproj create mode 100644 Sources/Tests/Harnesses/DocSamples/FSharpGenerator.cs create mode 100644 Sources/Tests/Harnesses/DocSamples/Generator.cs create mode 100644 Sources/Tests/Harnesses/DocSamples/Program.cs create mode 100644 Sources/Tests/Harnesses/DocSamples/README.md create mode 100644 Sources/Tests/Harnesses/DocSamples/Report.cs create mode 100644 Sources/Tests/Harnesses/DocSamples/Snippets.cs diff --git a/.github/workflows/Harnesses.yml b/.github/workflows/Harnesses.yml index ca8b7e2e3..a6fa25d47 100644 --- a/.github/workflows/Harnesses.yml +++ b/.github/workflows/Harnesses.yml @@ -13,11 +13,13 @@ on: - master paths: - 'Sources/AngouriMath/**' + - 'Sources/Wrappers/AngouriMath.FSharp/**' - 'Sources/Tests/Harnesses/**' - '.github/workflows/Harnesses.yml' pull_request: paths: - 'Sources/AngouriMath/**' + - 'Sources/Wrappers/AngouriMath.FSharp/**' - 'Sources/Tests/Harnesses/**' - '.github/workflows/Harnesses.yml' workflow_dispatch: @@ -116,3 +118,38 @@ jobs: with: name: crashcheck-report path: harness-reports + + # A job of its own: it checks the website out beside the library, clones the wiki, and builds a + # project per language from their samples. + DocSamples: + runs-on: ubuntu-latest + env: + HARNESS_REPORTS: ${{ github.workspace }}/harness-reports + + steps: + - uses: actions/checkout@v6 + + - name: Check out the website + uses: actions/checkout@v6 + with: + repository: asc-community/AngouriMathSite + path: site + + - name: Setup .NET 10 + uses: actions/setup-dotnet@v5 + with: + dotnet-version: '10.x' + dotnet-quality: 'preview' + + - name: Build + run: dotnet build -c Release Sources/Tests/Harnesses/DocSamples + + - name: DocSamples + run: dotnet Sources/Tests/Harnesses/DocSamples/bin/Release/net10.0/docsamples.dll --site=site + + - name: Report + if: always() + uses: actions/upload-artifact@v6 + with: + name: docsamples-report + path: harness-reports diff --git a/.gitignore b/.gitignore index ca86bf6f3..338229926 100644 --- a/.gitignore +++ b/.gitignore @@ -77,5 +77,10 @@ docsamples.md sympyparity.md egraph.md +# DocSamples clones the wiki beside itself and generates a project per language there. +Sources/Tests/Harnesses/DocSamples/wiki/ +Sources/Tests/Harnesses/DocSamples/generated/ +Sources/Tests/Harnesses/DocSamples/generated-fsharp/ + # Agent worktrees and local tool state. .claude/ diff --git a/CLAUDE.md b/CLAUDE.md index db737ffb5..ac38a7d41 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,13 +19,13 @@ nothing else is read: [`Contributing/SimplificationContract.md`](Sources/AngouriMath/Docs/Contributing/SimplificationContract.md). A rule states the assumptions under which it holds, or it is asserting there are none. -Nine measurement harnesses live in `Sources/Tests/Harnesses` and run in CI on every change to the +Ten measurement harnesses live in `Sources/Tests/Harnesses` and run in CI on every change to the library: the boundary checker, root completeness, the self-verifying solver corpus, the property checker, the simplification sweep, the rule-set checker, a crash harness that survives a stack -overflow, the canonical-form checker and the arm-order checker. Each fails on a defect, or the last -two on a change to the findings they list; the README says which. The rest are still in the analysis workspace -one directory up (`work/`), among them a checker for the documentation's code samples. Run them before claiming anything is -fixed. +overflow, the canonical-form checker, the arm-order checker, and a checker for the code samples in +the wiki and on the website. Each fails on a defect, or the canonical-form and arm-order checkers on +a change to the findings they list; the README says which. The rest are still in the analysis +workspace one directory up (`work/`). Run them before claiming anything is fixed. There is also a *gate*, which is not a harness: `Sources/Tests/UnitTests/Corpus` runs forty problems on every commit and reports **solved / unsolved diff --git a/Sources/Tests/Harnesses/DocSamples/DocSamples.csproj b/Sources/Tests/Harnesses/DocSamples/DocSamples.csproj new file mode 100644 index 000000000..b756aa26b --- /dev/null +++ b/Sources/Tests/Harnesses/DocSamples/DocSamples.csproj @@ -0,0 +1,33 @@ + + + + + + Exe + net10.0 + disable + preview + docsamples + DocSamples + true + false + false + + + + + + + + + + + diff --git a/Sources/Tests/Harnesses/DocSamples/FSharpGenerator.cs b/Sources/Tests/Harnesses/DocSamples/FSharpGenerator.cs new file mode 100644 index 000000000..7017a01a1 --- /dev/null +++ b/Sources/Tests/Harnesses/DocSamples/FSharpGenerator.cs @@ -0,0 +1,113 @@ +// +// Copyright (c) 2019-2026 Angouri. +// AngouriMath is licensed under MIT. +// Details: https://github.com/asc-community/AngouriMath/blob/master/LICENSE.md. +// Website: https://am.angouri.org. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace DocSamples; + +/// +/// The F# side of the same check. `AngouriMath.FSharp` is a published package, and the wiki +/// page for it is the only documentation of the wrapper's names, so a wrong name there -- +/// `dy/dx` for `d/dx` -- has nothing else to catch it. +/// +/// One file, one module per sample, so each sample keeps its own `open` directives. F# +/// wants `open` before anything else in a module, so they are hoisted out of the body. +/// +static class FSharpGenerator +{ + public static Dictionary Write(string dir, string wrapperFsproj, + IEnumerable snippets, ISet exclude) + { + var wanted = snippets.Where(s => s.Mode != Mode.Skip && !exclude.Contains(s.Id)).ToList(); + Directory.CreateDirectory(dir); + + var sb = new StringBuilder(); + var offsets = new Dictionary(); + sb.AppendLine("module DocSamples.Generated.FSharpSamples"); + sb.AppendLine(); + + foreach (var s in wanted) + { + var opens = s.Body.Where(l => l.TrimStart().StartsWith("open ")).ToList(); + sb.AppendLine($"module {s.Id} ="); + foreach (var o in opens) sb.AppendLine(" " + o.Trim()); + sb.AppendLine(" let run () ="); + offsets[s.Id] = sb.ToString().Count(c => c == '\n') + 1; + var wrote = false; + foreach (var line in s.Body) + { + if (opens.Contains(line)) { sb.AppendLine(); continue; } + sb.AppendLine(line.Trim().Length == 0 ? "" : " " + line); + if (line.Trim().Length > 0) wrote = true; + } + if (!wrote) sb.AppendLine(" ()"); + sb.AppendLine(); + } + + sb.AppendLine("module Runner ="); + sb.AppendLine(" open System"); + sb.AppendLine(" open System.IO"); + sb.AppendLine(" open System.Text.Json"); + sb.AppendLine(); + sb.AppendLine(" type Result = { Id: string; Status: string; Output: string }"); + sb.AppendLine(); + sb.AppendLine(" let samples : (string * (unit -> unit) * bool) list ="); + if (wanted.Count == 0) sb.AppendLine(" []"); + else + sb.AppendLine(" [ " + string.Join("\n ", + wanted.Select(s => $"(\"{s.Id}\", {s.Id}.run, {(s.Mode == Mode.Run ? "true" : "false")})")) + + " ]"); + sb.AppendLine(""" + + [] + let main argv = + let real = Console.Out + let results = + samples + |> List.map (fun (id, run, execute) -> + if not execute then { Id = id; Status = "compiled"; Output = "" } + else + let captured = new StringWriter() + Console.SetOut(captured) + let status = + try + run () + "ok" + with e -> "threw " + e.GetType().Name + ": " + e.Message.Split('\n').[0] + Console.SetOut(real) + { Id = id; Status = status; Output = captured.ToString() }) + let path = if argv.Length > 0 then argv.[0] else "results.json" + File.WriteAllText(path, JsonSerializer.Serialize(results)) + 0 + """); + + File.WriteAllText(Path.Combine(dir, "Samples.fs"), sb.ToString()); + File.WriteAllText(Path.Combine(dir, "generated-fsharp.fsproj"), $""" + + + Exe + net10.0 + generatedfsharp + false + 0 + FS0025;FS0049;FS0064;FS0193;FS1182 + + + + + + + + + """); + return offsets; + } +} diff --git a/Sources/Tests/Harnesses/DocSamples/Generator.cs b/Sources/Tests/Harnesses/DocSamples/Generator.cs new file mode 100644 index 000000000..eb6b741d2 --- /dev/null +++ b/Sources/Tests/Harnesses/DocSamples/Generator.cs @@ -0,0 +1,176 @@ +// +// Copyright (c) 2019-2026 Angouri. +// AngouriMath is licensed under MIT. +// Details: https://github.com/asc-community/AngouriMath/blob/master/LICENSE.md. +// Website: https://am.angouri.org. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace DocSamples; + +/// +/// Turns the extracted samples into one project that compiles them all, so a rename in the +/// library shows up as a compile error against the page that names it. +/// +/// Each sample becomes its own file, which keeps its `using` directives to itself: two +/// samples that import conflicting names both still compile. +/// +static class Generator +{ + /// What the wiki says is implied in every sample. Anything else a sample needs, it states. + static readonly string[] ImpliedUsings = + { + "using System;", + "using AngouriMath;", + "using static AngouriMath.MathS;", + "using static AngouriMath.Entity;", + }; + + /// Writes the project, and returns for each sample the generated file line its + /// body starts on, so a compile error can be pointed back at a line of the wiki. + public static Dictionary Write(string dir, string angouriMathCsproj, + IEnumerable snippets, ISet exclude) + { + var wanted = snippets.Where(s => s.Mode != Mode.Skip && !exclude.Contains(s.Id)).ToList(); + + Directory.CreateDirectory(dir); + var samplesDir = Path.Combine(dir, "Samples"); + if (Directory.Exists(samplesDir)) Directory.Delete(samplesDir, true); + Directory.CreateDirectory(samplesDir); + + var offsets = new Dictionary(); + foreach (var s in wanted) + { + File.WriteAllText(Path.Combine(samplesDir, s.Id + ".cs"), SampleFile(s, out var firstBodyLine)); + offsets[s.Id] = firstBodyLine; + } + + File.WriteAllText(Path.Combine(dir, "Runner.cs"), RunnerFile(wanted)); + File.WriteAllText(Path.Combine(dir, "generated.csproj"), Csproj(angouriMathCsproj)); + return offsets; + } + + static string SampleFile(Snippet s, out int firstBodyLine) + { + var sb = new StringBuilder(); + var ownUsings = s.Body.Where(l => l.TrimStart().StartsWith("using ") + && l.TrimEnd().EndsWith(";") + && !l.Contains('=')).ToList(); + + foreach (var u in ImpliedUsings) sb.AppendLine(u); + foreach (var u in ownUsings.Where(u => !ImpliedUsings.Contains(u.Trim()))) + sb.AppendLine(u.Trim()); + sb.AppendLine(); + sb.AppendLine("namespace DocSamples.Generated;"); + sb.AppendLine(); + sb.AppendLine($"static class {s.Id}"); + sb.AppendLine("{"); + sb.AppendLine(" public static void Run()"); + sb.AppendLine(" {"); + firstBodyLine = sb.ToString().Count(c => c == '\n') + 1; + foreach (var line in s.Body) + sb.AppendLine(ownUsings.Contains(line) ? "" : " " + line); + sb.AppendLine(" }"); + sb.AppendLine("}"); + return sb.ToString(); + } + + static string RunnerFile(List wanted) + { + var sb = new StringBuilder(); + sb.AppendLine(""" + using System; + using System.Collections.Generic; + using System.IO; + using System.Text.Json; + using System.Threading; + using System.Threading.Tasks; + + namespace DocSamples.Generated; + + record Result(string Id, string Status, string Output); + + static class Runner + { + // A sample that never returns must not take the run down with it: the cap is + // far above what any documented sample needs, so hitting it is a finding. + static readonly TimeSpan Cap = TimeSpan.FromSeconds(60); + + static int Main(string[] args) + { + var results = new List(); + var real = Console.Out; + foreach (var (id, run, execute) in Samples()) + { + if (!execute) { results.Add(new(id, "compiled", "")); continue; } + var captured = new StringWriter(); + var status = "ok"; + Console.SetOut(captured); + try + { + var task = Task.Run(run); + if (!task.Wait(Cap)) status = "timeout"; + else if (task.Exception is not null) throw task.Exception.InnerException; + } + catch (Exception e) + { + status = "threw " + e.GetType().Name + ": " + FirstLine(e.Message); + } + finally + { + Console.SetOut(real); + } + results.Add(new(id, status, captured.ToString())); + Console.Error.WriteLine($" {id}: {status}"); + } + File.WriteAllText(args.Length > 0 ? args[0] : "results.json", + JsonSerializer.Serialize(results)); + return 0; + } + + static string FirstLine(string s) + { + var i = s.IndexOfAny(new[] { '\r', '\n' }); + return i < 0 ? s : s.Substring(0, i); + } + + static IEnumerable<(string, Action, bool)> Samples() + { + """); + foreach (var s in wanted) + sb.AppendLine($" yield return (\"{s.Id}\", {s.Id}.Run, " + + (s.Mode == Mode.Run ? "true" : "false") + ");"); + sb.AppendLine(" }"); + sb.AppendLine("}"); + return sb.ToString(); + } + + static string Csproj(string angouriMathCsproj) => $""" + + + Exe + net10.0 + disable + preview + generated + DocSamples.Generated + true + false + + CS0168;CS0219;CS8019;CS0164;CS1717;CS0162 + + + + + + """; +} diff --git a/Sources/Tests/Harnesses/DocSamples/Program.cs b/Sources/Tests/Harnesses/DocSamples/Program.cs new file mode 100644 index 000000000..2bc6e9656 --- /dev/null +++ b/Sources/Tests/Harnesses/DocSamples/Program.cs @@ -0,0 +1,251 @@ +// +// Copyright (c) 2019-2026 Angouri. +// AngouriMath is licensed under MIT. +// Details: https://github.com/asc-community/AngouriMath/blob/master/LICENSE.md. +// Website: https://am.angouri.org. +// + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace DocSamples; + +static class Program +{ + const string WikiUrl = "https://github.com/asc-community/AngouriMath.wiki.git"; + + static int Main(string[] args) + { + // The harness's own project directory, from the assembly rather than the working + // directory, and the repository four levels above it. + var root = Directory.GetParent(AppContext.BaseDirectory).Parent.Parent.Parent.FullName; + var repository = Path.GetFullPath(Path.Combine(root, "..", "..", "..", "..")); + + var wiki = Arg(args, "--wiki") ?? Path.Combine(root, "wiki"); + var report = Arg(args, "--report") ?? Harness.Reports.PathFor("docsamples.md"); + var sources = Path.Combine(repository, "Sources"); + var csproj = Path.Combine(sources, "AngouriMath", "AngouriMath.csproj"); + var fsproj = Path.Combine(sources, "Wrappers", "AngouriMath.FSharp", "AngouriMath.FSharp.fsproj"); + + if (!Directory.Exists(wiki)) + { + Console.WriteLine($"Cloning the wiki into {wiki}"); + if (Run("git", $"clone --depth 1 {WikiUrl} \"{wiki}\"", repository, out var cloneLog) != 0) + { + Console.Error.WriteLine(cloneLog); + Console.Error.WriteLine("Could not clone the wiki. Pass --wiki= to use a local copy."); + return 2; + } + } + else if (Arg(args, "--wiki") is null && Directory.Exists(Path.Combine(wiki, ".git"))) + { + // The clone is kept between runs and brought up to date on each, so that a page + // fixed and pushed is read as it now stands. Only the clone this harness owns is + // touched; a path passed with --wiki belongs to the caller. + Console.WriteLine($"Updating the wiki clone at {wiki}"); + if (Run("git", "fetch --depth 1 origin", wiki, out var fetchLog) != 0 + || Run("git", "reset --hard FETCH_HEAD", wiki, out fetchLog) != 0) + { + Console.Error.WriteLine(fetchLog); + Console.Error.WriteLine( + "Could not update the wiki clone. Delete it, or pass --wiki=."); + return 2; + } + } + if (!File.Exists(csproj)) + { + Console.Error.WriteLine($"No library to check against at {csproj}"); + return 2; + } + + var snippets = Extractor.FromDirectory(wiki); + var pages = Directory.GetFiles(wiki, "*.md").Length; + var unannotated = 0; + + // The website is a second repository, and its quickstart is the page a new reader + // actually follows, so it is checked here too when a working copy is to hand. + var site = Arg(args, "--site"); + if (site is not null) + { + var content = Path.Combine(site, "src", "content"); + var siteRoot = Directory.Exists(content) ? content : site; + var fromSite = Extractor.FromHtmlDirectory(siteRoot, out unannotated); + Console.WriteLine($"{fromSite.Count} annotated samples in {siteRoot}, " + + $"{unannotated} code blocks not annotated"); + snippets.AddRange(fromSite); + pages += Directory.GetFiles(siteRoot, "*.html", SearchOption.AllDirectories).Length; + } + + var csharp = snippets.Where(s => !s.IsFSharp).ToList(); + var fsharp = snippets.Where(s => s.IsFSharp).ToList(); + Console.WriteLine($"{snippets.Count} samples ({csharp.Count} C#, {fsharp.Count} F#) in " + + $"{pages} pages"); + + var failed = new HashSet(); + var diagnostics = new Dictionary(); + var results = new Dictionary(); + + if (!Measure("C#", csharp, Path.Combine(root, "generated"), "generated.csproj", + (dir, exclude) => Generator.Write(dir, csproj, csharp, exclude), + ByFileName, failed, diagnostics, results)) + return 2; + + if (fsharp.Any(s => s.Mode != Mode.Skip) && File.Exists(fsproj) + && !Measure("F#", fsharp, Path.Combine(root, "generated-fsharp"), "generated-fsharp.fsproj", + (dir, exclude) => FSharpGenerator.Write(dir, fsproj, fsharp, exclude), + ByLineRange, failed, diagnostics, results)) + return 2; + + var findings = new List(); + foreach (var s in snippets) + { + if (s.Mode == Mode.Skip) + findings.Add(new Finding { Snippet = s, Verdict = Verdict.Skipped }); + else if (failed.Contains(s.Id)) + findings.Add(new Finding + { + Snippet = s, Verdict = Verdict.CompileError, + Detail = diagnostics.TryGetValue(s.Id, out var d) ? d : "(no diagnostic captured)", + }); + else if (!results.TryGetValue(s.Id, out var r)) + findings.Add(new Finding + { + Snippet = s, Verdict = Verdict.Skipped, + Detail = "no toolchain for this language here", + }); + else if (r.Status == "compiled") + findings.Add(new Finding { Snippet = s, Verdict = Verdict.Compiled }); + else if (r.Status == "timeout") + findings.Add(new Finding { Snippet = s, Verdict = Verdict.Timeout, Detail = "60 s cap" }); + else if (r.Status.StartsWith("threw")) + findings.Add(new Finding { Snippet = s, Verdict = Verdict.Threw, Detail = r.Status }); + else if (s.Expected is null) + findings.Add(new Finding { Snippet = s, Verdict = Verdict.Unchecked, Actual = r.Output }); + else if (Normalise(r.Output) == Normalise(s.Expected)) + findings.Add(new Finding { Snippet = s, Verdict = Verdict.Ok, Actual = r.Output }); + else + findings.Add(new Finding { Snippet = s, Verdict = Verdict.OutputMismatch, Actual = r.Output }); + } + + Report.Write(report, WikiUrl, repository, findings, unannotated); + + int n(Verdict v) => findings.Count(f => f.Verdict == v); + Console.WriteLine($"docsamples: {findings.Count} samples -- " + + $"{n(Verdict.CompileError)} compile errors, " + + $"{n(Verdict.OutputMismatch)} output mismatches, " + + $"{n(Verdict.Threw)} threw, {n(Verdict.Timeout)} did not finish; " + + $"{n(Verdict.Ok)} outputs verified, {n(Verdict.Unchecked)} outputs not stated, " + + $"{n(Verdict.Compiled)} compile-only, {n(Verdict.Skipped)} skipped"); + Console.WriteLine($"Wrote {report}"); + + // A sample that did not finish is reported and fails nothing: a shared runner is + // slower than the machine the cap was set on. + return n(Verdict.CompileError) + n(Verdict.OutputMismatch) + n(Verdict.Threw) == 0 ? 0 : 1; + } + + record RunResult(string Id, string Status, string Output); + + /// Builds one language's samples, dropping what does not compile until the rest + /// build, then runs them. A single broken sample must not stop the others from being + /// measured, so both numbers come out of one invocation. + static bool Measure(string language, List snippets, string dir, string projectFile, + Func, Dictionary> generate, + Func, List, Snippet> attribute, + HashSet failed, Dictionary diagnostics, + Dictionary results) + { + for (var pass = 1; ; pass++) + { + var offsets = generate(dir, failed); + var ok = Run("dotnet", $"build -c Release \"{Path.Combine(dir, projectFile)}\"", dir, out var log) == 0; + var round = new HashSet(); + foreach (Match m in Diagnostic.Matches(log)) + { + var s = attribute(m, offsets, snippets); + if (s is null) continue; + var generatedLine = int.Parse(m.Groups["line"].Value); + var wikiLine = s.Line + (generatedLine - offsets[s.Id]) + 1; + round.Add(s.Id); + var text = $"{s.At(wikiLine)}: error {m.Groups["rest"].Value.Trim()}"; + var had = diagnostics.GetValueOrDefault(s.Id); + if (had is null) diagnostics[s.Id] = text; + else if (!had.Split('\n').Contains(text)) diagnostics[s.Id] = had + "\n" + text; + } + if (ok) break; + var before = failed.Count; + failed.UnionWith(round); + if (failed.Count == before || pass > 20) + { + Console.Error.WriteLine($"The generated {language} project does not build, and " + + "dropping the samples the compiler names does not help, " + + "so this is the harness rather than the wiki:"); + Console.Error.WriteLine(log); + return false; + } + Console.WriteLine($"{language} pass {pass}: {failed.Count(f => snippets.Any(s => s.Id == f))} " + + "samples do not compile"); + } + + var resultsPath = Path.Combine(dir, "results.json"); + if (File.Exists(resultsPath)) File.Delete(resultsPath); + if (Run("dotnet", $"run -c Release --no-build --project \"{Path.Combine(dir, projectFile)}\" " + + $"-- \"{resultsPath}\"", dir, out var runLog) != 0 || !File.Exists(resultsPath)) + { + Console.Error.WriteLine(runLog); + Console.Error.WriteLine($"The {language} samples did not run to completion."); + return false; + } + + foreach (var r in JsonSerializer.Deserialize>(File.ReadAllText(resultsPath))) + results[r.Id] = r; + return true; + } + + /// One generated file per sample, so the file name is the sample. + static Snippet ByFileName(Match m, Dictionary offsets, List snippets) + { + var id = Path.GetFileNameWithoutExtension(m.Groups["file"].Value.Trim()); + return offsets.ContainsKey(id) ? snippets.FirstOrDefault(s => s.Id == id) : null; + } + + /// One generated file for all samples, so the line decides which one it is. + static Snippet ByLineRange(Match m, Dictionary offsets, List snippets) + { + var line = int.Parse(m.Groups["line"].Value); + return snippets.FirstOrDefault(s => offsets.TryGetValue(s.Id, out var start) + && line >= start && line < start + s.Body.Count + 1); + } + + static readonly Regex Diagnostic = new( + @"^(?[^(\r\n]+)\((?\d+),(?\d+)\):\s*error\s+(?.*)$", + RegexOptions.Compiled | RegexOptions.Multiline); + + /// Trailing whitespace and blank lines are formatting, not output. + static string Normalise(string s) => + string.Join("\n", (s ?? "").Replace("\r\n", "\n").Split('\n').Select(l => l.TrimEnd())) + .Trim('\n'); + + static string Arg(string[] args, string name) => + args.FirstOrDefault(a => a.StartsWith(name + "="))?.Substring(name.Length + 1); + + static int Run(string file, string arguments, string cwd, out string log) + { + var psi = new ProcessStartInfo(file, arguments) + { + WorkingDirectory = cwd, + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + using var p = Process.Start(psi); + var stdout = p.StandardOutput.ReadToEnd(); + var stderr = p.StandardError.ReadToEnd(); + p.WaitForExit(); + log = stdout + stderr; + return p.ExitCode; + } +} diff --git a/Sources/Tests/Harnesses/DocSamples/README.md b/Sources/Tests/Harnesses/DocSamples/README.md new file mode 100644 index 000000000..8a6111da0 --- /dev/null +++ b/Sources/Tests/Harnesses/DocSamples/README.md @@ -0,0 +1,83 @@ +# docsamples + +Compiles and runs every code sample in the [AngouriMath wiki](https://github.com/asc-community/AngouriMath/wiki), +and the annotated ones on the website, against the library in this checkout, and checks each +stated output against the one produced. + +```sh +dotnet run -c Release --project Sources/Tests/Harnesses/DocSamples +dotnet run -c Release --project Sources/Tests/Harnesses/DocSamples -- --wiki=/path/to/a/wiki/clone +dotnet run -c Release --project Sources/Tests/Harnesses/DocSamples -- --site=/path/to/AngouriMathSite +``` + +The wiki is a separate repository, so the first run clones it into `wiki/` beside this file, +which git ignores, and later runs bring that clone up to date. Point `--wiki=` at a working copy +to check edits before pushing them. The website is read only when `--site=` names a working copy +of [AngouriMathSite](https://github.com/asc-community/AngouriMathSite); CI checks it out and +passes it. + +Exit code is 0 when nothing fails, 1 when a sample does not compile, throws, or prints something +other than what its page says, and 2 when the harness itself could not run. A sample that does +not finish is reported and fails nothing. + +## Why + +The wiki is the library's documentation, and it lives in another repository. A rename lands in +the code while the pages keep the old name, and a reader finds out from a compile error. A stated +output goes stale the same way and more quietly: an integral that now carries its `+ C`, a +`Complex` that .NET 8 prints as `<9; 0>` rather than `(9, 0)`, a product whose factors print in +a new order. + +Both are mechanically checkable, so they are checked here rather than reported by a reader. + +## How a page is read + +A fenced block tagged `cs` or `fs` is a sample. `cs` samples are compiled against +`Sources/AngouriMath`, `fs` samples against `Sources/Wrappers/AngouriMath.FSharp`. + +Each C# sample becomes its own generated file, which keeps its `using` directives to +itself — two samples that import conflicting names both still compile. F# samples become +one module each in one file. The wiki states that `using AngouriMath;`, +`using static AngouriMath.MathS;` and `using static AngouriMath.Entity;` are implied in +every sample; those and `using System;` are supplied, and **anything else a sample needs it +must say**, which is how the missing `using System.Numerics;` and +`using AngouriMath.Extensions;` were found. + +**What a page claims it prints** is the next bare (untagged) fence, and only when the line +before that fence is `Output:` — or `Prints:`, `Should print:`, `Will print:`, `Returns:`. +A bare fence used for anything else is never mistaken for an expectation. A sample that +documents its output in trailing comments on its `Console.WriteLine` lines is read the same +way, as long as *every* printing line carries one; a partly annotated sample is reported as +unchecked rather than checked against half of its output. + +Four directives, written as HTML comments so they do not render: + +| | | +|---|---| +| `` | not compiled. For fragments and pseudo-code | +| `` | compiled but not run. For samples that throw on purpose | +| `` | appended to the sample before it, and its stated output appended to that sample's | +| `` | the fence that follows is prose, not an expectation | + +Every use of `skip` and `compile` is listed in the report with its reason, so what is not +being checked is visible rather than absent. + +## What it does not check + +- **Prose.** An API named in a sentence but never called by a sample is not checked. The + renames in `BREAKING-CHANGES.md` were grepped for by hand instead; a name-checker over + backticked identifiers would close this and does not exist yet. +- **The published package.** Samples compile against a *project reference* to the sources, so + anything that differs between the tree and the `.nupkg` — a member public in one and not the + other, a missing framework asset — is not covered here. Checked by hand once, on 2026-08-11, + when `2.0.0` reached nuget.org: a fresh `dotnet new console`, `dotnet add package AngouriMath + --version 2.0.0`, and the quickstart's own program compiled and printed `x + sin(y * x)` and + `1 + cos(y * x) * y`; likewise for F# with `AngouriMath.FSharp`. Doing that on every run would + mean waiting for a release, so it stays a manual check at release time. +- **The website's unannotated samples.** A `pre code` block on the site is checked only when an + `amcheck` comment marks it as a sample, since most of them are shell, CMake or notebook lines. + The report counts the blocks it did not check, so the coverage is not read as complete. +- **Ordering that happens to be stable.** `Alternate` sorts by a rate with ties, and the + order within a tie is whatever the sort produced. If that shifts, this reports it as a + mismatch, and the right response is to look at whether the sort should be made stable + rather than to edit the page. diff --git a/Sources/Tests/Harnesses/DocSamples/Report.cs b/Sources/Tests/Harnesses/DocSamples/Report.cs new file mode 100644 index 000000000..a09fdc823 --- /dev/null +++ b/Sources/Tests/Harnesses/DocSamples/Report.cs @@ -0,0 +1,134 @@ +// +// Copyright (c) 2019-2026 Angouri. +// AngouriMath is licensed under MIT. +// Details: https://github.com/asc-community/AngouriMath/blob/master/LICENSE.md. +// Website: https://am.angouri.org. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace DocSamples; + +enum Verdict { Ok, CompileError, Threw, Timeout, OutputMismatch, Unchecked, Compiled, Skipped } + +sealed class Finding +{ + public Snippet Snippet; + public Verdict Verdict; + /// The compiler diagnostics, the exception, or the two outputs -- whatever says what went wrong. + public string Detail = ""; + public string Actual = ""; +} + +static class Report +{ + public static void Write(string path, string wikiOrigin, string repository, + List findings, int unannotatedSiteBlocks = 0) + { + var sb = new StringBuilder(); + var counts = Enum.GetValues().ToDictionary(v => v, v => findings.Count(f => f.Verdict == v)); + + sb.AppendLine("# Wiki code samples against the built library"); + sb.AppendLine(); + sb.AppendLine($"Measured against `{Harness.Measured.Commit(repository)}`."); + sb.AppendLine(); + sb.AppendLine($"Generated by `Sources/Tests/Harnesses/DocSamples` from `{wikiOrigin}`."); + sb.AppendLine(); + sb.AppendLine("Every `cs` and `fs` block in the wiki is compiled against the library in this " + + "checkout, and run where it can be. A sample that names an API the library " + + "no longer has is a compile error here rather than a bug report from a reader. " + + "A sample that does not finish is listed and fails nothing."); + sb.AppendLine(); + sb.AppendLine("| | Samples |"); + sb.AppendLine("|---|---|"); + sb.AppendLine($"| Total | {findings.Count} |"); + sb.AppendLine($"| Compiled, ran, output matches the page | {counts[Verdict.Ok]} |"); + sb.AppendLine($"| Compiled and ran, page states no output | {counts[Verdict.Unchecked]} |"); + sb.AppendLine($"| Compiled, not run by directive | {counts[Verdict.Compiled]} |"); + sb.AppendLine($"| **Compile error** | **{counts[Verdict.CompileError]}** |"); + sb.AppendLine($"| **Output differs from the page** | **{counts[Verdict.OutputMismatch]}** |"); + sb.AppendLine($"| **Threw** | **{counts[Verdict.Threw]}** |"); + sb.AppendLine($"| **Did not finish** | **{counts[Verdict.Timeout]}** |"); + sb.AppendLine($"| Not compiled by directive | {counts[Verdict.Skipped]} |"); + sb.AppendLine(); + if (unannotatedSiteBlocks > 0) + { + sb.AppendLine($"{unannotatedSiteBlocks} `pre code` blocks on the website's pages carry no " + + "`amcheck` annotation and are therefore **not** checked — most are shell, " + + "CMake or notebook lines rather than a program, but the count is here so " + + "that the coverage is not read as complete."); + sb.AppendLine(); + } + + var bad = findings.Where(f => f.Verdict is Verdict.CompileError or Verdict.OutputMismatch + or Verdict.Threw or Verdict.Timeout).ToList(); + if (bad.Count == 0) + { + sb.AppendLine("No sample fails to compile, and every stated output is the one produced."); + } + else + { + sb.AppendLine("## What is wrong"); + sb.AppendLine(); + foreach (var f in bad) + { + sb.AppendLine($"### `{f.Snippet.Where}` — {Describe(f.Verdict)}"); + sb.AppendLine(); + sb.AppendLine("```" + f.Snippet.Language); + foreach (var line in f.Snippet.Body) sb.AppendLine(line); + sb.AppendLine("```"); + sb.AppendLine(); + if (f.Verdict == Verdict.OutputMismatch) + { + sb.AppendLine("The page says:"); + sb.AppendLine(); + sb.AppendLine("```"); + sb.AppendLine(f.Snippet.Expected); + sb.AppendLine("```"); + sb.AppendLine(); + sb.AppendLine("It prints:"); + sb.AppendLine(); + sb.AppendLine("```"); + sb.AppendLine(f.Actual.TrimEnd('\n')); + sb.AppendLine("```"); + } + else + { + sb.AppendLine("```"); + sb.AppendLine(f.Detail.TrimEnd('\n')); + sb.AppendLine("```"); + } + sb.AppendLine(); + } + } + + var skipped = findings.Where(f => f.Verdict is Verdict.Skipped or Verdict.Compiled).ToList(); + if (skipped.Count > 0) + { + sb.AppendLine("## Not checked in full, and why"); + sb.AppendLine(); + sb.AppendLine("| Sample | Treatment | Reason |"); + sb.AppendLine("|---|---|---|"); + foreach (var f in skipped) + sb.AppendLine($"| `{f.Snippet.Where}` | " + + (f.Verdict == Verdict.Skipped ? "not compiled" : "compiled, not run") + + $" | {(f.Snippet.Reason.Length == 0 ? "—" : f.Snippet.Reason)} |"); + sb.AppendLine(); + } + + File.WriteAllText(path, sb.ToString()); + } + + static string Describe(Verdict v) => v switch + { + Verdict.CompileError => "does not compile", + Verdict.OutputMismatch => "prints something else", + Verdict.Threw => "throws", + Verdict.Timeout => "does not finish", + _ => v.ToString(), + }; +} diff --git a/Sources/Tests/Harnesses/DocSamples/Snippets.cs b/Sources/Tests/Harnesses/DocSamples/Snippets.cs new file mode 100644 index 000000000..7b69e8eee --- /dev/null +++ b/Sources/Tests/Harnesses/DocSamples/Snippets.cs @@ -0,0 +1,276 @@ +// +// Copyright (c) 2019-2026 Angouri. +// AngouriMath is licensed under MIT. +// Details: https://github.com/asc-community/AngouriMath/blob/master/LICENSE.md. +// Website: https://am.angouri.org. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace DocSamples; + +enum Mode +{ + /// Compile it, run it, and -- if the page states one -- check the output. + Run, + /// Compile it only. For samples that throw on purpose, need input, or draw. + CompileOnly, + /// Not compiled. For fragments and pseudo-code. + Skip, +} + +sealed class Snippet +{ + public string Page; + public int Index; + public int Line; + public string Language; + public Mode Mode = Mode.Run; + public string Reason = ""; + /// Lines of code, in order; a `continues` block appends to the one before it. + public List Body = new(); + /// What the page says this prints, or null where the page does not say. + public string Expected; + + public string Id => $"{Ident(Page)}_{Index}"; + public string Where => At(Line); + public bool IsFSharp => Language is "fs" or "fsharp"; + /// Where in the source page a line of this sample sits. Site pages are HTML, wiki pages + /// are markdown, and the suffix is what makes the reference clickable in either. + public string At(int line) => Page.StartsWith("site:") ? $"{Page}:{line}" : $"{Page}.md:{line}"; + + public static string Ident(string page) + { + var sb = new StringBuilder(); + foreach (var c in page) + sb.Append(char.IsLetterOrDigit(c) ? c : '_'); + return sb.ToString(); + } +} + +/// +/// Pulls the code samples out of the wiki's markdown. +/// +/// A fenced block tagged `cs` is a sample. What the page claims it prints is the next +/// bare (untagged) fence, and only when the line before that fence is exactly `Output:` +/// -- so a bare fence used for anything else is never mistaken for an expectation. +/// A sample that documents its output in trailing `// ...` comments on its +/// `Console.WriteLine` lines is read the same way, as long as every such line carries one. +/// +/// Four directives, written as HTML comments so they do not render, override the default: +/// +/// <!-- amcheck:skip reason --> not compiled +/// <!-- amcheck:compile reason --> compiled but not run +/// <!-- amcheck:continues --> appended to the sample before it +/// <!-- amcheck:nooutput --> the fence that follows is not an expectation +/// +static class Extractor +{ + static readonly Regex Directive = new(@"", RegexOptions.Compiled); + static readonly Regex OutputLead = new(@"^(output|prints|should print|will print|returns)\s*:?\s*$", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + static readonly Regex WriteLineWithComment = + new(@"^\s*Console\.WriteLine\(.*\);\s*//\s*(?.*?)\s*$", RegexOptions.Compiled); + static readonly Regex WriteLineAny = new(@"Console\.Write(Line)?\s*\(", RegexOptions.Compiled); + + public static List FromDirectory(string dir) + { + var all = new List(); + foreach (var file in Directory.GetFiles(dir, "*.md").OrderBy(f => f, StringComparer.Ordinal)) + all.AddRange(FromFile(file)); + return all; + } + + public static List FromFile(string path) + { + var page = Path.GetFileNameWithoutExtension(path); + var lines = File.ReadAllLines(path); + var snippets = new List(); + + // Directives attach to the block that follows them, so they are collected as we walk. + Mode? pendingMode = null; + var pendingReason = ""; + var pendingContinues = false; + var pendingNoOutput = false; + var index = 0; + + for (var i = 0; i < lines.Length; i++) + { + var line = lines[i]; + + var m = Directive.Match(line); + if (m.Success) + { + switch (m.Groups[1].Value.ToLowerInvariant()) + { + case "skip": pendingMode = Mode.Skip; pendingReason = m.Groups[2].Value; break; + case "compile": pendingMode = Mode.CompileOnly; pendingReason = m.Groups[2].Value; break; + case "continues": pendingContinues = true; break; + case "nooutput": pendingNoOutput = true; break; + default: throw new Exception($"{page}.md:{i + 1}: unknown directive '{m.Groups[1].Value}'"); + } + continue; + } + + if (!IsFence(line, out var language)) + continue; + + var open = i; + var body = new List(); + for (i++; i < lines.Length && !IsFence(lines[i], out _); i++) + body.Add(lines[i]); + + if (language is not ("cs" or "csharp" or "fs" or "fsharp")) + { + // A bare fence outside a sample's expectation is prose -- output of a shell + // command, a rendered form. Only `cs` blocks are code we can check. + pendingMode = null; pendingReason = ""; pendingContinues = false; pendingNoOutput = false; + continue; + } + + var expected = pendingNoOutput ? null : FollowingOutputBlock(lines, i); + if (expected is null && !pendingNoOutput) + expected = OutputFromComments(body); + + if (pendingContinues && snippets.Count > 0) + { + var previous = snippets[^1]; + previous.Body.Add(""); + previous.Body.AddRange(body); + // A continuation prints after everything before it, so its stated output + // extends the expectation rather than replacing it. + if (expected is not null) + previous.Expected = previous.Expected is null + ? expected + : previous.Expected + "\n" + expected; + if (pendingMode is not null) previous.Mode = pendingMode.Value; + } + else + { + snippets.Add(new Snippet + { + Page = page, + Index = ++index, + Line = open + 1, + Language = language, + Mode = pendingMode ?? Mode.Run, + Reason = pendingReason, + Body = body, + Expected = expected, + }); + } + + pendingMode = null; pendingReason = ""; pendingContinues = false; pendingNoOutput = false; + } + + return snippets; + } + + static readonly Regex HtmlBlock = new( + @"\s*
(?.*?)
", + RegexOptions.Compiled | RegexOptions.Singleline); + static readonly Regex AnyHtmlBlock = new(@"
", RegexOptions.Compiled);
+
+    /// 
+    /// The website's pages are HTML, and a `pre code` block there could be C#, F#, shell or
+    /// CMake with nothing to say which. So a block is checked only where the page says to,
+    /// with a preceding `<!-- amcheck:cs -->` or `<!-- amcheck:fs -->`, and the count of
+    /// unannotated blocks is reported rather than passed over.
+    /// 
+    public static List FromHtmlDirectory(string dir, out int unannotated)
+    {
+        var all = new List();
+        var annotated = 0;
+        var total = 0;
+        foreach (var file in Directory.GetFiles(dir, "*.html", SearchOption.AllDirectories)
+                     .OrderBy(f => f, StringComparer.Ordinal))
+        {
+            var text = File.ReadAllText(file);
+            total += AnyHtmlBlock.Matches(text).Count;
+            var page = PageName(dir, file);
+            var index = 0;
+            foreach (Match m in HtmlBlock.Matches(text))
+            {
+                annotated++;
+                all.Add(new Snippet
+                {
+                    Page = page,
+                    Index = ++index,
+                    Line = text.Take(m.Index).Count(c => c == '\n') + 1,
+                    Language = m.Groups["lang"].Value,
+                    Mode = m.Groups["rest"].Value.Contains("compile") ? Mode.CompileOnly : Mode.Run,
+                    Reason = m.Groups["rest"].Value.Trim(),
+                    Body = Decode(m.Groups["body"].Value).Split('\n').Select(l => l.TrimEnd()).ToList(),
+                });
+            }
+        }
+        unannotated = total - annotated;
+        return all;
+    }
+
+    /// The page as a reader reaches it: `quickstart/index.html` is `quickstart`.
+    static string PageName(string root, string file)
+    {
+        var relative = Path.GetRelativePath(root, file).Replace('\\', '/');
+        if (relative.EndsWith("/index.html")) relative = relative[..^"/index.html".Length];
+        return "site:" + relative;
+    }
+
+    static string Decode(string body) => body
+        .Replace("<", "<").Replace(">", ">").Replace(""", "\"")
+        .Replace("'", "'").Replace(" ", " ").Replace("&", "&")
+        .Replace("\r\n", "\n").Trim('\n');
+
+    static bool IsFence(string line, out string language)
+    {
+        language = null;
+        var t = line.TrimStart();
+        if (!t.StartsWith("```")) return false;
+        language = t.Substring(3).Trim().ToLowerInvariant();
+        return true;
+    }
+
+    /// The bare fence after the sample, when the page introduces it with `Output:`.
+    static string FollowingOutputBlock(string[] lines, int afterCloseFence)
+    {
+        var lead = -1;
+        for (var i = afterCloseFence + 1; i < lines.Length; i++)
+        {
+            var t = lines[i].Trim();
+            if (t.Length == 0) continue;
+            if (OutputLead.IsMatch(t)) { lead = i; continue; }
+            if (!IsFence(lines[i], out var language)) return null;
+            if (lead < 0) return null;
+            if (language.Length != 0) return null;
+
+            var body = new List();
+            for (i++; i < lines.Length && !IsFence(lines[i], out _); i++)
+                body.Add(lines[i].TrimEnd());
+            return string.Join("\n", body).Trim('\n');
+        }
+        return null;
+    }
+
+    /// `Console.WriteLine(expr); // what it prints`, which the wiki uses for short outputs.
+    /// Read only when every printing line carries one, so a partially annotated sample is
+    /// reported as unchecked rather than checked against half its output.
+    static string OutputFromComments(List body)
+    {
+        var texts = new List();
+        var printers = 0;
+        foreach (var line in body)
+        {
+            if (!WriteLineAny.IsMatch(line)) continue;
+            printers++;
+            var m = WriteLineWithComment.Match(line);
+            if (!m.Success) return null;
+            texts.Add(m.Groups["text"].Value);
+        }
+        return printers > 0 && texts.Count == printers ? string.Join("\n", texts) : null;
+    }
+}
diff --git a/Sources/Tests/Harnesses/README.md b/Sources/Tests/Harnesses/README.md
index 7b28169c3..0ed966ee7 100644
--- a/Sources/Tests/Harnesses/README.md
+++ b/Sources/Tests/Harnesses/README.md
@@ -16,6 +16,7 @@ Each exits non-zero when it finds a defect, or when a list of findings it keeps
 | `PropCheck` | does each transformation satisfy a property it must: `Simplify`, `Expand` and `Factorize` keep the value, `Differentiate` agrees with a difference quotient, `Integrate` differentiates back | any property that does not hold |
 | `CanonCheck` | is there a **canonical form**: idempotence, order independence over commutative operators, and agreement between writings, for `InnerSimplified` and `Simplify` alike | a change to its findings, listed in `canoncheck-baseline.tsv` |
 | `Confluence` | where two arms of one rule set both fire at a node, do they agree, or is the order of the arms load-bearing | a change to its conflicting pairs, listed in `confluence-baseline.tsv` |
+| `DocSamples` | does every code sample in the wiki, and every annotated one on the website, compile, run, and print what its page says | a sample that does not compile, throws, or prints something else |
 
 A timeout fails none of them: a shared runner is slower than the machine a budget was set on.
 
diff --git a/Sources/Tests/Harnesses/Shared/MeasuredCommit.cs b/Sources/Tests/Harnesses/Shared/MeasuredCommit.cs
index 5779128b3..776a50478 100644
--- a/Sources/Tests/Harnesses/Shared/MeasuredCommit.cs
+++ b/Sources/Tests/Harnesses/Shared/MeasuredCommit.cs
@@ -55,6 +55,12 @@ internal static string Commit()
             catch { return "unknown build"; }
         }
 
+        /// 
+        /// The commit of the repository that contains , for a harness
+        /// that builds the library itself rather than referencing it.
+        /// 
+        internal static string Commit(string path) => At(path);
+
         /// 
         /// The commit of the repository that contains .
         ///