From 9c5e7b265eb6d3af5d1f02f3d3e5e5071fbaeda1 Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 16:18:21 +0100 Subject: [PATCH 01/15] Add the global tool registry: store, promotion gate, harness and checks Piece 1 of the tool registry plan. A reusable tool is a manifest, a script and its cases, kept on this machine under /tools. ToolRegistry.Root() is the only place that location is decided (D1). What is here: - Models (src/Loadout.Models/Tools): ToolRecord, ToolVersion, ToolCase, ToolUsage - ownership, version, capabilities, dependencies, compatibility, test status, usage and lifecycle state. - ToolRegistry: drafts, verify, promote, set-active, deprecate, usage and an audit trail. A promoted version is written once; only a known-good version whose fingerprint still matches can be active, so a failed or tampered refinement never replaces the last known-good one. Deprecation needs a replacement or a reason. - ToolHarness: runs every case in a fresh temp directory and refuses a set of cases missing any of success, failure, edge or invalid-input. Running goes through RemedyCeiling with kind tool-test and asks a person by default (D2); the fingerprint is RemedyCeiling.Fingerprint, reused. - ToolPromotion: the manifest must name purpose, inputs, outputs, error behaviour, examples and origin; a draft is gated on its own cases and on every case of the active known-good version. - ToolCompatibility: a removed required input, a new required input with no default, or a changed exit meaning is a break, allowed only when declared with a major bump and a migration. - ToolGenericity: refuses project detail (absolute paths, repository URLs, e-mail addresses, GUIDs, known project and team names) and reports secrets by type, never by value. - ToolOverlap: duplicate by normalised-script fingerprint, overlap by a 0.6 score over description words and script shingles. - Stop rule: two stand-downs with no new signal and a tool is left alone. Verification: the Tool filter passes 56 of 56 and the full suite passes 3158 of 3180, with 22 skipped. Every one of the 19 new tests was mutation-checked: 20 compilable mutations, each failed its named test and was then reverted. Reverting meant a matching edit, checked against a staged copy of every file, not git checkout. Generic_text_passes had survived every plausible mutation, because its input contained nothing that came close to matching any check. It now contains the near-miss "alpha-shopfront", and dropping the known-name regex's trailing boundary fails it. Left out on purpose: a registry lock (write-once is a Directory.Exists check, which is not atomic), the nightly overlap pass, and any CLI or MCP surface. --- src/Loadout.Core/ServiceRegistration.cs | 5 + src/Loadout.Core/Tools/ToolAudit.cs | 81 ++ src/Loadout.Core/Tools/ToolCompatibility.cs | 84 ++ src/Loadout.Core/Tools/ToolGenericity.cs | 128 +++ src/Loadout.Core/Tools/ToolHarness.cs | 239 +++++ src/Loadout.Core/Tools/ToolOverlap.cs | 128 +++ src/Loadout.Core/Tools/ToolPromotion.cs | 157 +++ src/Loadout.Core/Tools/ToolRegistry.cs | 916 ++++++++++++++++++ src/Loadout.Models/Tools/ToolCase.cs | 68 ++ src/Loadout.Models/Tools/ToolRecord.cs | 113 +++ src/Loadout.Models/Tools/ToolUsage.cs | 42 + src/Loadout.Models/Tools/ToolVersion.cs | 163 ++++ .../Unit/ToolCompatibilityTests.cs | 34 + .../Loadout.Tests/Unit/ToolGenericityTests.cs | 43 + tests/Loadout.Tests/Unit/ToolHarnessTests.cs | 71 ++ tests/Loadout.Tests/Unit/ToolOverlapTests.cs | 55 ++ .../Loadout.Tests/Unit/ToolPromotionTests.cs | 73 ++ tests/Loadout.Tests/Unit/ToolRegistryTests.cs | 130 +++ tests/Loadout.Tests/Unit/ToolStopRuleTests.cs | 35 + tests/Loadout.Tests/Unit/ToolStoreFixture.cs | 154 +++ 20 files changed, 2719 insertions(+) create mode 100644 src/Loadout.Core/Tools/ToolAudit.cs create mode 100644 src/Loadout.Core/Tools/ToolCompatibility.cs create mode 100644 src/Loadout.Core/Tools/ToolGenericity.cs create mode 100644 src/Loadout.Core/Tools/ToolHarness.cs create mode 100644 src/Loadout.Core/Tools/ToolOverlap.cs create mode 100644 src/Loadout.Core/Tools/ToolPromotion.cs create mode 100644 src/Loadout.Core/Tools/ToolRegistry.cs create mode 100644 src/Loadout.Models/Tools/ToolCase.cs create mode 100644 src/Loadout.Models/Tools/ToolRecord.cs create mode 100644 src/Loadout.Models/Tools/ToolUsage.cs create mode 100644 src/Loadout.Models/Tools/ToolVersion.cs create mode 100644 tests/Loadout.Tests/Unit/ToolCompatibilityTests.cs create mode 100644 tests/Loadout.Tests/Unit/ToolGenericityTests.cs create mode 100644 tests/Loadout.Tests/Unit/ToolHarnessTests.cs create mode 100644 tests/Loadout.Tests/Unit/ToolOverlapTests.cs create mode 100644 tests/Loadout.Tests/Unit/ToolPromotionTests.cs create mode 100644 tests/Loadout.Tests/Unit/ToolRegistryTests.cs create mode 100644 tests/Loadout.Tests/Unit/ToolStopRuleTests.cs create mode 100644 tests/Loadout.Tests/Unit/ToolStoreFixture.cs diff --git a/src/Loadout.Core/ServiceRegistration.cs b/src/Loadout.Core/ServiceRegistration.cs index 41aca6bb..efd10d2c 100644 --- a/src/Loadout.Core/ServiceRegistration.cs +++ b/src/Loadout.Core/ServiceRegistration.cs @@ -161,6 +161,11 @@ public static IServiceCollection AddCoreServices(this IServiceCollection service services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(provider => new Tools.ToolRegistry( + provider.GetRequiredService(), + provider.GetRequiredService(), + provider.GetRequiredService())); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Loadout.Core/Tools/ToolAudit.cs b/src/Loadout.Core/Tools/ToolAudit.cs new file mode 100644 index 00000000..3d2356ef --- /dev/null +++ b/src/Loadout.Core/Tools/ToolAudit.cs @@ -0,0 +1,81 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Loadout.Core.Tools; + +/// One line of the audit log. +/// When. +/// submit, verify, reject, promote, activate, used, deprecate, retire, stand-down. +/// The tool, or empty for a submission about none. +/// The version, where there is one. +/// Who: a node, a person, the gate. +/// The run, where there is one. +/// What happened, never a secret. +public sealed record ToolAuditEntry( + DateTimeOffset At, + string Action, + string Tool, + string? Version = null, + string? Actor = null, + string? Run = null, + string? Note = null); + +/// +/// The registry's append-only log: everything that changed it, and who did. +/// +/// +/// A line per event, so an interrupted write costs one line and not the file, +/// and so reading it back needs no parser beyond one line at a time. Activity +/// goes here and not into any team's brief, which is what keeps it from +/// flooding everybody else's context. +/// +public static class ToolAudit +{ + private static readonly JsonSerializerOptions Json = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; + + /// Adds one entry to the end of the log. + public static void Append(string file, ToolAuditEntry entry) + { + ArgumentNullException.ThrowIfNull(entry); + + Directory.CreateDirectory(Path.GetDirectoryName(file)!); + File.AppendAllText(file, JsonSerializer.Serialize(entry, Json) + "\n"); + } + + /// Every entry, oldest first. A line that cannot be read is skipped, not fatal. + public static IReadOnlyList Read(string file) + { + if (!File.Exists(file)) + { + return []; + } + + var entries = new List(); + + foreach (var line in File.ReadLines(file)) + { + if (line.Length == 0) + { + continue; + } + + try + { + if (JsonSerializer.Deserialize(line, Json) is { } entry) + { + entries.Add(entry); + } + } + catch (JsonException) + { + // One damaged line is one lost event, and the rest still read. + } + } + + return entries; + } +} diff --git a/src/Loadout.Core/Tools/ToolCompatibility.cs b/src/Loadout.Core/Tools/ToolCompatibility.cs new file mode 100644 index 00000000..c9256cd2 --- /dev/null +++ b/src/Loadout.Core/Tools/ToolCompatibility.cs @@ -0,0 +1,84 @@ +using Loadout.Models.Tools; + +namespace Loadout.Core.Tools; + +/// +/// Whether a version breaks the callers of the one before it, and whether it +/// has owned up to it. +/// +/// +/// Computed rather than declared, because the agent writing a version is the +/// one with a reason to say it breaks nothing. Removing or renaming a required +/// input, adding one with no default, and changing what an exit code means all +/// break somebody's call that worked yesterday. +/// +public static class ToolCompatibility +{ + /// How a version breaks its predecessor's callers. Empty when it does not. + public static IReadOnlyList Breaks(ToolVersion previous, ToolVersion next) + { + ArgumentNullException.ThrowIfNull(previous); + ArgumentNullException.ThrowIfNull(next); + + var found = new List(); + var inputs = next.Inputs.ToDictionary(one => one.Name, StringComparer.OrdinalIgnoreCase); + + foreach (var input in previous.Inputs.Where(one => one.Required)) + { + if (!inputs.ContainsKey(input.Name)) + { + found.Add($"The required input '{input.Name}' is gone."); + } + } + + var before = previous.Inputs.Select(one => one.Name).ToHashSet(StringComparer.OrdinalIgnoreCase); + + foreach (var input in next.Inputs.Where(one => one.Required && one.Default is null && !before.Contains(one.Name))) + { + found.Add($"'{input.Name}' is a new required input with no default."); + } + + foreach (var (code, meaning) in previous.Outputs.Exit) + { + if (!next.Outputs.Exit.TryGetValue(code, out var now) + || !string.Equals(now.Trim(), meaning.Trim(), StringComparison.OrdinalIgnoreCase)) + { + found.Add($"Exit code {code} no longer means '{meaning}'."); + } + } + + return found; + } + + /// + /// Why a version may not follow its predecessor, or null where it may. + /// + /// + /// A break is allowed only when declared, with a major bump and the words a + /// caller needs to move over. + /// + public static string? Refusal(ToolVersion previous, ToolVersion next) + { + var breaks = Breaks(previous, next); + + if (breaks.Count == 0 || Declared(previous, next)) + { + return null; + } + + return "This version breaks callers of " + previous.Version + " without saying so: " + + string.Join(" ", breaks) + + " Declare compatibility.breaks, bump the major version and write the migration."; + } + + /// Whether a break has been declared the only way that counts. + public static bool Declared(ToolVersion previous, ToolVersion next) => + next.Compatibility.Breaks + && Major(next.Version) > Major(previous.Version) + && next.Compatibility.Migration is { Length: > 0 } said + && !string.IsNullOrWhiteSpace(said); + + /// The major part of major.minor, or -1 where it is not a number. + public static int Major(string? version) => + int.TryParse((version ?? string.Empty).Split('.')[0], out var major) ? major : -1; +} diff --git a/src/Loadout.Core/Tools/ToolGenericity.cs b/src/Loadout.Core/Tools/ToolGenericity.cs new file mode 100644 index 00000000..3168abbf --- /dev/null +++ b/src/Loadout.Core/Tools/ToolGenericity.cs @@ -0,0 +1,128 @@ +using System.Text.RegularExpressions; +using Loadout.Core.Security; +using Loadout.Models.Tools; + +namespace Loadout.Core.Tools; + +/// +/// Whether a tool still carries the project it was written for. +/// +/// +/// +/// A tool is shared by every team on this machine. A script with somebody's +/// drive, repository or address in it works for the project it came from and +/// quietly does the wrong thing everywhere else, so those values have to become +/// inputs before anything is promoted. +/// +/// +/// Pattern-based, and the limit is stated: a project detail with no +/// recognisable shape passes unless it is a known project or team name. A +/// secret is reported by the name of its pattern and never by its value, for +/// the reason gives. +/// +/// +public static partial class ToolGenericity +{ + /// + /// What in the text ties it to one project, one finding per line. Empty + /// when it reads as generic. + /// + /// What to check. + /// Project slugs and team names this machine knows. + public static IReadOnlyList Check(string? text, IEnumerable? known = null) + { + if (string.IsNullOrWhiteSpace(text)) + { + return []; + } + + var found = new List(); + + // Secrets first, and by type only. + foreach (var name in SecretScanner.Match(text)) + { + found.Add($"a credential ({name})"); + } + + Look(found, "an absolute path", WindowsPath(), text); + Look(found, "an absolute path", UnixPath(), text); + Look(found, "a repository URL", RepositoryUrl(), text); + Look(found, "an e-mail address", Email(), text); + Look(found, "a GUID", Guid(), text); + + foreach (var name in (known ?? []).Where(one => one is { Length: >= 3 }).Distinct(StringComparer.OrdinalIgnoreCase)) + { + if (Regex.IsMatch( + text, + $@"(?Everything about a version that is shared: its manifest, script and cases. + public static IReadOnlyList Check( + ToolVersion version, + string script, + IEnumerable cases, + IEnumerable? known = null) + { + ArgumentNullException.ThrowIfNull(version); + + var parts = new List + { + version.Purpose, + version.Origin, + version.ErrorBehaviour, + version.Outputs.Stdout, + script, + }; + + parts.AddRange(version.Constraints); + parts.AddRange(version.Dependencies); + parts.AddRange(version.Examples.SelectMany(one => new[] { one.Command, one.Expect })); + parts.AddRange(version.Inputs.SelectMany(one => new[] { one.Default, one.Describe })); + parts.AddRange(cases.SelectMany(one => one.Args.Values.Concat(one.Setup.Files.Keys))); + + return [.. Check(string.Join('\n', parts.Where(one => one is { Length: > 0 })), known).Distinct()]; + } + + private static void Look(List found, string what, Regex pattern, string text) + { + try + { + foreach (Match match in pattern.Matches(text)) + { + found.Add($"{what}: '{match.Value.Trim()}'"); + } + } + catch (RegexMatchTimeoutException) + { + // Unchecked is not the same as clean. + found.Add($"{what} (the check did not complete)"); + } + } + + [GeneratedRegex(@"(?How one case went. +/// Its name. +/// Its class. +/// Whether every expectation held. +/// What did not hold, or empty. +public sealed record ToolCaseResult(string Case, string Class, bool Passed, string Why); + +/// What this machine says about running a harness, from its own configuration. +/// The machine's rule for kind tool-test, or null for none, which is asking. +/// What a person at this machine has agreed to. +public sealed record ToolTestConsent(string? Rule, IReadOnlyList? Trusted); + +/// +/// Runs a version's cases, each in a directory of its own. +/// +/// +/// +/// A harness run executes a script an agent wrote, so whether it may run at +/// all is 's to say, with kind +/// , exactly as for a remedy. Nobody has decided yet that +/// it may run unattended, so a machine that says nothing is asked. +/// +/// +/// A fresh temporary directory per case, so one case cannot pass on what the +/// last one left behind. +/// +/// +public sealed class ToolHarness +{ + /// The kind a machine sets its rule against for harness runs. + public const string Kind = "tool-test"; + + /// The placeholder a case uses for its own directory. + public const string Tmp = "{tmp}"; + + private static readonly TimeSpan Timeout = TimeSpan.FromMinutes(2); + + private static readonly ISerializer Writer = new SerializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .Build(); + + private readonly IProcessLauncher _launcher; + + public ToolHarness(IProcessLauncher launcher) => _launcher = launcher; + + /// The case classes a set of cases does not cover. + public static IReadOnlyList MissingClasses(IEnumerable cases) + { + var have = cases.Select(one => one.Class.Trim().ToLowerInvariant()).ToHashSet(StringComparer.Ordinal); + + return [.. ToolCaseClass.All.Where(one => !have.Contains(one))]; + } + + /// What a person agrees to when they agree to a harness run: the script and its cases. + public static string Subject(string script, IEnumerable cases) => + script + "\n---\n" + string.Join( + "\n---\n", + cases.OrderBy(one => one.Name, StringComparer.Ordinal).Select(Writer.Serialize)); + + /// The fingerprint a person's agreement to a harness run is recorded against. + public static string Fingerprint(string script, IEnumerable cases) => + RemedyCeiling.Fingerprint(Subject(script, cases)); + + /// The name a harness run is agreed to under. + public static string Named(ToolVersion draft) => $"{Kind}:{draft.Name}@{draft.Version}"; + + /// Whether this machine lets the harness run these cases against this script. + public static RemedyCeiling.Decision May( + ToolVersion draft, + string script, + IReadOnlyList cases, + ToolTestConsent consent) + { + ArgumentNullException.ThrowIfNull(draft); + ArgumentNullException.ThrowIfNull(consent); + + return RemedyCeiling.Decide( + new Remedy { Name = Named(draft), Kind = Kind }, + consent.Rule, + Subject(script, cases), + consent.Trusted); + } + + /// Runs every case, in order. + public async Task> RunAllAsync( + string scriptPath, + IEnumerable cases, + CancellationToken ct = default) + { + var results = new List(); + + foreach (var one in cases) + { + results.Add(await RunAsync(scriptPath, one, ct).ConfigureAwait(false)); + } + + return results; + } + + /// Runs one case in a directory of its own, and checks what it expects. + public async Task RunAsync(string scriptPath, ToolCase toolCase, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(toolCase); + + ToolCaseResult Fail(string why) => new(toolCase.Name, toolCase.Class, false, why); + + // Project detail in a case is project detail all the same. + foreach (var (name, value) in toolCase.Args) + { + if (!value.Contains(Tmp, StringComparison.Ordinal) && Path.IsPathRooted(value)) + { + return Fail($"'{name}' is an absolute path. Cases may only use {Tmp}."); + } + } + + var directory = Path.Combine(Path.GetTempPath(), "loadout-tool-case-" + Guid.NewGuid().ToString("N")); + + try + { + Directory.CreateDirectory(directory); + + string Expand(string value) => value.Replace(Tmp, directory, StringComparison.Ordinal); + + foreach (var (file, content) in toolCase.Setup.Files) + { + var at = Under(directory, Expand(file)); + + if (at is null) + { + return Fail($"The setup file '{file}' is outside {Tmp}."); + } + + Directory.CreateDirectory(Path.GetDirectoryName(at)!); + await File.WriteAllTextAsync(at, content, ct).ConfigureAwait(false); + } + + var arguments = new List { "-NoProfile", "-NonInteractive", "-File", scriptPath }; + + foreach (var (name, value) in toolCase.Args) + { + arguments.Add("-" + name); + arguments.Add(Expand(value)); + } + + var ran = await _launcher.RunAsync( + new ProcessRequest("pwsh", arguments, directory), + Timeout, + ct).ConfigureAwait(false); + + if (ran.Failed) + { + return Fail("It could not be run: " + ran.Error); + } + + var outcome = ran.Value!; + var expect = toolCase.Expect; + + if (expect.Exit is { } exit && outcome.ExitCode != exit) + { + return Fail($"Exit code {outcome.ExitCode}, expected {exit}."); + } + + if (!Matches(expect.StdoutMatches, outcome.StandardOutput)) + { + return Fail($"Standard output did not match '{expect.StdoutMatches}'."); + } + + if (!Matches(expect.StderrMatches, outcome.StandardError)) + { + return Fail($"Standard error did not match '{expect.StderrMatches}'."); + } + + foreach (var file in expect.FilesPresent) + { + if (Under(directory, Expand(file)) is not { } at || !File.Exists(at)) + { + return Fail($"'{file}' should be there and is not."); + } + } + + foreach (var file in expect.FilesAbsent) + { + if (Under(directory, Expand(file)) is { } at && File.Exists(at)) + { + return Fail($"'{file}' should be gone and is not."); + } + } + + return new ToolCaseResult(toolCase.Name, toolCase.Class, true, string.Empty); + } + finally + { + try + { + Directory.Delete(directory, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // A temporary directory left behind is untidy, not wrong. + } + } + } + + private static bool Matches(string pattern, string text) + { + if (string.IsNullOrEmpty(pattern)) + { + return true; + } + + try + { + return Regex.IsMatch(text, pattern, RegexOptions.None, TimeSpan.FromSeconds(1)); + } + catch (Exception ex) when (ex is ArgumentException or RegexMatchTimeoutException) + { + return false; + } + } + + private static string? Under(string directory, string path) + { + var at = Path.GetFullPath(Path.IsPathRooted(path) ? path : Path.Combine(directory, path)); + var root = Path.GetFullPath(directory); + + return at.StartsWith(root + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase) ? at : null; + } +} diff --git a/src/Loadout.Core/Tools/ToolOverlap.cs b/src/Loadout.Core/Tools/ToolOverlap.cs new file mode 100644 index 00000000..84672877 --- /dev/null +++ b/src/Loadout.Core/Tools/ToolOverlap.cs @@ -0,0 +1,128 @@ +using System.Text.RegularExpressions; +using Loadout.Core.Teams; + +namespace Loadout.Core.Tools; + +/// What overlap is measured on: what a tool says it is, and what it does. +/// Its search terms. +/// Its one-line summary. +/// Its script text. +public sealed record ToolShape(IReadOnlyList Capabilities, string Summary, string Script); + +/// How much two tools overlap. +/// 0 to 1; 1 for a duplicate. +/// The same script once comments and spacing are taken out. +public sealed record ToolOverlapScore(double Score, bool Duplicate) +{ + /// Whether this is enough to ask the submitter to extend rather than add. + public bool Overlaps => Duplicate || Score >= ToolOverlap.Threshold; +} + +/// +/// Whether a new tool is one the catalogue already has. +/// +/// +/// Deterministic on purpose: the same two tools always score the same, so a +/// refusal can be argued with. Half on what the tools say they are, half on +/// what their scripts do, because either alone is fooled by a rename. +/// +public static partial class ToolOverlap +{ + /// The score at which two tools are flagged. A starting value, to be judged on real tools. + public const double Threshold = 0.6; + + private const int Shingle = 5; + + /// Scores two tools against each other. + public static ToolOverlapScore Score(ToolShape a, ToolShape b) + { + ArgumentNullException.ThrowIfNull(a); + ArgumentNullException.ThrowIfNull(b); + + var left = Normalise(a.Script); + var right = Normalise(b.Script); + + if (left.Length > 0 + && string.Equals(RemedyCeiling.Fingerprint(left), RemedyCeiling.Fingerprint(right), StringComparison.Ordinal)) + { + return new ToolOverlapScore(1, true); + } + + var described = Jaccard(Words(a), Words(b)); + var scripted = Jaccard(Shingles(left), Shingles(right)); + + return new ToolOverlapScore(0.5 * described + 0.5 * scripted, false); + } + + /// A script with comments and spacing taken out, so layout is not difference. + public static string Normalise(string? script) + { + if (string.IsNullOrEmpty(script)) + { + return string.Empty; + } + + var text = script.Replace("\r\n", "\n", StringComparison.Ordinal); + text = BlockComment().Replace(text, " "); + text = LineComment().Replace(text, string.Empty); + + return Space().Replace(text, " ").Trim(); + } + + private static HashSet Words(ToolShape shape) => + [ + .. shape.Capabilities + .Concat(Word().Matches(shape.Summary ?? string.Empty).Select(one => one.Value)) + .Select(one => one.Trim().ToLowerInvariant()) + .Where(one => one.Length > 2), + ]; + + private static HashSet Shingles(string normalised) + { + var tokens = Word().Matches(normalised).Select(one => one.Value.ToLowerInvariant()).ToList(); + + var set = new HashSet(StringComparer.Ordinal); + + if (tokens.Count < Shingle) + { + if (tokens.Count > 0) + { + set.Add(string.Join(' ', tokens)); + } + + return set; + } + + for (var i = 0; i + Shingle <= tokens.Count; i++) + { + set.Add(string.Join(' ', tokens.Skip(i).Take(Shingle))); + } + + return set; + } + + private static double Jaccard(HashSet a, HashSet b) + { + if (a.Count == 0 && b.Count == 0) + { + return 0; + } + + var union = new HashSet(a, a.Comparer); + union.UnionWith(b); + + return (double)a.Count(b.Contains) / union.Count; + } + + [GeneratedRegex(@"<#.*?#>", RegexOptions.Singleline, 1000)] + private static partial Regex BlockComment(); + + [GeneratedRegex(@"#[^\n]*", RegexOptions.None, 1000)] + private static partial Regex LineComment(); + + [GeneratedRegex(@"\s+", RegexOptions.None, 1000)] + private static partial Regex Space(); + + [GeneratedRegex(@"[A-Za-z0-9_$-]+", RegexOptions.None, 1000)] + private static partial Regex Word(); +} diff --git a/src/Loadout.Core/Tools/ToolPromotion.cs b/src/Loadout.Core/Tools/ToolPromotion.cs new file mode 100644 index 00000000..51220a04 --- /dev/null +++ b/src/Loadout.Core/Tools/ToolPromotion.cs @@ -0,0 +1,157 @@ +using Loadout.Models.Tools; + +namespace Loadout.Core.Tools; + +/// What the regression gate found. +/// Whether the draft may be promoted. +/// The draft's own cases. +/// The active known-good version's cases, rerun against the draft. +/// Known-good cases the draft newly fails, and has not retired. +/// Known-good cases the draft fails under a declared break. +public sealed record ToolGateResult( + bool Passed, + IReadOnlyList Own, + IReadOnlyList Regression, + IReadOnlyList Regressions, + IReadOnlyList Retired) +{ + /// Why, in a sentence. + public string Because => + Passed + ? "Every case passed" + (Retired.Count > 0 ? $", with {string.Join(", ", Retired)} retired by a declared break." : ".") + : string.Join( + " ", + Own.Where(one => !one.Passed).Select(one => $"'{one.Case}' failed: {one.Why}") + .Concat(Regressions.Select(one => $"'{one}' passed on the known-good version and fails now."))); +} + +/// +/// What a draft has to show before it can replace a known-good version. +/// +/// +/// A failed refinement must not replace what worked. The gate reruns every case +/// the active version passed against the draft, and a case that newly fails +/// rejects the draft, unless the draft owns up: a declared break, a major bump, +/// migration text, and the case named as retired. +/// +public static class ToolPromotion +{ + /// The manifest fields a version must fill before it can be promoted. + public static IReadOnlyList Missing(ToolVersion version) + { + ArgumentNullException.ThrowIfNull(version); + + var missing = new List(); + + if (string.IsNullOrWhiteSpace(version.Purpose)) + { + missing.Add("purpose"); + } + + if (version.Inputs.Count == 0) + { + missing.Add("inputs"); + } + + if (string.IsNullOrWhiteSpace(version.Outputs.Stdout) && version.Outputs.Exit.Count == 0) + { + missing.Add("outputs"); + } + + if (version.Dependencies.Count == 0) + { + missing.Add("dependencies"); + } + + if (version.Constraints.Count == 0) + { + missing.Add("constraints"); + } + + if (string.IsNullOrWhiteSpace(version.ErrorBehaviour)) + { + missing.Add("error_behaviour"); + } + + if (version.Examples.Count == 0) + { + missing.Add("examples"); + } + + if (string.IsNullOrWhiteSpace(version.Origin)) + { + missing.Add("origin"); + } + + return missing; + } + + /// + /// Runs the draft's cases and the active version's cases against the draft's script. + /// + /// The draft. + /// The draft's script, where the runner can find it. + /// The draft's own cases. + /// The active known-good version, or null for a first version. + /// Its cases. + /// Runs one case against one script: the harness, in anything but a test. + /// Cancellation. + public static async Task GateAsync( + ToolVersion next, + string nextScript, + IReadOnlyList nextCases, + ToolVersion? active, + IReadOnlyList activeCases, + Func> run, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(next); + ArgumentNullException.ThrowIfNull(run); + + var own = new List(); + + foreach (var one in nextCases) + { + own.Add(await run(nextScript, one, ct).ConfigureAwait(false)); + } + + var regression = new List(); + var regressions = new List(); + var retired = new List(); + + if (active is not null) + { + foreach (var one in activeCases) + { + var result = await run(nextScript, one, ct).ConfigureAwait(false); + regression.Add(result); + + if (result.Passed) + { + continue; + } + + if (Retires(active, next, one.Name)) + { + retired.Add(one.Name); + } + else + { + regressions.Add(one.Name); + } + } + } + + return new ToolGateResult( + own.All(one => one.Passed) && regressions.Count == 0, + own, + regression, + regressions, + retired); + } + + /// Whether a draft has retired a known-good case the only way that counts. + private static bool Retires(ToolVersion active, ToolVersion next, string name) => + ToolCompatibility.Declared(active, next) + && next.Compatibility.RetiredCases.Contains(name, StringComparer.OrdinalIgnoreCase); +} diff --git a/src/Loadout.Core/Tools/ToolRegistry.cs b/src/Loadout.Core/Tools/ToolRegistry.cs new file mode 100644 index 00000000..e6ad3ba9 --- /dev/null +++ b/src/Loadout.Core/Tools/ToolRegistry.cs @@ -0,0 +1,916 @@ +using System.Text.Json; +using System.Text.RegularExpressions; +using Loadout.Core.Security; +using Loadout.Core.Teams; +using Loadout.Models; +using Loadout.Models.Results; +using Loadout.Models.Teams; +using Loadout.Models.Tools; +using Loadout.Platform.Abstractions; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace Loadout.Core.Tools; + +/// A tool as somebody asking about it sees it. +/// Its head. +/// The active version's manifest, or null where there is none or it no longer matches. +/// Every known-good version and its standing: known-good, tampered or missing. +public sealed record ToolShown( + ToolRecord Record, + ToolVersion? Active, + IReadOnlyDictionary Versions); + +/// A submission to the catalogue's inbox. +/// candidate, idea, bug or lesson. +/// What it says. +/// The tool it is about, or null. +/// Who sent it. +/// The run it came from, where there was one. +/// A candidate's script, where there is one, for the overlap check. +/// A candidate's search terms. +/// A candidate's one-line summary. +public sealed record ToolSubmission( + string Kind, + string Text, + string? Tool = null, + string? By = null, + string? Run = null, + string? Script = null, + IReadOnlyList? Capabilities = null, + string? Summary = null); + +/// What a submission became. +/// The inbox item. +/// Active tools it overlaps, which it should extend rather than duplicate. +public sealed record ToolSubmitted(string Id, IReadOnlyList Overlapping); + +/// What a verify came to. +/// Whether the harness was allowed to run. +/// Why, in a sentence. +/// What the gate found, or null where it did not run. +public sealed record ToolVerification(RemedyRuling Ruling, string Because, ToolGateResult? Gate); + +/// Who is promoting, and why the version exists. +/// lesson, requirement, bug, idea, nomination or consolidation. +/// The submission or run it came from. +/// Who asked. +/// The run, where there is one. +/// The owning team, for a new tool. +/// The kind, for a new tool. +/// The summary, for a new tool or to replace it. +/// The search terms, for a new tool or to replace them. +public sealed record ToolPromotionRequest( + string Because, + string Source, + string? Actor = null, + string? Run = null, + string? Owner = null, + string? Kind = null, + string? Summary = null, + IReadOnlyList? Capabilities = null); + +/// The machine's shared catalogue of tools. +public interface IToolRegistry +{ + /// Where the catalogue is kept. + string Root(); + + /// Tools matching any of the words, best first. Deprecated and retired only with . + IReadOnlyList Search(string words, bool all = false); + + /// One tool, its active version and the standing of each version. + OperationResult Show(string name); + + /// Puts a candidate, idea, bug or lesson in the inbox, after the screens. + OperationResult Submit(ToolSubmission submission); + + /// Records one use. + OperationResult RecordUsage(ToolUsage usage); + + /// The audit log, oldest first. + IReadOnlyList Audit(string? tool = null, DateTimeOffset? since = null); + + /// Runs a draft's harness and the regression gate, where this machine allows it. + Task> VerifyAsync( + string draft, + ToolTestConsent consent, + CancellationToken ct = default); + + /// Writes a verified draft into its version directory, once, and makes it active. + OperationResult Promote(string draft, ToolPromotionRequest request); + + /// Moves the active version to another known-good one. + OperationResult SetActive(string name, string version); + + /// Deprecates a tool, with a replacement or a reason. + OperationResult Deprecate(string name, string? replacement, string? reason); + + /// Retires a deprecated tool nobody has used lately. + OperationResult Retire(string name); + + /// Whether a promoted version's files still match what was promoted. + string Standing(string name, string version); + + /// Records that a refinement was considered and not worth making. + OperationResult StandDown(string name, string reason, string? actor = null, string? run = null); + + /// Whether a tool is worth another look. + bool NeedsRefining(string name); +} + +/// +/// The machine's shared catalogue of tools, in files under . +/// +/// +/// +/// Files, like a team's remedies, because what is kept is a script somebody +/// may want to open and read. The rules that matter are enforced here rather +/// than trusted to whoever writes the files: a version directory is written +/// once, only a known-good version whose files still match can be active, and +/// a failed refinement cannot move what is active. +/// +/// +/// Nothing here decides whether anything runs. That is this machine's +/// configuration and a person's, as for remedies. +/// +/// +public sealed partial class ToolRegistry : IToolRegistry +{ + /// How long a deprecated tool has to go unused before it can be retired. + public static readonly TimeSpan RetireAfter = TimeSpan.FromDays(30); + + private static readonly IDeserializer Yaml = new DeserializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + private static readonly ISerializer Writer = new SerializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitNull) + .Build(); + + private static readonly JsonSerializerOptions Json = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + }; + + private static readonly HashSet Kinds = new(StringComparer.Ordinal) { "candidate", "idea", "bug", "lesson" }; + + private readonly IPlatformPaths _paths; + private readonly ToolHarness _harness; + private readonly TimeProvider _clock; + private readonly Func> _known; + + /// Where this machine keeps its state. + /// Runs cases. + /// What time it is. + /// + /// Project slugs and team names a tool must not carry. By default, the + /// team directories and the workspace's projects on this machine. + /// + public ToolRegistry( + IPlatformPaths paths, + ToolHarness harness, + TimeProvider clock, + Func>? known = null) + { + _paths = paths; + _harness = harness; + _clock = clock; + _known = known ?? KnownOnThisMachine; + } + + /// + /// + /// The one place the location is decided. Whether the catalogue belongs to + /// the machine or to the workspace is not settled yet, and moving it should + /// be a change to this line and nothing else. + /// + public string Root() => Path.Combine(_paths.Paths.State, "tools"); + + private string AuditFile => Path.Combine(Root(), "audit.jsonl"); + + private string ToolDirectory(string name) => Path.Combine(Root(), RemedyBook.Slug(name)); + + private string HeadFile(string name) => Path.Combine(ToolDirectory(name), "tool.yaml"); + + private string UsageFile(string name) => Path.Combine(ToolDirectory(name), "usage.jsonl"); + + private string VersionDirectory(string name, string version) => + Path.Combine(ToolDirectory(name), "versions", version); + + /// + public IReadOnlyList Search(string words, bool all = false) + { + var terms = Word().Matches(words ?? string.Empty).Select(one => one.Value.ToLowerInvariant()).ToList(); + + return + [ + .. Heads() + .Where(one => all || one.Lifecycle is not (ToolLifecycle.Deprecated or ToolLifecycle.Retired)) + .Select(one => (Record: one, Score: Score(one, terms))) + .Where(one => terms.Count == 0 || one.Score > 0) + .OrderByDescending(one => one.Score) + .ThenBy(one => one.Record.Name, StringComparer.Ordinal) + .Select(one => one.Record), + ]; + } + + /// + public OperationResult Show(string name) + { + if (ReadHead(name) is not { } head) + { + return OperationResult.Fail($"There is no tool called '{name}'.", ExitCode.ProjectNotFound); + } + + var versions = head.KnownGood.ToDictionary(one => one, one => Standing(name, one), StringComparer.Ordinal); + + return OperationResult.Ok(new ToolShown(head, ActiveVersion(head), versions)); + } + + /// + public OperationResult Submit(ToolSubmission submission) + { + ArgumentNullException.ThrowIfNull(submission); + + var kind = (submission.Kind ?? string.Empty).Trim().ToLowerInvariant(); + + if (!Kinds.Contains(kind)) + { + return OperationResult.Fail( + $"'{submission.Kind}' is not something the inbox takes: candidate, idea, bug or lesson.", + ExitCode.InvalidArguments); + } + + var everything = submission.Text + "\n" + submission.Summary + "\n" + submission.Script; + + // By type and never by value, for the reason the scanner gives. + if (SecretScanner.Match(everything) is { Count: > 0 } secrets) + { + return OperationResult.Fail( + "That looks like it contains a credential (" + string.Join(", ", secrets) + "), so it was not written.", + ExitCode.PolicyViolation); + } + + if (kind == "candidate" && ToolGenericity.Check(everything, _known()) is { Count: > 0 } specific) + { + return OperationResult.Fail( + "A candidate has to work for any project, and this carries one: " + + string.Join("; ", specific) + ". Make each of them an input.", + ExitCode.PolicyViolation); + } + + var overlapping = new List(); + + if (submission.Script is { Length: > 0 } script) + { + var shape = new ToolShape(submission.Capabilities ?? [], submission.Summary ?? submission.Text, script); + + foreach (var (record, active) in ActiveShapes()) + { + if (ToolOverlap.Score(shape, active).Overlaps) + { + overlapping.Add(record.Name); + } + } + } + + var now = _clock.GetUtcNow(); + var id = $"{now:yyyy-MM-dd}-{Guid.NewGuid():N}"[..19]; + var item = new Dictionary + { + ["id"] = id, + ["kind"] = kind, + ["tool"] = submission.Tool, + ["by"] = submission.By, + ["run"] = submission.Run, + ["at"] = now, + ["text"] = submission.Text, + ["summary"] = submission.Summary, + ["capabilities"] = submission.Capabilities, + ["script"] = submission.Script, + ["extends"] = overlapping.Count > 0 ? overlapping : null, + }; + + try + { + var inbox = Path.Combine(Root(), "inbox"); + Directory.CreateDirectory(inbox); + File.WriteAllText(Path.Combine(inbox, id + ".yaml"), Writer.Serialize(item)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return OperationResult.Fail($"That submission could not be written: {ex.Message}"); + } + + Record("submit", submission.Tool ?? string.Empty, null, submission.By, submission.Run, $"{kind} {id}"); + + return OperationResult.Ok(new ToolSubmitted(id, overlapping)); + } + + /// + public OperationResult RecordUsage(ToolUsage usage) + { + ArgumentNullException.ThrowIfNull(usage); + + if (ReadHead(usage.Tool) is not { } head) + { + return OperationResult.Fail($"There is no tool called '{usage.Tool}'.", ExitCode.ProjectNotFound); + } + + if (!ToolOutcome.All.Contains(usage.Outcome)) + { + return OperationResult.Fail( + $"'{usage.Outcome}' is not an outcome: ok, failed or workaround.", ExitCode.InvalidArguments); + } + + if (SecretScanner.Match(usage.Note) is { Count: > 0 } secrets) + { + return OperationResult.Fail( + "That note looks like it contains a credential (" + string.Join(", ", secrets) + ").", + ExitCode.PolicyViolation); + } + + usage.At ??= _clock.GetUtcNow(); + + try + { + Directory.CreateDirectory(ToolDirectory(head.Name)); + File.AppendAllText(UsageFile(head.Name), JsonSerializer.Serialize(usage, Json) + "\n"); + + var all = Usage(head.Name); + head.UsageSummary = new ToolUsageSummary + { + Runs = all.Count, + Ok = all.Count(one => one.Outcome == ToolOutcome.Ok), + Failed = all.Count(one => one.Outcome == ToolOutcome.Failed), + Workaround = all.Count(one => one.Outcome == ToolOutcome.Workaround), + Teams = all.Select(one => one.Team).Where(one => one.Length > 0) + .Distinct(StringComparer.OrdinalIgnoreCase).Count(), + }; + + WriteHead(head); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return OperationResult.Fail($"That use could not be recorded: {ex.Message}"); + } + + Record("used", head.Name, usage.Version, usage.Team, usage.Run, usage.Outcome); + + return OperationResult.Ok(); + } + + /// + public IReadOnlyList Audit(string? tool = null, DateTimeOffset? since = null) => + [ + .. ToolAudit.Read(AuditFile).Where(one => + (tool is null || string.Equals(one.Tool, tool, StringComparison.OrdinalIgnoreCase)) + && (since is null || one.At >= since)), + ]; + + /// + public async Task> VerifyAsync( + string draft, + ToolTestConsent consent, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(consent); + + var read = ReadDraft(draft); + + if (read.Failed) + { + return OperationResult.Fail(read.Error!, ExitCode.InvalidArguments); + } + + var (version, script, scriptPath, cases) = read.Value!; + + if (ToolHarness.MissingClasses(cases) is { Count: > 0 } missing) + { + return OperationResult.Fail( + "A version needs a case of every class, and this has none of: " + string.Join(", ", missing) + ".", + ExitCode.InvalidArguments); + } + + var may = ToolHarness.May(version, script, cases, consent); + + if (may.Ruling != RemedyRuling.Run) + { + Record("verify", version.Name, version.Version, null, null, $"held: {may.Because}"); + + return OperationResult.Ok(new ToolVerification(may.Ruling, may.Because, null)); + } + + ToolVersion? active = null; + IReadOnlyList activeCases = []; + + if (ReadHead(version.Name) is { } head && ActiveVersion(head) is { } known) + { + active = known; + activeCases = ReadCases(Path.Combine(VersionDirectory(head.Name, known.Version), "cases")); + } + + var gate = await ToolPromotion.GateAsync( + version, + scriptPath, + cases, + active, + activeCases, + (path, one, token) => _harness.RunAsync(path, one, token), + ct).ConfigureAwait(false); + + version.Status = gate.Passed ? ToolVersionStatus.Verified : ToolVersionStatus.Rejected; + version.Tests = new ToolTestRecord + { + RanAt = _clock.GetUtcNow(), + Passed = gate.Own.Count(one => one.Passed) + gate.Regression.Count(one => one.Passed), + Failed = gate.Own.Count(one => !one.Passed) + gate.Regression.Count(one => !one.Passed), + RegressionAgainst = active is null ? [] : [active.Version], + Fingerprint = RemedyCeiling.Fingerprint(script), + CasesFingerprint = CasesFingerprint(Path.Combine(draft, "cases")), + }; + + File.WriteAllText(Path.Combine(draft, "manifest.yaml"), Writer.Serialize(version)); + Record(gate.Passed ? "verify" : "reject", version.Name, version.Version, null, null, gate.Because); + + return OperationResult.Ok(new ToolVerification(RemedyRuling.Run, gate.Because, gate)); + } + + /// + public OperationResult Promote(string draft, ToolPromotionRequest request) + { + ArgumentNullException.ThrowIfNull(request); + + var read = ReadDraft(draft); + + if (read.Failed) + { + return OperationResult.Fail(read.Error!, ExitCode.InvalidArguments); + } + + var (version, script, scriptPath, cases) = read.Value!; + + OperationResult Refuse(string why) => + OperationResult.Fail(why, ExitCode.PolicyViolation); + + if (!string.Equals(version.Name, RemedyBook.Slug(version.Name), StringComparison.Ordinal)) + { + return Refuse($"'{version.Name}' is not a tool name: lowercase and hyphens only."); + } + + if (!VersionShape().IsMatch(version.Version)) + { + return Refuse($"'{version.Version}' is not a version: major.minor."); + } + + // Written once. A version that has been promoted is what somebody + // trusted, and writing over it would spend that trust on something else. + if (Directory.Exists(VersionDirectory(version.Name, version.Version))) + { + return Refuse($"{version.Name}@{version.Version} has already been promoted, and a version is written once."); + } + + if (ToolPromotion.Missing(version) is { Count: > 0 } missing) + { + return Refuse("The manifest is missing " + string.Join(", ", missing) + "."); + } + + if (version.Status != ToolVersionStatus.Verified || version.Tests is not { } tests) + { + return Refuse($"{version.Name}@{version.Version} has not passed verify."); + } + + var casesFingerprint = CasesFingerprint(Path.Combine(draft, "cases")); + + if (!string.Equals(tests.Fingerprint, RemedyCeiling.Fingerprint(script), StringComparison.Ordinal) + || !string.Equals(tests.CasesFingerprint, casesFingerprint, StringComparison.Ordinal)) + { + return Refuse("The script or its cases have changed since they were verified. Verify again."); + } + + if (ToolGenericity.Check(version, script, cases, _known()) is { Count: > 0 } specific) + { + return Refuse("This carries a project: " + string.Join("; ", specific) + ". Make each of them an input."); + } + + var head = ReadHead(version.Name); + + if (head is not null && ActiveVersion(head) is { } previous + && ToolCompatibility.Refusal(previous, version) is { } broken) + { + return Refuse(broken); + } + + var shape = new ToolShape( + request.Capabilities ?? head?.Capabilities ?? [], + request.Summary ?? head?.Summary ?? version.Purpose, + script); + + foreach (var (record, active) in ActiveShapes()) + { + if (!string.Equals(record.Name, version.Name, StringComparison.Ordinal) + && !version.Compatibility.Replaces.Contains(record.Name, StringComparer.OrdinalIgnoreCase) + && ToolOverlap.Score(shape, active).Overlaps) + { + return Refuse( + $"This overlaps '{record.Name}'. Extend it, or name it in compatibility.replaces."); + } + } + + var target = VersionDirectory(version.Name, version.Version); + var file = $"{version.Name}.v{version.Version}.ps1"; + + try + { + Directory.CreateDirectory(Path.Combine(target, "cases")); + File.Copy(scriptPath, Path.Combine(target, file)); + + foreach (var one in Directory.EnumerateFiles(Path.Combine(draft, "cases"), "*.yaml")) + { + File.Copy(one, Path.Combine(target, "cases", Path.GetFileName(one))); + } + + version.Status = ToolVersionStatus.KnownGood; + version.Script = file; + version.Fingerprint = RemedyCeiling.Fingerprint(script); + version.CasesFingerprint = casesFingerprint; + File.WriteAllText(Path.Combine(target, "manifest.yaml"), Writer.Serialize(version)); + + head ??= new ToolRecord { Name = version.Name }; + head.Owner = request.Owner ?? head.Owner; + head.Kind = request.Kind ?? head.Kind; + head.Summary = request.Summary ?? (head.Summary.Length > 0 ? head.Summary : version.Purpose); + head.Capabilities = request.Capabilities?.ToList() ?? head.Capabilities; + head.KnownGood.Add(version.Version); + head.Active = version.Version; + + if (head.Lifecycle == ToolLifecycle.Candidate) + { + head.Lifecycle = ToolLifecycle.Active; + } + + head.Lineage.Add(new ToolLineage + { + Version = version.Version, + Because = request.Because, + Source = request.Source, + }); + + WriteHead(head); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return OperationResult.Fail($"That version could not be written: {ex.Message}"); + } + + Record("promote", version.Name, version.Version, request.Actor, request.Run, version.Fingerprint); + + return OperationResult.Ok(version); + } + + /// + public OperationResult SetActive(string name, string version) + { + if (ReadHead(name) is not { } head) + { + return OperationResult.Fail($"There is no tool called '{name}'.", ExitCode.ProjectNotFound); + } + + if (!head.KnownGood.Contains(version, StringComparer.Ordinal)) + { + return OperationResult.Fail( + $"{name}@{version} never passed the gate, so it cannot be active.", ExitCode.PolicyViolation); + } + + if (Standing(name, version) != ToolVersionStatus.KnownGood) + { + return OperationResult.Fail( + $"{name}@{version} no longer matches what was promoted, so it cannot be active.", + ExitCode.PolicyViolation); + } + + head.Active = version; + WriteHead(head); + Record("activate", name, version, null, null, null); + + return OperationResult.Ok(); + } + + /// + public OperationResult Deprecate(string name, string? replacement, string? reason) + { + if (ReadHead(name) is not { } head) + { + return OperationResult.Fail($"There is no tool called '{name}'.", ExitCode.ProjectNotFound); + } + + if (string.IsNullOrWhiteSpace(replacement) && string.IsNullOrWhiteSpace(reason)) + { + return OperationResult.Fail( + "A tool is deprecated with a replacement or a reason, so whoever used it knows what to do instead.", + ExitCode.InvalidArguments); + } + + if (replacement is { Length: > 0 } instead + && (string.Equals(instead, name, StringComparison.OrdinalIgnoreCase) + || ReadHead(instead) is not { Lifecycle: not ToolLifecycle.Retired })) + { + return OperationResult.Fail( + $"'{instead}' is not a tool anybody could use instead.", ExitCode.InvalidArguments); + } + + head.Lifecycle = ToolLifecycle.Deprecated; + head.Deprecated = new ToolDeprecation + { + Replacement = replacement ?? string.Empty, + Reason = reason ?? string.Empty, + At = _clock.GetUtcNow(), + }; + + WriteHead(head); + Record("deprecate", name, head.Active, null, null, replacement is { Length: > 0 } ? "replacement " + replacement : reason); + + return OperationResult.Ok(); + } + + /// + public OperationResult Retire(string name) + { + if (ReadHead(name) is not { } head) + { + return OperationResult.Fail($"There is no tool called '{name}'.", ExitCode.ProjectNotFound); + } + + if (head.Lifecycle != ToolLifecycle.Deprecated) + { + return OperationResult.Fail($"'{name}' is retired only once it is deprecated.", ExitCode.PolicyViolation); + } + + var since = _clock.GetUtcNow() - RetireAfter; + + if (Usage(name).Any(one => one.At >= since)) + { + return OperationResult.Fail( + $"'{name}' has been used in the last {RetireAfter.TotalDays:0} days.", ExitCode.PolicyViolation); + } + + head.Lifecycle = ToolLifecycle.Retired; + WriteHead(head); + Record("retire", name, head.Active, null, null, null); + + return OperationResult.Ok(); + } + + /// + /// + /// Recomputed on every read, because the files are on a disk a node with + /// Bash can write to. A version that no longer matches is refused, never + /// trusted. + /// + public string Standing(string name, string version) + { + var directory = VersionDirectory(name, version); + var manifest = ReadYaml(Path.Combine(directory, "manifest.yaml")); + + if (manifest is null || manifest.Script is not { Length: > 0 }) + { + return "missing"; + } + + var script = Path.Combine(directory, manifest.Script); + + if (!File.Exists(script) + || !string.Equals(RemedyCeiling.Fingerprint(File.ReadAllText(script)), manifest.Fingerprint, StringComparison.OrdinalIgnoreCase) + || !string.Equals(CasesFingerprint(Path.Combine(directory, "cases")), manifest.CasesFingerprint, StringComparison.OrdinalIgnoreCase)) + { + return ToolVersionStatus.Tampered; + } + + return ToolVersionStatus.KnownGood; + } + + /// + public OperationResult StandDown(string name, string reason, string? actor = null, string? run = null) + { + if (ReadHead(name) is null) + { + return OperationResult.Fail($"There is no tool called '{name}'.", ExitCode.ProjectNotFound); + } + + Record("stand-down", name, null, actor, run, reason); + + return OperationResult.Ok(); + } + + /// + /// + /// Two stand-downs in a row with nothing new between them mean the last two + /// looks found nothing worth changing, and a third would find the same. + /// Something new - a promotion, a submission about the tool, a use that + /// failed or was worked around - starts the count again. + /// + public bool NeedsRefining(string name) + { + if (ReadHead(name) is not { } head || head.Lifecycle == ToolLifecycle.Retired) + { + return false; + } + + var standDowns = 0; + + foreach (var entry in Audit(head.Name).Reverse()) + { + if (entry.Action == "stand-down") + { + standDowns++; + } + else if (IsSignal(entry)) + { + break; + } + } + + return standDowns < 2; + } + + private static bool IsSignal(ToolAuditEntry entry) => + entry.Action is "promote" or "submit" + || (entry.Action == "used" && entry.Note is ToolOutcome.Failed or ToolOutcome.Workaround); + + private ToolVersion? ActiveVersion(ToolRecord head) + { + if (head.Active is not { Length: > 0 } active + || !head.KnownGood.Contains(active, StringComparer.Ordinal) + || Standing(head.Name, active) != ToolVersionStatus.KnownGood) + { + return null; + } + + return ReadYaml(Path.Combine(VersionDirectory(head.Name, active), "manifest.yaml")); + } + + private IEnumerable<(ToolRecord Record, ToolShape Shape)> ActiveShapes() + { + foreach (var head in Heads().Where(one => one.Lifecycle == ToolLifecycle.Active)) + { + if (ActiveVersion(head) is { } version) + { + var script = Path.Combine(VersionDirectory(head.Name, version.Version), version.Script); + + yield return (head, new ToolShape(head.Capabilities, head.Summary, File.ReadAllText(script))); + } + } + } + + private IEnumerable Heads() + { + if (!Directory.Exists(Root())) + { + yield break; + } + + foreach (var directory in Directory.EnumerateDirectories(Root()).OrderBy(one => one, StringComparer.Ordinal)) + { + if (ReadYaml(Path.Combine(directory, "tool.yaml")) is { } head) + { + yield return head; + } + } + } + + private ToolRecord? ReadHead(string name) => ReadYaml(HeadFile(name)); + + private void WriteHead(ToolRecord head) + { + Directory.CreateDirectory(ToolDirectory(head.Name)); + File.WriteAllText(HeadFile(head.Name), Writer.Serialize(head)); + } + + private List Usage(string name) + { + var file = UsageFile(name); + + if (!File.Exists(file)) + { + return []; + } + + var all = new List(); + + foreach (var line in File.ReadLines(file).Where(one => one.Length > 0)) + { + try + { + if (JsonSerializer.Deserialize(line, Json) is { } usage) + { + all.Add(usage); + } + } + catch (JsonException) + { + // A damaged line is one lost use, not a lost log. + } + } + + return all; + } + + private void Record(string action, string tool, string? version, string? actor, string? run, string? note) => + ToolAudit.Append(AuditFile, new ToolAuditEntry(_clock.GetUtcNow(), action, tool, version, actor, run, note)); + + private static OperationResult<(ToolVersion Version, string Script, string ScriptPath, IReadOnlyList Cases)> ReadDraft(string draft) + { + if (ReadYaml(Path.Combine(draft, "manifest.yaml")) is not { } version) + { + return OperationResult<(ToolVersion, string, string, IReadOnlyList)>.Fail( + $"There is no readable manifest.yaml in {draft}."); + } + + var scriptPath = Path.GetFullPath(Path.Combine(draft, version.Script)); + + if (version.Script is not { Length: > 0 } + || !scriptPath.StartsWith(Path.GetFullPath(draft) + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase) + || !File.Exists(scriptPath)) + { + return OperationResult<(ToolVersion, string, string, IReadOnlyList)>.Fail( + $"The manifest names no script beside it in {draft}."); + } + + return OperationResult<(ToolVersion, string, string, IReadOnlyList)>.Ok( + (version, File.ReadAllText(scriptPath), scriptPath, ReadCases(Path.Combine(draft, "cases")))); + } + + private static List ReadCases(string directory) + { + if (!Directory.Exists(directory)) + { + return []; + } + + return + [ + .. Directory.EnumerateFiles(directory, "*.yaml") + .OrderBy(one => one, StringComparer.Ordinal) + .Select(ReadYaml) + .OfType(), + ]; + } + + /// The fingerprint over a directory of cases, in name order. + internal static string CasesFingerprint(string directory) + { + if (!Directory.Exists(directory)) + { + return RemedyCeiling.Fingerprint(string.Empty); + } + + var text = string.Concat( + Directory.EnumerateFiles(directory, "*.yaml") + .OrderBy(one => Path.GetFileName(one), StringComparer.Ordinal) + .Select(one => Path.GetFileName(one) + "\n" + File.ReadAllText(one) + "\n")); + + return RemedyCeiling.Fingerprint(text); + } + + private static T? ReadYaml(string file) + where T : class + { + try + { + return File.Exists(file) ? Yaml.Deserialize(File.ReadAllText(file)) : null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or YamlDotNet.Core.YamlException) + { + return null; + } + } + + private static int Score(ToolRecord record, List terms) + { + var haystack = (record.Name + " " + record.Summary + " " + string.Join(' ', record.Capabilities)).ToLowerInvariant(); + + return terms.Count(haystack.Contains); + } + + private IEnumerable KnownOnThisMachine() + { + var places = new[] + { + Path.Combine(_paths.Paths.State, "teams", "work"), + Path.Combine(_paths.Paths.WorkspaceClone, "projects"), + }; + + return places.Where(Directory.Exists) + .SelectMany(one => Directory.EnumerateDirectories(one)) + .Select(one => Path.GetFileName(one)); + } + + [GeneratedRegex(@"[A-Za-z0-9-]+", RegexOptions.None, 1000)] + private static partial Regex Word(); + + [GeneratedRegex(@"^\d+\.\d+$", RegexOptions.None, 1000)] + private static partial Regex VersionShape(); +} diff --git a/src/Loadout.Models/Tools/ToolCase.cs b/src/Loadout.Models/Tools/ToolCase.cs new file mode 100644 index 00000000..2065fc8e --- /dev/null +++ b/src/Loadout.Models/Tools/ToolCase.cs @@ -0,0 +1,68 @@ +namespace Loadout.Models.Tools; + +/// One harness case, cases/<name>.yaml. +public sealed class ToolCase +{ + /// What it is called, which is how a migration names it. + public string Name { get; set; } = string.Empty; + + /// One of . + public string Class { get; set; } = string.Empty; + + /// + /// The tool's inputs by name. Placeholders only for paths: {tmp} + /// is the case's own fresh directory. + /// + public Dictionary Args { get; set; } = []; + + /// What to create before the run. + public ToolCaseSetup Setup { get; set; } = new(); + + /// What must be true afterwards. + public ToolCaseExpect Expect { get; set; } = new(); +} + +/// The four classes a version must cover. +public static class ToolCaseClass +{ + /// It does its job. + public const string Success = "success"; + + /// It cannot, and says so. + public const string Failure = "failure"; + + /// A boundary: empty, huge, already done. + public const string Edge = "edge"; + + /// It is given something it must refuse. + public const string InvalidInput = "invalid-input"; + + /// All four, in the order they are documented. + public static IReadOnlyList All { get; } = [Success, Failure, Edge, InvalidInput]; +} + +/// What a case creates first. +public sealed class ToolCaseSetup +{ + /// Files by path under {tmp}, and their content. + public Dictionary Files { get; set; } = []; +} + +/// What a case checks. +public sealed class ToolCaseExpect +{ + /// The exit code, or null for any. + public int? Exit { get; set; } + + /// A regular expression standard output must match, or empty. + public string StdoutMatches { get; set; } = string.Empty; + + /// A regular expression standard error must match, or empty. + public string StderrMatches { get; set; } = string.Empty; + + /// Paths that must not exist afterwards. + public List FilesAbsent { get; set; } = []; + + /// Paths that must exist afterwards. + public List FilesPresent { get; set; } = []; +} diff --git a/src/Loadout.Models/Tools/ToolRecord.cs b/src/Loadout.Models/Tools/ToolRecord.cs new file mode 100644 index 00000000..f4a4f282 --- /dev/null +++ b/src/Loadout.Models/Tools/ToolRecord.cs @@ -0,0 +1,113 @@ +namespace Loadout.Models.Tools; + +/// +/// One tool in the machine's shared catalogue: the head, kept in +/// tool.yaml, which is the only part of a tool that changes after it is +/// written. +/// +/// +/// +/// Mutable with settable properties because it deserialises from YAML, like a +/// remedy record. What runs is never decided here: the versions under it are +/// written once, and trust lives in this machine's configuration. +/// +/// +public sealed class ToolRecord +{ + /// The name it is searched and asked for by. Lowercase, hyphenated, unique. + public string Name { get; set; } = string.Empty; + + /// The team that maintains it. + public string Owner { get; set; } = string.Empty; + + /// The sort of task, which is what a machine sets a rule against. + public string Kind { get; set; } = string.Empty; + + /// What it does, in a sentence somebody searching would recognise. + public string Summary { get; set; } = string.Empty; + + /// Search terms, and half of what overlap is measured on. + public List Capabilities { get; set; } = []; + + /// Where it is in its life. One of . + public string Lifecycle { get; set; } = ToolLifecycle.Candidate; + + /// + /// The version in use, which is only ever one in + /// whose files still match what was promoted. + /// + public string? Active { get; set; } + + /// Every version that passed the gate. Never removed from. + public List KnownGood { get; set; } = []; + + /// Present only when the tool has been deprecated. + public ToolDeprecation? Deprecated { get; set; } + + /// Every version, and what caused it. + public List Lineage { get; set; } = []; + + /// Recomputed from the usage log; never edited by hand. + public ToolUsageSummary UsageSummary { get; set; } = new(); +} + +/// The states a tool passes through. +public static class ToolLifecycle +{ + /// Submitted, never promoted. + public const string Candidate = "candidate"; + + /// Has a known-good version in use. + public const string Active = "active"; + + /// Still usable, with a replacement or a reason not to. + public const string Deprecated = "deprecated"; + + /// No longer offered. Nothing on disk is removed. + public const string Retired = "retired"; +} + +/// Why a tool is deprecated. +public sealed class ToolDeprecation +{ + /// The tool to use instead, or empty. + public string Replacement { get; set; } = string.Empty; + + /// Why, where there is no replacement or as well as one. + public string Reason { get; set; } = string.Empty; + + /// When. + public DateTimeOffset? At { get; set; } +} + +/// What caused one version. +public sealed class ToolLineage +{ + /// The version. + public string Version { get; set; } = string.Empty; + + /// lesson, requirement, bug, idea, nomination or consolidation. + public string Because { get; set; } = string.Empty; + + /// The submission or run it came from. + public string Source { get; set; } = string.Empty; +} + +/// How a tool has fared, counted from its usage log. +public sealed class ToolUsageSummary +{ + /// Every recorded use. + public int Runs { get; set; } + + /// Uses that worked. + public int Ok { get; set; } + + /// Uses that failed. + public int Failed { get; set; } + + /// Uses that needed working around. + public int Workaround { get; set; } + + /// Distinct teams, counted rather than named. + public int Teams { get; set; } +} diff --git a/src/Loadout.Models/Tools/ToolUsage.cs b/src/Loadout.Models/Tools/ToolUsage.cs new file mode 100644 index 00000000..b5619c0e --- /dev/null +++ b/src/Loadout.Models/Tools/ToolUsage.cs @@ -0,0 +1,42 @@ +namespace Loadout.Models.Tools; + +/// One recorded use of a tool, a line of usage.jsonl. +public sealed class ToolUsage +{ + /// The tool. + public string Tool { get; set; } = string.Empty; + + /// The version used. + public string Version { get; set; } = string.Empty; + + /// One of . + public string Outcome { get; set; } = ToolOutcome.Ok; + + /// The team that used it, counted but never shown. + public string Team { get; set; } = string.Empty; + + /// The run it was used in, where there was one. + public string Run { get; set; } = string.Empty; + + /// What happened, in the user's words. + public string Note { get; set; } = string.Empty; + + /// When. + public DateTimeOffset? At { get; set; } +} + +/// How a use went. +public static class ToolOutcome +{ + /// It worked. + public const string Ok = "ok"; + + /// It did not. + public const string Failed = "failed"; + + /// It worked once something was done around it. + public const string Workaround = "workaround"; + + /// Every outcome. + public static IReadOnlyList All { get; } = [Ok, Failed, Workaround]; +} diff --git a/src/Loadout.Models/Tools/ToolVersion.cs b/src/Loadout.Models/Tools/ToolVersion.cs new file mode 100644 index 00000000..5becec9b --- /dev/null +++ b/src/Loadout.Models/Tools/ToolVersion.cs @@ -0,0 +1,163 @@ +namespace Loadout.Models.Tools; + +/// +/// One version of a tool: its manifest, versions/<v>/manifest.yaml, +/// written once when it is promoted and never again. +/// +/// +/// The same shape serves a draft, which is freely rewritten until it is +/// promoted. What makes a version immutable is the store refusing to write +/// its directory twice, not anything on this class. +/// +public sealed class ToolVersion +{ + /// The tool this is a version of. + public string Name { get; set; } = string.Empty; + + /// major.minor. + public string Version { get; set; } = string.Empty; + + /// One of . + public string Status { get; set; } = ToolVersionStatus.Draft; + + /// The script's file name, beside this manifest. + public string Script { get; set; } = string.Empty; + + /// The script's fingerprint, as a remedy's is taken. + public string Fingerprint { get; set; } = string.Empty; + + /// The fingerprint over the cases, in name order. + public string CasesFingerprint { get; set; } = string.Empty; + + /// What it is for, in one sentence naming no project. + public string Purpose { get; set; } = string.Empty; + + /// What it takes. + public List Inputs { get; set; } = []; + + /// What it gives back. + public ToolOutputs Outputs { get; set; } = new(); + + /// What must be present for it to run. + public List Dependencies { get; set; } = []; + + /// What it will never do. + public List Constraints { get; set; } = []; + + /// What it does when it cannot do its job. + public string ErrorBehaviour { get; set; } = string.Empty; + + /// How it is called, and what to expect. + public List Examples { get; set; } = []; + + /// The problem it was made to solve, described without the project. + public string Origin { get; set; } = string.Empty; + + /// Where it runs, and whether it breaks what came before. + public ToolCompatibilityInfo Compatibility { get; set; } = new(); + + /// What the gate found. Written by the gate, not by an agent. + public ToolTestRecord? Tests { get; set; } +} + +/// The states a version passes through. +public static class ToolVersionStatus +{ + /// Being written. + public const string Draft = "draft"; + + /// Its harness and the regression gate passed, against the fingerprints recorded. + public const string Verified = "verified"; + + /// Failed the harness or the regression gate. Never promoted. + public const string Rejected = "rejected"; + + /// Promoted. + public const string KnownGood = "known-good"; + + /// Promoted, but its files no longer match. Shown, never active. + public const string Tampered = "tampered"; +} + +/// One input a tool takes. +public sealed class ToolInput +{ + /// The parameter name. + public string Name { get; set; } = string.Empty; + + /// path, int, string, and so on. + public string Type { get; set; } = string.Empty; + + /// Whether a caller must give it. + public bool Required { get; set; } + + /// What it is when not given, or null for nothing. + public string? Default { get; set; } + + /// What it means. + public string Describe { get; set; } = string.Empty; +} + +/// What a tool gives back. +public sealed class ToolOutputs +{ + /// What it writes to standard output. + public string Stdout { get; set; } = string.Empty; + + /// Each exit code and what it means. + public Dictionary Exit { get; set; } = []; +} + +/// One example call. +public sealed class ToolExample +{ + /// The command. + public string Command { get; set; } = string.Empty; + + /// What it should do. + public string Expect { get; set; } = string.Empty; +} + +/// Where a version runs, and what it breaks. +public sealed class ToolCompatibilityInfo +{ + /// windows, linux, macos. + public List Platforms { get; set; } = []; + + /// The interpreter it needs. + public string Shell { get; set; } = string.Empty; + + /// Whether it deliberately breaks the previous version's callers. + public bool Breaks { get; set; } + + /// What a caller of the previous version has to change. + public string Migration { get; set; } = string.Empty; + + /// Previous known-good cases this version deliberately no longer passes. + public List RetiredCases { get; set; } = []; + + /// Tools or versions this one consolidates. + public List Replaces { get; set; } = []; +} + +/// What the gate found when it last ran. +public sealed class ToolTestRecord +{ + /// When. + public DateTimeOffset? RanAt { get; set; } + + /// Cases that passed, the version's own and the regression set. + public int Passed { get; set; } + + /// Cases that failed. + public int Failed { get; set; } + + /// The known-good versions whose cases were rerun. + public List RegressionAgainst { get; set; } = []; + + /// The script's fingerprint the run was made against. + public string Fingerprint { get; set; } = string.Empty; + + /// The cases' fingerprint the run was made against. + public string CasesFingerprint { get; set; } = string.Empty; +} diff --git a/tests/Loadout.Tests/Unit/ToolCompatibilityTests.cs b/tests/Loadout.Tests/Unit/ToolCompatibilityTests.cs new file mode 100644 index 00000000..97557ac9 --- /dev/null +++ b/tests/Loadout.Tests/Unit/ToolCompatibilityTests.cs @@ -0,0 +1,34 @@ +using FluentAssertions; +using Loadout.Core.Tools; +using Loadout.Models.Tools; +using Xunit; + +namespace Loadout.Tests.Unit; + +/// Whether a version breaks the callers of the one before it. +public sealed class ToolCompatibilityTests +{ + [Fact] + public void Removing_a_required_input_is_breaking() + { + var previous = ToolStoreFixture.Manifest("free-cache", "1.0"); + var next = ToolStoreFixture.Manifest("free-cache", "1.1"); + next.Inputs = [new ToolInput { Name = "Path", Type = "path", Required = true, Default = "." }]; + + ToolCompatibility.Breaks(previous, next).Should().Contain(one => one.Contains("CachePath", StringComparison.Ordinal)); + ToolCompatibility.Refusal(previous, next).Should().NotBeNull(); + + next.Version = "2.0"; + next.Compatibility = new ToolCompatibilityInfo { Breaks = true, Migration = "Pass -Path instead of -CachePath." }; + + ToolCompatibility.Refusal(previous, next).Should().BeNull(); + } + + [Fact] + public void An_unchanged_interface_is_not_breaking() + { + ToolCompatibility.Breaks( + ToolStoreFixture.Manifest("free-cache", "1.0"), + ToolStoreFixture.Manifest("free-cache", "1.1")).Should().BeEmpty(); + } +} diff --git a/tests/Loadout.Tests/Unit/ToolGenericityTests.cs b/tests/Loadout.Tests/Unit/ToolGenericityTests.cs new file mode 100644 index 00000000..64fd91e8 --- /dev/null +++ b/tests/Loadout.Tests/Unit/ToolGenericityTests.cs @@ -0,0 +1,43 @@ +using FluentAssertions; +using Loadout.Core.Tools; +using Xunit; + +namespace Loadout.Tests.Unit; + +/// What ties a tool to the project it was written for. +public sealed class ToolGenericityTests +{ + private static readonly string[] Known = ["alpha-shop", "system-watch"]; + + [Theory] + [InlineData(@"Remove-Item D:\alpha\build\cache -Recurse", "an absolute path")] + [InlineData("Get-ChildItem /home/someone/cache", "an absolute path")] + [InlineData("Clears the cache for alpha-shop builds.", "a project or team name")] + [InlineData("Learned by system-watch last week.", "a project or team name")] + [InlineData("git clone https://github.com/someone/thing", "a repository URL")] + [InlineData("Ask nobody@example.com first.", "an e-mail address")] + [InlineData("Subscription 3f2504e0-4f89-11d3-9a0c-0305e82c3301", "a GUID")] + public void Project_detail_is_refused(string text, string expected) + { + ToolGenericity.Check(text, Known).Should().Contain(one => one.StartsWith(expected, StringComparison.Ordinal)); + } + + [Fact] + public void A_secret_is_reported_by_type_and_never_by_value() + { + var token = "ghp_" + new string('a', 36); + + var found = ToolGenericity.Check("$token = '" + token + "'", Known); + + found.Should().Contain(one => one.Contains("GitHub token", StringComparison.Ordinal)); + found.Should().NotContain(one => one.Contains(token, StringComparison.Ordinal)); + } + + [Fact] + public void Generic_text_passes() + { + ToolGenericity.Check( + "Clears {tmp}/cache of files older than OlderThanDays, as an alpha-shopfront would. Run: pwsh -File tool.ps1 -CachePath ./cache", + Known).Should().BeEmpty(); + } +} diff --git a/tests/Loadout.Tests/Unit/ToolHarnessTests.cs b/tests/Loadout.Tests/Unit/ToolHarnessTests.cs new file mode 100644 index 00000000..94eca2fa --- /dev/null +++ b/tests/Loadout.Tests/Unit/ToolHarnessTests.cs @@ -0,0 +1,71 @@ +using FluentAssertions; +using Loadout.Core.Tools; +using Loadout.Models.Teams; +using Loadout.Models.Tools; +using Loadout.Tests.Fakes; +using Xunit; + +namespace Loadout.Tests.Unit; + +/// +/// The harness, through so that no test +/// here needs pwsh on the machine. +/// +public sealed class ToolHarnessTests : IDisposable +{ + private const string Script = "param([string]$CachePath)\nGet-ChildItem $CachePath\n"; + + private readonly ToolStoreFixture _store = new(); + + public void Dispose() => _store.Dispose(); + + [Fact] + public async Task Verify_refuses_without_all_four_case_classes() + { + var (registry, launcher) = _store.Registry(); + var manifest = ToolStoreFixture.Manifest("free-cache", "1.0"); + var cases = ToolStoreFixture.Cases().Where(one => one.Class != ToolCaseClass.Edge).ToList(); + var draft = _store.Draft(manifest, Script, cases); + + var verified = await registry.VerifyAsync(draft, ToolStoreFixture.Agreed(manifest, Script, cases)); + + verified.Failed.Should().BeTrue(); + verified.Error.Should().Contain(ToolCaseClass.Edge); + launcher.Requests.Should().BeEmpty(); + } + + [Fact] + public async Task Cases_run_in_a_fresh_temp_directory() + { + var launcher = new StubProcessLauncher(string.Empty); + var harness = new ToolHarness(launcher); + var cases = new List + { + new() { Name = "a", Class = ToolCaseClass.Success, Args = new() { ["CachePath"] = "{tmp}/cache" } }, + new() { Name = "b", Class = ToolCaseClass.Success, Args = new() { ["CachePath"] = "{tmp}/cache" } }, + }; + + var results = await harness.RunAllAsync("tool.ps1", cases); + + results.Should().OnlyContain(one => one.Passed); + var directories = launcher.Requests.Select(one => one.WorkingDirectory).ToList(); + directories.Should().HaveCount(2).And.OnlyHaveUniqueItems().And.NotContainNulls(); + launcher.Requests[0].Arguments.Should().Contain(directories[0] + "/cache"); + directories.Should().OnlyContain(one => !Directory.Exists(one)); + } + + [Fact] + public async Task Running_is_held_for_a_person_by_default() + { + var (registry, launcher) = _store.Registry(); + var manifest = ToolStoreFixture.Manifest("free-cache", "1.0"); + var cases = ToolStoreFixture.Cases(); + var draft = _store.Draft(manifest, Script, cases); + + var verified = await registry.VerifyAsync(draft, new ToolTestConsent(null, null)); + + verified.Value!.Ruling.Should().Be(RemedyRuling.Ask); + verified.Value.Gate.Should().BeNull(); + launcher.Requests.Should().BeEmpty(); + } +} diff --git a/tests/Loadout.Tests/Unit/ToolOverlapTests.cs b/tests/Loadout.Tests/Unit/ToolOverlapTests.cs new file mode 100644 index 00000000..637bb318 --- /dev/null +++ b/tests/Loadout.Tests/Unit/ToolOverlapTests.cs @@ -0,0 +1,55 @@ +using FluentAssertions; +using Loadout.Core.Tools; +using Xunit; + +namespace Loadout.Tests.Unit; + +/// Whether a new tool is one the catalogue already has. +public sealed class ToolOverlapTests +{ + private const string Clear = + "param([string]$CachePath, [int]$OlderThanDays = 7)\n" + + "$cutoff = (Get-Date).AddDays(-$OlderThanDays)\n" + + "Get-ChildItem -Path $CachePath -Recurse -File | Where-Object { $_.LastWriteTime -lt $cutoff } | Remove-Item -Force\n" + + "Write-Output 'freed'\n"; + + [Fact] + public void Identical_scripts_are_duplicates() + { + var a = new ToolShape(["cache"], "Clears a cache.", Clear); + var b = new ToolShape(["disk"], "Something else entirely.", "# a comment\n" + Clear.Replace("\n", "\r\n ", StringComparison.Ordinal)); + + var score = ToolOverlap.Score(a, b); + + score.Duplicate.Should().BeTrue(); + score.Overlaps.Should().BeTrue(); + } + + [Fact] + public void Threshold_flags_overlap() + { + var a = new ToolShape(["cache", "disk", "cleanup"], "Clears old files from a cache directory.", Clear); + var b = new ToolShape( + ["cache", "disk", "cleanup"], + "Clears old files from a cache directory.", + Clear.Replace("Write-Output 'freed'", "Write-Output 'done'", StringComparison.Ordinal)); + + var score = ToolOverlap.Score(a, b); + + score.Duplicate.Should().BeFalse(); + score.Score.Should().BeGreaterThanOrEqualTo(ToolOverlap.Threshold); + score.Overlaps.Should().BeTrue(); + } + + [Fact] + public void Unrelated_tools_do_not() + { + var a = new ToolShape(["cache", "disk"], "Clears old files from a cache directory.", Clear); + var b = new ToolShape( + ["service", "restart"], + "Restarts a stopped service and waits for it.", + "param([string]$Name)\nRestart-Service -Name $Name\nStart-Sleep -Seconds 5\nGet-Service $Name\n"); + + ToolOverlap.Score(a, b).Overlaps.Should().BeFalse(); + } +} diff --git a/tests/Loadout.Tests/Unit/ToolPromotionTests.cs b/tests/Loadout.Tests/Unit/ToolPromotionTests.cs new file mode 100644 index 00000000..092bdeaa --- /dev/null +++ b/tests/Loadout.Tests/Unit/ToolPromotionTests.cs @@ -0,0 +1,73 @@ +using FluentAssertions; +using Loadout.Core.Tools; +using Loadout.Models.Tools; +using Xunit; + +namespace Loadout.Tests.Unit; + +/// +/// The regression gate: a failed refinement cannot replace what worked. +/// +public sealed class ToolPromotionTests : IDisposable +{ + private const string Script = "param([string]$CachePath)\nGet-ChildItem $CachePath | Remove-Item\n"; + + private readonly ToolStoreFixture _store = new(); + + public void Dispose() => _store.Dispose(); + + [Fact] + public async Task A_draft_failing_a_previous_known_good_case_is_rejected_and_active_does_not_move() + { + var (first, _) = _store.Registry(exit: 0); + await _store.PromoteAsync(first, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases(0, "v1-")); + + // The refinement exits 3 everywhere. Its own cases say that is right; + // the known-good cases say 0. + var (second, _) = _store.Registry(exit: 3); + var next = ToolStoreFixture.Manifest("free-cache", "1.1"); + var cases = ToolStoreFixture.Cases(3, "v11-"); + var changed = Script + "exit 3\n"; + var draft = _store.Draft(next, changed, cases); + + var verified = await second.VerifyAsync(draft, ToolStoreFixture.Agreed(next, changed, cases)); + + verified.Value!.Gate!.Passed.Should().BeFalse(); + verified.Value.Gate.Regressions.Should().Contain("v1-success"); + second.Promote(draft, new("idea", "inbox/refine")).Failed.Should().BeTrue(); + second.Show("free-cache").Value!.Record.Active.Should().Be("1.0"); + Directory.Exists(Path.Combine(second.Root(), "free-cache", "versions", "1.1")).Should().BeFalse(); + } + + [Fact] + public async Task A_declared_break_with_migration_may_retire_a_case() + { + var active = ToolStoreFixture.Manifest("free-cache", "1.0"); + var old = new List { new() { Name = "old-flag", Class = ToolCaseClass.Success } }; + + Task Run(string script, ToolCase one, CancellationToken ct) => + Task.FromResult(new ToolCaseResult(one.Name, one.Class, one.Name != "old-flag", string.Empty)); + + var declared = ToolStoreFixture.Manifest("free-cache", "2.0"); + declared.Compatibility = new ToolCompatibilityInfo + { + Breaks = true, + Migration = "Pass -CachePath instead of -Path.", + RetiredCases = ["old-flag"], + }; + + var allowed = await ToolPromotion.GateAsync(declared, "next.ps1", ToolStoreFixture.Cases(), active, old, Run); + + allowed.Passed.Should().BeTrue(); + allowed.Retired.Should().Equal("old-flag"); + + // The same claim without a major bump is not a declared break. + var minor = ToolStoreFixture.Manifest("free-cache", "1.1"); + minor.Compatibility = declared.Compatibility; + + var refused = await ToolPromotion.GateAsync(minor, "next.ps1", ToolStoreFixture.Cases(), active, old, Run); + + refused.Passed.Should().BeFalse(); + refused.Regressions.Should().Equal("old-flag"); + } +} diff --git a/tests/Loadout.Tests/Unit/ToolRegistryTests.cs b/tests/Loadout.Tests/Unit/ToolRegistryTests.cs new file mode 100644 index 00000000..34d76368 --- /dev/null +++ b/tests/Loadout.Tests/Unit/ToolRegistryTests.cs @@ -0,0 +1,130 @@ +using FluentAssertions; +using Loadout.Models.Tools; +using Xunit; + +namespace Loadout.Tests.Unit; + +/// +/// The rules the catalogue enforces itself rather than trusting whoever writes +/// its files: written once, active only when known-good, and never active once +/// the files stop matching. +/// +public sealed class ToolRegistryTests : IDisposable +{ + private const string Script = "param([string]$CachePath)\nGet-ChildItem $CachePath | Remove-Item\n"; + + private readonly ToolStoreFixture _store = new(); + + public void Dispose() => _store.Dispose(); + + [Fact] + public async Task Promote_refuses_an_existing_version_directory() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + + var again = ToolStoreFixture.Manifest("free-cache", "1.0"); + var cases = ToolStoreFixture.Cases(); + var draft = _store.Draft(again, Script + "# changed\n", cases); + (await registry.VerifyAsync(draft, ToolStoreFixture.Agreed(again, Script + "# changed\n", cases))).Succeeded.Should().BeTrue(); + + var promoted = registry.Promote(draft, new("bug", "inbox/again")); + + promoted.Failed.Should().BeTrue(); + promoted.Error.Should().Contain("written once"); + } + + [Fact] + public async Task SetActive_refuses_a_version_not_known_good() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + + // A version directory that matches its own manifest but never went + // through the gate: somebody copied it in by hand. + var versions = Path.Combine(registry.Root(), "free-cache", "versions"); + Copy(Path.Combine(versions, "1.0"), Path.Combine(versions, "1.1")); + + var set = registry.SetActive("free-cache", "1.1"); + + set.Failed.Should().BeTrue(); + registry.Show("free-cache").Value!.Record.Active.Should().Be("1.0"); + } + + [Fact] + public async Task A_tampered_version_is_never_active() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + registry.Show("free-cache").Value!.Active.Should().NotBeNull(); + + var script = Path.Combine(registry.Root(), "free-cache", "versions", "1.0", "free-cache.v1.0.ps1"); + File.AppendAllText(script, "Remove-Item -Recurse /\n"); + + var shown = registry.Show("free-cache").Value!; + + shown.Active.Should().BeNull(); + shown.Versions["1.0"].Should().Be(ToolVersionStatus.Tampered); + registry.SetActive("free-cache", "1.0").Failed.Should().BeTrue(); + } + + [Theory] + [InlineData("purpose")] + [InlineData("inputs")] + [InlineData("outputs")] + [InlineData("dependencies")] + [InlineData("constraints")] + [InlineData("error_behaviour")] + [InlineData("examples")] + [InlineData("origin")] + public async Task Promote_refuses_a_manifest_missing_a_field(string field) + { + var (registry, _) = _store.Registry(); + var manifest = ToolStoreFixture.Manifest("free-cache", "1.0"); + + switch (field) + { + case "purpose": manifest.Purpose = string.Empty; break; + case "inputs": manifest.Inputs = []; break; + case "outputs": manifest.Outputs = new ToolOutputs(); break; + case "dependencies": manifest.Dependencies = []; break; + case "constraints": manifest.Constraints = []; break; + case "error_behaviour": manifest.ErrorBehaviour = string.Empty; break; + case "examples": manifest.Examples = []; break; + case "origin": manifest.Origin = string.Empty; break; + } + + var cases = ToolStoreFixture.Cases(); + var draft = _store.Draft(manifest, Script, cases); + await registry.VerifyAsync(draft, ToolStoreFixture.Agreed(manifest, Script, cases)); + + var promoted = registry.Promote(draft, new("lesson", "inbox/test")); + + promoted.Failed.Should().BeTrue(); + promoted.Error.Should().Contain(field); + Directory.Exists(Path.Combine(registry.Root(), "free-cache", "versions", "1.0")).Should().BeFalse(); + } + + [Fact] + public async Task Deprecate_needs_a_replacement_or_a_reason() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + + registry.Deprecate("free-cache", null, " ").Failed.Should().BeTrue(); + registry.Show("free-cache").Value!.Record.Lifecycle.Should().Be(ToolLifecycle.Active); + + registry.Deprecate("free-cache", null, "Nothing uses a cache like this any more.").Succeeded.Should().BeTrue(); + registry.Show("free-cache").Value!.Record.Lifecycle.Should().Be(ToolLifecycle.Deprecated); + } + + private static void Copy(string from, string to) + { + foreach (var file in Directory.EnumerateFiles(from, "*", SearchOption.AllDirectories)) + { + var target = Path.Combine(to, Path.GetRelativePath(from, file)); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + File.Copy(file, target); + } + } +} diff --git a/tests/Loadout.Tests/Unit/ToolStopRuleTests.cs b/tests/Loadout.Tests/Unit/ToolStopRuleTests.cs new file mode 100644 index 00000000..d963e8b1 --- /dev/null +++ b/tests/Loadout.Tests/Unit/ToolStopRuleTests.cs @@ -0,0 +1,35 @@ +using FluentAssertions; +using Loadout.Models.Tools; +using Xunit; + +namespace Loadout.Tests.Unit; + +/// When the Refiner stops looking at a tool. +public sealed class ToolStopRuleTests : IDisposable +{ + private const string Script = "param([string]$CachePath)\nGet-ChildItem $CachePath | Remove-Item\n"; + + private readonly ToolStoreFixture _store = new(); + + public void Dispose() => _store.Dispose(); + + [Fact] + public async Task Two_stand_downs_without_new_signal_skip_the_tool() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + + registry.NeedsRefining("free-cache").Should().BeTrue(); + + registry.StandDown("free-cache", "Nothing measurably better."); + registry.NeedsRefining("free-cache").Should().BeTrue(); + + // An ordinary successful use is not something new to refine against. + registry.RecordUsage(new ToolUsage { Tool = "free-cache", Version = "1.0", Outcome = ToolOutcome.Ok, Team = "t" }); + registry.StandDown("free-cache", "Still nothing."); + registry.NeedsRefining("free-cache").Should().BeFalse(); + + registry.RecordUsage(new ToolUsage { Tool = "free-cache", Version = "1.0", Outcome = ToolOutcome.Failed, Team = "t" }); + registry.NeedsRefining("free-cache").Should().BeTrue(); + } +} diff --git a/tests/Loadout.Tests/Unit/ToolStoreFixture.cs b/tests/Loadout.Tests/Unit/ToolStoreFixture.cs new file mode 100644 index 00000000..244f85d6 --- /dev/null +++ b/tests/Loadout.Tests/Unit/ToolStoreFixture.cs @@ -0,0 +1,154 @@ +using Loadout.Core.Tools; +using Loadout.Models.Configuration; +using Loadout.Models.Platform; +using Loadout.Models.Tools; +using Loadout.Platform.Abstractions; +using Loadout.Platform.Linux; +using Loadout.Tests.Fakes; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace Loadout.Tests.Unit; + +/// +/// A tool catalogue in a temporary state directory, and drafts to put in it. +/// +/// +/// Every run goes through , so nothing here +/// needs pwsh on the machine running the tests. +/// +internal sealed class ToolStoreFixture : IDisposable +{ + private static readonly ISerializer Writer = new SerializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitNull) + .Build(); + + private readonly string _root; + + public ToolStoreFixture(params string[] known) + { + _root = Path.Combine(Path.GetTempPath(), "loadout-tools-" + Guid.NewGuid().ToString("N")); + + Paths = new LinuxPaths( + new FakeEnvironmentProvider( + Path.Combine(_root, "home"), + new Dictionary + { + ["XDG_CONFIG_HOME"] = Path.Combine(_root, "config"), + ["XDG_DATA_HOME"] = Path.Combine(_root, "data"), + ["XDG_STATE_HOME"] = Path.Combine(_root, "state"), + ["XDG_CACHE_HOME"] = Path.Combine(_root, "cache"), + }), + new NoOpFilePermissions(), + new HostPlatform( + HostOperatingSystem.Linux, + System.Runtime.InteropServices.Architecture.X64, + "test", + "TEST")); + + Paths.EnsureDirectoriesExist(); + Known = known; + } + + public IPlatformPaths Paths { get; } + + public IReadOnlyList Known { get; } + + /// A registry whose every run exits with . + public (ToolRegistry Registry, StubProcessLauncher Launcher) Registry(int exit = 0) + { + var launcher = new StubProcessLauncher(string.Empty, exit); + + return (new ToolRegistry(Paths, new ToolHarness(launcher), TimeProvider.System, () => Known), launcher); + } + + /// A manifest with every field filled. + public static ToolVersion Manifest(string name, string version) => new() + { + Name = name, + Version = version, + Script = "tool.ps1", + Purpose = "Clears a named cache directory of files older than a given age.", + Inputs = [new ToolInput { Name = "CachePath", Type = "path", Required = true, Describe = "The directory." }], + Outputs = new ToolOutputs { Stdout = "freed files", Exit = new Dictionary { [0] = "success" } }, + Dependencies = ["pwsh>=7"], + Constraints = ["Deletes only below CachePath."], + ErrorBehaviour = "Exits non-zero with one line on stderr.", + Examples = [new ToolExample { Command = "pwsh -File tool.ps1 -CachePath ./cache", Expect = "exit 0" }], + Origin = "A build cache kept filling the disk.", + }; + + /// One case of each class, each expecting . + public static List Cases(int exit = 0, string prefix = "") => + [ + .. ToolCaseClass.All.Select(one => new ToolCase + { + Name = prefix + one, + Class = one, + Args = new Dictionary { ["CachePath"] = "{tmp}/cache" }, + Expect = new ToolCaseExpect { Exit = exit }, + }), + ]; + + /// Writes a draft directory and returns where it is. + public string Draft(ToolVersion manifest, string script, IEnumerable cases) + { + var directory = Path.Combine(_root, "drafts", manifest.Name, manifest.Version + "-" + Guid.NewGuid().ToString("N")[..6]); + + Directory.CreateDirectory(Path.Combine(directory, "cases")); + File.WriteAllText(Path.Combine(directory, "manifest.yaml"), Writer.Serialize(manifest)); + File.WriteAllText(Path.Combine(directory, manifest.Script), script); + + foreach (var one in cases) + { + File.WriteAllText(Path.Combine(directory, "cases", one.Name + ".yaml"), Writer.Serialize(one)); + } + + return directory; + } + + /// A person's agreement to run this draft's harness, on a machine that lets tool-test run. + public static ToolTestConsent Agreed(ToolVersion manifest, string script, IReadOnlyList cases) => + new( + "trusted", + [ + new TrustedRemedy + { + Remedy = ToolHarness.Named(manifest), + Fingerprint = ToolHarness.Fingerprint(script, cases), + }, + ]); + + /// Writes, verifies and promotes a version, failing the test if any step does. + public async Task PromoteAsync(ToolRegistry registry, ToolVersion manifest, string script, List cases) + { + var draft = Draft(manifest, script, cases); + var verified = await registry.VerifyAsync(draft, Agreed(manifest, script, cases)); + + if (verified.Failed || verified.Value!.Gate is not { Passed: true }) + { + throw new InvalidOperationException("Verify did not pass: " + (verified.Error ?? verified.Value!.Because)); + } + + var promoted = registry.Promote(draft, new ToolPromotionRequest("lesson", "inbox/test", Summary: "Clears a cache.", Capabilities: ["cache", "disk"])); + + if (promoted.Failed) + { + throw new InvalidOperationException("Promote did not pass: " + promoted.Error); + } + + return draft; + } + + public void Dispose() + { + try + { + Directory.Delete(_root, recursive: true); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + } + } +} From ae9ece2da1dff45ed9d08b3d6108b55f0cfd1741 Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 16:41:53 +0100 Subject: [PATCH 02/15] Stop the tool registry trusting files an agent can write The review of 9c5e7b2 found four places where the registry believed a file that anything with Bash can rewrite, and no lock around its writes. - Verify is no longer forgeable. Promote read status and fingerprints from the draft's own manifest.yaml, so a draft written by hand as "verified", or a rejected draft edited to say so, was promoted. A passing verify now writes verified//.json under Root(), outside the drafts; a rejected one deletes it. Promote compares the draft's current script and cases fingerprints with that record and refuses without it. - A secret is never echoed. ToolGenericity quoted each match, and the repository URL pattern captured user-info, so a token in a clone URL came back in the refusal, twice (once as a URL, once read as an e-mail address). Once any secret pattern matches, no value is quoted; URL user-info is stripped before a URL is quoted; a value that itself matches a secret pattern is not quoted; and an address no longer starts after ':' or '/'. - Drafts are confined to Root()/drafts, after the path is resolved. Verify wrote manifest.yaml into whatever path it was given. The test fixture now writes its drafts there too. - Standing checks a version's files against the fingerprints recorded in the promote audit entry ("script cases "), not against the manifest beside them, so rewriting the script and the manifest's fingerprint together reads as tampered. - registry.lock, as the plan's section 1 says: an exclusive FileStream under Root(), waited for with back-off up to LockWait (10s), held across Promote and every read-modify-write of a head (usage, set active, deprecate, retire) and around verify's writes. A writer that cannot get it is refused with a sentence saying so. Before it, twelve promotions at once failed four of twelve. Minor items: - Promote writes into a dot-named staging directory beside the target and moves it into place, deleting it on failure, so a failed copy leaves no half-written versions//. Not covered by a test: forcing a copy to fail after the draft has been read needs a seam that does not exist. - ToolHarness refuses any argument with a '..' segment, which covers '{tmp}/..', '{tmp}\..' and relative '../'. - A missing tool still returns ExitCode.ProjectNotFound (3): there is no better code, and adding one changes the public exit-code contract. - Correction to 9c5e7b2's message: promotion requires eight manifest fields, not six - purpose, inputs, outputs, dependencies, constraints, error behaviour, examples and origin, as ToolPromotion.Missing and the theory in ToolRegistryTests check. Tests, each seen failing before the fix and failing again under a compilable mutation of it, restored from a copy: ToolRegistrySafetyTests (8) - A_draft_claiming_verified_without_a_verify_ record_is_refused, A_rejected_draft_edited_to_verified_is_refused, A_token_inside_a_url_never_reaches_the_refusal_text, A_password_in_a_ url_is_not_quoted_even_when_no_secret_pattern_knows_it, A_draft_outside_ the_drafts_directory_is_refused, Rewriting_script_and_manifest_ fingerprint_together_reads_as_tampered, A_second_writer_is_refused_while_ the_lock_is_held, Promotions_at_once_lose_no_known_good_version; and ToolHarnessTests.An_argument_climbing_out_of_tmp_is_refused_without_ running (4 cases). dotnet build Loadout.slnx: 0 Warning(s), 0 Error(s). Tool filter: 68 passed of 68. Full suite: 3170 passed, 0 failed, 22 skipped, of 3192. --- src/Loadout.Core/Tools/ToolGenericity.cs | 32 ++- src/Loadout.Core/Tools/ToolHarness.cs | 7 + src/Loadout.Core/Tools/ToolRegistry.cs | 247 ++++++++++++++++-- tests/Loadout.Tests/Unit/ToolHarnessTests.cs | 19 ++ .../Unit/ToolRegistrySafetyTests.cs | 208 +++++++++++++++ tests/Loadout.Tests/Unit/ToolStoreFixture.cs | 11 +- 6 files changed, 487 insertions(+), 37 deletions(-) create mode 100644 tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs diff --git a/src/Loadout.Core/Tools/ToolGenericity.cs b/src/Loadout.Core/Tools/ToolGenericity.cs index 3168abbf..fa3bf0b0 100644 --- a/src/Loadout.Core/Tools/ToolGenericity.cs +++ b/src/Loadout.Core/Tools/ToolGenericity.cs @@ -37,18 +37,24 @@ public static IReadOnlyList Check(string? text, IEnumerable? kno } var found = new List(); + var secrets = SecretScanner.Match(text); // Secrets first, and by type only. - foreach (var name in SecretScanner.Match(text)) + foreach (var name in secrets) { found.Add($"a credential ({name})"); } - Look(found, "an absolute path", WindowsPath(), text); - Look(found, "an absolute path", UnixPath(), text); - Look(found, "a repository URL", RepositoryUrl(), text); - Look(found, "an e-mail address", Email(), text); - Look(found, "a GUID", Guid(), text); + // Once anything in the text is a credential, no value is quoted at + // all: a path, URL or address can carry it, and a pattern only has to + // miss one shape of token for the refusal to print it. + var quote = secrets.Count == 0; + + Look(found, "an absolute path", WindowsPath(), text, quote); + Look(found, "an absolute path", UnixPath(), text, quote); + Look(found, "a repository URL", RepositoryUrl(), text, quote); + Look(found, "an e-mail address", Email(), text, quote); + Look(found, "a GUID", Guid(), text, quote); foreach (var name in (known ?? []).Where(one => one is { Length: >= 3 }).Distinct(StringComparer.OrdinalIgnoreCase)) { @@ -92,13 +98,17 @@ public static IReadOnlyList Check( return [.. Check(string.Join('\n', parts.Where(one => one is { Length: > 0 })), known).Distinct()]; } - private static void Look(List found, string what, Regex pattern, string text) + private static void Look(List found, string what, Regex pattern, string text, bool quote) { try { foreach (Match match in pattern.Matches(text)) { - found.Add($"{what}: '{match.Value.Trim()}'"); + // A URL's user-info is a name and a password, and neither is + // anybody else's business, whether or not a pattern knew it. + var value = UserInfo().Replace(match.Value.Trim(), "://"); + + found.Add(quote && SecretScanner.Match(value).Count == 0 ? $"{what}: '{value}'" : $"{what} (not quoted)"); } } catch (RegexMatchTimeoutException) @@ -120,7 +130,11 @@ private static void Look(List found, string what, Regex pattern, string 1000)] private static partial Regex RepositoryUrl(); - [GeneratedRegex(@"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}\b", RegexOptions.None, 1000)] + [GeneratedRegex(@"://[^\s'""/@]+@", RegexOptions.None, 1000)] + private static partial Regex UserInfo(); + + // Not after ':' or '/', so a URL's password is never read as the start of an address. + [GeneratedRegex(@"(? RunAsync(string scriptPath, ToolCase toolCase, { return Fail($"'{name}' is an absolute path. Cases may only use {Tmp}."); } + + // {tmp}/.. is the directory every case shares, and ../ from the + // case's own directory is the same place. + if (value.Split('/', '\\').Contains("..")) + { + return Fail($"'{name}' climbs out of {Tmp} with '..'. Cases may only use {Tmp}."); + } } var directory = Path.Combine(Path.GetTempPath(), "loadout-tool-case-" + Guid.NewGuid().ToString("N")); diff --git a/src/Loadout.Core/Tools/ToolRegistry.cs b/src/Loadout.Core/Tools/ToolRegistry.cs index e6ad3ba9..a9668a64 100644 --- a/src/Loadout.Core/Tools/ToolRegistry.cs +++ b/src/Loadout.Core/Tools/ToolRegistry.cs @@ -189,8 +189,21 @@ public ToolRegistry( /// public string Root() => Path.Combine(_paths.Paths.State, "tools"); + /// How long a writer waits for registry.lock before giving up. + internal TimeSpan LockWait { get; set; } = TimeSpan.FromSeconds(10); + private string AuditFile => Path.Combine(Root(), "audit.jsonl"); + private string LockFile => Path.Combine(Root(), "registry.lock"); + + private string DraftsRoot => Path.Combine(Root(), "drafts"); + + private string VerifiedFile(string name, string version) => + Path.Combine(Root(), "verified", RemedyBook.Slug(name), version + ".json"); + + private const string Busy = + "Something else is writing the tool catalogue and holds registry.lock. Try again when it has finished."; + private string ToolDirectory(string name) => Path.Combine(Root(), RemedyBook.Slug(name)); private string HeadFile(string name) => Path.Combine(ToolDirectory(name), "tool.yaml"); @@ -315,6 +328,13 @@ public OperationResult RecordUsage(ToolUsage usage) { ArgumentNullException.ThrowIfNull(usage); + using var held = Lock(); + + if (held is null) + { + return OperationResult.Fail(Busy); + } + if (ReadHead(usage.Tool) is not { } head) { return OperationResult.Fail($"There is no tool called '{usage.Tool}'.", ExitCode.ProjectNotFound); @@ -386,7 +406,17 @@ public async Task> VerifyAsync( return OperationResult.Fail(read.Error!, ExitCode.InvalidArguments); } - var (version, script, scriptPath, cases) = read.Value!; + var (directory, version, script, scriptPath, cases) = read.Value!; + + // Checked here as well as at promotion, because the version names the + // file the verify record is written to. + if (!string.Equals(version.Name, RemedyBook.Slug(version.Name), StringComparison.Ordinal) + || !VersionShape().IsMatch(version.Version)) + { + return OperationResult.Fail( + $"'{version.Name}@{version.Version}' is not a tool name and version: lowercase and hyphens, then major.minor.", + ExitCode.InvalidArguments); + } if (ToolHarness.MissingClasses(cases) is { Count: > 0 } missing) { @@ -430,10 +460,44 @@ public async Task> VerifyAsync( Failed = gate.Own.Count(one => !one.Passed) + gate.Regression.Count(one => !one.Passed), RegressionAgainst = active is null ? [] : [active.Version], Fingerprint = RemedyCeiling.Fingerprint(script), - CasesFingerprint = CasesFingerprint(Path.Combine(draft, "cases")), + CasesFingerprint = CasesFingerprint(Path.Combine(directory, "cases")), }; - File.WriteAllText(Path.Combine(draft, "manifest.yaml"), Writer.Serialize(version)); + using (var held = Lock()) + { + if (held is null) + { + return OperationResult.Fail(Busy); + } + + // The draft's manifest is for whoever reads the draft. What + // promotion believes is the record under verified/, outside the + // drafts, because anything that can write a draft can write + // "status: verified" into it. + var proof = VerifiedFile(version.Name, version.Version); + + try + { + if (gate.Passed) + { + Directory.CreateDirectory(Path.GetDirectoryName(proof)!); + File.WriteAllText(proof, JsonSerializer.Serialize( + new VerifyRecord(version.Tests.Fingerprint, version.Tests.CasesFingerprint, version.Tests.RanAt), + Json)); + } + else if (File.Exists(proof)) + { + File.Delete(proof); + } + + File.WriteAllText(Path.Combine(directory, "manifest.yaml"), Writer.Serialize(version)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return OperationResult.Fail($"What verify found could not be written: {ex.Message}"); + } + } + Record(gate.Passed ? "verify" : "reject", version.Name, version.Version, null, null, gate.Because); return OperationResult.Ok(new ToolVerification(RemedyRuling.Run, gate.Because, gate)); @@ -444,6 +508,16 @@ public OperationResult Promote(string draft, ToolPromotionRequest r { ArgumentNullException.ThrowIfNull(request); + // Held from the first read to the last write, so two promotions of one + // tool cannot both read the head and each write back only their own + // version. + using var held = Lock(); + + return held is null ? OperationResult.Fail(Busy) : PromoteHeld(draft, request); + } + + private OperationResult PromoteHeld(string draft, ToolPromotionRequest request) + { var read = ReadDraft(draft); if (read.Failed) @@ -451,7 +525,7 @@ public OperationResult Promote(string draft, ToolPromotionRequest r return OperationResult.Fail(read.Error!, ExitCode.InvalidArguments); } - var (version, script, scriptPath, cases) = read.Value!; + var (directory, version, script, scriptPath, cases) = read.Value!; OperationResult Refuse(string why) => OperationResult.Fail(why, ExitCode.PolicyViolation); @@ -478,15 +552,17 @@ OperationResult Refuse(string why) => return Refuse("The manifest is missing " + string.Join(", ", missing) + "."); } - if (version.Status != ToolVersionStatus.Verified || version.Tests is not { } tests) + // The draft's own status is not evidence: the draft is writable by + // whoever wrote it. Only the record verify leaves outside it is. + if (ReadVerified(version.Name, version.Version) is not { } proof) { return Refuse($"{version.Name}@{version.Version} has not passed verify."); } - var casesFingerprint = CasesFingerprint(Path.Combine(draft, "cases")); + var casesFingerprint = CasesFingerprint(Path.Combine(directory, "cases")); - if (!string.Equals(tests.Fingerprint, RemedyCeiling.Fingerprint(script), StringComparison.Ordinal) - || !string.Equals(tests.CasesFingerprint, casesFingerprint, StringComparison.Ordinal)) + if (!string.Equals(proof.Script, RemedyCeiling.Fingerprint(script), StringComparison.Ordinal) + || !string.Equals(proof.Cases, casesFingerprint, StringComparison.Ordinal)) { return Refuse("The script or its cases have changed since they were verified. Verify again."); } @@ -523,21 +599,26 @@ OperationResult Refuse(string why) => var target = VersionDirectory(version.Name, version.Version); var file = $"{version.Name}.v{version.Version}.ps1"; + // Written beside the target and moved into place, so a copy that fails + // partway leaves no half a version where a whole one is expected. + var staging = Path.Combine(Path.GetDirectoryName(target)!, "." + version.Version + "-" + Guid.NewGuid().ToString("N")[..8]); + try { - Directory.CreateDirectory(Path.Combine(target, "cases")); - File.Copy(scriptPath, Path.Combine(target, file)); + Directory.CreateDirectory(Path.Combine(staging, "cases")); + File.Copy(scriptPath, Path.Combine(staging, file)); - foreach (var one in Directory.EnumerateFiles(Path.Combine(draft, "cases"), "*.yaml")) + foreach (var one in Directory.EnumerateFiles(Path.Combine(directory, "cases"), "*.yaml")) { - File.Copy(one, Path.Combine(target, "cases", Path.GetFileName(one))); + File.Copy(one, Path.Combine(staging, "cases", Path.GetFileName(one))); } version.Status = ToolVersionStatus.KnownGood; version.Script = file; version.Fingerprint = RemedyCeiling.Fingerprint(script); version.CasesFingerprint = casesFingerprint; - File.WriteAllText(Path.Combine(target, "manifest.yaml"), Writer.Serialize(version)); + File.WriteAllText(Path.Combine(staging, "manifest.yaml"), Writer.Serialize(version)); + Directory.Move(staging, target); head ??= new ToolRecord { Name = version.Name }; head.Owner = request.Owner ?? head.Owner; @@ -563,10 +644,24 @@ OperationResult Refuse(string why) => } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { + try + { + if (Directory.Exists(staging)) + { + Directory.Delete(staging, recursive: true); + } + } + catch (Exception again) when (again is IOException or UnauthorizedAccessException) + { + // Named with a leading dot and a random suffix, so nothing reads it as a version. + } + return OperationResult.Fail($"That version could not be written: {ex.Message}"); } - Record("promote", version.Name, version.Version, request.Actor, request.Run, version.Fingerprint); + // What Standing checks the version's files against, rather than the + // version's own manifest, which sits beside the files it vouches for. + Record("promote", version.Name, version.Version, request.Actor, request.Run, Promoted(version.Fingerprint, casesFingerprint)); return OperationResult.Ok(version); } @@ -574,6 +669,13 @@ OperationResult Refuse(string why) => /// public OperationResult SetActive(string name, string version) { + using var held = Lock(); + + if (held is null) + { + return OperationResult.Fail(Busy); + } + if (ReadHead(name) is not { } head) { return OperationResult.Fail($"There is no tool called '{name}'.", ExitCode.ProjectNotFound); @@ -602,6 +704,13 @@ public OperationResult SetActive(string name, string version) /// public OperationResult Deprecate(string name, string? replacement, string? reason) { + using var held = Lock(); + + if (held is null) + { + return OperationResult.Fail(Busy); + } + if (ReadHead(name) is not { } head) { return OperationResult.Fail($"There is no tool called '{name}'.", ExitCode.ProjectNotFound); @@ -639,6 +748,13 @@ public OperationResult Deprecate(string name, string? replacement, string? reaso /// public OperationResult Retire(string name) { + using var held = Lock(); + + if (held is null) + { + return OperationResult.Fail(Busy); + } + if (ReadHead(name) is not { } head) { return OperationResult.Fail($"There is no tool called '{name}'.", ExitCode.ProjectNotFound); @@ -668,7 +784,9 @@ public OperationResult Retire(string name) /// /// Recomputed on every read, because the files are on a disk a node with /// Bash can write to. A version that no longer matches is refused, never - /// trusted. + /// trusted. What the files are checked against is the fingerprint the + /// promotion recorded in the audit log, not the version's own manifest: + /// whoever can rewrite the script can rewrite the manifest beside it. /// public string Standing(string name, string version) { @@ -681,10 +799,14 @@ public string Standing(string name, string version) } var script = Path.Combine(directory, manifest.Script); - - if (!File.Exists(script) - || !string.Equals(RemedyCeiling.Fingerprint(File.ReadAllText(script)), manifest.Fingerprint, StringComparison.OrdinalIgnoreCase) - || !string.Equals(CasesFingerprint(Path.Combine(directory, "cases")), manifest.CasesFingerprint, StringComparison.OrdinalIgnoreCase)) + var promoted = Audit(name).LastOrDefault(one => + one.Action == "promote" && string.Equals(one.Version, version, StringComparison.Ordinal)); + var (scriptPrint, casesPrint) = ReadPromoted(promoted?.Note); + + if (scriptPrint is null + || !File.Exists(script) + || !string.Equals(RemedyCeiling.Fingerprint(File.ReadAllText(script)), scriptPrint, StringComparison.OrdinalIgnoreCase) + || !string.Equals(CasesFingerprint(Path.Combine(directory, "cases")), casesPrint, StringComparison.OrdinalIgnoreCase)) { return ToolVersionStatus.Tampered; } @@ -821,26 +943,97 @@ private List Usage(string name) private void Record(string action, string tool, string? version, string? actor, string? run, string? note) => ToolAudit.Append(AuditFile, new ToolAuditEntry(_clock.GetUtcNow(), action, tool, version, actor, run, note)); - private static OperationResult<(ToolVersion Version, string Script, string ScriptPath, IReadOnlyList Cases)> ReadDraft(string draft) + private OperationResult<(string Directory, ToolVersion Version, string Script, string ScriptPath, IReadOnlyList Cases)> ReadDraft(string draft) { - if (ReadYaml(Path.Combine(draft, "manifest.yaml")) is not { } version) + // Only under drafts/, after the path is resolved, because verify + // writes into a draft and nothing asked it to write anywhere else. + var full = Path.GetFullPath(draft); + + if (!full.StartsWith(Path.GetFullPath(DraftsRoot) + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase)) { - return OperationResult<(ToolVersion, string, string, IReadOnlyList)>.Fail( + return OperationResult<(string, ToolVersion, string, string, IReadOnlyList)>.Fail( + $"A draft is read only from {DraftsRoot}, and {draft} is not in it."); + } + + if (ReadYaml(Path.Combine(full, "manifest.yaml")) is not { } version) + { + return OperationResult<(string, ToolVersion, string, string, IReadOnlyList)>.Fail( $"There is no readable manifest.yaml in {draft}."); } - var scriptPath = Path.GetFullPath(Path.Combine(draft, version.Script)); + var scriptPath = Path.GetFullPath(Path.Combine(full, version.Script ?? string.Empty)); if (version.Script is not { Length: > 0 } - || !scriptPath.StartsWith(Path.GetFullPath(draft) + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase) + || !scriptPath.StartsWith(full + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase) || !File.Exists(scriptPath)) { - return OperationResult<(ToolVersion, string, string, IReadOnlyList)>.Fail( + return OperationResult<(string, ToolVersion, string, string, IReadOnlyList)>.Fail( $"The manifest names no script beside it in {draft}."); } - return OperationResult<(ToolVersion, string, string, IReadOnlyList)>.Ok( - (version, File.ReadAllText(scriptPath), scriptPath, ReadCases(Path.Combine(draft, "cases")))); + return OperationResult<(string, ToolVersion, string, string, IReadOnlyList)>.Ok( + (full, version, File.ReadAllText(scriptPath), scriptPath, ReadCases(Path.Combine(full, "cases")))); + } + + /// What a passing verify leaves outside the drafts, for promotion to check. + private sealed record VerifyRecord(string Script, string Cases, DateTimeOffset? At); + + private VerifyRecord? ReadVerified(string name, string version) + { + try + { + var file = VerifiedFile(name, version); + + return File.Exists(file) ? JsonSerializer.Deserialize(File.ReadAllText(file), Json) : null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or JsonException) + { + return null; + } + } + + private static string Promoted(string script, string cases) => $"script {script} cases {cases}"; + + private static (string? Script, string? Cases) ReadPromoted(string? note) + { + var parts = (note ?? string.Empty).Split(' '); + + return parts is ["script", var script, "cases", var cases] ? (script, cases) : (null, null); + } + + /// + /// Takes registry.lock, waiting up to , or + /// returns null when something else holds it for longer. + /// + /// + /// Every read-modify-write of a head, and every promotion, holds it. Without + /// it two writers each read the head, add their own change and write it + /// back, and the first one's change is gone: twelve promotions at once lost + /// versions from known_good before this was here. + /// + private FileStream? Lock() + { + Directory.CreateDirectory(Root()); + var deadline = DateTime.UtcNow + LockWait; + var wait = TimeSpan.FromMilliseconds(10); + + while (true) + { + try + { + return new FileStream(LockFile, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + if (DateTime.UtcNow >= deadline) + { + return null; + } + + Thread.Sleep(wait); + wait = TimeSpan.FromMilliseconds(Math.Min(wait.TotalMilliseconds * 2, 250)); + } + } } private static List ReadCases(string directory) diff --git a/tests/Loadout.Tests/Unit/ToolHarnessTests.cs b/tests/Loadout.Tests/Unit/ToolHarnessTests.cs index 94eca2fa..af05715b 100644 --- a/tests/Loadout.Tests/Unit/ToolHarnessTests.cs +++ b/tests/Loadout.Tests/Unit/ToolHarnessTests.cs @@ -34,6 +34,25 @@ public async Task Verify_refuses_without_all_four_case_classes() launcher.Requests.Should().BeEmpty(); } + [Theory] + [InlineData("{tmp}/../escape")] + [InlineData("{tmp}\\..\\escape")] + [InlineData("../escape")] + [InlineData("cache/../../escape")] + public async Task An_argument_climbing_out_of_tmp_is_refused_without_running(string value) + { + var launcher = new StubProcessLauncher(string.Empty); + var harness = new ToolHarness(launcher); + + var result = await harness.RunAsync( + "tool.ps1", + new ToolCase { Name = "a", Class = ToolCaseClass.Success, Args = new() { ["CachePath"] = value } }); + + result.Passed.Should().BeFalse(); + result.Why.Should().Contain(".."); + launcher.Requests.Should().BeEmpty(); + } + [Fact] public async Task Cases_run_in_a_fresh_temp_directory() { diff --git a/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs b/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs new file mode 100644 index 00000000..33597d76 --- /dev/null +++ b/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs @@ -0,0 +1,208 @@ +using FluentAssertions; +using Loadout.Core.Teams; +using Loadout.Core.Tools; +using Loadout.Models.Tools; +using Xunit; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace Loadout.Tests.Unit; + +/// +/// What the catalogue must not take on trust from files an agent can write: a +/// draft's claim to have passed verify, a version's own fingerprint, a draft +/// somewhere else on the disk, and a second writer. +/// +public sealed class ToolRegistrySafetyTests : IDisposable +{ + private const string Script = "param([string]$CachePath)\nGet-ChildItem $CachePath | Remove-Item\n"; + + private static readonly ISerializer Writer = new SerializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitNull) + .Build(); + + private static readonly IDeserializer Yaml = new DeserializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + private readonly ToolStoreFixture _store = new(); + + public void Dispose() => _store.Dispose(); + + [Fact] + public void A_draft_claiming_verified_without_a_verify_record_is_refused() + { + var (registry, _) = _store.Registry(); + var cases = ToolStoreFixture.Cases(); + var manifest = ToolStoreFixture.Manifest("free-cache", "1.0"); + var draft = _store.Draft(manifest, Script, cases); + + // Written by hand, with every fingerprint right, and never verified. + manifest.Status = ToolVersionStatus.Verified; + manifest.Tests = new ToolTestRecord + { + RanAt = DateTimeOffset.UtcNow, + Passed = 4, + Fingerprint = RemedyCeiling.Fingerprint(Script), + CasesFingerprint = ToolRegistry.CasesFingerprint(Path.Combine(draft, "cases")), + }; + File.WriteAllText(Path.Combine(draft, "manifest.yaml"), Writer.Serialize(manifest)); + + var promoted = registry.Promote(draft, new("lesson", "inbox/test")); + + promoted.Failed.Should().BeTrue(); + Directory.Exists(Path.Combine(registry.Root(), "free-cache", "versions", "1.0")).Should().BeFalse(); + } + + [Fact] + public async Task A_rejected_draft_edited_to_verified_is_refused() + { + var (registry, _) = _store.Registry(exit: 1); + var cases = ToolStoreFixture.Cases(); + var manifest = ToolStoreFixture.Manifest("free-cache", "1.0"); + var draft = _store.Draft(manifest, Script, cases); + + var verified = await registry.VerifyAsync(draft, ToolStoreFixture.Agreed(manifest, Script, cases)); + verified.Value!.Gate!.Passed.Should().BeFalse(); + + var file = Path.Combine(draft, "manifest.yaml"); + var rejected = Yaml.Deserialize(File.ReadAllText(file)); + rejected.Status.Should().Be(ToolVersionStatus.Rejected); + rejected.Status = ToolVersionStatus.Verified; + File.WriteAllText(file, Writer.Serialize(rejected)); + + var promoted = registry.Promote(draft, new("lesson", "inbox/test")); + + promoted.Failed.Should().BeTrue(); + promoted.Error.Should().Contain("verify"); + } + + [Fact] + public async Task A_token_inside_a_url_never_reaches_the_refusal_text() + { + var token = "ghp_" + new string('b', 36); + var script = Script + "git clone https://x-access-token:" + token + "@github.com/someone/thing\n"; + var (registry, _) = _store.Registry(); + var cases = ToolStoreFixture.Cases(); + var manifest = ToolStoreFixture.Manifest("free-cache", "1.0"); + var draft = _store.Draft(manifest, script, cases); + await registry.VerifyAsync(draft, ToolStoreFixture.Agreed(manifest, script, cases)); + + var promoted = registry.Promote(draft, new("lesson", "inbox/test")); + + promoted.Failed.Should().BeTrue(); + promoted.Error.Should().Contain("GitHub token"); + promoted.Error.Should().NotContain(token); + File.ReadAllText(Path.Combine(registry.Root(), "audit.jsonl")).Should().NotContain(token); + registry.Audit().Should().NotContain(one => (one.Note ?? string.Empty).Contains(token, StringComparison.Ordinal)); + } + + [Fact] + public void A_password_in_a_url_is_not_quoted_even_when_no_secret_pattern_knows_it() + { + var found = ToolGenericity.Check("git clone https://someone:hunter2hunter2@github.com/someone/thing"); + + found.Should().Contain(one => one.StartsWith("a repository URL", StringComparison.Ordinal)); + found.Should().NotContain(one => one.Contains("hunter2", StringComparison.Ordinal)); + } + + [Fact] + public async Task A_draft_outside_the_drafts_directory_is_refused() + { + var (registry, _) = _store.Registry(); + var cases = ToolStoreFixture.Cases(); + var manifest = ToolStoreFixture.Manifest("free-cache", "1.0"); + var outside = Path.Combine(_store.Paths.Paths.State, "elsewhere", "free-cache"); + ToolStoreFixture.Write(outside, manifest, Script, cases); + var before = File.ReadAllText(Path.Combine(outside, "manifest.yaml")); + + // Climbing out of drafts by name is the same as starting outside it. + var climbing = Path.Combine(_store.Drafts, "..", "elsewhere", "free-cache"); + + foreach (var draft in new[] { outside, climbing }) + { + var verified = await registry.VerifyAsync(draft, ToolStoreFixture.Agreed(manifest, Script, cases)); + verified.Failed.Should().BeTrue(); + verified.Error.Should().Contain("drafts"); + registry.Promote(draft, new("lesson", "inbox/test")).Failed.Should().BeTrue(); + } + + File.ReadAllText(Path.Combine(outside, "manifest.yaml")).Should().Be(before); + } + + [Fact] + public async Task Rewriting_script_and_manifest_fingerprint_together_reads_as_tampered() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + + var directory = Path.Combine(registry.Root(), "free-cache", "versions", "1.0"); + var changed = Script + "Remove-Item -Recurse /\n"; + File.WriteAllText(Path.Combine(directory, "free-cache.v1.0.ps1"), changed); + + var file = Path.Combine(directory, "manifest.yaml"); + var manifest = Yaml.Deserialize(File.ReadAllText(file)); + manifest.Fingerprint = RemedyCeiling.Fingerprint(changed); + File.WriteAllText(file, Writer.Serialize(manifest)); + + var shown = registry.Show("free-cache").Value!; + + shown.Versions["1.0"].Should().Be(ToolVersionStatus.Tampered); + shown.Active.Should().BeNull(); + } + + [Fact] + public async Task A_second_writer_is_refused_while_the_lock_is_held() + { + var (registry, _) = _store.Registry(); + registry.LockWait = TimeSpan.FromMilliseconds(100); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + + using (new FileStream(Path.Combine(registry.Root(), "registry.lock"), FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None)) + { + var used = registry.RecordUsage(new ToolUsage { Tool = "free-cache", Version = "1.0", Outcome = ToolOutcome.Ok }); + + used.Failed.Should().BeTrue(); + used.Error.Should().Contain("lock"); + } + + registry.RecordUsage(new ToolUsage { Tool = "free-cache", Version = "1.0", Outcome = ToolOutcome.Ok }) + .Succeeded.Should().BeTrue(); + } + + [Fact] + public async Task Promotions_at_once_lose_no_known_good_version() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + + var drafts = new List(); + + for (var minor = 1; minor <= 12; minor++) + { + var manifest = ToolStoreFixture.Manifest("free-cache", "1." + minor); + var script = Script + "# " + minor + "\n"; + var cases = ToolStoreFixture.Cases(); + var draft = _store.Draft(manifest, script, cases); + (await registry.VerifyAsync(draft, ToolStoreFixture.Agreed(manifest, script, cases))).Value!.Gate!.Passed.Should().BeTrue(); + drafts.Add(draft); + } + + var results = new System.Collections.Concurrent.ConcurrentBag(); + using var start = new ManualResetEventSlim(); + var threads = drafts.Select(one => new Thread(() => + { + start.Wait(); + results.Add(registry.Promote(one, new("lesson", "inbox/test")).Succeeded); + })).ToList(); + + threads.ForEach(one => one.Start()); + start.Set(); + threads.ForEach(one => one.Join()); + + results.Should().OnlyContain(one => one); + registry.Show("free-cache").Value!.Record.KnownGood.Should().HaveCount(13); + } +} diff --git a/tests/Loadout.Tests/Unit/ToolStoreFixture.cs b/tests/Loadout.Tests/Unit/ToolStoreFixture.cs index 244f85d6..49c8cadd 100644 --- a/tests/Loadout.Tests/Unit/ToolStoreFixture.cs +++ b/tests/Loadout.Tests/Unit/ToolStoreFixture.cs @@ -94,8 +94,17 @@ public static List Cases(int exit = 0, string prefix = "") => /// Writes a draft directory and returns where it is. public string Draft(ToolVersion manifest, string script, IEnumerable cases) { - var directory = Path.Combine(_root, "drafts", manifest.Name, manifest.Version + "-" + Guid.NewGuid().ToString("N")[..6]); + var directory = Path.Combine(Drafts, manifest.Name, manifest.Version + "-" + Guid.NewGuid().ToString("N")[..6]); + return Write(directory, manifest, script, cases); + } + + /// Where the registry keeps drafts, and the only place it reads them from. + public string Drafts => Path.Combine(Paths.Paths.State, "tools", "drafts"); + + /// Writes a draft into , wherever that is. + public static string Write(string directory, ToolVersion manifest, string script, IEnumerable cases) + { Directory.CreateDirectory(Path.Combine(directory, "cases")); File.WriteAllText(Path.Combine(directory, "manifest.yaml"), Writer.Serialize(manifest)); File.WriteAllText(Path.Combine(directory, manifest.Script), script); From 6c853a4c7278418d3d36627e8694950e784d82f5 Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 16:43:57 +0100 Subject: [PATCH 03/15] Refuse tool names that are the catalogue's own directories A tool called drafts, inbox or verified would share a directory with the catalogue's working files: a tool named drafts would put its promoted versions where agents are allowed to write, and one named verified would sit among the proofs promotion trusts. Verify and promote now both refuse those names. --- src/Loadout.Core/Tools/ToolRegistry.cs | 14 ++++++++++++++ .../Unit/ToolRegistrySafetyTests.cs | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/Loadout.Core/Tools/ToolRegistry.cs b/src/Loadout.Core/Tools/ToolRegistry.cs index a9668a64..a075a260 100644 --- a/src/Loadout.Core/Tools/ToolRegistry.cs +++ b/src/Loadout.Core/Tools/ToolRegistry.cs @@ -157,6 +157,14 @@ public sealed partial class ToolRegistry : IToolRegistry private static readonly HashSet Kinds = new(StringComparer.Ordinal) { "candidate", "idea", "bug", "lesson" }; + /// + /// The directories the catalogue keeps beside its tools, which a tool of the + /// same name would share: a tool called drafts would put its versions + /// where agents can write. + /// + public static readonly IReadOnlySet Reserved = + new HashSet(StringComparer.OrdinalIgnoreCase) { "drafts", "inbox", "verified" }; + private readonly IPlatformPaths _paths; private readonly ToolHarness _harness; private readonly TimeProvider _clock; @@ -411,6 +419,7 @@ public async Task> VerifyAsync( // Checked here as well as at promotion, because the version names the // file the verify record is written to. if (!string.Equals(version.Name, RemedyBook.Slug(version.Name), StringComparison.Ordinal) + || Reserved.Contains(version.Name) || !VersionShape().IsMatch(version.Version)) { return OperationResult.Fail( @@ -535,6 +544,11 @@ OperationResult Refuse(string why) => return Refuse($"'{version.Name}' is not a tool name: lowercase and hyphens only."); } + if (Reserved.Contains(version.Name)) + { + return Refuse($"'{version.Name}' is where the catalogue keeps its own files, so no tool can have it."); + } + if (!VersionShape().IsMatch(version.Version)) { return Refuse($"'{version.Version}' is not a version: major.minor."); diff --git a/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs b/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs index 33597d76..f087b624 100644 --- a/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs +++ b/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs @@ -31,6 +31,25 @@ public sealed class ToolRegistrySafetyTests : IDisposable public void Dispose() => _store.Dispose(); + [Theory] + [InlineData("drafts")] + [InlineData("inbox")] + [InlineData("verified")] + public async Task A_tool_named_after_the_catalogues_own_directories_is_refused(string name) + { + var (registry, _) = _store.Registry(); + var cases = ToolStoreFixture.Cases(); + var manifest = ToolStoreFixture.Manifest(name, "1.0"); + var draft = _store.Draft(manifest, Script, cases); + + var verified = await registry.VerifyAsync(draft, ToolStoreFixture.Agreed(manifest, Script, cases)); + var promoted = registry.Promote(draft, new("lesson", "inbox/test")); + + verified.Failed.Should().BeTrue(); + promoted.Failed.Should().BeTrue(); + File.Exists(Path.Combine(registry.Root(), name, "tool.yaml")).Should().BeFalse(); + } + [Fact] public void A_draft_claiming_verified_without_a_verify_record_is_refused() { From d0c4be4b564a85c4fd5520236397f82561203f79 Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 16:52:40 +0100 Subject: [PATCH 04/15] Check a version against its first promotion, and reserve 'versions' Standing read the LAST "promote" line in the audit log for a version. The log is append-only and a version is written once, so any later promote line for the same version is somebody vouching for files the gate never saw: edit versions/1.0's script, append a promote line carrying the new fingerprint, and the version read as known-good. It now reads the first. 'versions' joins drafts, inbox and verified as a name no tool may take, since a tool called that would share its directory name with every tool's own version store. The registry's remarks now state the precondition the whole design rests on: a node may write only Root()/drafts and Root()/inbox. Tests: Appending_a_forged_promote_entry_leaves_the_version_tampered and a "versions" row in A_tool_named_after_the_catalogues_own_directories_is_refused both failed before the change and pass after. --- src/Loadout.Core/Tools/ToolRegistry.cs | 13 +++++++++-- .../Unit/ToolRegistrySafetyTests.cs | 23 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/Loadout.Core/Tools/ToolRegistry.cs b/src/Loadout.Core/Tools/ToolRegistry.cs index a075a260..8c730c37 100644 --- a/src/Loadout.Core/Tools/ToolRegistry.cs +++ b/src/Loadout.Core/Tools/ToolRegistry.cs @@ -134,6 +134,12 @@ Task> VerifyAsync( /// Nothing here decides whether anything runs. That is this machine's /// configuration and a person's, as for remedies. /// +/// +/// Precondition: a node may write only Root()/drafts and +/// Root()/inbox. The verify records, the audit log, the heads and the +/// version directories are safe only while that holds; a node given the whole +/// root could forge any of them. +/// /// public sealed partial class ToolRegistry : IToolRegistry { @@ -163,7 +169,7 @@ public sealed partial class ToolRegistry : IToolRegistry /// where agents can write. /// public static readonly IReadOnlySet Reserved = - new HashSet(StringComparer.OrdinalIgnoreCase) { "drafts", "inbox", "verified" }; + new HashSet(StringComparer.OrdinalIgnoreCase) { "drafts", "inbox", "verified", "versions" }; private readonly IPlatformPaths _paths; private readonly ToolHarness _harness; @@ -813,7 +819,10 @@ public string Standing(string name, string version) } var script = Path.Combine(directory, manifest.Script); - var promoted = Audit(name).LastOrDefault(one => + // The first promotion, because a version is written once: the log is + // only ever appended to, so a later "promote" line for the same + // version is somebody vouching for files the gate never saw. + var promoted = Audit(name).FirstOrDefault(one => one.Action == "promote" && string.Equals(one.Version, version, StringComparison.Ordinal)); var (scriptPrint, casesPrint) = ReadPromoted(promoted?.Note); diff --git a/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs b/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs index f087b624..6ee403f7 100644 --- a/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs +++ b/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs @@ -35,6 +35,7 @@ public sealed class ToolRegistrySafetyTests : IDisposable [InlineData("drafts")] [InlineData("inbox")] [InlineData("verified")] + [InlineData("versions")] public async Task A_tool_named_after_the_catalogues_own_directories_is_refused(string name) { var (registry, _) = _store.Registry(); @@ -172,6 +173,28 @@ public async Task Rewriting_script_and_manifest_fingerprint_together_reads_as_ta shown.Active.Should().BeNull(); } + [Fact] + public async Task Appending_a_forged_promote_entry_leaves_the_version_tampered() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + + var directory = Path.Combine(registry.Root(), "free-cache", "versions", "1.0"); + var changed = Script + "Remove-Item -Recurse /\n"; + File.WriteAllText(Path.Combine(directory, "free-cache.v1.0.ps1"), changed); + + // The audit log is appended to, so a later promote line for the same + // version is exactly what somebody rewriting the script would add. + var audit = Path.Combine(registry.Root(), "audit.jsonl"); + var promote = File.ReadAllLines(audit).Single(line => line.Contains("\"promote\"", StringComparison.Ordinal)); + var forged = promote.Replace( + RemedyCeiling.Fingerprint(Script), RemedyCeiling.Fingerprint(changed), StringComparison.OrdinalIgnoreCase); + forged.Should().NotBe(promote); + File.AppendAllText(audit, forged + "\n"); + + registry.Standing("free-cache", "1.0").Should().Be(ToolVersionStatus.Tampered); + } + [Fact] public async Task A_second_writer_is_refused_while_the_lock_is_held() { From e0ef2d7fe254b4e421771b016b8dffa40ca12e7e Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 17:05:21 +0100 Subject: [PATCH 05/15] Let teams find, submit to and run the shared tool catalogue Piece 2 of the tool registry: the catalogue becomes reachable from where agents and people work, without any of it deciding what may run. - 'loadout tools' search, show, submit, used, audit, verify, promote, deprecate, retire and trust, with --json and --dry-run. - Four MCP tools making the same calls: loadout_tools_search, _show, _submit and _used. Promote, trust and deprecate are deliberately absent: promotion is the gate's and trust is a person's. - Every brief carries a fixed pointer to the catalogue and never lists tools, so a growing catalogue costs no brief anything. - A remediator is offered active, known-good tool versions through the same RemedyCeiling ruling remedies use, named tool:@ and matched by the versioned script file name. Agreement lives in MachineTeams.TrustedTools, fingerprinting the script ScriptOf reads from disk, so a record in the catalogue claiming trust decides nothing and a changed script is asked about again. Tests cover the brief pointer and its fixed length, the --json shapes and that --dry-run changes nothing, trust by fingerprint, offering only to the remediator and never a candidate, secret screening on MCP submit, and a guard that no node can reach the catalogue outside drafts and inbox. Each new test was mutation-checked against a compiling mutation and failed. docs/commands.md gains the rows for 'loadout tools' and the four MCP tools. --- docs/commands.md | 9 +- src/Loadout.Agents/ServiceRegistration.cs | 3 +- src/Loadout.Agents/Teams/TeamRunner.cs | 33 +- src/Loadout.Cli/Commands/McpServeCommand.cs | 78 +- src/Loadout.Cli/Commands/TeamCommands.cs | 1 + src/Loadout.Cli/Commands/TeamResumeCommand.cs | 1 + src/Loadout.Cli/Commands/ToolCommands.cs | 966 ++++++++++++++++++ src/Loadout.Cli/Program.cs | 18 + src/Loadout.Core/Tools/ToolOffer.cs | 96 ++ src/Loadout.Core/Tools/ToolRegistry.cs | 76 ++ .../Configuration/MachineConfig.cs | 35 + .../Contract/ToolCommandsContractTests.cs | 176 ++++ .../Integration/AgentToolsTests.cs | 3 +- .../Unit/LoadoutToolsCatalogueTests.cs | 73 ++ tests/Loadout.Tests/Unit/RemedyGateTests.cs | 61 ++ tests/Loadout.Tests/Unit/TeamGoalTests.cs | 27 + tests/Loadout.Tests/Unit/TeamRunnerTests.cs | 34 + tests/Loadout.Tests/Unit/TeamsToolTests.cs | 3 +- tests/Loadout.Tests/Unit/ToolTrustTests.cs | 88 ++ 19 files changed, 1773 insertions(+), 8 deletions(-) create mode 100644 src/Loadout.Cli/Commands/ToolCommands.cs create mode 100644 src/Loadout.Core/Tools/ToolOffer.cs create mode 100644 tests/Loadout.Tests/Contract/ToolCommandsContractTests.cs create mode 100644 tests/Loadout.Tests/Unit/LoadoutToolsCatalogueTests.cs create mode 100644 tests/Loadout.Tests/Unit/ToolTrustTests.cs diff --git a/docs/commands.md b/docs/commands.md index a6a73824..d66468d7 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -69,6 +69,9 @@ | `loadout list` | List registered projects | | `loadout running` | The sessions running now, and how long each has been quiet | | `loadout task list\|declare\|remove` | Record what is being worked on, and check it against the repository | +| `loadout tools search\|show\|submit\|used` | The tools this machine shares between every team: find one before building it, send in a candidate, idea, bug or lesson, and say how a use went | +| `loadout tools audit\|verify\|promote\|deprecate\|retire` | Look after the catalogue: what happened to it, whether its files still match, and a tool's way in and out | +| `loadout tools trust [--revoke]` | Agree that one version's script may run on this machine, as it is now | | `loadout team list\|show\|run` | Run a team of agents against a project: a lead that briefs workers and reports to you | | `loadout team new\|edit\|remove` | Write a team of your own, open its file, or delete it | | `loadout team run "" --done-when` | Say what the run is judged on, one per use. The lead must report a verdict and evidence for every one | @@ -401,7 +404,7 @@ Every launch declares Loadout itself as an MCP server, so the handoff runs both ways: a session can ask the launcher things rather than parse console output written for a person. -Eleven tools, each making the same call its command makes: +Fifteen tools, each making the same call its command makes: | | | |---|---| @@ -416,6 +419,10 @@ Eleven tools, each making the same call its command makes: | `loadout_mode` | Change the posture for the rest of the session, and get what that changes | | `loadout_progress` | For a node of a team run: say what you are doing, in your own words | | `loadout_teams` | What the team runs on this machine are doing, and which have stopped to ask | +| `loadout_tools_search` | The machine's shared tools matching some words, as `tools search` finds them | +| `loadout_tools_show` | One shared tool in full: its purpose, inputs, outputs, versions and why it exists | +| `loadout_tools_submit` | Send the catalogue a candidate, idea, bug or lesson, screened for credentials, as `tools submit` does | +| `loadout_tools_used` | Say how a use of a shared tool went, as `tools used` does | `loadout_recall` exists because only the memory index reaches the context — one line per topic — and a session deciding from that alone either opens six files diff --git a/src/Loadout.Agents/ServiceRegistration.cs b/src/Loadout.Agents/ServiceRegistration.cs index e23422c3..55f419b5 100644 --- a/src/Loadout.Agents/ServiceRegistration.cs +++ b/src/Loadout.Agents/ServiceRegistration.cs @@ -42,7 +42,8 @@ public static IServiceCollection AddAgentServices(this IServiceCollection servic provider.GetRequiredService(), provider.GetRequiredService(), provider.GetRequiredService(), - provider.GetRequiredService())); + provider.GetRequiredService(), + provider.GetRequiredService())); services.AddSingleton(); return services; diff --git a/src/Loadout.Agents/Teams/TeamRunner.cs b/src/Loadout.Agents/Teams/TeamRunner.cs index b9e7d52a..40819ac2 100644 --- a/src/Loadout.Agents/Teams/TeamRunner.cs +++ b/src/Loadout.Agents/Teams/TeamRunner.cs @@ -36,6 +36,10 @@ namespace Loadout.Agents.Teams; /// The remedies this machine has agreed may run, from its own configuration. /// Never read from a team's directory, which its own nodes write in. /// +/// +/// The catalogue tool versions this machine has agreed may run, from its own +/// configuration. Never read from the catalogue, which agents write in. +/// /// /// What this run is judged on, each one checkable, or null for a run with /// nothing but its goal. @@ -90,7 +94,8 @@ public sealed record TeamRunRequest( IReadOnlyList? Criteria = null, string? Resuming = null, string? ResumeMessage = null, - TimeSpan? TakeRecommendationAfter = null); + TimeSpan? TakeRecommendationAfter = null, + IReadOnlyList? TrustedTools = null); /// How a run ended. /// The run's identifier, which names its directory under the state root. @@ -389,7 +394,8 @@ public TeamRunner( Core.Projects.IProjectService? projects = null, Core.Git.IGitManager? git = null, IChildLifetime? lifetime = null, - Core.Tasks.ITaskService? tasks = null) + Core.Tasks.ITaskService? tasks = null, + Core.Tools.IToolRegistry? tools = null) { _launcher = launcher; _paths = paths; @@ -398,8 +404,11 @@ public TeamRunner( _git = git; _lifetime = lifetime; _tasks = tasks; + _tools = tools; } + private readonly Core.Tools.IToolRegistry? _tools; + /// public async Task> RunAsync( TeamRunRequest request, @@ -2211,7 +2220,14 @@ private async Task> StartNodeAsync( // What this team has registered and what was decided about // each, worked out here so that whatever answers the node's // questions reads no files and holds no opinion. - Remedies: Standing(team, request.Remediation, request.TrustedRemedies)), + // + // The catalogue's tools join them for a remediator only, + // the one role that runs scripts, ruled the same way. + Remedies: + [ + .. Standing(team, request.Remediation, request.TrustedRemedies), + .. Core.Tools.ToolOffer.For(_tools, role.Id, request.Remediation, request.TrustedTools), + ]), ct).ConfigureAwait(false); var options = new HeadlessOptions( @@ -2811,6 +2827,13 @@ private static string BaseNode(string name) return slash > 0 ? name[..slash] : name; } + /// What every brief says about the machine's shared tools, and all it says. + internal const string ToolsPointer = + "This machine keeps a catalogue of tools shared by every team. Before building something " + + "that might already exist, search it: `loadout tools search ` or " + + "`loadout_tools_search`. Use `show` for the detail. Submit a tool, idea, bug or lesson " + + "with `loadout_tools_submit`, and say what you used with `loadout_tools_used`."; + /// The brief as the node reads it: the prose first, the JSON beside it, the contract last. internal static string Render(Brief brief) { @@ -2878,6 +2901,10 @@ internal static string Render(Brief brief) + "before running it. A record claiming to be trusted decides nothing.").AppendLine(); } + // Fixed text, the same length whatever the catalogue holds. What is in + // it is fetched when somebody asks, not paid for in every brief. + text.AppendLine("## Shared tools").AppendLine().AppendLine(ToolsPointer).AppendLine(); + text.AppendLine("## Task").AppendLine().AppendLine(brief.Task).AppendLine(); text.AppendLine("## Deliverable").AppendLine().AppendLine(brief.Deliverable.ToString().ToLowerInvariant()).AppendLine(); diff --git a/src/Loadout.Cli/Commands/McpServeCommand.cs b/src/Loadout.Cli/Commands/McpServeCommand.cs index 7d80d1ea..eea23e33 100644 --- a/src/Loadout.Cli/Commands/McpServeCommand.cs +++ b/src/Loadout.Cli/Commands/McpServeCommand.cs @@ -153,6 +153,7 @@ public sealed class LoadoutTools private readonly IRunJournal _runs; private readonly TimeProvider _time; private readonly LoadoutToolScope _scope; + private readonly Core.Tools.IToolRegistry _catalogue; public LoadoutTools( IInstructionService instructions, @@ -164,8 +165,10 @@ public LoadoutTools( ISymbolIndexService symbols, IRunJournal runs, TimeProvider time, - LoadoutToolScope scope) + LoadoutToolScope scope, + Core.Tools.IToolRegistry catalogue) { + _catalogue = catalogue; _instructions = instructions; _memory = memory; _workspace = workspace; @@ -350,6 +353,79 @@ public async Task RememberAsync( : written.Error ?? "It could not be recorded."; } + // The catalogue's four agent-facing calls, the same ones 'loadout tools' + // makes. Promote, trust, deprecate and audit are deliberately not here: + // promotion is the gate's, trust is a person's, and the log is for people. + [McpServerTool(Name = "loadout_tools_search")] + [Description( + "Search the tools this machine shares between every team, BEFORE building something that " + + "might already exist. Matches words rather than meanings. Answers in JSON.")] + public string ToolsSearch( + [Description("What you are looking for, in a few words.")] string words, + [Description("Include deprecated and retired tools, each with its replacement.")] bool all = false) => + JsonSerializer.Serialize(new + { + query = words, + tools = _catalogue.Search(words ?? string.Empty, all).Select(ToolShapes.Found), + }); + + [McpServerTool(Name = "loadout_tools_show")] + [Description( + "One shared tool in full: what it is for, its inputs and outputs, its versions, how it has " + + "gone for others, and why it exists. Answers in JSON.")] + public string ToolsShow([Description("The tool's name.")] string name) + { + var shown = _catalogue.Show(ToolShapes.Split(name).Name); + + return shown.Failed + ? shown.Error ?? "It could not be read." + : JsonSerializer.Serialize(ToolShapes.Shown(shown.Value!, shown.Value!.Record.Active, trusted: false)); + } + + [McpServerTool(Name = "loadout_tools_submit")] + [Description( + "Send the shared catalogue a candidate tool, an idea, a bug in a tool, or a lesson worth " + + "turning into one. Not for anything secret: the text is screened and a credential is refused.")] + public string ToolsSubmit( + [Description("candidate, idea, bug or lesson.")] string kind, + [Description("What it is, in a sentence or a paragraph.")] string text, + [Description("The tool it is about, where it is about one.")] string? tool = null, + [Description("A candidate's script, where there is one.")] string? script = null, + [Description("A candidate's one-line summary.")] string? summary = null) + { + // The same call 'loadout tools submit' makes, screening included. + var submitted = _catalogue.Submit(new Core.Tools.ToolSubmission( + kind, text, tool, "agent", null, script, null, summary)); + + return submitted.Failed + ? submitted.Error ?? "It could not be submitted." + : $"Submitted as {submitted.Value!.Id}." + + (submitted.Value!.Overlapping.Count > 0 + ? $" It overlaps {string.Join(", ", submitted.Value!.Overlapping)}; extending that is usually better." + : string.Empty); + } + + [McpServerTool(Name = "loadout_tools_used")] + [Description( + "Say how a use of a shared tool went, so whoever looks after it knows: ok, failed, or " + + "workaround when you had to work round it.")] + public string ToolsUsed( + [Description("The tool and the version used, as name@version.")] string tool, + [Description("ok, failed or workaround.")] string outcome, + [Description("What happened, where it did not simply work.")] string? note = null) + { + var (name, version) = ToolShapes.Split(tool); + var recorded = _catalogue.RecordUsage(new Models.Tools.ToolUsage + { + Tool = name, + Version = version ?? string.Empty, + Outcome = outcome, + Note = note ?? string.Empty, + }); + + return recorded.Failed ? recorded.Error ?? "It could not be recorded." : "Recorded."; + } + [McpServerTool(Name = "loadout_locate")] [Description( "Where a type or member is declared, as file and line, from an index kept in step with " diff --git a/src/Loadout.Cli/Commands/TeamCommands.cs b/src/Loadout.Cli/Commands/TeamCommands.cs index 303467bb..7138cd8a 100644 --- a/src/Loadout.Cli/Commands/TeamCommands.cs +++ b/src/Loadout.Cli/Commands/TeamCommands.cs @@ -1408,6 +1408,7 @@ internal static TeamRunRequest Requesting( // Never read from the team's directory, which its own nodes write // in. Trust lives on this machine or it is not trust. TrustedRemedies: machine?.Teams.TrustedRemedies, + TrustedTools: machine?.Teams.TrustedTools, // Blank ones dropped rather than passed through: an empty criterion // is one the lead can never report a verdict on, so it would refuse diff --git a/src/Loadout.Cli/Commands/TeamResumeCommand.cs b/src/Loadout.Cli/Commands/TeamResumeCommand.cs index 5e1f4bda..3349d80e 100644 --- a/src/Loadout.Cli/Commands/TeamResumeCommand.cs +++ b/src/Loadout.Cli/Commands/TeamResumeCommand.cs @@ -207,6 +207,7 @@ protected override async Task ExecuteAsync( OutwardAllowed: ceiling.Allowed, Remediation: machine.Value?.Teams.Remediation, TrustedRemedies: machine.Value?.Teams.TrustedRemedies, + TrustedTools: machine.Value?.Teams.TrustedTools, Resuming: run, ResumeMessage: settings.Message is { Length: > 0 } said ? said.Trim() : null, diff --git a/src/Loadout.Cli/Commands/ToolCommands.cs b/src/Loadout.Cli/Commands/ToolCommands.cs new file mode 100644 index 00000000..c96d2e53 --- /dev/null +++ b/src/Loadout.Cli/Commands/ToolCommands.cs @@ -0,0 +1,966 @@ +using System.ComponentModel; +using System.Globalization; +using Loadout.Cli.Infrastructure; +using Loadout.Core.Configuration; +using Loadout.Core.Teams; +using Loadout.Core.Tools; +using Loadout.Models; +using Loadout.Models.Configuration; +using Loadout.Models.Tools; +using Loadout.Tui; +using Spectre.Console; +using Spectre.Console.Cli; + +namespace Loadout.Cli.Commands; + +/// What every tools command shares. +public class ToolSettings : GlobalSettings +{ +} + +/// +/// Parsing and shaping shared by the tools commands and the MCP tools +/// that make the same calls. +/// +/// +/// One place for the shapes, so what an agent is told over MCP and what a +/// person reads with --json cannot drift apart. +/// +public static class ToolShapes +{ + /// Splits name@version; the version is null where none was given. + public static (string Name, string? Version) Split(string named) + { + var at = (named ?? string.Empty).IndexOf('@', StringComparison.Ordinal); + + return at < 0 ? (named ?? string.Empty, null) : (named![..at], named[(at + 1)..]); + } + + /// One tool as a search result. + public static object Found(ToolRecord one) => new + { + name = one.Name, + kind = one.Kind, + summary = one.Summary, + lifecycle = one.Lifecycle, + active = one.Active, + capabilities = one.Capabilities, + replacement = one.Deprecated?.Replacement is { Length: > 0 } instead ? instead : null, + }; + + /// One tool in full, and whether this machine trusts the version asked about. + public static object Shown(ToolShown shown, string? version, bool trusted) => new + { + name = shown.Record.Name, + kind = shown.Record.Kind, + owner = shown.Record.Owner, + summary = shown.Record.Summary, + lifecycle = shown.Record.Lifecycle, + active = shown.Record.Active, + version = version ?? shown.Record.Active, + versions = shown.Versions, + trusted, + deprecated = shown.Record.Deprecated, + lineage = shown.Record.Lineage, + usage = shown.Record.UsageSummary, + manifest = shown.Active, + }; + + /// Capabilities from a comma-separated list, or null for none. + public static IReadOnlyList? Words(string? list) => + list is { Length: > 0 } + ? [.. list.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)] + : null; + + /// Whether this machine agreed to that version, at the script it has now. + public static bool Trusted(IToolRegistry registry, MachineConfig? machine, string name, string? version) => + version is { Length: > 0 } + && registry.ScriptOf(name, version) is { } script + && (machine?.Teams.TrustedTools ?? []).Any(one => + string.Equals(one.Tool, name, StringComparison.OrdinalIgnoreCase) + && string.Equals(one.Version, version, StringComparison.Ordinal) + && string.Equals(one.Fingerprint, RemedyCeiling.Fingerprint(script), StringComparison.OrdinalIgnoreCase)); +} + +/// Search the machine's shared tools. +[Description("Search the tools this machine shares between teams.")] +[CommandMeta(CommandCategory.Start, + Intent = "tools search catalogue registry shared reusable find existing", + Example = "loadout tools search cache disk")] +public sealed class ToolSearchCommand : Command +{ + private readonly IToolRegistry _registry; + private readonly IAnsiConsole _console; + + public ToolSearchCommand(IToolRegistry registry, IAnsiConsole console) + { + _registry = registry; + _console = console; + } + + public sealed class Settings : ToolSettings + { + [CommandArgument(0, "[WORDS]")] + [Description("What you are looking for. Every tool when omitted.")] + public string[] Words { get; init; } = []; + + [CommandOption("--kind ")] + [Description("Only tools of this kind, such as disk or service.")] + public string Kind { get; init; } = string.Empty; + + [CommandOption("--all")] + [Description("Include deprecated and retired tools, each with what replaces it.")] + public bool All { get; init; } + } + + /// + protected override int Execute(CommandContext context, Settings settings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(settings); + + var output = new CommandOutput(_console, settings); + var words = string.Join(' ', settings.Words); + var found = _registry.Search(words, settings.All) + .Where(one => settings.Kind.Length == 0 || string.Equals(one.Kind, settings.Kind, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + if (output.IsJson) + { + output.WriteJson(new { query = words, root = _registry.Root(), tools = found.Select(ToolShapes.Found) }); + + return CommandOutput.Success(); + } + + if (found.Count == 0) + { + output.WriteLine("[dim]Nothing in the catalogue matched. It matches words, not meanings.[/]"); + + return CommandOutput.Success(); + } + + foreach (var one in found) + { + output.WriteLine( + $"[bold]{Markup.Escape(one.Name)}[/] [dim]{Markup.Escape(one.Kind)} {Markup.Escape(one.Lifecycle)}" + + (one.Active is { Length: > 0 } active ? $" v{Markup.Escape(active)}" : string.Empty) + "[/]"); + output.WriteLine($" {Shown.Safely(one.Summary)}"); + } + + return CommandOutput.Success(); + } +} + +/// One shared tool in full. +[Description("Show one shared tool: its manifest, versions, usage, lineage and trust here.")] +[CommandMeta(CommandCategory.Start, + Intent = "tools show manifest version lineage usage trusted", + Example = "loadout tools show free-disk-by-cache@1.2")] +public sealed class ToolShowCommand : AsyncCommand +{ + private readonly IToolRegistry _registry; + private readonly IConfigurationService _configuration; + private readonly IAnsiConsole _console; + + public ToolShowCommand(IToolRegistry registry, IConfigurationService configuration, IAnsiConsole console) + { + _registry = registry; + _configuration = configuration; + _console = console; + } + + public sealed class Settings : ToolSettings + { + [CommandArgument(0, "")] + [Description("The tool, as name or name@version.")] + public string Tool { get; init; } = string.Empty; + } + + /// + protected override async Task ExecuteAsync(CommandContext context, Settings settings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(settings); + + var output = new CommandOutput(_console, settings); + var (name, version) = ToolShapes.Split(settings.Tool); + var shown = _registry.Show(name); + + if (shown.Failed) + { + return output.Fail(shown); + } + + var machine = await _configuration.LoadMachineAsync(cancellationToken).ConfigureAwait(false); + var asked = version ?? shown.Value!.Record.Active; + var trusted = ToolShapes.Trusted(_registry, machine.Value, name, asked); + + if (output.IsJson) + { + output.WriteJson(ToolShapes.Shown(shown.Value!, asked, trusted)); + + return CommandOutput.Success(); + } + + var head = shown.Value!.Record; + output.WriteLine($"[bold]{Markup.Escape(head.Name)}[/] [dim]{Markup.Escape(head.Kind)} {Markup.Escape(head.Lifecycle)}[/]"); + output.WriteLine($" {Shown.Safely(head.Summary)}"); + + foreach (var (one, standing) in shown.Value!.Versions) + { + output.WriteLine($" v{Markup.Escape(one)} [dim]{Markup.Escape(standing)}{(one == head.Active ? ", active" : string.Empty)}[/]"); + } + + if (shown.Value!.Active is { } active) + { + output.WriteLine($" [dim]Purpose:[/] {Shown.Safely(active.Purpose)}"); + output.WriteLine($" [dim]Origin:[/] {Shown.Safely(active.Origin)}"); + } + + output.WriteLine( + $" [dim]Used {head.UsageSummary.Runs} times: {head.UsageSummary.Ok} ok, {head.UsageSummary.Failed} failed, " + + $"{head.UsageSummary.Workaround} worked around.[/]"); + output.WriteLine(trusted + ? $" [green]v{Markup.Escape(asked ?? string.Empty)} is trusted on this machine.[/]" + : " [dim]Not trusted on this machine. A remediator asks before running it.[/]"); + + return CommandOutput.Success(); + } +} + +/// Sending a candidate, idea, bug or lesson to the catalogue. +[Description("Submit a candidate tool, an idea, a bug or a lesson to the shared catalogue's inbox.")] +[CommandMeta(CommandCategory.Start, + Intent = "tools submit candidate idea bug lesson inbox propose", Mutates = true, + Example = "loadout tools submit --kind lesson --text \"Clearing the build cache fixed a full disk twice.\"")] +public sealed class ToolSubmitCommand : Command +{ + private readonly IToolRegistry _registry; + private readonly IAnsiConsole _console; + + public ToolSubmitCommand(IToolRegistry registry, IAnsiConsole console) + { + _registry = registry; + _console = console; + } + + public sealed class Settings : ToolSettings + { + [CommandOption("--kind ")] + [Description("candidate, idea, bug or lesson.")] + public string Kind { get; init; } = string.Empty; + + [CommandOption("--text ")] + [Description("What it says. Screened for credentials, and refused if it holds one.")] + public string Text { get; init; } = string.Empty; + + [CommandOption("--tool ")] + [Description("The tool it is about, where it is about one.")] + public string Tool { get; init; } = string.Empty; + + [CommandOption("--from-run ")] + [Description("The run it came from.")] + public string Run { get; init; } = string.Empty; + + [CommandOption("--script ")] + [Description("A candidate's script, checked for overlap with the tools already here.")] + public string Script { get; init; } = string.Empty; + + [CommandOption("--summary ")] + [Description("A candidate's one-line summary.")] + public string Summary { get; init; } = string.Empty; + + [CommandOption("--capabilities ")] + [Description("A candidate's search terms, separated by commas.")] + public string Capabilities { get; init; } = string.Empty; + } + + /// + protected override int Execute(CommandContext context, Settings settings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(settings); + + var output = new CommandOutput(_console, settings); + + if (settings.Text is not { Length: > 0 }) + { + return output.Fail("Say what it is with --text.", ExitCode.InvalidArguments); + } + + string? script = null; + + if (settings.Script is { Length: > 0 } file) + { + try + { + script = File.ReadAllText(file); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return output.Fail($"{file} could not be read: {ex.Message}", ExitCode.InvalidArguments); + } + } + + if (settings.DryRun) + { + if (output.IsJson) + { + output.WriteJson(new { dry_run = true, kind = settings.Kind, tool = settings.Tool }); + } + else + { + output.WriteLine($"[dim]Dry run: nothing was changed.[/] A {Markup.Escape(settings.Kind)} would be submitted."); + } + + return CommandOutput.Success(); + } + + var submitted = _registry.Submit(new ToolSubmission( + settings.Kind, + settings.Text, + settings.Tool is { Length: > 0 } tool ? tool : null, + Environment.UserName, + settings.Run is { Length: > 0 } run ? run : null, + script, + ToolShapes.Words(settings.Capabilities), + settings.Summary is { Length: > 0 } summary ? summary : null)); + + if (submitted.Failed) + { + return output.Fail(submitted); + } + + if (output.IsJson) + { + output.WriteJson(new { id = submitted.Value!.Id, kind = settings.Kind.Trim().ToLowerInvariant(), overlapping = submitted.Value!.Overlapping }); + + return CommandOutput.Success(); + } + + output.WriteLine($"[green]Submitted as {Markup.Escape(submitted.Value!.Id)}.[/]"); + + if (submitted.Value!.Overlapping.Count > 0) + { + output.WriteLine( + $"[yellow]It overlaps {Markup.Escape(string.Join(", ", submitted.Value!.Overlapping))}.[/] " + + "Extending that is usually better than a second tool beside it."); + } + + return CommandOutput.Success(); + } +} + +/// Saying how a use of a shared tool went. +[Description("Record a use of a shared tool: whether it worked, failed, or needed working around.")] +[CommandMeta(CommandCategory.Start, + Intent = "tools used usage outcome failed workaround record", Mutates = true, + Example = "loadout tools used free-disk-by-cache@1.2 --outcome ok")] +public sealed class ToolUsedCommand : Command +{ + private readonly IToolRegistry _registry; + private readonly IAnsiConsole _console; + + public ToolUsedCommand(IToolRegistry registry, IAnsiConsole console) + { + _registry = registry; + _console = console; + } + + public sealed class Settings : ToolSettings + { + [CommandArgument(0, "")] + [Description("The tool and the version used, as name@version.")] + public string Tool { get; init; } = string.Empty; + + [CommandOption("--outcome ")] + [Description("ok, failed or workaround.")] + public string Outcome { get; init; } = ToolOutcome.Ok; + + [CommandOption("--run ")] + [Description("The run it was used in.")] + public string Run { get; init; } = string.Empty; + + [CommandOption("--team ")] + [Description("The team that used it.")] + public string Team { get; init; } = string.Empty; + + [CommandOption("--note ")] + [Description("What happened, where it did not simply work.")] + public string Note { get; init; } = string.Empty; + } + + /// + protected override int Execute(CommandContext context, Settings settings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(settings); + + var output = new CommandOutput(_console, settings); + var (name, version) = ToolShapes.Split(settings.Tool); + + if (settings.DryRun) + { + if (output.IsJson) + { + output.WriteJson(new { dry_run = true, tool = name, version, outcome = settings.Outcome }); + } + else + { + output.WriteLine($"[dim]Dry run: nothing was changed.[/] A use of {Markup.Escape(settings.Tool)} would be recorded."); + } + + return CommandOutput.Success(); + } + + var recorded = _registry.RecordUsage(new ToolUsage + { + Tool = name, + Version = version ?? string.Empty, + Outcome = settings.Outcome, + Run = settings.Run, + Team = settings.Team, + Note = settings.Note, + }); + + if (recorded.Failed) + { + return output.Fail(recorded); + } + + if (output.IsJson) + { + output.WriteJson(new { tool = name, version, outcome = settings.Outcome, recorded = true }); + } + else + { + output.WriteLine($"[green]Recorded: {Markup.Escape(settings.Tool)} {Markup.Escape(settings.Outcome)}.[/]"); + } + + return CommandOutput.Success(); + } +} + +/// What has happened to the catalogue. +[Description("Read the shared catalogue's audit log: submissions, verifies, promotions, deprecations, trust.")] +[CommandMeta(CommandCategory.Start, + Intent = "tools audit log history promote deprecate who when", + Example = "loadout tools audit --since 7d")] +public sealed class ToolAuditCommand : Command +{ + private readonly IToolRegistry _registry; + private readonly IAnsiConsole _console; + + public ToolAuditCommand(IToolRegistry registry, IAnsiConsole console) + { + _registry = registry; + _console = console; + } + + public sealed class Settings : ToolSettings + { + [CommandOption("--tool ")] + [Description("Only this tool.")] + public string Tool { get; init; } = string.Empty; + + [CommandOption("--since ")] + [Description("Only entries this recent, in days, such as 7d.")] + public string Since { get; init; } = string.Empty; + } + + /// + protected override int Execute(CommandContext context, Settings settings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(settings); + + var output = new CommandOutput(_console, settings); + DateTimeOffset? since = null; + + if (settings.Since is { Length: > 0 } age) + { + if (!int.TryParse(age.TrimEnd('d', 'D'), NumberStyles.None, CultureInfo.InvariantCulture, out var days)) + { + return output.Fail($"'{age}' is not an age in days, such as 7d.", ExitCode.InvalidArguments); + } + + since = DateTimeOffset.UtcNow.AddDays(-days); + } + + var entries = _registry.Audit(settings.Tool is { Length: > 0 } tool ? tool : null, since); + + if (output.IsJson) + { + output.WriteJson(new { entries }); + + return CommandOutput.Success(); + } + + foreach (var one in entries) + { + output.WriteLine( + $"[dim]{one.At:yyyy-MM-dd HH:mm}[/] {Markup.Escape(one.Action)} {Markup.Escape(one.Tool)}" + + (one.Version is { Length: > 0 } v ? $"@{Markup.Escape(v)}" : string.Empty) + + (one.Note is { Length: > 0 } note ? $" [dim]{Shown.Safely(note)}[/]" : string.Empty)); + } + + return CommandOutput.Success(); + } +} + +/// Running a draft's harness and the regression gate. +[Description("Verify a draft: run its harness and the known-good cases, where this machine allows it.")] +[CommandMeta(CommandCategory.Start, + Intent = "tools verify harness test regression gate draft", Mutates = true, + Example = "loadout tools verify /tools/drafts/free-disk-by-cache/1")] +public sealed class ToolVerifyCommand : AsyncCommand +{ + private readonly IToolRegistry _registry; + private readonly IConfigurationService _configuration; + private readonly IAnsiConsole _console; + + public ToolVerifyCommand(IToolRegistry registry, IConfigurationService configuration, IAnsiConsole console) + { + _registry = registry; + _configuration = configuration; + _console = console; + } + + public sealed class Settings : ToolSettings + { + [CommandArgument(0, "")] + [Description("The draft's directory, under the catalogue's drafts.")] + public string Draft { get; init; } = string.Empty; + } + + /// + protected override async Task ExecuteAsync(CommandContext context, Settings settings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(settings); + + var output = new CommandOutput(_console, settings); + + if (settings.DryRun) + { + if (output.IsJson) + { + output.WriteJson(new { dry_run = true, draft = settings.Draft }); + } + else + { + output.WriteLine($"[dim]Dry run: nothing was run or changed.[/] {Markup.Escape(settings.Draft)} would be verified."); + } + + return CommandOutput.Success(); + } + + // What this machine says about running harnesses, and what a person + // agreed to, from this machine's configuration and nowhere else. + var machine = await _configuration.LoadMachineAsync(cancellationToken).ConfigureAwait(false); + + if (machine.Failed) + { + return output.Fail(machine); + } + + var teams = machine.Value!.Teams; + var consent = new ToolTestConsent( + teams.Remediation.TryGetValue(ToolHarness.Kind, out var rule) ? rule : null, + teams.TrustedRemedies); + + var verified = await _registry.VerifyAsync(settings.Draft, consent, cancellationToken).ConfigureAwait(false); + + if (verified.Failed) + { + return output.Fail(verified); + } + + var result = verified.Value!; + + if (output.IsJson) + { + output.WriteJson(new + { + ruling = result.Ruling.ToString().ToLowerInvariant(), + because = result.Because, + passed = result.Gate?.Passed, + }); + + return CommandOutput.Success(); + } + + output.WriteLine(result.Gate is { Passed: true } + ? $"[green]Verified.[/] {Shown.Safely(result.Because)}" + : $"[yellow]Not verified.[/] {Shown.Safely(result.Because)}"); + + return CommandOutput.Success(); + } +} + +/// Making a verified draft a version. +[Description("Promote a verified draft to a written-once version and make it active.")] +[CommandMeta(CommandCategory.Start, + Intent = "tools promote version active known-good release draft", Mutates = true, + Example = "loadout tools promote --because lesson --source inbox/2026-09-23-7f3a")] +public sealed class ToolPromoteCommand : Command +{ + private readonly IToolRegistry _registry; + private readonly IAnsiConsole _console; + + public ToolPromoteCommand(IToolRegistry registry, IAnsiConsole console) + { + _registry = registry; + _console = console; + } + + public sealed class Settings : ToolSettings + { + [CommandArgument(0, "")] + [Description("The draft's directory, under the catalogue's drafts.")] + public string Draft { get; init; } = string.Empty; + + [CommandOption("--because ")] + [Description("lesson, requirement, bug, idea, nomination or consolidation.")] + public string Because { get; init; } = string.Empty; + + [CommandOption("--source ")] + [Description("The submission or run this version came from.")] + public string Source { get; init; } = string.Empty; + + [CommandOption("--owner ")] + [Description("The team that maintains it, for a new tool.")] + public string Owner { get; init; } = string.Empty; + + [CommandOption("--kind ")] + [Description("The kind this machine's remediation rules decide on, for a new tool.")] + public string Kind { get; init; } = string.Empty; + + [CommandOption("--summary ")] + [Description("One line saying what it is for.")] + public string Summary { get; init; } = string.Empty; + + [CommandOption("--capabilities ")] + [Description("Search terms, separated by commas.")] + public string Capabilities { get; init; } = string.Empty; + } + + /// + protected override int Execute(CommandContext context, Settings settings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(settings); + + var output = new CommandOutput(_console, settings); + + if (settings.Because is not { Length: > 0 } || settings.Source is not { Length: > 0 }) + { + return output.Fail( + "Say why the version exists with --because and where it came from with --source.", + ExitCode.InvalidArguments); + } + + if (settings.DryRun) + { + if (output.IsJson) + { + output.WriteJson(new { dry_run = true, draft = settings.Draft }); + } + else + { + output.WriteLine($"[dim]Dry run: nothing was changed.[/] {Markup.Escape(settings.Draft)} would be promoted."); + } + + return CommandOutput.Success(); + } + + var promoted = _registry.Promote(settings.Draft, new ToolPromotionRequest( + settings.Because, + settings.Source, + Environment.UserName, + Owner: settings.Owner is { Length: > 0 } owner ? owner : null, + Kind: settings.Kind is { Length: > 0 } kind ? kind : null, + Summary: settings.Summary is { Length: > 0 } summary ? summary : null, + Capabilities: ToolShapes.Words(settings.Capabilities))); + + if (promoted.Failed) + { + return output.Fail(promoted); + } + + if (output.IsJson) + { + output.WriteJson(new { name = promoted.Value!.Name, version = promoted.Value!.Version, fingerprint = promoted.Value!.Fingerprint }); + } + else + { + output.WriteLine($"[green]{Markup.Escape(promoted.Value!.Name)}@{Markup.Escape(promoted.Value!.Version)} is promoted and active.[/]"); + output.WriteLine("[dim]Nobody has trusted it yet: a remediator asks before running it.[/]"); + } + + return CommandOutput.Success(); + } +} + +/// Marking a tool as not to be used any more. +[Description("Deprecate a shared tool, naming what replaces it or why.")] +[CommandMeta(CommandCategory.Start, + Intent = "tools deprecate replace obsolete lifecycle", Mutates = true, + Example = "loadout tools deprecate free-disk-by-cache --replacement cache-sweeper")] +public sealed class ToolDeprecateCommand : Command +{ + private readonly IToolRegistry _registry; + private readonly IAnsiConsole _console; + + public ToolDeprecateCommand(IToolRegistry registry, IAnsiConsole console) + { + _registry = registry; + _console = console; + } + + public sealed class Settings : ToolSettings + { + [CommandArgument(0, "")] + [Description("The tool.")] + public string Tool { get; init; } = string.Empty; + + [CommandOption("--replacement ")] + [Description("The tool to use instead.")] + public string Replacement { get; init; } = string.Empty; + + [CommandOption("--reason ")] + [Description("Why, where nothing replaces it.")] + public string Reason { get; init; } = string.Empty; + } + + /// + protected override int Execute(CommandContext context, Settings settings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(settings); + + var output = new CommandOutput(_console, settings); + + if (settings.DryRun) + { + return ToolLifecycleOutput.DryRun(output, settings.Tool, "deprecated"); + } + + var done = _registry.Deprecate( + settings.Tool, + settings.Replacement is { Length: > 0 } instead ? instead : null, + settings.Reason is { Length: > 0 } why ? why : null); + + return done.Failed ? output.Fail(done) : ToolLifecycleOutput.Done(output, settings.Tool, ToolLifecycle.Deprecated); + } +} + +/// Retiring a deprecated tool. +[Description("Retire a deprecated shared tool nobody has used for thirty days.")] +[CommandMeta(CommandCategory.Start, + Intent = "tools retire remove lifecycle unused", Mutates = true, + Example = "loadout tools retire free-disk-by-cache")] +public sealed class ToolRetireCommand : Command +{ + private readonly IToolRegistry _registry; + private readonly IAnsiConsole _console; + + public ToolRetireCommand(IToolRegistry registry, IAnsiConsole console) + { + _registry = registry; + _console = console; + } + + public sealed class Settings : ToolSettings + { + [CommandArgument(0, "")] + [Description("The tool.")] + public string Tool { get; init; } = string.Empty; + } + + /// + protected override int Execute(CommandContext context, Settings settings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(settings); + + var output = new CommandOutput(_console, settings); + + if (settings.DryRun) + { + return ToolLifecycleOutput.DryRun(output, settings.Tool, "retired"); + } + + var done = _registry.Retire(settings.Tool); + + return done.Failed ? output.Fail(done) : ToolLifecycleOutput.Done(output, settings.Tool, ToolLifecycle.Retired); + } +} + +/// What deprecate and retire say. +internal static class ToolLifecycleOutput +{ + public static int DryRun(CommandOutput output, string tool, string would) + { + if (output.IsJson) + { + output.WriteJson(new { dry_run = true, tool }); + } + else + { + output.WriteLine($"[dim]Dry run: nothing was changed.[/] {Markup.Escape(tool)} would be {would}."); + } + + return CommandOutput.Success(); + } + + public static int Done(CommandOutput output, string tool, string lifecycle) + { + if (output.IsJson) + { + output.WriteJson(new { tool, lifecycle }); + } + else + { + output.WriteLine($"[green]{Markup.Escape(tool)} is {Markup.Escape(lifecycle)}.[/]"); + } + + return CommandOutput.Success(); + } +} + +/// +/// Saying that a remediator may run one version of a shared tool on this +/// machine. +/// +/// +/// The key an agent can never turn, as for a team's remedies: it is kept in +/// this machine's configuration, against the exact script, and a changed +/// script is asked about again. +/// +[Description("Trust one version of a shared tool so a remediator may run it, or take that back.")] +[CommandMeta(CommandCategory.Start, + Intent = "tools trust approve allow script version remediator", Mutates = true, + Example = "loadout tools trust free-disk-by-cache@1.2")] +public sealed class ToolTrustCommand : AsyncCommand +{ + private readonly IToolRegistry _registry; + private readonly IConfigurationService _configuration; + private readonly IAnsiConsole _console; + + public ToolTrustCommand(IToolRegistry registry, IConfigurationService configuration, IAnsiConsole console) + { + _registry = registry; + _configuration = configuration; + _console = console; + } + + public sealed class Settings : ToolSettings + { + [CommandArgument(0, "")] + [Description("The tool and version, as name@version.")] + public string Tool { get; init; } = string.Empty; + + [CommandOption("--revoke")] + [Description("Take the agreement back.")] + public bool Revoke { get; init; } + + [CommandOption("--by ")] + [Description("Who is saying so, for the record. Your user name when omitted.")] + public string By { get; init; } = string.Empty; + } + + /// + protected override async Task ExecuteAsync(CommandContext context, Settings settings, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(settings); + + var output = new CommandOutput(_console, settings); + var (name, version) = ToolShapes.Split(settings.Tool); + + if (version is not { Length: > 0 }) + { + return output.Fail("Trust is given to one version: name it as name@version.", ExitCode.InvalidArguments); + } + + var machine = await _configuration.LoadMachineAsync(cancellationToken).ConfigureAwait(false); + + if (machine.Failed) + { + return output.Fail(machine); + } + + var config = machine.Value!; + var already = config.Teams.TrustedTools.FirstOrDefault(one => + string.Equals(one.Tool, name, StringComparison.OrdinalIgnoreCase) + && string.Equals(one.Version, version, StringComparison.Ordinal)); + + string? fingerprint = null; + + if (!settings.Revoke) + { + // Only a version whose files still match what was promoted. A + // person agreeing to a script that has been changed since would be + // agreeing to something the gate never saw. + if (_registry.ScriptOf(name, version) is not { } script) + { + return output.Fail( + $"{name}@{version} is not a promoted version whose files still match, so there is nothing to agree to.", + ExitCode.PolicyViolation); + } + + fingerprint = RemedyCeiling.Fingerprint(script); + } + + if (settings.DryRun) + { + if (output.IsJson) + { + output.WriteJson(new { dry_run = true, tool = name, version, revoke = settings.Revoke }); + } + else + { + output.WriteLine( + $"[dim]Dry run: nothing was changed.[/] {Markup.Escape(settings.Tool)} would " + + (settings.Revoke ? "stop being agreed to." : "be agreed to.")); + } + + return CommandOutput.Success(); + } + + if (already is not null) + { + config.Teams.TrustedTools.Remove(already); + } + + if (!settings.Revoke) + { + config.Teams.TrustedTools.Add(new TrustedTool + { + Tool = name, + Version = version, + Fingerprint = fingerprint!, + By = settings.By is { Length: > 0 } who ? who : Environment.UserName, + At = DateTimeOffset.UtcNow, + }); + } + + var saved = await _configuration.SaveMachineAsync(config, cancellationToken).ConfigureAwait(false); + + if (saved.Failed) + { + return output.Fail(saved); + } + + _registry.RecordTrust(name, version, settings.Revoke, settings.By is { Length: > 0 } by ? by : Environment.UserName); + + if (output.IsJson) + { + output.WriteJson(new { tool = name, version, trusted = !settings.Revoke, fingerprint }); + } + else if (settings.Revoke) + { + output.WriteLine($"[yellow]{Markup.Escape(settings.Tool)} is no longer agreed to.[/]"); + } + else + { + output.WriteLine($"[green]{Markup.Escape(settings.Tool)} is agreed to on this machine.[/]"); + output.WriteLine( + "[dim]A remediator may run it where this machine lets a trusted remedy of its kind run. " + + "Changing the script takes this back.[/]"); + } + + return CommandOutput.Success(); + } +} diff --git a/src/Loadout.Cli/Program.cs b/src/Loadout.Cli/Program.cs index 1fe736d9..7edcdf84 100644 --- a/src/Loadout.Cli/Program.cs +++ b/src/Loadout.Cli/Program.cs @@ -628,6 +628,24 @@ private static void Configure(IConfigurator config, bool showFullExceptions, boo task.AddCommand("remove"); }); + TopBranch(config, "tools", tools => + { + tools.Describe( + "The machine's catalogue of tools shared by every team: search it, submit to it, and decide what may run.", + CommandCategory.Start, + "tools catalogue registry shared reusable global tool creator refiner"); + tools.AddCommand("search"); + tools.AddCommand("show"); + tools.AddCommand("submit"); + tools.AddCommand("used"); + tools.AddCommand("audit"); + tools.AddCommand("verify"); + tools.AddCommand("promote"); + tools.AddCommand("deprecate"); + tools.AddCommand("retire"); + tools.AddCommand("trust"); + }); + TopBranch(config, "team", team => { team.Describe( diff --git a/src/Loadout.Core/Tools/ToolOffer.cs b/src/Loadout.Core/Tools/ToolOffer.cs new file mode 100644 index 00000000..070f5895 --- /dev/null +++ b/src/Loadout.Core/Tools/ToolOffer.cs @@ -0,0 +1,96 @@ +using Loadout.Core.Teams; +using Loadout.Models.Configuration; +using Loadout.Models.Teams; + +namespace Loadout.Core.Tools; + +/// +/// The catalogue's tools as a remediator is offered them: each active version +/// shown to the remedy ruling as though it were a remedy, and decided the same +/// way. +/// +/// +/// +/// The ruling is unchanged. A tool is shown +/// to it as a remedy named tool:<name>@<version> of the +/// tool's kind, and what this machine agreed to is read from +/// , never from the catalogue, which +/// agents write. +/// +/// +/// Named by the versioned file name, so the permission check that matches a +/// call by the script's file name matches one version and not the next. +/// +/// +public static class ToolOffer +{ + /// The one role that runs scripts, and so the one role offered a tool. + public const string Remediator = "role.remediator"; + + /// What a tool version is called where a remedy's name would be. + public static string Named(string tool, string version) => $"tool:{tool}@{version}"; + + /// + /// What a node in is offered from the catalogue, + /// with what this machine decided about each. + /// + public static IReadOnlyList For( + IToolRegistry? registry, + string role, + IReadOnlyDictionary? rules, + IReadOnlyList? trusted) + { + if (registry is null || !string.Equals(role, Remediator, StringComparison.OrdinalIgnoreCase)) + { + return []; + } + + var standing = new List(); + + foreach (var (record, version, script) in registry.Offerable()) + { + var remedy = new Remedy + { + Name = Named(record.Name, version.Version), + Kind = record.Kind, + What = record.Summary, + Assumes = string.Join("; ", version.Dependencies), + Proves = version.Outputs.Stdout, + Script = version.Script, + }; + + var rule = rules is not null + && rules.TryGetValue(record.Kind is { Length: > 0 } kind ? kind.Trim() : "unclassified", out var said) + ? said + : RemedyRules.Default; + + var decided = RemedyCeiling.Decide(remedy, rule, script, Agreed(trusted, record.Name, version.Version)); + + standing.Add(new RemedyStanding( + remedy.Name, + remedy.Script, + decided.Ruling.ToString().ToLowerInvariant(), + decided.Because, + remedy.What, + remedy.Assumes, + remedy.Proves)); + } + + return standing; + } + + /// This machine's agreements about one version, in the shape the ruling reads. + private static IReadOnlyList Agreed(IReadOnlyList? trusted, string tool, string version) => + [ + .. (trusted ?? []) + .Where(one => string.Equals(one.Tool, tool, StringComparison.OrdinalIgnoreCase) + && string.Equals(one.Version, version, StringComparison.Ordinal)) + .Select(one => new TrustedRemedy + { + Remedy = Named(tool, version), + Fingerprint = one.Fingerprint, + By = one.By, + At = one.At, + }), + ]; +} diff --git a/src/Loadout.Core/Tools/ToolRegistry.cs b/src/Loadout.Core/Tools/ToolRegistry.cs index 8c730c37..d46432a8 100644 --- a/src/Loadout.Core/Tools/ToolRegistry.cs +++ b/src/Loadout.Core/Tools/ToolRegistry.cs @@ -117,8 +117,29 @@ Task> VerifyAsync( /// Whether a tool is worth another look. bool NeedsRefining(string name); + + /// + /// Every active tool whose active version is known-good and still matches + /// what was promoted, with its script as it is now. + /// + IReadOnlyList Offerable(); + + /// + /// A promoted version's script, or null where it is missing or no longer + /// matches what was promoted. + /// + string? ScriptOf(string name, string version); + + /// Records in the audit log that a person trusted a version, or took it back. + void RecordTrust(string name, string version, bool revoked, string by); } +/// An active tool a remediator could be offered. +/// Its head. +/// The active version's manifest. +/// The script's text, as it is on disk now. +public sealed record ToolOffered(ToolRecord Record, ToolVersion Version, string Script); + /// /// The machine's shared catalogue of tools, in files under . /// @@ -881,6 +902,61 @@ public bool NeedsRefining(string name) return standDowns < 2; } + /// + public IReadOnlyList Offerable() + { + var offered = new List(); + + foreach (var head in Heads().Where(one => one.Lifecycle == ToolLifecycle.Active)) + { + if (ActiveVersion(head) is not { } version) + { + continue; + } + + try + { + offered.Add(new ToolOffered( + head, + version, + File.ReadAllText(Path.Combine(VersionDirectory(head.Name, version.Version), version.Script)))); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // A script that cannot be read is not offered: nobody could + // tell whether it is the one that was agreed to. + } + } + + return offered; + } + + /// + public string? ScriptOf(string name, string version) + { + if (Standing(name, version) != ToolVersionStatus.KnownGood) + { + return null; + } + + var directory = VersionDirectory(name, version); + + try + { + return ReadYaml(Path.Combine(directory, "manifest.yaml")) is { Script: { Length: > 0 } script } + ? File.ReadAllText(Path.Combine(directory, script)) + : null; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return null; + } + } + + /// + public void RecordTrust(string name, string version, bool revoked, string by) => + Record(revoked ? "untrust" : "trust", name, version, by, null, null); + private static bool IsSignal(ToolAuditEntry entry) => entry.Action is "promote" or "submit" || (entry.Action == "used" && entry.Note is ToolOutcome.Failed or ToolOutcome.Workaround); diff --git a/src/Loadout.Models/Configuration/MachineConfig.cs b/src/Loadout.Models/Configuration/MachineConfig.cs index 48578e72..59515f13 100644 --- a/src/Loadout.Models/Configuration/MachineConfig.cs +++ b/src/Loadout.Models/Configuration/MachineConfig.cs @@ -65,6 +65,30 @@ public sealed class TrustedRemedy public DateTimeOffset? At { get; set; } } +/// One version of a catalogue tool somebody at this machine agreed may run. +/// +/// A version rather than a tool, and a fingerprint as well, for the reason a +/// remedy carries one: trust is granted to a script, and a tool whose next +/// version is better is still a script nobody has read yet. +/// +public sealed class TrustedTool +{ + /// The tool, by name. + public string Tool { get; set; } = string.Empty; + + /// The version agreed to, major.minor. + public string Version { get; set; } = string.Empty; + + /// The script as it was when it was agreed to. + public string Fingerprint { get; set; } = string.Empty; + + /// Who said so. + public string By { get; set; } = string.Empty; + + /// When. + public DateTimeOffset? At { get; set; } +} + public sealed class MachineTeams { /// @@ -134,6 +158,17 @@ public sealed class MachineTeams /// public List TrustedRemedies { get; set; } = []; + /// + /// Catalogue tools somebody at this machine has said may run, each naming + /// the exact version and script they agreed to. + /// + /// + /// Written only by loadout tools trust, and here for the reason + /// is: the catalogue's files are written by + /// agents, so nothing in them can say what may run. + /// + public List TrustedTools { get; set; } = []; + /// /// The address the dashboard and its webhook listen on. /// diff --git a/tests/Loadout.Tests/Contract/ToolCommandsContractTests.cs b/tests/Loadout.Tests/Contract/ToolCommandsContractTests.cs new file mode 100644 index 00000000..5a476110 --- /dev/null +++ b/tests/Loadout.Tests/Contract/ToolCommandsContractTests.cs @@ -0,0 +1,176 @@ +using System.Text.Json; +using FluentAssertions; +using Loadout.Core.Teams; +using Loadout.Tests.Unit; +using Xunit; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace Loadout.Tests.Contract; + +/// +/// The shapes loadout tools ... --json prints, which scripts and the +/// MCP tools' callers read, and that --dry-run changes nothing. +/// +/// +/// The catalogue is built in a temporary directory by the registry itself, +/// promoted through the gate, and copied to where the built launcher keeps it, +/// which the launcher names in tools search --json. +/// +[Collection(ContractCollection.Name)] +public sealed class ToolCommandsContractTests +{ + private const string Script = "param([string]$CachePath)\nGet-ChildItem $CachePath | Remove-Item\n"; + + private static async Task<(LoadoutProcess Loadout, string Root)> CatalogueAsync() + { + var loadout = new LoadoutProcess(); + var root = (await loadout.RunAsync("tools", "search", "x", "--json")).Json().GetProperty("root").GetString()!; + + using var store = new ToolStoreFixture(); + var (registry, _) = store.Registry(); + await store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + + Copy(registry.Root(), root); + + return (loadout, root); + } + + private static void Copy(string from, string to) + { + foreach (var directory in Directory.EnumerateDirectories(from, "*", SearchOption.AllDirectories)) + { + Directory.CreateDirectory(Path.Combine(to, Path.GetRelativePath(from, directory))); + } + + Directory.CreateDirectory(to); + + foreach (var file in Directory.EnumerateFiles(from, "*", SearchOption.AllDirectories)) + { + File.Copy(file, Path.Combine(to, Path.GetRelativePath(from, file)), overwrite: true); + } + } + + private static Dictionary Snapshot(string root) => + Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories) + .Where(file => !file.EndsWith("registry.lock", StringComparison.Ordinal)) + .ToDictionary(file => Path.GetRelativePath(root, file), File.ReadAllText, StringComparer.Ordinal); + + [BuiltCliFact] + public async Task Search_json_names_the_query_the_root_and_each_tool() + { + var (loadout, root) = await CatalogueAsync(); + using var _ = loadout; + + var run = await loadout.RunAsync("tools", "search", "cache", "--json"); + + run.ExitCode.Should().Be(0, run.StandardError); + var json = run.Json(); + json.GetProperty("query").GetString().Should().Be("cache"); + json.GetProperty("root").GetString().Should().Be(root); + var tool = json.GetProperty("tools").EnumerateArray().Should().ContainSingle().Subject; + tool.GetProperty("name").GetString().Should().Be("free-cache"); + tool.GetProperty("active").GetString().Should().Be("1.0"); + tool.GetProperty("lifecycle").GetString().Should().Be("active"); + } + + [BuiltCliFact] + public async Task Show_json_carries_versions_and_trust() + { + var (loadout, _) = await CatalogueAsync(); + using var _ = loadout; + + var run = await loadout.RunAsync("tools", "show", "free-cache", "--json"); + + run.ExitCode.Should().Be(0, run.StandardError); + var json = run.Json(); + json.GetProperty("name").GetString().Should().Be("free-cache"); + json.GetProperty("version").GetString().Should().Be("1.0"); + json.GetProperty("versions").GetProperty("1.0").GetString().Should().Be("known-good"); + json.GetProperty("trusted").GetBoolean().Should().BeFalse(); + json.GetProperty("manifest").ValueKind.Should().Be(JsonValueKind.Object); + } + + [BuiltCliFact] + public async Task Submit_json_names_the_inbox_item() + { + using var loadout = new LoadoutProcess(); + + var run = await loadout.RunAsync( + "tools", "submit", "--kind", "idea", "--text", "A tool that clears a named cache directory.", "--json"); + + run.ExitCode.Should().Be(0, run.StandardError); + var json = run.Json(); + json.GetProperty("id").GetString().Should().NotBeNullOrEmpty(); + json.GetProperty("kind").GetString().Should().Be("idea"); + json.GetProperty("overlapping").ValueKind.Should().Be(JsonValueKind.Array); + } + + [BuiltCliFact] + public async Task Used_json_says_what_was_recorded() + { + var (loadout, _) = await CatalogueAsync(); + using var _ = loadout; + + var run = await loadout.RunAsync("tools", "used", "free-cache@1.0", "--outcome", "ok", "--json"); + + run.ExitCode.Should().Be(0, run.StandardError); + var json = run.Json(); + json.GetProperty("tool").GetString().Should().Be("free-cache"); + json.GetProperty("version").GetString().Should().Be("1.0"); + json.GetProperty("outcome").GetString().Should().Be("ok"); + json.GetProperty("recorded").GetBoolean().Should().BeTrue(); + } + + [BuiltCliFact] + public async Task Dry_run_changes_nothing() + { + var (loadout, root) = await CatalogueAsync(); + using var _ = loadout; + var before = Snapshot(root); + + string[][] asked = + [ + ["tools", "submit", "--kind", "idea", "--text", "Something new.", "--dry-run"], + ["tools", "used", "free-cache@1.0", "--outcome", "failed", "--dry-run"], + ["tools", "trust", "free-cache@1.0", "--dry-run"], + ["tools", "deprecate", "free-cache", "--reason", "Replaced.", "--dry-run"], + ["tools", "retire", "free-cache", "--dry-run"], + ]; + + foreach (var one in asked) + { + var run = await loadout.RunAsync(one); + (run.StandardOutput + run.StandardError).Should().Contain("Dry run", string.Join(' ', one)); + } + + Snapshot(root).Should().BeEquivalentTo(before); + (await loadout.RunAsync("tools", "show", "free-cache", "--json")).Json() + .GetProperty("trusted").GetBoolean().Should().BeFalse("a dry-run trust wrote no agreement"); + } + + [BuiltCliFact] + public async Task Trust_fingerprints_the_script_on_disk_not_the_manifest_field() + { + var (loadout, root) = await CatalogueAsync(); + using var _ = loadout; + + // The manifest sits beside the script and anything that can write one + // can write the other: a fingerprint read from it would be whatever the + // writer wanted agreed to. + var manifest = Path.Combine(root, "free-cache", "versions", "1.0", "manifest.yaml"); + var yaml = new DeserializerBuilder().WithNamingConvention(UnderscoredNamingConvention.Instance) + .IgnoreUnmatchedProperties().Build().Deserialize(File.ReadAllText(manifest)); + var forged = RemedyCeiling.Fingerprint("Remove-Item -Recurse /\n"); + yaml.Fingerprint = forged; + File.WriteAllText(manifest, new SerializerBuilder().WithNamingConvention(UnderscoredNamingConvention.Instance) + .ConfigureDefaultValuesHandling(DefaultValuesHandling.OmitNull).Build().Serialize(yaml)); + + var run = await loadout.RunAsync("tools", "trust", "free-cache@1.0", "--by", "tester", "--json"); + + run.ExitCode.Should().Be(0, run.StandardError); + var fingerprint = run.Json().GetProperty("fingerprint").GetString(); + fingerprint.Should().Be(RemedyCeiling.Fingerprint(Script)); + fingerprint.Should().NotBe(forged); + } +} diff --git a/tests/Loadout.Tests/Integration/AgentToolsTests.cs b/tests/Loadout.Tests/Integration/AgentToolsTests.cs index aae18e5c..99657b7d 100644 --- a/tests/Loadout.Tests/Integration/AgentToolsTests.cs +++ b/tests/Loadout.Tests/Integration/AgentToolsTests.cs @@ -170,7 +170,8 @@ private LoadoutTools Tools(string slug) => symbols: null!, runs: null!, TimeProvider.System, - new LoadoutToolScope(slug)); + new LoadoutToolScope(slug), + catalogue: null!); private async Task RegisterCsharpRepositoryAsync(string name) { diff --git a/tests/Loadout.Tests/Unit/LoadoutToolsCatalogueTests.cs b/tests/Loadout.Tests/Unit/LoadoutToolsCatalogueTests.cs new file mode 100644 index 00000000..0bd10253 --- /dev/null +++ b/tests/Loadout.Tests/Unit/LoadoutToolsCatalogueTests.cs @@ -0,0 +1,73 @@ +using FluentAssertions; +using Loadout.Cli.Commands; +using Loadout.Cli.Infrastructure; +using Xunit; + +namespace Loadout.Tests.Unit; + +/// +/// The catalogue's MCP calls, against a real catalogue in a temporary state +/// directory, because what matters is what reaches its files. +/// +public sealed class LoadoutToolsCatalogueTests : IDisposable +{ + private readonly ToolStoreFixture _store = new(); + + public void Dispose() => _store.Dispose(); + + /// + /// Only the catalogue is real. The rest are not reached on this path. + /// + private static LoadoutTools Tools(Loadout.Core.Tools.IToolRegistry catalogue) => + new( + instructions: null!, + memory: null!, + workspace: null!, + projects: null!, + tasks: null!, + git: null!, + symbols: null!, + runs: null!, + TimeProvider.System, + new LoadoutToolScope(null), + catalogue); + + [Fact] + public void loadout_tools_submit_screens_secrets() + { + var (registry, _) = _store.Registry(); + var token = "ghp_" + new string('c', 36); + + var answer = Tools(registry).ToolsSubmit( + "lesson", + "Cloning needed a token, so I used " + token + " and it worked."); + + answer.Should().NotStartWith("Submitted"); + answer.Should().NotContain(token); + + var inbox = Path.Combine(registry.Root(), "inbox"); + + (Directory.Exists(inbox) ? Directory.EnumerateFiles(inbox, "*", SearchOption.AllDirectories) : []) + .Should().BeEmpty("a submission carrying a credential is refused, not stored"); + + if (Directory.Exists(registry.Root())) + { + foreach (var file in Directory.EnumerateFiles(registry.Root(), "*", SearchOption.AllDirectories)) + { + File.ReadAllText(file).Should().NotContain(token); + } + } + } + + [Fact] + public void A_clean_submission_reaches_the_inbox() + { + var (registry, _) = _store.Registry(); + + var answer = Tools(registry).ToolsSubmit("idea", "A tool that clears a named cache directory."); + + answer.Should().StartWith("Submitted as "); + Directory.EnumerateFiles(Path.Combine(registry.Root(), "inbox"), "*", SearchOption.AllDirectories) + .Should().NotBeEmpty(); + } +} diff --git a/tests/Loadout.Tests/Unit/RemedyGateTests.cs b/tests/Loadout.Tests/Unit/RemedyGateTests.cs index f1ebd94f..4c36004c 100644 --- a/tests/Loadout.Tests/Unit/RemedyGateTests.cs +++ b/tests/Loadout.Tests/Unit/RemedyGateTests.cs @@ -1,6 +1,7 @@ using FluentAssertions; using Loadout.Core.Instructions; using Loadout.Core.Teams; +using Loadout.Core.Tools; using Xunit; namespace Loadout.Tests.Unit; @@ -212,6 +213,66 @@ bool CanRunAScript(string id) => Allowed("role.remediator").Should().NotContain("Write").And.NotContain("Edit"); } + private const string ToolScript = "param([string]$CachePath)\nGet-ChildItem $CachePath | Remove-Item\n"; + + private static readonly Dictionary TrustedKind = new() { ["unclassified"] = "trusted" }; + + private static async Task CatalogueAsync(ToolStoreFixture store) + { + var (registry, _) = store.Registry(); + await store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), ToolScript, ToolStoreFixture.Cases()); + + return registry; + } + + private static IReadOnlyList AgreedTo(string version) => + [ + new() { Tool = "free-cache", Version = version, Fingerprint = RemedyCeiling.Fingerprint(ToolScript) }, + ]; + + [Fact] + public async Task An_active_global_tool_is_matched_by_its_versioned_file_name() + { + using var store = new ToolStoreFixture(); + var registry = await CatalogueAsync(store); + var policy = new NodePolicy( + "r", "remediator/1", ToolOffer.Remediator, ["Bash"], [], Ask: true, + Remedies: ToolOffer.For(registry, ToolOffer.Remediator, TrustedKind, AgreedTo("1.0"))); + + var one = NodePermissions.Decide(policy, "Bash", Calling("pwsh ./free-cache.v1.0.ps1")); + var next = NodePermissions.Decide(policy, "Bash", Calling("pwsh ./free-cache.v1.1.ps1")); + + one.Allowed.Should().BeTrue(); + one.Reason.Should().Contain("tool:free-cache@1.0"); + next.Reason.Should().NotContain("tool:free-cache@1.0", "a trust given to one version names that version's file"); + } + + [Fact] + public async Task A_candidate_tool_is_never_offered() + { + using var store = new ToolStoreFixture(); + var registry = await CatalogueAsync(store); + var head = Path.Combine(registry.Root(), "free-cache", "tool.yaml"); + File.WriteAllText(head, File.ReadAllText(head).Replace("lifecycle: active", "lifecycle: candidate", StringComparison.Ordinal)); + + registry.Show("free-cache").Value!.Record.Lifecycle.Should().Be("candidate"); + ToolOffer.For(registry, ToolOffer.Remediator, TrustedKind, AgreedTo("1.0")).Should().BeEmpty(); + } + + [Fact] + public async Task A_global_tool_is_offered_only_to_the_remediator() + { + using var store = new ToolStoreFixture(); + var registry = await CatalogueAsync(store); + + ToolOffer.For(registry, ToolOffer.Remediator, TrustedKind, AgreedTo("1.0")).Should().ContainSingle(); + + foreach (var other in new[] { "role.fixer", "role.investigator", "role.implementer", "role.project-lead" }) + { + ToolOffer.For(registry, other, TrustedKind, AgreedTo("1.0")).Should().BeEmpty(other + " does not run scripts"); + } + } + [Fact] public void A_team_with_nothing_registered_is_unaffected() { diff --git a/tests/Loadout.Tests/Unit/TeamGoalTests.cs b/tests/Loadout.Tests/Unit/TeamGoalTests.cs index f8049f7c..fd5b1b3e 100644 --- a/tests/Loadout.Tests/Unit/TeamGoalTests.cs +++ b/tests/Loadout.Tests/Unit/TeamGoalTests.cs @@ -157,6 +157,33 @@ public void A_node_is_told_what_registering_something_actually_means() .And.Contain("claiming to be trusted decides nothing"); } + [Fact] + public void Brief_carries_the_tools_pointer_and_no_tool_names() + { + var read = TeamRunner.Render(Briefed()); + + read.Should().Contain("## Shared tools"); + read.Should().Contain(TeamRunner.ToolsPointer); + read.Should().Contain("loadout_tools_search"); + + // A tool is offered to a remediator's permissions as tool:@, + // never listed to a node in its brief. + read.Should().NotContain("tool:"); + } + + [Fact] + public void Brief_length_does_not_grow_with_the_registry() + { + // The section is the pointer and nothing else: whatever the catalogue + // holds is fetched when a node asks, so no listing of tools can grow + // here and be paid for in every brief. + var read = TeamRunner.Render(Briefed()); + var start = read.IndexOf("## Shared tools", StringComparison.Ordinal); + var end = read.IndexOf("## Task", start, StringComparison.Ordinal); + + read[start..end].Trim().Should().Be("## Shared tools" + Environment.NewLine + Environment.NewLine + TeamRunner.ToolsPointer); + } + [Fact] public void A_team_with_nothing_standing_to_say_says_nothing() { diff --git a/tests/Loadout.Tests/Unit/TeamRunnerTests.cs b/tests/Loadout.Tests/Unit/TeamRunnerTests.cs index 8b231a84..b7b5851e 100644 --- a/tests/Loadout.Tests/Unit/TeamRunnerTests.cs +++ b/tests/Loadout.Tests/Unit/TeamRunnerTests.cs @@ -1078,6 +1078,40 @@ public async Task A_node_can_reach_the_directory_its_brief_tells_it_to_write_in( .Which.Should().EndWith(Path.Combine("teams", "work", team.Name)); } + [Fact] + public async Task A_node_can_reach_no_part_of_the_tool_catalogue_but_drafts_and_inbox() + { + // The catalogue's safety rests on this: verify records, the audit log, + // the heads and the version directories are all files, and a node that + // could write them could forge any of them. + var team = await IteratingProjectAsync(); + + _launcher.Script("role.project-lead", Init("lead-1"), Result(LeadDone(), 0.05m)); + + (await RunAsync(team)).Succeeded.Should().BeTrue(); + + var catalogue = Path.GetFullPath(Path.Combine(_paths.Paths.State, "tools")); + var allowed = new[] { Path.Combine(catalogue, "drafts"), Path.Combine(catalogue, "inbox") }; + + foreach (var reached in _launcher.Requests.SelectMany(one => one.Request.ReachableDirectories ?? [])) + { + var full = Path.GetFullPath(reached); + var inside = (full + Path.DirectorySeparatorChar).StartsWith( + catalogue + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase); + var covers = (catalogue + Path.DirectorySeparatorChar).StartsWith( + full + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase); + + covers.Should().BeFalse($"{reached} holds the whole catalogue"); + + if (inside) + { + allowed.Should().Contain(one => (full + Path.DirectorySeparatorChar).StartsWith( + one + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase), + $"{reached} is in the catalogue outside drafts and inbox"); + } + } + } + [Fact] public async Task A_dry_run_names_the_team_directory_and_makes_nothing() { diff --git a/tests/Loadout.Tests/Unit/TeamsToolTests.cs b/tests/Loadout.Tests/Unit/TeamsToolTests.cs index 9b429fd3..f316f862 100644 --- a/tests/Loadout.Tests/Unit/TeamsToolTests.cs +++ b/tests/Loadout.Tests/Unit/TeamsToolTests.cs @@ -43,7 +43,8 @@ private static LoadoutTools Tools(IRunJournal runs) => symbols: null!, runs, new Clock(Noon.AddMinutes(10)), - new LoadoutToolScope(null)); + new LoadoutToolScope(null), + catalogue: null!); [Fact] public void A_run_that_wants_somebody_says_so_in_its_first_line() diff --git a/tests/Loadout.Tests/Unit/ToolTrustTests.cs b/tests/Loadout.Tests/Unit/ToolTrustTests.cs new file mode 100644 index 00000000..fd420079 --- /dev/null +++ b/tests/Loadout.Tests/Unit/ToolTrustTests.cs @@ -0,0 +1,88 @@ +using FluentAssertions; +using Loadout.Core.Teams; +using Loadout.Core.Tools; +using Loadout.Models.Configuration; +using Loadout.Models.Teams; +using Xunit; + +namespace Loadout.Tests.Unit; + +/// +/// Trust in a catalogue tool is a person's agreement to one script, kept in +/// this machine's configuration. Nothing in the catalogue can stand in for it. +/// +public sealed class ToolTrustTests : IDisposable +{ + private const string Script = "param([string]$CachePath)\nGet-ChildItem $CachePath | Remove-Item\n"; + + private static readonly Dictionary Rules = new() { ["unclassified"] = RemedyRules.Trusted }; + + private readonly ToolStoreFixture _store = new(); + + public void Dispose() => _store.Dispose(); + + private static TrustedTool Agreed(string version, string script) => new() + { + Tool = "free-cache", + Version = version, + Fingerprint = RemedyCeiling.Fingerprint(script), + By = "someone", + }; + + [Fact] + public async Task A_changed_script_is_asked_about_again() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + + // Agreed to 1.0, and to "1.1" at 1.0's script, as a person carrying + // their agreement forward without reading the new one would. + IReadOnlyList trusted = [Agreed("1.0", Script), Agreed("1.1", Script)]; + + ToolOffer.For(registry, ToolOffer.Remediator, Rules, trusted) + .Should().ContainSingle().Which.Ruling.Should().Be("run"); + + var changed = Script + "Write-Output done\n"; + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.1"), changed, ToolStoreFixture.Cases()); + + var offered = ToolOffer.For(registry, ToolOffer.Remediator, Rules, trusted).Should().ContainSingle().Subject; + + offered.Name.Should().Be(ToolOffer.Named("free-cache", "1.1")); + offered.Ruling.Should().Be("ask"); + } + + [Fact] + public async Task Trust_fingerprints_the_script_ScriptOf_reads_and_nothing_once_it_changes() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + + RemedyCeiling.Fingerprint(registry.ScriptOf("free-cache", "1.0")!) + .Should().Be(RemedyCeiling.Fingerprint(Script)); + + // A script edited after promotion is not what the gate saw, so there + // is nothing to fingerprint and nothing for a person to agree to. + var file = Directory.EnumerateFiles( + Path.Combine(registry.Root(), "free-cache", "versions", "1.0"), "*.ps1").Single(); + File.AppendAllText(file, "Remove-Item -Recurse /\n"); + + registry.ScriptOf("free-cache", "1.0").Should().BeNull(); + } + + [Fact] + public async Task A_record_claiming_trust_decides_nothing() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Script, ToolStoreFixture.Cases()); + + // Everything the catalogue could say: the audit line trust leaves, and + // a file beside the tools claiming the agreement outright. + registry.RecordTrust("free-cache", "1.0", revoked: false, by: "someone"); + File.WriteAllText( + Path.Combine(registry.Root(), "free-cache", "trusted.yaml"), + $"tool: free-cache\nversion: '1.0'\nfingerprint: {RemedyCeiling.Fingerprint(Script)}\n"); + + ToolOffer.For(registry, ToolOffer.Remediator, Rules, trusted: null) + .Should().ContainSingle().Which.Ruling.Should().Be("ask"); + } +} From b0d89015f21a91fc7a922c7cfdeee7d0715d6673 Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 17:09:11 +0100 Subject: [PATCH 06/15] Make the reserved-name theory prove promote's own check Verify already refuses a tool named after the catalogue's own directories, so promote failed for want of a verify record whether or not its reserved-name check was there. The theory now asserts promote's own refusal text. Deleting the check at ToolRegistry.cs:574 fails all four cases; restored afterwards. --- tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs b/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs index 6ee403f7..50b9b25a 100644 --- a/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs +++ b/tests/Loadout.Tests/Unit/ToolRegistrySafetyTests.cs @@ -48,6 +48,9 @@ public async Task A_tool_named_after_the_catalogues_own_directories_is_refused(s verified.Failed.Should().BeTrue(); promoted.Failed.Should().BeTrue(); + // Verify refused too, so promote would also fail for want of a verify + // record. The message is what shows promote's own check refused it. + promoted.Error.Should().Contain("catalogue keeps its own files"); File.Exists(Path.Combine(registry.Root(), name, "tool.yaml")).Should().BeFalse(); } From 7e65cd4cd5378be25446d5012137ed8153f7067b Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 17:15:29 +0100 Subject: [PATCH 07/15] Add the tool-creator and tool-refiner roles and the tool-works team The content comes from 9a7d57f on teams-20260923-1216-be60-implementer-3, which was cut from main rather than from the registry pieces. It is applied here on top of piece 2 unchanged: the 'loadout tools' subcommands the roles name (search, show, submit, used, audit, verify, deprecate, retire) all exist in ToolCommands.cs, so SpecialistCommandTests, which failed on 9a7d57f alone, now passes. Neither role can promote or trust a tool. Each allowed subcommand is named individually and promote and trust are denied as well; the refiner alone may deprecate and retire. tool-works is role.project-lead delegating to creator and refiner, autonomous, $3 and 30 turns per node, outward: ask, stop_when goal_met, budget_spent, no_progress_2_rounds. Specialist counts go 101 -> 103, roles 23 -> 25. Mutation checks: writing the creator's allow list as the plan's blanket 'Bash(loadout tools:*)' without the promote deny fails Creator_and_refiner_roles_cannot_promote_or_trust; a $5 budget fails tool_works_team_validates. Both restored from 9a7d57f. Suite: 3202 passed, 0 failed, 22 skipped. --- README.md | 2 +- docs/features.md | 2 +- docs/specialists.md | 4 +- .../Specialists/role/tool-creator.md | 102 ++++++++++++++++++ .../Specialists/role/tool-refiner.md | 100 +++++++++++++++++ .../Teams/Catalogue/tool-works.yaml | 42 ++++++++ tests/Loadout.Tests/Unit/RoleLibraryTests.cs | 25 +++++ .../Loadout.Tests/Unit/TeamCatalogueTests.cs | 24 ++++- 8 files changed, 296 insertions(+), 5 deletions(-) create mode 100644 src/Loadout.Core/Specialists/role/tool-creator.md create mode 100644 src/Loadout.Core/Specialists/role/tool-refiner.md create mode 100644 src/Loadout.Core/Teams/Catalogue/tool-works.yaml diff --git a/README.md b/README.md index 0b74e385..591837e8 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ most wants one. ### Specialists and context: the right instructions, with the cost shown Loadout doesn't hand your agent one enormous prompt that's mostly irrelevant. -There are 101 specialists compiled in, including 16 skills, and it works out +There are 103 specialists compiled in, including 16 skills, and it works out which ones apply from what your repository is made of and the sentence you typed. It tells you why it picked each one. diff --git a/docs/features.md b/docs/features.md index b042d59c..b610afcd 100644 --- a/docs/features.md +++ b/docs/features.md @@ -5,7 +5,7 @@ the commands. ## Instructions picked for the job -There are 101 specialists built into the binary: foundations, modes, languages, +There are 103 specialists built into the binary: foundations, modes, languages, frameworks, databases, platforms, clouds, functional areas, skills and the roles a team run gives its nodes. Instead of one enormous prompt that's mostly irrelevant, Loadout works out which ones diff --git a/docs/specialists.md b/docs/specialists.md index 41f4b738..9608a0bd 100644 --- a/docs/specialists.md +++ b/docs/specialists.md @@ -240,8 +240,8 @@ loadout instructions list # everything available to this project loadout instructions list --kind language ``` -The library ships 101 specialists — 5 foundations, 5 modes, 10 languages, 8 -frameworks, 4 databases, 5 platforms, 3 clouds, 22 functions, 16 skills and 23 +The library ships 103 specialists — 5 foundations, 5 modes, 10 languages, 8 +frameworks, 4 databases, 5 platforms, 3 clouds, 22 functions, 16 skills and 25 roles. They are embedded in the binary rather than kept on disk, so the command is the way to read them; there is no directory to browse. diff --git a/src/Loadout.Core/Specialists/role/tool-creator.md b/src/Loadout.Core/Specialists/role/tool-creator.md new file mode 100644 index 00000000..2b56ceb5 --- /dev/null +++ b/src/Loadout.Core/Specialists/role/tool-creator.md @@ -0,0 +1,102 @@ +--- +id: role.tool-creator +kind: role +mode: implement +deliverable: file +contract: report/1 +tools: + allowed: + - 'Read' + - 'Grep' + - 'Glob' + - 'Write' + - 'Edit' + - 'Bash(loadout tools search:*)' + - 'Bash(loadout tools show:*)' + - 'Bash(loadout tools submit:*)' + - 'Bash(loadout tools used:*)' + - 'Bash(loadout tools audit:*)' + - 'Bash(loadout tools verify:*)' + - 'Bash(loadout memory find:*)' + - 'Bash(pwsh -NoProfile -File:*)' + denied: + - 'Bash(loadout tools promote:*)' + - 'Bash(loadout tools trust:*)' + - 'Bash(git push:*)' + - 'Bash(git rebase:*)' + - 'Bash(git reset:*)' + - 'Bash(git commit --amend:*)' + - 'Bash(git branch -D:*)' + - 'Bash(git tag -d:*)' + - 'Bash(git push --force:*)' + - 'Bash(rm:*)' + - 'Bash(del:*)' + - 'Bash(format:*)' +title: Tool Creator +summary: Turns a fix that has worked more than once into a draft tool every team can use, or says why it is not one yet. +requires: + - role.member +modes: + - implement +probe: + summary: searched the tool catalogue before drafting anything + pattern: 'loadout tools search' +--- + +## Job + +You read what other teams have finished - nominations, remedies, lessons, +submissions - and decide whether any of it should become a tool that every +team on this machine can find. Your `deliverable` is a `file`: a draft tool +with its harness cases, submitted through `loadout tools submit`, or nothing, +with the reason. + +You propose. You do not promote and you do not trust. Promotion is the gate's, +and whether a script may run unattended is a person's, so neither command is +in your tools. A draft you think is ready is still a draft. + +## Rules + +- **Search first.** You MUST run `loadout tools search` with the candidate's + capabilities and the words of its purpose before drafting anything, and + quote the queries and what they returned as `evidence`. Where an existing + tool overlaps, you MUST draft a new version of that tool rather than a new + tool, unless you say why extending it would break the callers it has. +- **The promotion bar.** You MAY draft a new tool only when every one of these + holds, and you MUST say in `summary` which evidence meets each: + 1. Its purpose fits in one sentence that names no project, team, repository, + host or person. + 2. There are at least two independent uses: two runs, two teams, or one run + and an explicit submission saying it recurs. A single success is a + nomination, not a tool. + 3. It has been run and seen to work at least once, with the command and its + result in a report or in a remedy's `proves` line. + 4. Its inputs can be named and its outputs checked by a harness case. + + Where one does not hold, report `done` with no draft and say which, so the + nomination is not picked up again for the same reason. +- **Strip it before you submit it.** Every project-specific value becomes a + named input with no project default: absolute paths, drive letters, + repository URLs, project slugs, team names, run ids, host names, e-mail + addresses, GUIDs, ports and credentials. The `origin` describes the problem, + never the project: "a build cache filled the disk", not the name of whose + disk. The registry checks this in code when you submit and again at + promotion, so a draft that still carries one is refused, and the refusal + says what it found. +- You MUST write the harness cases with the draft: at least one each of + success, failure, edge and invalid-input. A draft without all four is + refused by `verify`. +- You MUST NOT run the draft against anything but its own harness, and only + through `loadout tools verify`, which goes through the same gate as every + other script on this machine. Where `verify` is held for a person, wait for + the answer; being held is an outcome, not an obstacle. +- You MUST NOT touch anything outside the registry's drafts and inbox. A team's + remedy that inspired a tool stays the team's. + +## Report + +`summary`: which nominations you read, which met the bar and which did not, +and why. `deliverables`: each draft submitted, as `name@version`. `evidence`: +the searches and their results, the submit result, and the verify result. +`next`: nominations left for another round, and anything the Refiner should +look at. diff --git a/src/Loadout.Core/Specialists/role/tool-refiner.md b/src/Loadout.Core/Specialists/role/tool-refiner.md new file mode 100644 index 00000000..491c30bb --- /dev/null +++ b/src/Loadout.Core/Specialists/role/tool-refiner.md @@ -0,0 +1,100 @@ +--- +id: role.tool-refiner +kind: role +mode: implement +deliverable: file +contract: report/1 +tools: + allowed: + - 'Read' + - 'Grep' + - 'Glob' + - 'Write' + - 'Edit' + - 'Bash(loadout tools search:*)' + - 'Bash(loadout tools show:*)' + - 'Bash(loadout tools submit:*)' + - 'Bash(loadout tools used:*)' + - 'Bash(loadout tools audit:*)' + - 'Bash(loadout tools verify:*)' + - 'Bash(loadout tools deprecate:*)' + - 'Bash(loadout tools retire:*)' + - 'Bash(loadout memory find:*)' + - 'Bash(pwsh -NoProfile -File:*)' + denied: + - 'Bash(loadout tools promote:*)' + - 'Bash(loadout tools trust:*)' + - 'Bash(git push:*)' + - 'Bash(git rebase:*)' + - 'Bash(git reset:*)' + - 'Bash(git commit --amend:*)' + - 'Bash(git branch -D:*)' + - 'Bash(git tag -d:*)' + - 'Bash(git push --force:*)' + - 'Bash(rm:*)' + - 'Bash(del:*)' + - 'Bash(format:*)' +title: Tool Refiner +summary: Improves, merges, splits or retires the machine's shared tools when the evidence says it would help, and stops when it would not. +requires: + - role.member +modes: + - implement +probe: + summary: named the signal behind the change and the gain it measured + pattern: 'stand-down|signal' +--- + +## Job + +You keep the shared tool catalogue worth using. Your `deliverable` is a `file`: +a draft version of an existing tool, a deprecation, a retirement, or a +stand-down with its reason. + +Like the Creator, you propose. You can change a tool's lifecycle - deprecate +it, retire it - because that changes nothing that runs. You cannot promote a +version and you cannot trust one; those are the gate's and a person's. + +## Rules + +- **Read the signals first.** You MUST read, for each tool you look at: its + usage (outcomes `failed` and `workaround`, with their notes), the inbox's + `bug` and `idea` items for it, its lineage, team remedies whose script + overlaps it (a sign the tool did not fit), how often those remedies were + revised, and lesson topics in memory that match its capabilities. Quote the + ones that led to a change as `evidence`. +- **What you may do.** Extend, simplify, refactor, consolidate (the survivor + names what it `replaces`, and the absorbed tool is deprecated with a + `replacement`), split (two drafts, and the original deprecated with a + replacement for each use), deprecate (with a replacement or a reason; the + registry refuses one with neither) and retire (only a deprecated tool, with no + use in 30 days and nothing depending on it). +- **The stop rule.** You MAY propose a change only where it shows at least one + measured gain against the active version: + 1. a case that failed before, or a new case from a signal, now passes; or + 2. the failure-and-workaround rate over the recent uses would have been + lower, with those uses replayed as cases; or + 3. complexity falls - script lines, inputs and dependencies - with every + known-good case still passing. + + Where a change raises complexity and none of those improves, you MUST NOT + propose it. Write a stand-down for that tool with the reason instead. After + two stand-downs in a row with no new signal between them, the tool is left + alone until a new signal arrives; the registry records this and will not + offer it to you again. Improving something indefinitely is not refinement. +- **Compatibility.** Removing or renaming a required input, adding a required + input with no default, or changing what an exit code means is a break. A + breaking draft MUST say so, take a major version, and carry a migration, or + promotion refuses it. Every case the current known-good version passes is run + against your draft, and a draft that fails one does not replace it. +- **Strip it, as the Creator does.** A draft carries no paths, slugs, team + names, run ids, hosts, addresses, GUIDs, ports or credentials; the registry + refuses one that does. +- You MUST NOT run a draft except through `loadout tools verify`. + +## Report + +`summary`: which tools you looked at, the signal behind each change, and the +gain it measured - or the stand-down and why. `deliverables`: each draft as +`name@version`, and each deprecation or retirement. `evidence`: the signals +read, and the verify results. `next`: tools that need a person's decision. diff --git a/src/Loadout.Core/Teams/Catalogue/tool-works.yaml b/src/Loadout.Core/Teams/Catalogue/tool-works.yaml new file mode 100644 index 00000000..049a6874 --- /dev/null +++ b/src/Loadout.Core/Teams/Catalogue/tool-works.yaml @@ -0,0 +1,42 @@ +name: tool-works +description: Turns fixes that worked in more than one run into tools every team on this machine can find, and keeps those tools worth using. + +# A standing team: the daemon starts it on a schedule and when another team's +# run finishes, never from inside somebody else's run. It reads only runs that +# have ended, so it cannot disrupt one that has not. +goal: > + Every team on this machine should find a working tool for a problem another + team has already solved, without knowing which team solved it. Nothing + becomes a tool until it has worked twice, and nothing is kept once it has + stopped being worth using. + +declarations: + - Search the tool catalogue before drafting anything, and quote what the + search returned. + - Draft a tool only when its purpose names no project, it has at least two + independent uses, it has been seen to work, and a harness case can check it. + - Every project-specific value in a draft becomes a named input with no + project default. + - Propose a refinement only with a measured gain against the active version; + otherwise record a stand-down with the reason. + - Nothing here promotes a tool or trusts one. Promotion is the gate's and + trust is a person's. + +lead: lead +nodes: + lead: + role: role.project-lead + delegates: [creator, refiner] + creator: + role: role.tool-creator + refiner: + role: role.tool-refiner +rules: + autonomy: autonomous + budget: + usd: 3 + turns_per_node: 30 + wall_clock: 30m + gates: + outward: ask + stop_when: [goal_met, budget_spent, no_progress_2_rounds] diff --git a/tests/Loadout.Tests/Unit/RoleLibraryTests.cs b/tests/Loadout.Tests/Unit/RoleLibraryTests.cs index 19a11afd..76dde150 100644 --- a/tests/Loadout.Tests/Unit/RoleLibraryTests.cs +++ b/tests/Loadout.Tests/Unit/RoleLibraryTests.cs @@ -87,6 +87,31 @@ public async Task Every_role_reads_its_coordinator_facing_fields_from_its_frontm } } + [Theory] + [InlineData("role.tool-creator")] + [InlineData("role.tool-refiner")] + public async Task Creator_and_refiner_roles_cannot_promote_or_trust(string id) + { + // They propose; promotion is the gate's and trust is a person's. A + // blanket 'loadout tools' allow would have let both through, so this + // goes through the same gate a node's call would. + var role = (await LibraryAsync()).Find(id)!.Role!; + var policy = new Loadout.Core.Teams.NodePolicy("run", "node", id, role.AllowedTools, role.DeniedTools); + + bool Allowed(string command) => Loadout.Core.Teams.NodePermissions + .Decide(policy, "Bash", $$"""{"command":{{System.Text.Json.JsonSerializer.Serialize(command)}}}""") + .Allowed; + + Allowed("loadout tools promote free-disk-by-cache@1.0").Should().BeFalse(); + Allowed("loadout tools trust free-disk-by-cache@1.0").Should().BeFalse(); + Allowed("loadout tools search disk cache").Should().BeTrue(); + Allowed("loadout tools submit --kind candidate --text x").Should().BeTrue(); + + // Lifecycle is the Refiner's alone, and changes nothing that runs. + Allowed("loadout tools deprecate free-disk-by-cache --reason unused") + .Should().Be(id == "role.tool-refiner"); + } + [Fact] public async Task No_role_is_reachable_by_evidence() { diff --git a/tests/Loadout.Tests/Unit/TeamCatalogueTests.cs b/tests/Loadout.Tests/Unit/TeamCatalogueTests.cs index cf272d5c..dc46e953 100644 --- a/tests/Loadout.Tests/Unit/TeamCatalogueTests.cs +++ b/tests/Loadout.Tests/Unit/TeamCatalogueTests.cs @@ -52,7 +52,7 @@ public async Task The_built_in_teams_load_and_every_one_of_them_checks_out() catalogue.Teams.Keys.Should().BeEquivalentTo( ["iterating-project", "bug-hunt", "release-crew", "docs-crew", "dependency-sweep", - "marketing-studio", "product-company", "system-watch"]); + "marketing-studio", "product-company", "system-watch", "tool-works"]); catalogue.Findings.Should().BeEmpty( "a shipped team naming a role that does not ship is a shipped defect"); @@ -77,6 +77,28 @@ public async Task The_iterating_project_is_wired_as_the_design_says() team.Template.Should().BeFalse(); } + [Fact] + public async Task tool_works_team_validates() + { + // The standing team behind the global tool catalogue, with the rules + // the person accepted: cheap, unattended, and asking before anything + // leaves the machine. + var catalogue = await LoadAsync(); + var team = catalogue.Find("tool-works")!; + + catalogue.Findings.Should().NotContain(f => f.Rule == "tool-works"); + + team.Nodes["lead"].Role.Should().Be("role.project-lead"); + team.Nodes["lead"].Delegates.Should().Equal("creator", "refiner"); + team.Nodes["creator"].Role.Should().Be("role.tool-creator"); + team.Nodes["refiner"].Role.Should().Be("role.tool-refiner"); + team.Rules.Autonomy.Should().Be("autonomous"); + team.Rules.Budget.Usd.Should().Be(3m); + team.Rules.Budget.TurnsPerNode.Should().Be(30); + team.Rules.Gates.Outward.Should().Be("ask"); + team.Rules.StopWhen.Should().Equal("goal_met", "budget_spent", "no_progress_2_rounds"); + } + [Fact] public async Task The_company_is_a_template_and_the_release_crew_may_push_a_tag_only_when_autonomous() { From b215176d56e2558aa8e81e2e5b2dec0c095bc364 Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 17:27:59 +0100 Subject: [PATCH 08/15] Nominate recurring work for the tool catalogue, and start tool-works when runs finish ToolNominator reads finished, non-tool-works runs (reports through RunDocuments.In, Kind == "report") and every team's remedy shelf, and files a candidate when: (1) one remedy, by fingerprint or overlap >= 0.6, sits on two or more team shelves; (2) a remedy revised at least once passed in two or more runs; (3) one command shape, arguments replaced, passed in runs of two or more teams; (4) a lesson names a command that passed. It never nominates what an active tool already covers at overlap >= 0.6, and files each nomination once, keyed in the audit log. It writes through a new IToolRegistry.Nominate, which is not on the CLI or MCP. It skips the genericity check, because a nomination quotes the project it came from so the Creator can find it. It keeps the secret screen and the audit record ("nominate"). ScheduleService.Events gains run-finished. RunFinished is a pure rule over RunSummary: the run must be finished, from a team that is neither tool-works nor the schedule's own, and ordinally after the watermark kept in LastCommit, with LastRun more than an hour ago. The first look records a baseline and does not fire. A run held back by the hour is not written off, because the watermark moves only when the schedule fires. The daemon's ReadyAsync applies it, and SeenAsync skips these schedules so it cannot overwrite the watermark with a git head. TeamRunner gives role.tool-creator and role.tool-refiner nodes /tools/drafts and inbox, and nothing else under /tools. The new tool-works test runs a creator node, so the drafts-and-inbox branch of the catalogue guard now actually runs. Before, it launched only a lead. Mutation checks: each new test failed under a compilable mutation of its own guard, except the tool-works schedule rule. Its test's schedule belonged to tool-works, so the own-team check hid the mutation. The schedule now belongs to another team; that fix was not re-mutated. --- src/Loadout.Agents/Teams/TeamRunner.cs | 42 ++- src/Loadout.Cli/Commands/TeamDaemonCommand.cs | 27 +- src/Loadout.Core/Teams/ScheduleService.cs | 67 +++- src/Loadout.Core/Tools/ToolNominator.cs | 346 ++++++++++++++++++ src/Loadout.Core/Tools/ToolRegistry.cs | 33 +- .../Unit/RunFinishedScheduleTests.cs | 72 ++++ tests/Loadout.Tests/Unit/TeamRunnerTests.cs | 36 ++ .../Loadout.Tests/Unit/ToolNominatorTests.cs | 136 +++++++ 8 files changed, 753 insertions(+), 6 deletions(-) create mode 100644 src/Loadout.Core/Tools/ToolNominator.cs create mode 100644 tests/Loadout.Tests/Unit/RunFinishedScheduleTests.cs create mode 100644 tests/Loadout.Tests/Unit/ToolNominatorTests.cs diff --git a/src/Loadout.Agents/Teams/TeamRunner.cs b/src/Loadout.Agents/Teams/TeamRunner.cs index 40819ac2..7d2ed9d2 100644 --- a/src/Loadout.Agents/Teams/TeamRunner.cs +++ b/src/Loadout.Agents/Teams/TeamRunner.cs @@ -2275,11 +2275,51 @@ .. Core.Tools.ToolOffer.For(_tools, role.Id, request.Remediation, request.Truste // in. Without it the agent refuses every write there, which is // what it did: the directory existed, the path in the brief was // right, the declaration was clear, and nothing could be written. - ReachableDirectories: brief.TeamDirectory is { Length: > 0 } kept ? [kept] : null); + ReachableDirectories: Reachable(brief, role.Id, dryRun)); return await _launcher.StartHeadlessAsync(launch, options, ct).ConfigureAwait(false); } + /// The roles that write drafts and read the inbox of the tool catalogue. + private static readonly HashSet CatalogueWriters = + new(StringComparer.Ordinal) { "role.tool-creator", "role.tool-refiner" }; + + /// + /// The directories a node may write outside its tree: the team's own, and + /// for the catalogue's creator and refiner its drafts and inbox. + /// + /// + /// Those two and never the catalogue's root. The verify records, the audit + /// log, the heads and the versions sit beside them, and a node that could + /// write those could forge any of them. + /// + private List? Reachable(Brief brief, string role, bool dryRun) + { + var reach = new List(); + + if (brief.TeamDirectory is { Length: > 0 } kept) + { + reach.Add(kept); + } + + if (CatalogueWriters.Contains(role)) + { + foreach (var one in new[] { "drafts", "inbox" }) + { + var directory = Path.Combine(_paths.Paths.State, "tools", one); + + if (!dryRun) + { + Directory.CreateDirectory(directory); + } + + reach.Add(directory); + } + } + + return reach.Count > 0 ? reach : null; + } + /// Where a run keeps everything it writes. private string RunDirectory(string runId) => Path.Combine(_paths.Paths.State, "teams", "runs", runId); diff --git a/src/Loadout.Cli/Commands/TeamDaemonCommand.cs b/src/Loadout.Cli/Commands/TeamDaemonCommand.cs index eb6745ec..34921c7e 100644 --- a/src/Loadout.Cli/Commands/TeamDaemonCommand.cs +++ b/src/Loadout.Cli/Commands/TeamDaemonCommand.cs @@ -984,6 +984,28 @@ private async Task> ReadyAsync( continue; } + if (string.Equals(schedule.On, ScheduleService.RunFinishedEvent, StringComparison.OrdinalIgnoreCase)) + { + var runs = _journal.List(50) + .Select(_journal.Summarise) + .Where(one => one.Succeeded) + .Select(one => one.Value!) + .ToList(); + var (fire, seen) = ScheduleService.RunFinished(schedule, runs, now); + + if (record && seen is not null) + { + await _schedules.SawAsync(schedule.Id, seen, ct).ConfigureAwait(false); + } + + if (fire) + { + ready.Add(schedule); + } + + continue; + } + if (await MovedAsync(schedule, record, ct).ConfigureAwait(false)) { ready.Add(schedule); @@ -1005,7 +1027,10 @@ private async Task> ReadyAsync( /// private async Task SeenAsync(TeamSchedule schedule, CancellationToken ct) { - if (schedule.On.Length == 0) + // A run-finished schedule keeps a run identifier where this would + // write a commit, and its watermark was written when it fired. + if (schedule.On.Length == 0 + || string.Equals(schedule.On, ScheduleService.RunFinishedEvent, StringComparison.OrdinalIgnoreCase)) { return; } diff --git a/src/Loadout.Core/Teams/ScheduleService.cs b/src/Loadout.Core/Teams/ScheduleService.cs index 30de0515..0ffc9428 100644 --- a/src/Loadout.Core/Teams/ScheduleService.cs +++ b/src/Loadout.Core/Teams/ScheduleService.cs @@ -309,7 +309,72 @@ public static bool Moved(TeamSchedule schedule, string? head) /// the daemon answers it; this only says the word is one that means /// something. /// - public static IReadOnlyList Events { get; } = ["commit"]; + public static IReadOnlyList Events { get; } = ["commit", RunFinishedEvent]; + + /// The event for another team's run reaching its end. + public const string RunFinishedEvent = "run-finished"; + + /// The standing team that looks after the tool catalogue. + public const string ToolWorksTeam = "tool-works"; + + /// The least time between two starts on run-finished. + public static readonly TimeSpan RunFinishedQuiet = TimeSpan.FromHours(1); + + /// + /// Whether a schedule waiting for a finished run should start, and the + /// newest finished run to write down as seen, or null to write nothing. + /// + /// + /// + /// The watermark is the newest run identifier seen, kept where a commit + /// watcher keeps its commit; identifiers start with the time a run began, + /// so ordinal order is the order they started in. The first look writes it + /// down and does not fire, for the reason gives. + /// + /// + /// Never on a tool-works run, nor on the schedule's own team: either would + /// start a run whose finishing starts another. At most once an hour, and a + /// run that finished inside the hour is not lost - the watermark is not + /// moved until the schedule fires, so it is seen at the next look after. + /// + /// + public static (bool Fire, string? Seen) RunFinished( + TeamSchedule schedule, + IEnumerable runs, + DateTimeOffset now) + { + ArgumentNullException.ThrowIfNull(schedule); + ArgumentNullException.ThrowIfNull(runs); + + if (!schedule.Enabled || !string.Equals(schedule.On, RunFinishedEvent, StringComparison.OrdinalIgnoreCase)) + { + return (false, null); + } + + var newest = runs + .Where(one => one.Finished is not null + && !string.Equals(one.Team, ToolWorksTeam, StringComparison.OrdinalIgnoreCase) + && !string.Equals(one.Team, schedule.Team, StringComparison.OrdinalIgnoreCase)) + .Select(one => one.RunId) + .OrderByDescending(one => one, StringComparer.Ordinal) + .FirstOrDefault(); + + if (schedule.LastCommit.Length == 0) + { + // Below every run identifier, so a baseline taken before any run + // has finished still lets the first one fire. + return (false, newest ?? "0"); + } + + if (newest is null + || string.CompareOrdinal(newest, schedule.LastCommit) <= 0 + || (schedule.LastRun is { } last && now - last < RunFinishedQuiet)) + { + return (false, null); + } + + return (true, newest); + } /// What is wrong with a schedule, or null when nothing is. private static string? Check(TeamSchedule schedule) diff --git a/src/Loadout.Core/Tools/ToolNominator.cs b/src/Loadout.Core/Tools/ToolNominator.cs new file mode 100644 index 00000000..b16b96d1 --- /dev/null +++ b/src/Loadout.Core/Tools/ToolNominator.cs @@ -0,0 +1,346 @@ +using System.Text.RegularExpressions; +using Loadout.Core.Teams; +using Loadout.Models.Results; +using Loadout.Models.Teams; +using Loadout.Platform.Abstractions; + +namespace Loadout.Core.Tools; + +/// Something the nominator thinks is worth the Creator's look. +/// Which of the four rules found it. +/// What makes it this nomination and no other, so it is filed once. +/// One line saying what it is. +/// Where it was seen, for the Creator to follow up. +/// The script, where it is a remedy. +public sealed record ToolNomination(int Rule, string Key, string Summary, string Text, string? Script); + +/// +/// Finds what finished work keeps doing again, and files it for the Creator. +/// +/// +/// +/// A nomination is not a tool. It says something recurred and where; whether +/// it is general enough to become one is the Creator's question, answered +/// against the promotion bar. That is why a nomination skips the genericity +/// check a submission gets - it quotes the project it came from, on purpose, +/// so the Creator can find it - and why it still goes through the secret +/// screen, which no reason overrides. +/// +/// +/// Reads only runs that have finished and never a tool-works run, so it +/// cannot disturb work in progress or nominate its own output. +/// +/// +public sealed partial class ToolNominator +{ + private readonly IToolRegistry _registry; + private readonly IRunJournal _journal; + private readonly IRemedyBook _remedies; + private readonly IPlatformPaths _paths; + + public ToolNominator(IToolRegistry registry, IRunJournal journal, IRemedyBook remedies, IPlatformPaths paths) + { + _registry = registry; + _journal = journal; + _remedies = remedies; + _paths = paths; + } + + /// How many runs back it reads. + public int Depth { get; init; } = 200; + + /// Finds and files every nomination not already filed. + /// The text of every memory topic of kind lesson. + /// Each nomination found, with what filing it came to; null where it was filed before. + public IReadOnlyList<(ToolNomination Nomination, OperationResult? Filed)> Scan(IReadOnlyList lessons) + { + var filed = _registry.Audit() + .Where(one => one.Action == "nominate") + .Select(one => one.Note ?? string.Empty) + .ToList(); + + return + [ + .. Find(lessons).Select(one => + { + var key = KeyOf(one); + + return (one, filed.Any(note => note.EndsWith(" key " + key, StringComparison.Ordinal)) + ? null + : _registry.Nominate( + new ToolSubmission("candidate", one.Text, By: "nominator", Script: one.Script, Summary: one.Summary), + key)); + }), + ]; + } + + /// Every nomination the rules find, leaving out what an active tool already covers. + public IReadOnlyList Find(IReadOnlyList lessons) + { + ArgumentNullException.ThrowIfNull(lessons); + + var shelves = Shelves(); + var evidence = Evidence(); + var found = new List(); + + found.AddRange(SameOnTwoShelves(shelves)); + found.AddRange(RevisedAndRunTwice(shelves, evidence)); + found.AddRange(Commands(evidence, lessons)); + + var active = _registry.Offerable() + .Select(one => new ToolShape(one.Record.Capabilities, one.Record.Summary, one.Script)) + .ToList(); + + // What an active tool already does is the Refiner's to hear about, + // not the Creator's to build again. + return + [ + .. found.Where(one => + { + var shape = new ToolShape([], one.Summary, one.Script ?? one.Text); + + return !active.Any(tool => ToolOverlap.Score(shape, tool).Overlaps); + }), + ]; + } + + /// A command with its arguments replaced, so two runs of it compare equal. + public static string Shape(string command) + { + var words = (command ?? string.Empty).Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + + if (words.Length == 0) + { + return string.Empty; + } + + var shaped = new List { Path.GetFileName(words[0]).ToLowerInvariant() }; + + foreach (var word in words.Skip(1)) + { + if (Flag().Match(word) is { Success: true } flag) + { + shaped.Add(flag.Groups[1].Value + (word.Contains('=', StringComparison.Ordinal) ? "=" : string.Empty)); + } + else + { + shaped.Add(Plain().IsMatch(word) ? word : ""); + } + } + + return string.Join(' ', shaped); + } + + /// The words a lesson has to name for a command shape to count as named. + private static string? Named(string shape) + { + var head = shape.Split(' ').TakeWhile(one => one != "" && !one.StartsWith('-')).ToList(); + + // A bare program name is in too many lessons to mean anything. + return head.Count >= 2 ? string.Join(' ', head) : null; + } + + private static string KeyOf(ToolNomination nomination) => + RemedyCeiling.Fingerprint(nomination.Key)[..16]; + + private sealed record Shelved(string Team, Remedy Remedy, string Script); + + private sealed record Seen(string Run, string Team, ReportEvidence Evidence); + + private List Shelves() + { + var work = Path.Combine(_paths.Paths.State, "teams", "work"); + var shelved = new List(); + + if (!Directory.Exists(work)) + { + return shelved; + } + + foreach (var team in Directory.EnumerateDirectories(work).Select(Path.GetFileName).OfType().Order(StringComparer.Ordinal)) + { + if (string.Equals(team, ScheduleService.ToolWorksTeam, StringComparison.OrdinalIgnoreCase) + || _remedies.All(team) is not { Succeeded: true } all) + { + continue; + } + + foreach (var remedy in all.Value!) + { + if (_remedies.ScriptOf(team, remedy) is { Succeeded: true } script) + { + shelved.Add(new Shelved(team, remedy, script.Value!)); + } + } + } + + return shelved; + } + + private List Evidence() + { + var seen = new List(); + + foreach (var id in _journal.List(Depth)) + { + if (_journal.Summarise(id) is not { Succeeded: true } summarised + || summarised.Value! is not { Finished: not null } run + || string.Equals(run.Team, ScheduleService.ToolWorksTeam, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + foreach (var document in RunDocuments.In(run.Directory).Where(one => one.Kind == "report")) + { + string text; + + try + { + text = File.ReadAllText(Path.Combine(run.Directory, document.Name)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + continue; + } + + if (ReportReader.Read(text) is { Succeeded: true } report) + { + seen.AddRange(report.Value!.Evidence + .Where(one => one.Result == EvidenceResult.Pass) + .Select(one => new Seen(run.RunId, run.Team, one))); + } + } + } + + return seen; + } + + /// Rule 1: one remedy, or near enough, kept by two or more teams. + private static IEnumerable SameOnTwoShelves(List shelves) + { + var group = Enumerable.Range(0, shelves.Count).ToArray(); + + int Root(int one) => group[one] == one ? one : group[one] = Root(group[one]); + + for (var a = 0; a < shelves.Count; a++) + { + for (var b = a + 1; b < shelves.Count; b++) + { + if (string.Equals(shelves[a].Team, shelves[b].Team, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var same = string.Equals( + RemedyCeiling.Fingerprint(shelves[a].Script), + RemedyCeiling.Fingerprint(shelves[b].Script), + StringComparison.Ordinal) + || ToolOverlap.Score(ShapeOf(shelves[a]), ShapeOf(shelves[b])).Overlaps; + + if (same) + { + group[Root(b)] = Root(a); + } + } + } + + foreach (var cluster in Enumerable.Range(0, shelves.Count).GroupBy(Root)) + { + var members = cluster.Select(one => shelves[one]).ToList(); + + if (members.Select(one => one.Team).Distinct(StringComparer.OrdinalIgnoreCase).Count() < 2) + { + continue; + } + + var names = members.Select(one => one.Team + "/" + one.Remedy.Name).Order(StringComparer.Ordinal).ToList(); + + yield return new ToolNomination( + 1, + "1 " + string.Join(' ', names), + members[0].Remedy.What, + $"The same remedy is kept on {names.Count} team shelves: {string.Join(", ", names)}.", + members[0].Script); + } + } + + /// Rule 2: a remedy improved at least once and seen to pass in two or more runs. + private static IEnumerable RevisedAndRunTwice(List shelves, List evidence) + { + foreach (var shelved in shelves.Where(one => one.Remedy.Revision >= 1)) + { + var file = Path.GetFileName(shelved.Remedy.Script); + + if (file.Length == 0) + { + continue; + } + + var runs = evidence + .Where(one => (one.Evidence.Ref + " " + one.Evidence.Note).Contains(file, StringComparison.OrdinalIgnoreCase)) + .Select(one => one.Run) + .Distinct(StringComparer.Ordinal) + .Order(StringComparer.Ordinal) + .ToList(); + + if (runs.Count >= 2) + { + yield return new ToolNomination( + 2, + $"2 {shelved.Team}/{shelved.Remedy.Name}", + shelved.Remedy.What, + $"{shelved.Team}/{shelved.Remedy.Name}, revised {shelved.Remedy.Revision} times, passed in runs {string.Join(", ", runs)}.", + shelved.Script); + } + } + } + + /// + /// Rule 3: one command shape passing in runs of two or more teams. Rule 4: + /// a lesson naming a command that passed, which is a second use the runs + /// alone did not show. + /// + private static IEnumerable Commands(List evidence, IReadOnlyList lessons) + { + var shapes = evidence + .Where(one => one.Evidence.Kind == EvidenceKind.Command) + .GroupBy(one => Shape(one.Evidence.Ref), StringComparer.Ordinal) + .Where(one => one.Key.Length > 0) + .OrderBy(one => one.Key, StringComparer.Ordinal); + + foreach (var shape in shapes) + { + var teams = shape.Select(one => one.Team).Distinct(StringComparer.OrdinalIgnoreCase).Count(); + var runs = shape.Select(one => one.Run).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToList(); + + if (teams >= 2 && runs.Count >= 2) + { + yield return new ToolNomination( + 3, + "3 " + shape.Key, + $"Runs '{shape.Key}'.", + $"'{shape.Key}' passed in runs of {teams} teams: {string.Join(", ", runs)}.", + null); + } + else if (Named(shape.Key) is { } named + && lessons.Any(one => one.Contains(named, StringComparison.OrdinalIgnoreCase))) + { + yield return new ToolNomination( + 4, + "4 " + shape.Key, + $"Runs '{shape.Key}'.", + $"A lesson names '{named}', and '{shape.Key}' passed in runs {string.Join(", ", runs)}.", + null); + } + } + } + + private static ToolShape ShapeOf(Shelved shelved) => + new([shelved.Remedy.Kind], shelved.Remedy.What, shelved.Script); + + [GeneratedRegex(@"^(--?[A-Za-z][A-Za-z0-9-]*)(=.*)?$", RegexOptions.None, 1000)] + private static partial Regex Flag(); + + [GeneratedRegex(@"^[a-z][a-z-]*$", RegexOptions.None, 1000)] + private static partial Regex Plain(); +} diff --git a/src/Loadout.Core/Tools/ToolRegistry.cs b/src/Loadout.Core/Tools/ToolRegistry.cs index d46432a8..5439b116 100644 --- a/src/Loadout.Core/Tools/ToolRegistry.cs +++ b/src/Loadout.Core/Tools/ToolRegistry.cs @@ -85,6 +85,15 @@ public interface IToolRegistry /// Puts a candidate, idea, bug or lesson in the inbox, after the screens. OperationResult Submit(ToolSubmission submission); + /// + /// Files the nominator's candidate: screened for secrets and audited, but + /// not for genericity, because a nomination quotes where it was seen. + /// Not on the command line or MCP; only calls it. + /// + /// What was found. + /// What makes it this nomination, recorded so it is filed once. + OperationResult Nominate(ToolSubmission submission, string key); + /// Records one use. OperationResult RecordUsage(ToolUsage usage); @@ -279,7 +288,19 @@ public OperationResult Show(string name) } /// - public OperationResult Submit(ToolSubmission submission) + public OperationResult Submit(ToolSubmission submission) => + Put(submission, generic: true, key: null); + + /// + public OperationResult Nominate(ToolSubmission submission, string key) + { + ArgumentNullException.ThrowIfNull(submission); + ArgumentException.ThrowIfNullOrWhiteSpace(key); + + return Put(submission with { Kind = "candidate", By = "nominator" }, generic: false, key); + } + + private OperationResult Put(ToolSubmission submission, bool generic, string? key) { ArgumentNullException.ThrowIfNull(submission); @@ -302,7 +323,7 @@ public OperationResult Submit(ToolSubmission submission) ExitCode.PolicyViolation); } - if (kind == "candidate" && ToolGenericity.Check(everything, _known()) is { Count: > 0 } specific) + if (generic && kind == "candidate" && ToolGenericity.Check(everything, _known()) is { Count: > 0 } specific) { return OperationResult.Fail( "A candidate has to work for any project, and this carries one: " @@ -353,7 +374,13 @@ public OperationResult Submit(ToolSubmission submission) return OperationResult.Fail($"That submission could not be written: {ex.Message}"); } - Record("submit", submission.Tool ?? string.Empty, null, submission.By, submission.Run, $"{kind} {id}"); + Record( + key is null ? "submit" : "nominate", + submission.Tool ?? string.Empty, + null, + submission.By, + submission.Run, + key is null ? $"{kind} {id}" : $"{kind} {id} key {key}"); return OperationResult.Ok(new ToolSubmitted(id, overlapping)); } diff --git a/tests/Loadout.Tests/Unit/RunFinishedScheduleTests.cs b/tests/Loadout.Tests/Unit/RunFinishedScheduleTests.cs new file mode 100644 index 00000000..48998435 --- /dev/null +++ b/tests/Loadout.Tests/Unit/RunFinishedScheduleTests.cs @@ -0,0 +1,72 @@ +using FluentAssertions; +using Loadout.Core.Teams; +using Loadout.Models.Teams; +using Xunit; + +namespace Loadout.Tests.Unit; + +/// The run-finished schedule event: when another team's run ending starts one. +public sealed class RunFinishedScheduleTests +{ + private static readonly DateTimeOffset Now = new(2026, 9, 23, 12, 0, 0, TimeSpan.Zero); + + [Fact] + public void Run_finished_fires_once_per_finished_run_and_never_for_tool_works() + { + var schedule = Watching(seen: "20260901-0000-0000"); + var runs = new List { Finished("20260920-1000-a001", "tool-works") }; + + ScheduleService.RunFinished(schedule, runs, Now).Fire.Should().BeFalse("a tool-works run never starts one"); + + runs.Add(Finished("20260921-1000-b001", "alpha")); + var (fire, seen) = ScheduleService.RunFinished(schedule, runs, Now); + + fire.Should().BeTrue(); + seen.Should().Be("20260921-1000-b001"); + + schedule.LastCommit = seen!; + schedule.LastRun = Now; + + ScheduleService.RunFinished(schedule, runs, Now.AddHours(3)).Fire.Should().BeFalse("that run has been seen"); + } + + [Fact] + public void First_look_records_a_baseline_without_firing() + { + var (fire, seen) = ScheduleService.RunFinished( + Watching(seen: string.Empty), [Finished("20260921-1000-b001", "alpha")], Now); + + fire.Should().BeFalse(); + seen.Should().Be("20260921-1000-b001"); + } + + [Fact] + public void Debounced_to_once_an_hour() + { + var schedule = Watching(seen: "20260901-0000-0000"); + schedule.LastRun = Now.AddMinutes(-30); + var runs = new[] { Finished("20260921-1000-b001", "alpha") }; + + var inside = ScheduleService.RunFinished(schedule, runs, Now); + inside.Fire.Should().BeFalse(); + inside.Seen.Should().BeNull("a run held back by the hour is not written off"); + + ScheduleService.RunFinished(schedule, runs, Now.AddMinutes(31)).Fire.Should().BeTrue(); + } + + private static TeamSchedule Watching(string seen) => new() + { + Id = "tools-on-finish", + // Not tool-works itself, so the own-team rule cannot stand in for the tool-works one. + Team = "tidy-up", + Project = "demo", + Goal = "Look at what finished.", + On = ScheduleService.RunFinishedEvent, + Enabled = true, + LastCommit = seen, + }; + + private static RunSummary Finished(string runId, string team) => new( + runId, string.Empty, team, "goal", "autonomous", + Now.AddHours(-5), Now.AddHours(-4), "done", 0m, 1, [], [], []); +} diff --git a/tests/Loadout.Tests/Unit/TeamRunnerTests.cs b/tests/Loadout.Tests/Unit/TeamRunnerTests.cs index b7b5851e..ebf7ea62 100644 --- a/tests/Loadout.Tests/Unit/TeamRunnerTests.cs +++ b/tests/Loadout.Tests/Unit/TeamRunnerTests.cs @@ -1090,6 +1090,42 @@ public async Task A_node_can_reach_no_part_of_the_tool_catalogue_but_drafts_and_ (await RunAsync(team)).Succeeded.Should().BeTrue(); + CatalogueReachIsOnlyDraftsAndInbox(); + } + + [Fact] + public async Task A_tool_works_node_reaches_drafts_and_inbox_and_nothing_else_of_the_catalogue() + { + // The guard above on the team that has catalogue writers, so the + // drafts-and-inbox branch actually runs rather than passing vacuously. + var team = (await new TeamCatalogue().LoadAsync(null, null, await SpecialistsAsync())).Find("tool-works")!; + var creator = new Report( + "creator", ReportStatus.Done, "Drafted nothing; searched first.", + [new ReportDeliverable(DeliverableKind.File, "drafts/none")], [Passed], []); + + _launcher.Script( + "role.project-lead", + Init("lead-1"), + Result(LeadRequests(new ReportRequest("creator", "Look at the inbox.", DeliverableKind.File, [])), 0.05m), + Result(LeadDone(), 0.09m), + Result(LeadDone(), 0.09m)); + _launcher.Script("role.tool-creator", Init("creator-1"), Result(creator, 0.02m)); + + (await RunAsync(team)).Succeeded.Should().BeTrue(); + + var catalogue = Path.GetFullPath(Path.Combine(_paths.Paths.State, "tools")); + var reached = _launcher.Requests + .Where(one => one.Request.Specialists?.Contains("role.tool-creator") == true) + .SelectMany(one => one.Request.ReachableDirectories ?? []) + .Select(Path.GetFullPath) + .ToList(); + + reached.Should().Contain(Path.Combine(catalogue, "drafts")).And.Contain(Path.Combine(catalogue, "inbox")); + CatalogueReachIsOnlyDraftsAndInbox(); + } + + private void CatalogueReachIsOnlyDraftsAndInbox() + { var catalogue = Path.GetFullPath(Path.Combine(_paths.Paths.State, "tools")); var allowed = new[] { Path.Combine(catalogue, "drafts"), Path.Combine(catalogue, "inbox") }; diff --git a/tests/Loadout.Tests/Unit/ToolNominatorTests.cs b/tests/Loadout.Tests/Unit/ToolNominatorTests.cs new file mode 100644 index 00000000..887fcade --- /dev/null +++ b/tests/Loadout.Tests/Unit/ToolNominatorTests.cs @@ -0,0 +1,136 @@ +using FluentAssertions; +using Loadout.Core.Teams; +using Loadout.Core.Tools; +using Loadout.Models.Teams; +using Xunit; + +namespace Loadout.Tests.Unit; + +/// +/// The four ways finished work is recognised as worth a tool, read from runs +/// and shelves written the way a real run writes them. +/// +public sealed class ToolNominatorTests : IDisposable +{ + private const string Cache = "param([string]$CachePath)\nGet-ChildItem $CachePath -Recurse | Remove-Item -Force\nWrite-Output 'freed'\n"; + + private readonly ToolStoreFixture _store = new(); + + public void Dispose() => _store.Dispose(); + + [Fact] + public void Rule_1_the_same_remedy_on_two_team_shelves_is_nominated() + { + Shelve("alpha", "clear-cache", Cache); + Shelve("beta", "free-disk", Cache); + + var scanned = Nominator().Scan([]); + + scanned.Should().ContainSingle(one => one.Nomination.Rule == 1); + scanned.Single().Filed!.Succeeded.Should().BeTrue(); + Directory.EnumerateFiles(Path.Combine(_store.Paths.Paths.State, "tools", "inbox")).Should().ContainSingle(); + } + + [Fact] + public void Rule_2_a_revised_remedy_that_passed_in_two_runs_is_nominated() + { + Shelve("alpha", "clear-cache", Cache, revision: 1); + Run("20260901-1000-a001", "alpha", Command("pwsh -NoProfile -File remedies/clear-cache.ps1")); + Run("20260902-1000-a002", "alpha", Command("pwsh -NoProfile -File remedies/clear-cache.ps1")); + + Nominator().Find([]).Should().ContainSingle(one => one.Rule == 2); + } + + [Fact] + public void Rule_3_one_command_shape_passing_for_two_teams_is_nominated() + { + Run("20260901-1000-a001", "alpha", Command("dotnet test --filter Alpha.Tests")); + Run("20260902-1000-b001", "beta", Command("dotnet test --filter Beta.Tests")); + + Nominator().Find([]).Should().ContainSingle(one => one.Rule == 3 && one.Key == "3 dotnet test --filter "); + } + + [Fact] + public void Rule_4_a_lesson_naming_a_command_that_passed_is_nominated() + { + Run("20260901-1000-a001", "alpha", Command("docker system prune --force")); + + Nominator().Find(["When the disk fills, docker system prune clears the build layers."]) + .Should().ContainSingle(one => one.Rule == 4); + } + + [Fact] + public async Task Covered_by_an_active_tool_is_not_nominated() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Cache, ToolStoreFixture.Cases()); + + Shelve("alpha", "clear-cache", Cache); + Shelve("beta", "free-disk", Cache); + + Nominator().Find([]).Should().BeEmpty(); + } + + [Fact] + public void A_nomination_never_carries_a_secret_value() + { + const string Key = "AKIAABCDEFGHIJKLMNOP"; + var script = Cache + "$key = '" + Key + "'\n"; + + Shelve("alpha", "clear-cache", script); + Shelve("beta", "free-disk", script); + + var scanned = Nominator().Scan([]); + + scanned.Should().ContainSingle(one => one.Filed != null && one.Filed.Failed); + + var catalogue = Path.Combine(_store.Paths.Paths.State, "tools"); + var written = Directory.Exists(catalogue) + ? Directory.EnumerateFiles(catalogue, "*", SearchOption.AllDirectories) + : []; + + foreach (var file in written) + { + File.ReadAllText(file).Should().NotContain(Key, $"{file} was written by the nominator"); + } + } + + private ToolNominator Nominator() => + new(_store.Registry().Registry, new RunJournal(_store.Paths), new RemedyBook(_store.Paths), _store.Paths); + + private static ReportEvidence Command(string command) => + new(EvidenceKind.Command, command, EvidenceResult.Pass, "exit 0"); + + /// A remedy on a team's shelf, as a node registers one. + private void Shelve(string team, string name, string script, int revision = 0) + { + var shelf = Path.Combine(_store.Paths.Paths.State, "teams", "work", team, "remedies"); + Directory.CreateDirectory(shelf); + + File.WriteAllText(Path.Combine(shelf, name + ".ps1"), script); + File.WriteAllText( + Path.Combine(shelf, name + ".yaml"), + $"name: {name}\nkind: disk\nwhat: Clears a cache that fills the disk.\nassumes: pwsh\nproves: the disk has room\n" + + $"script: {name}.ps1\nrevision: {revision}\n"); + } + + /// A finished run of a team, with one report carrying this evidence. + private void Run(string runId, string team, params ReportEvidence[] evidence) + { + var journal = new RunJournal(_store.Paths); + var directory = journal.DirectoryOf(runId); + Directory.CreateDirectory(directory); + + File.WriteAllLines(Path.Combine(directory, "journal.jsonl"), + [ + """{"at":"2026-09-01T10:00:00+00:00","kind":"run.started","data":{"team":""" + "\"" + team + "\"" + + ""","goal":"keep it up","autonomy":"autonomous"}}""", + """{"at":"2026-09-01T10:30:00+00:00","kind":"run.finished","data":{"ended":"done","outcome":"done"}}""", + ]); + + var report = new Report("remediator", ReportStatus.Done, "Fixed it.", [], evidence, []); + File.WriteAllText(Path.Combine(directory, "report-remediator-1.json"), ReportReader.Write(report)); + + journal.Summarise(runId).Value!.Finished.Should().NotBeNull("the fixture is a finished run"); + } +} From a2a83c7df5a4f3a5c1daa0b98196d07b22685545 Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 17:39:58 +0100 Subject: [PATCH 09/15] Refuse an unreadable draft on a dry run, and stop tool roles running scripts directly 'loadout tools verify --dry-run' and 'promote --dry-run' reported that they would act on a draft the real run could not even read. Both now call IToolRegistry.CheckDraft first, which is ReadDraft's answer without the side effects, and fail with InvalidArguments exactly as the real run would. A contract test runs both against a draft that does not exist and expects a non-zero exit, "no readable manifest.yaml", and no "would be". The tool-creator and tool-refiner roles no longer allow 'pwsh -NoProfile -File'. A draft runs only through 'loadout tools verify', which goes through the same gate as every other script; a direct pwsh call went round it. RoleLibraryTests pins that for both roles: restoring the refiner's allow fails the role.tool-refiner case. The installing guide's doctor sample now counts 103 specialists, the number the library loads since the two tool roles were added. --- docs/guides/installing.md | 2 +- src/Loadout.Cli/Commands/ToolCommands.cs | 12 ++++++++++ .../Specialists/role/tool-creator.md | 1 - .../Specialists/role/tool-refiner.md | 1 - src/Loadout.Core/Tools/ToolRegistry.cs | 13 +++++++++++ .../Contract/ToolCommandsContractTests.cs | 23 +++++++++++++++++++ tests/Loadout.Tests/Unit/RoleLibraryTests.cs | 4 ++++ 7 files changed, 53 insertions(+), 3 deletions(-) diff --git a/docs/guides/installing.md b/docs/guides/installing.md index 352cf513..0de609cc 100644 --- a/docs/guides/installing.md +++ b/docs/guides/installing.md @@ -139,7 +139,7 @@ Instructions + Tasks nobody is shown: storefront 2 task(s) are recorded for storefront and its sessions are not shown any of them. Carrying them costs a heading and a line each. (fixable) -+ Specialist library 101 specialists loaded and valid. ++ Specialist library 103 specialists loaded and valid. Editor + code C:\Program Files\Microsoft VS Code\bin\code.cmd diff --git a/src/Loadout.Cli/Commands/ToolCommands.cs b/src/Loadout.Cli/Commands/ToolCommands.cs index c96d2e53..530f6632 100644 --- a/src/Loadout.Cli/Commands/ToolCommands.cs +++ b/src/Loadout.Cli/Commands/ToolCommands.cs @@ -537,6 +537,13 @@ protected override async Task ExecuteAsync(CommandContext context, Settings if (settings.DryRun) { + // A preview that says it would verify a draft the real run + // cannot even read is the optimism --dry-run is meant to prevent. + if (_registry.CheckDraft(settings.Draft) is { Failed: true } unreadable) + { + return output.Fail(unreadable); + } + if (output.IsJson) { output.WriteJson(new { dry_run = true, draft = settings.Draft }); @@ -655,6 +662,11 @@ protected override int Execute(CommandContext context, Settings settings, Cancel if (settings.DryRun) { + if (_registry.CheckDraft(settings.Draft) is { Failed: true } unreadable) + { + return output.Fail(unreadable); + } + if (output.IsJson) { output.WriteJson(new { dry_run = true, draft = settings.Draft }); diff --git a/src/Loadout.Core/Specialists/role/tool-creator.md b/src/Loadout.Core/Specialists/role/tool-creator.md index 2b56ceb5..8a721d6a 100644 --- a/src/Loadout.Core/Specialists/role/tool-creator.md +++ b/src/Loadout.Core/Specialists/role/tool-creator.md @@ -18,7 +18,6 @@ tools: - 'Bash(loadout tools audit:*)' - 'Bash(loadout tools verify:*)' - 'Bash(loadout memory find:*)' - - 'Bash(pwsh -NoProfile -File:*)' denied: - 'Bash(loadout tools promote:*)' - 'Bash(loadout tools trust:*)' diff --git a/src/Loadout.Core/Specialists/role/tool-refiner.md b/src/Loadout.Core/Specialists/role/tool-refiner.md index 491c30bb..0394ceef 100644 --- a/src/Loadout.Core/Specialists/role/tool-refiner.md +++ b/src/Loadout.Core/Specialists/role/tool-refiner.md @@ -20,7 +20,6 @@ tools: - 'Bash(loadout tools deprecate:*)' - 'Bash(loadout tools retire:*)' - 'Bash(loadout memory find:*)' - - 'Bash(pwsh -NoProfile -File:*)' denied: - 'Bash(loadout tools promote:*)' - 'Bash(loadout tools trust:*)' diff --git a/src/Loadout.Core/Tools/ToolRegistry.cs b/src/Loadout.Core/Tools/ToolRegistry.cs index 5439b116..39a7f839 100644 --- a/src/Loadout.Core/Tools/ToolRegistry.cs +++ b/src/Loadout.Core/Tools/ToolRegistry.cs @@ -100,6 +100,13 @@ public interface IToolRegistry /// The audit log, oldest first. IReadOnlyList Audit(string? tool = null, DateTimeOffset? since = null); + /// + /// Whether a draft can be read at all: under drafts/, with a manifest and + /// the script it names. What a dry run of verify or promote checks, so a + /// preview refuses what the real run would. + /// + OperationResult CheckDraft(string draft); + /// Runs a draft's harness and the regression gate, where this machine allows it. Task> VerifyAsync( string draft, @@ -453,6 +460,12 @@ .. ToolAudit.Read(AuditFile).Where(one => && (since is null || one.At >= since)), ]; + /// + public OperationResult CheckDraft(string draft) => + ReadDraft(draft) is { Failed: true } read + ? OperationResult.Fail(read.Error!, ExitCode.InvalidArguments) + : OperationResult.Ok(); + /// public async Task> VerifyAsync( string draft, diff --git a/tests/Loadout.Tests/Contract/ToolCommandsContractTests.cs b/tests/Loadout.Tests/Contract/ToolCommandsContractTests.cs index 5a476110..aa569c7f 100644 --- a/tests/Loadout.Tests/Contract/ToolCommandsContractTests.cs +++ b/tests/Loadout.Tests/Contract/ToolCommandsContractTests.cs @@ -149,6 +149,29 @@ public async Task Dry_run_changes_nothing() .GetProperty("trusted").GetBoolean().Should().BeFalse("a dry-run trust wrote no agreement"); } + [BuiltCliFact] + public async Task Dry_run_on_a_missing_draft_says_so() + { + var (loadout, root) = await CatalogueAsync(); + using var _ = loadout; + var missing = Path.Combine(root, "drafts", "no-such-tool", "1"); + + string[][] asked = + [ + ["tools", "verify", missing, "--dry-run"], + ["tools", "promote", missing, "--because", "lesson", "--source", "inbox/x", "--dry-run"], + ]; + + foreach (var one in asked) + { + var run = await loadout.RunAsync(one); + + run.ExitCode.Should().NotBe(0, string.Join(' ', one)); + (run.StandardOutput + run.StandardError).Should().Contain("no readable manifest.yaml", string.Join(' ', one)); + (run.StandardOutput + run.StandardError).Should().NotContain("would be", string.Join(' ', one)); + } + } + [BuiltCliFact] public async Task Trust_fingerprints_the_script_on_disk_not_the_manifest_field() { diff --git a/tests/Loadout.Tests/Unit/RoleLibraryTests.cs b/tests/Loadout.Tests/Unit/RoleLibraryTests.cs index 76dde150..9516b61f 100644 --- a/tests/Loadout.Tests/Unit/RoleLibraryTests.cs +++ b/tests/Loadout.Tests/Unit/RoleLibraryTests.cs @@ -107,6 +107,10 @@ bool Allowed(string command) => Loadout.Core.Teams.NodePermissions Allowed("loadout tools search disk cache").Should().BeTrue(); Allowed("loadout tools submit --kind candidate --text x").Should().BeTrue(); + // A draft runs only through 'loadout tools verify', which goes through + // the same gate as every other script; running it directly would not. + Allowed("pwsh -NoProfile -File x.ps1").Should().BeFalse(); + // Lifecycle is the Refiner's alone, and changes nothing that runs. Allowed("loadout tools deprecate free-disk-by-cache --reason unused") .Should().Be(id == "role.tool-refiner"); From dd24997cee334e71956e0928be3ac262577c3458 Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 17:40:05 +0100 Subject: [PATCH 10/15] Run the tool nominator before a run-finished or tool-works schedule starts ToolNominator existed but nothing called it. ToolNominationPass runs it over the run journal, the remedy shelves and every lesson topic of every registered project, and the daemon calls it just before starting a schedule that fires on run-finished or starts tool-works, so the inbox the Creator reads already holds what finished work nominated. A project whose memory cannot be read is passed over rather than failing the pass. The daemon prints how many it filed when there were any. Nothing else calls the pass: a nomination nobody reads until the next tool-works run can wait until then to be filed. ToolNominationPassTests cover both triggering schedules and the ones that do not; making Precedes return false fails the two triggering cases. DaemonControlTests and DaemonStartsWorkTests pass the new constructor argument. --- src/Loadout.Cli/Commands/TeamDaemonCommand.cs | 14 +- src/Loadout.Core/ServiceRegistration.cs | 1 + src/Loadout.Core/Tools/ToolNominationPass.cs | 110 +++++++++++++ .../Loadout.Tests/Unit/DaemonControlTests.cs | 3 +- .../Unit/DaemonStartsWorkTests.cs | 3 +- .../Unit/ToolNominationPassTests.cs | 148 ++++++++++++++++++ 6 files changed, 276 insertions(+), 3 deletions(-) create mode 100644 src/Loadout.Core/Tools/ToolNominationPass.cs create mode 100644 tests/Loadout.Tests/Unit/ToolNominationPassTests.cs diff --git a/src/Loadout.Cli/Commands/TeamDaemonCommand.cs b/src/Loadout.Cli/Commands/TeamDaemonCommand.cs index 34921c7e..f31991a1 100644 --- a/src/Loadout.Cli/Commands/TeamDaemonCommand.cs +++ b/src/Loadout.Cli/Commands/TeamDaemonCommand.cs @@ -81,6 +81,7 @@ public sealed class TeamDaemonCommand : AsyncCommand private readonly Loadout.Core.Workspace.IWorkspaceManager _workspace; private readonly Loadout.Agents.IAgentRegistry _agents; private readonly IProcessLauncher _launcher; + private readonly Loadout.Core.Tools.ToolNominationPass _nominations; /// /// Every command this daemon runs, counted while it runs. @@ -112,9 +113,11 @@ public TeamDaemonCommand( Loadout.Core.Instructions.ISpecialistLibrary library, Loadout.Core.Workspace.IWorkspaceManager workspace, Loadout.Agents.IAgentRegistry agents, - IProcessLauncher launcher) + IProcessLauncher launcher, + Loadout.Core.Tools.ToolNominationPass nominations) { _launcher = launcher; + _nominations = nominations; _inFlight = new InFlight(commands); commands = _inFlight; _teams = teams; @@ -840,6 +843,15 @@ private async Task FireAsync(CommandOutput output, CancellationToken ct) $"[dim]{now.ToLocalTime():HH:mm}[/] starting {Markup.Escape(schedule.Id)}: " + $"{Markup.Escape(schedule.Team)} on {Markup.Escape(schedule.Project)}"); + // Before the team starts, so what finished work nominated is + // in the inbox the Creator is about to read. + var nominated = await _nominations.BeforeAsync(schedule, ct).ConfigureAwait(false); + + if (nominated.Count(one => one.Filed is { Succeeded: true }) is > 0 and var filed) + { + output.WriteLine($"[dim]{now.ToLocalTime():HH:mm}[/] nominated {filed} for the tool catalogue"); + } + var code = await _commands.RunAsync( "team run", [ diff --git a/src/Loadout.Core/ServiceRegistration.cs b/src/Loadout.Core/ServiceRegistration.cs index efd10d2c..37d8cbf3 100644 --- a/src/Loadout.Core/ServiceRegistration.cs +++ b/src/Loadout.Core/ServiceRegistration.cs @@ -166,6 +166,7 @@ public static IServiceCollection AddCoreServices(this IServiceCollection service provider.GetRequiredService(), provider.GetRequiredService(), provider.GetRequiredService())); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Loadout.Core/Tools/ToolNominationPass.cs b/src/Loadout.Core/Tools/ToolNominationPass.cs new file mode 100644 index 00000000..06745870 --- /dev/null +++ b/src/Loadout.Core/Tools/ToolNominationPass.cs @@ -0,0 +1,110 @@ +using Loadout.Core.Instructions; +using Loadout.Core.Projects; +using Loadout.Core.Teams; +using Loadout.Core.Workspace; +using Loadout.Models.Instructions; +using Loadout.Models.Results; +using Loadout.Models.Teams; +using Loadout.Platform.Abstractions; + +namespace Loadout.Core.Tools; + +/// +/// The nominator's pass over finished work, run when something is about to +/// look at the tool catalogue. +/// +/// +/// +/// Run before a run-finished schedule starts its team, and before tool-works +/// starts on its own schedule, so the nominations the Creator reads are the +/// ones finished work produced up to that moment. Nothing else calls it: a +/// nomination nobody reads until tomorrow can wait until tomorrow to be filed. +/// +/// +/// The lessons are every lesson topic in the memory of every registered +/// project. A project whose memory cannot be read is passed over rather than +/// failing the pass; the runs and shelves are still worth reading without it. +/// +/// +public sealed class ToolNominationPass +{ + private readonly IToolRegistry _registry; + private readonly IRunJournal _journal; + private readonly IRemedyBook _remedies; + private readonly IPlatformPaths _paths; + private readonly IMemoryService _memory; + private readonly IProjectService _projects; + private readonly IWorkspaceManager _workspace; + + public ToolNominationPass( + IToolRegistry registry, + IRunJournal journal, + IRemedyBook remedies, + IPlatformPaths paths, + IMemoryService memory, + IProjectService projects, + IWorkspaceManager workspace) + { + _registry = registry; + _journal = journal; + _remedies = remedies; + _paths = paths; + _memory = memory; + _projects = projects; + _workspace = workspace; + } + + /// Whether starting this schedule should be preceded by a pass. + public static bool Precedes(TeamSchedule schedule) + { + ArgumentNullException.ThrowIfNull(schedule); + + return string.Equals(schedule.On, ScheduleService.RunFinishedEvent, StringComparison.OrdinalIgnoreCase) + || string.Equals(schedule.Team, ScheduleService.ToolWorksTeam, StringComparison.OrdinalIgnoreCase); + } + + /// Files what the nominator finds, when this schedule is one that reads it. + /// Each nomination found, as gives it; empty when none is due. + public async Task? Filed)>> BeforeAsync( + TeamSchedule schedule, + CancellationToken ct = default) + { + if (!Precedes(schedule)) + { + return []; + } + + var lessons = await LessonsAsync(ct).ConfigureAwait(false); + + return new ToolNominator(_registry, _journal, _remedies, _paths).Scan(lessons); + } + + /// The text of every lesson topic of every registered project. + private async Task> LessonsAsync(CancellationToken ct) + { + var listed = await _projects.ListAsync(ct).ConfigureAwait(false); + + if (listed.Failed) + { + return []; + } + + var lessons = new List(); + + foreach (var project in listed.Value!) + { + var topics = await _memory.ListAsync(_workspace.LocalPath, project.Entry.Slug, ct).ConfigureAwait(false); + + if (topics.Failed) + { + continue; + } + + lessons.AddRange(topics.Value! + .Where(one => one.Kind == MemoryKind.Lesson) + .Select(one => string.Join('\n', [one.Description, .. one.Facts]))); + } + + return lessons; + } +} diff --git a/tests/Loadout.Tests/Unit/DaemonControlTests.cs b/tests/Loadout.Tests/Unit/DaemonControlTests.cs index 0b0c27b0..56f361a4 100644 --- a/tests/Loadout.Tests/Unit/DaemonControlTests.cs +++ b/tests/Loadout.Tests/Unit/DaemonControlTests.cs @@ -258,7 +258,8 @@ private TeamDaemonCommand Daemon(ICommandCatalogue commands, IProcessLauncher? l library: null!, workspace: null!, agents: null!, - launcher ?? new StubProcessLauncher(string.Empty)); + launcher ?? new StubProcessLauncher(string.Empty), + nominations: null!); private CommandOutput Output() => new( diff --git a/tests/Loadout.Tests/Unit/DaemonStartsWorkTests.cs b/tests/Loadout.Tests/Unit/DaemonStartsWorkTests.cs index b0695651..288029ab 100644 --- a/tests/Loadout.Tests/Unit/DaemonStartsWorkTests.cs +++ b/tests/Loadout.Tests/Unit/DaemonStartsWorkTests.cs @@ -139,7 +139,8 @@ private static TeamDaemonCommand Daemon(ICommandCatalogue commands) => library: null!, workspace: null!, agents: null!, - launcher: null!); + launcher: null!, + nominations: null!); private static CommandOutput Output() => new(Quiet(), new GlobalSettings()); diff --git a/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs b/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs new file mode 100644 index 00000000..7f6e2004 --- /dev/null +++ b/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs @@ -0,0 +1,148 @@ +using FluentAssertions; +using Loadout.Core.Configuration; +using Loadout.Core.Instructions; +using Loadout.Core.Teams; +using Loadout.Core.Tools; +using Loadout.Core.Workspace; +using Loadout.Models.Instructions; +using Loadout.Models.Results; +using Loadout.Models.Teams; +using Loadout.Tests.Fakes; +using Xunit; + +namespace Loadout.Tests.Unit; + +/// +/// What calls the nominator: a run-finished schedule firing, and tool-works +/// starting on its own, each before its team is started. +/// +public sealed class ToolNominationPassTests : IDisposable +{ + private static readonly DateTimeOffset Now = new(2026, 9, 23, 12, 0, 0, TimeSpan.Zero); + + private const string Lesson = "When the disk fills, docker system prune clears the build layers."; + + private readonly ToolStoreFixture _store = new(); + + public void Dispose() => _store.Dispose(); + + [Fact] + public async Task A_fired_run_finished_schedule_files_a_nomination_from_a_finished_run() + { + FinishedRun("20260923-1000-a001", "alpha", "docker system prune --force"); + var schedule = Schedule("tools-on-finish", "tidy-up", ScheduleService.RunFinishedEvent); + var journal = new RunJournal(_store.Paths); + var runs = journal.List(50).Select(journal.Summarise).Select(one => one.Value!).ToList(); + + ScheduleService.RunFinished(schedule, runs, Now).Fire.Should().BeTrue("the fixture run is one it has not seen"); + + var nominated = await Pass().BeforeAsync(schedule); + + nominated.Should().ContainSingle(one => one.Nomination.Rule == 4 && one.Filed!.Succeeded, + "the lesson in memory names the command the finished run passed with"); + Directory.EnumerateFiles(Path.Combine(_store.Paths.Paths.State, "tools", "inbox")).Should().ContainSingle(); + } + + [Fact] + public async Task Tool_works_on_its_own_schedule_nominates_first() + { + FinishedRun("20260923-1000-a001", "alpha", "docker system prune --force"); + + (await Pass().BeforeAsync(Schedule("tool-works-daily", ScheduleService.ToolWorksTeam, string.Empty))) + .Should().ContainSingle(one => one.Filed!.Succeeded); + } + + [Fact] + public async Task Any_other_schedule_nominates_nothing() + { + FinishedRun("20260923-1000-a001", "alpha", "docker system prune --force"); + + (await Pass().BeforeAsync(Schedule("nightly", "alpha", "commit"))).Should().BeEmpty(); + Directory.Exists(Path.Combine(_store.Paths.Paths.State, "tools", "inbox")).Should().BeFalse(); + } + + private ToolNominationPass Pass() + { + var workspace = new WorkspaceManager( + _store.Paths, new FakeGit(_store.Paths.Paths.State), new YamlStore(new NoOpFilePermissions()), TimeProvider.System); + + return new ToolNominationPass( + _store.Registry().Registry, + new RunJournal(_store.Paths), + new RemedyBook(_store.Paths), + _store.Paths, + new LessonMemory(Lesson), + new FakeProjects("demo", _store.Paths.Paths.State), + workspace); + } + + private static TeamSchedule Schedule(string id, string team, string on) => new() + { + Id = id, + Team = team, + Project = "demo", + Goal = "Look at what finished.", + On = on, + Enabled = true, + LastCommit = "20260901-0000-0000", + }; + + /// A finished run of a team, with one report carrying a command that passed. + private void FinishedRun(string runId, string team, string command) + { + var journal = new RunJournal(_store.Paths); + var directory = journal.DirectoryOf(runId); + Directory.CreateDirectory(directory); + + File.WriteAllLines(Path.Combine(directory, "journal.jsonl"), + [ + """{"at":"2026-09-23T10:00:00+00:00","kind":"run.started","data":{"team":""" + "\"" + team + "\"" + + ""","goal":"keep it up","autonomy":"autonomous"}}""", + """{"at":"2026-09-23T10:30:00+00:00","kind":"run.finished","data":{"ended":"done","outcome":"done"}}""", + ]); + + var report = new Report("remediator", ReportStatus.Done, "Fixed it.", [], + [new ReportEvidence(EvidenceKind.Command, command, EvidenceResult.Pass, "exit 0")], []); + File.WriteAllText(Path.Combine(directory, "report-remediator-1.json"), ReportReader.Write(report)); + } + + /// Memory holding one lesson topic, and nothing else a nominator asks for. + private sealed class LessonMemory(string lesson) : IMemoryService + { + public Task>> ListAsync( + string workspaceRoot, string slug, CancellationToken ct = default) => + Task.FromResult(OperationResult>.Ok( + [ + new MemoryTopic("disk-fills", "disk-fills.md", "The build disk fills.", MemoryKind.Lesson, + [lesson], [], 0, Now), + new MemoryTopic("unrelated", "unrelated.md", "Not a lesson.", MemoryKind.Project, + ["docker system prune is mentioned here too, but this is not a lesson."], [], 0, Now), + ])); + + public Task> AuditAsync( + string workspaceRoot, string slug, int staleMonths = 6, CancellationToken ct = default) => + throw new NotSupportedException(); + + public Task> WriteAsync( + string workspaceRoot, string slug, string name, string description, MemoryKind kind, + IReadOnlyList facts, bool acknowledgedSimilar = false, + MemoryScope scope = MemoryScope.Project, CancellationToken ct = default) => + throw new NotSupportedException(); + + public OperationResult ValidateWrite(string name, string description, IReadOnlyList facts) => + throw new NotSupportedException(); + + public Task RebuildIndexAsync(string workspaceRoot, string slug, CancellationToken ct = default) => + throw new NotSupportedException(); + + public Task> CleanAsync( + string workspaceRoot, string slug, bool apply, CancellationToken ct = default) => + throw new NotSupportedException(); + + public IReadOnlyList CleanupPaths(string workspaceRoot, string slug) => + throw new NotSupportedException(); + + public Task> ReadIndexAsync(string workspaceRoot, string slug, CancellationToken ct = default) => + throw new NotSupportedException(); + } +} From 542b08f63a3aa0e72f082098decedd9ffbaa9184 Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 17:54:21 +0100 Subject: [PATCH 11/15] Fix the review of b215176: finish-time watermark, refiner hints, exact rule-2 match The review of b215176 returned it with five findings. Each is fixed here with a test that pins it. (a) run-finished kept the newest run id as its watermark, and a run id is ordered by when the run began. A long run that began first and finished after a shorter one already seen sorted below the watermark and never fired. The watermark is now the latest finish time, in round-trip form; a watermark that does not parse as a time (one an older build wrote) is treated as a first look and re-baselined. Test: A_run_that_finishes_after_a_later_starting_run_still_fires. (b) A hit an active tool already covers was dropped, where plan section 6 says it goes to the Refiner as a usage hint. The nominator now files it through ToolRegistry.Nominate naming the tool, and Nominate files a submission that names a tool as an idea about that tool rather than as a candidate. Test: A_covered_hit_becomes_a_refiner_hint. (c) For rules 3 and 4 there is no script, so coverage compared prose with script shingles and could almost never reach the overlap threshold. A command is now covered where a tool's examples run the same shape, or its script or capabilities name the command. Test: A_command_an_active_tool_wraps_is_not_nominated. (d) Rule 2 matched evidence by file-name substring from any team, so another team's clear-cache.ps1 counted and fix.ps1 matched prefix.ps1. It now matches the whole file name, in the owning team's runs only. Tests: Another_teams_same_named_script_does_not_count, fix_ps1_does_not_match_prefix_ps1. (e) Rule 1's key listed every shelf in the cluster, so a third team shelving the same remedy changed the key and filed the cluster again. The key is now the scripts' fingerprints, with the other fingerprints kept as alternative keys checked before filing. Test: A_third_team_joining_a_cluster_is_not_filed_again. --- src/Loadout.Core/Teams/ScheduleService.cs | 35 +++--- src/Loadout.Core/Tools/ToolNominator.cs | 115 ++++++++++++++---- src/Loadout.Core/Tools/ToolRegistry.cs | 8 +- .../Unit/RunFinishedScheduleTests.cs | 34 +++++- .../Unit/ToolNominationPassTests.cs | 2 +- .../Loadout.Tests/Unit/ToolNominatorTests.cs | 61 ++++++++++ 6 files changed, 206 insertions(+), 49 deletions(-) diff --git a/src/Loadout.Core/Teams/ScheduleService.cs b/src/Loadout.Core/Teams/ScheduleService.cs index 0ffc9428..4a11e746 100644 --- a/src/Loadout.Core/Teams/ScheduleService.cs +++ b/src/Loadout.Core/Teams/ScheduleService.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Loadout.Core.Configuration; using Loadout.Models; using Loadout.Models.Results; @@ -326,10 +327,13 @@ public static bool Moved(TeamSchedule schedule, string? head) /// /// /// - /// The watermark is the newest run identifier seen, kept where a commit - /// watcher keeps its commit; identifiers start with the time a run began, - /// so ordinal order is the order they started in. The first look writes it - /// down and does not fire, for the reason gives. + /// The watermark is the latest time a run was seen to finish, kept where a + /// commit watcher keeps its commit. Not the run identifier: that starts + /// with the time a run began, so a long run that began first and finished + /// last sorts below a short one already seen, and would never fire. The + /// first look writes it down and does not fire, for the reason + /// gives; so does a watermark that is not a time, + /// which is one an older build wrote. /// /// /// Never on a tool-works run, nor on the schedule's own team: either would @@ -355,27 +359,30 @@ public static (bool Fire, string? Seen) RunFinished( .Where(one => one.Finished is not null && !string.Equals(one.Team, ToolWorksTeam, StringComparison.OrdinalIgnoreCase) && !string.Equals(one.Team, schedule.Team, StringComparison.OrdinalIgnoreCase)) - .Select(one => one.RunId) - .OrderByDescending(one => one, StringComparer.Ordinal) - .FirstOrDefault(); + .Select(one => one.Finished) + .Max(); - if (schedule.LastCommit.Length == 0) + if (!DateTimeOffset.TryParseExact( + schedule.LastCommit, "o", CultureInfo.InvariantCulture, DateTimeStyles.None, out var watermark)) { - // Below every run identifier, so a baseline taken before any run - // has finished still lets the first one fire. - return (false, newest ?? "0"); + // Before every finish, so a baseline taken before any run has + // finished still lets the first one fire. + return (false, Watermark(newest ?? DateTimeOffset.MinValue)); } - if (newest is null - || string.CompareOrdinal(newest, schedule.LastCommit) <= 0 + if (newest is not { } latest + || latest <= watermark || (schedule.LastRun is { } last && now - last < RunFinishedQuiet)) { return (false, null); } - return (true, newest); + return (true, Watermark(latest)); } + private static string Watermark(DateTimeOffset finished) => + finished.ToString("o", CultureInfo.InvariantCulture); + /// What is wrong with a schedule, or null when nothing is. private static string? Check(TeamSchedule schedule) { diff --git a/src/Loadout.Core/Tools/ToolNominator.cs b/src/Loadout.Core/Tools/ToolNominator.cs index b16b96d1..2f6eb2de 100644 --- a/src/Loadout.Core/Tools/ToolNominator.cs +++ b/src/Loadout.Core/Tools/ToolNominator.cs @@ -12,7 +12,17 @@ namespace Loadout.Core.Tools; /// One line saying what it is. /// Where it was seen, for the Creator to follow up. /// The script, where it is a remedy. -public sealed record ToolNomination(int Rule, string Key, string Summary, string Text, string? Script); +/// +/// Other keys it may have been filed under before, so a cluster that grows by +/// a shelf is still filed once. +/// +public sealed record ToolNomination( + int Rule, + string Key, + string Summary, + string Text, + string? Script, + IReadOnlyList? Also = null); /// /// Finds what finished work keeps doing again, and files it for the Creator. @@ -49,7 +59,11 @@ public ToolNominator(IToolRegistry registry, IRunJournal journal, IRemedyBook re /// How many runs back it reads. public int Depth { get; init; } = 200; - /// Finds and files every nomination not already filed. + /// + /// Files every nomination not already filed: a candidate for the Creator, + /// or, where an active tool already covers it, an idea about that tool for + /// the Refiner, as a use it can weigh. + /// /// The text of every memory topic of kind lesson. /// Each nomination found, with what filing it came to; null where it was filed before. public IReadOnlyList<(ToolNomination Nomination, OperationResult? Filed)> Scan(IReadOnlyList lessons) @@ -59,23 +73,37 @@ public ToolNominator(IToolRegistry registry, IRunJournal journal, IRemedyBook re .Select(one => one.Note ?? string.Empty) .ToList(); + bool Filed(string key) => filed.Any(note => note.EndsWith(" key " + KeyOf(key), StringComparison.Ordinal)); + return [ - .. Find(lessons).Select(one => + .. Sorted(lessons).Select(sorted => { - var key = KeyOf(one); + var (one, tool) = sorted; + var prefix = tool is null ? string.Empty : "hint " + tool + " "; + + if (new[] { one.Key }.Concat(one.Also ?? []).Any(key => Filed(prefix + key))) + { + return (one, (OperationResult?)null); + } + + var text = tool is null + ? one.Text + : $"{one.Text} The active tool {tool} already covers this, so it is a use of that tool rather than a new one."; - return (one, filed.Any(note => note.EndsWith(" key " + key, StringComparison.Ordinal)) - ? null - : _registry.Nominate( - new ToolSubmission("candidate", one.Text, By: "nominator", Script: one.Script, Summary: one.Summary), - key)); + return (one, _registry.Nominate( + new ToolSubmission("candidate", text, Tool: tool, By: "nominator", Script: one.Script, Summary: one.Summary), + KeyOf(prefix + one.Key))); }), ]; } /// Every nomination the rules find, leaving out what an active tool already covers. - public IReadOnlyList Find(IReadOnlyList lessons) + public IReadOnlyList Find(IReadOnlyList lessons) => + [.. Sorted(lessons).Where(one => one.CoveredBy is null).Select(one => one.Nomination)]; + + /// Every nomination the rules find, with the active tool that covers it, if one does. + private List<(ToolNomination Nomination, string? CoveredBy)> Sorted(IReadOnlyList lessons) { ArgumentNullException.ThrowIfNull(lessons); @@ -87,21 +115,35 @@ public IReadOnlyList Find(IReadOnlyList lessons) found.AddRange(RevisedAndRunTwice(shelves, evidence)); found.AddRange(Commands(evidence, lessons)); - var active = _registry.Offerable() - .Select(one => new ToolShape(one.Record.Capabilities, one.Record.Summary, one.Script)) - .ToList(); + var active = _registry.Offerable(); // What an active tool already does is the Refiner's to hear about, // not the Creator's to build again. - return - [ - .. found.Where(one => - { - var shape = new ToolShape([], one.Summary, one.Script ?? one.Text); + return [.. found.Select(one => (one, active.FirstOrDefault(tool => Covers(tool, one))?.Record.Name))]; + } - return !active.Any(tool => ToolOverlap.Score(shape, tool).Overlaps); - }), - ]; + /// Whether an active tool already does what a nomination found. + /// + /// A remedy is compared script to script. A command has no script, and a + /// sentence about it shares too little with a tool's script to overlap, + /// so it is covered where the tool's examples run the same shape, or its + /// script or capabilities name the command. + /// + private static bool Covers(ToolOffered tool, ToolNomination nomination) + { + if (nomination.Script is { } script) + { + return ToolOverlap.Score( + new ToolShape([], nomination.Summary, script), + new ToolShape(tool.Record.Capabilities, tool.Record.Summary, tool.Script)).Overlaps; + } + + var shape = nomination.Key[(nomination.Key.IndexOf(' ', StringComparison.Ordinal) + 1)..]; + var named = Named(shape) ?? shape; + + return tool.Version.Examples.Any(one => string.Equals(Shape(one.Command), shape, StringComparison.Ordinal)) + || tool.Script.Contains(named, StringComparison.OrdinalIgnoreCase) + || tool.Record.Capabilities.Any(one => string.Equals(one, named, StringComparison.OrdinalIgnoreCase)); } /// A command with its arguments replaced, so two runs of it compare equal. @@ -140,8 +182,8 @@ public static string Shape(string command) return head.Count >= 2 ? string.Join(' ', head) : null; } - private static string KeyOf(ToolNomination nomination) => - RemedyCeiling.Fingerprint(nomination.Key)[..16]; + private static string KeyOf(string key) => + RemedyCeiling.Fingerprint(key)[..16]; private sealed record Shelved(string Team, Remedy Remedy, string Script); @@ -255,16 +297,29 @@ private static IEnumerable SameOnTwoShelves(List shelve var names = members.Select(one => one.Team + "/" + one.Remedy.Name).Order(StringComparer.Ordinal).ToList(); + // Keyed by what the scripts are rather than who keeps them, so a + // third shelf joining the cluster is the same nomination. + var keys = members + .Select(one => "1 " + RemedyCeiling.Fingerprint(one.Script)) + .Distinct(StringComparer.Ordinal) + .Order(StringComparer.Ordinal) + .ToList(); + yield return new ToolNomination( 1, - "1 " + string.Join(' ', names), + keys[0], members[0].Remedy.What, $"The same remedy is kept on {names.Count} team shelves: {string.Join(", ", names)}.", - members[0].Script); + members[0].Script, + keys[1..]); } } - /// Rule 2: a remedy improved at least once and seen to pass in two or more runs. + /// Rule 2: a remedy improved at least once and seen to pass in two or more of its team's runs. + /// + /// The file by its whole name, and only in the owning team's runs: another + /// team's fix.ps1 is another script, and so is this team's prefix.ps1. + /// private static IEnumerable RevisedAndRunTwice(List shelves, List evidence) { foreach (var shelved in shelves.Where(one => one.Remedy.Revision >= 1)) @@ -277,7 +332,9 @@ private static IEnumerable RevisedAndRunTwice(List shel } var runs = evidence - .Where(one => (one.Evidence.Ref + " " + one.Evidence.Note).Contains(file, StringComparison.OrdinalIgnoreCase)) + .Where(one => string.Equals(one.Team, shelved.Team, StringComparison.OrdinalIgnoreCase) + && Words(one.Evidence.Ref + " " + one.Evidence.Note) + .Any(word => string.Equals(Path.GetFileName(word), file, StringComparison.OrdinalIgnoreCase))) .Select(one => one.Run) .Distinct(StringComparer.Ordinal) .Order(StringComparer.Ordinal) @@ -335,6 +392,10 @@ private static IEnumerable Commands(List evidence, IReadOn } } + /// The words of a command line, without the quotes around a path. + private static IEnumerable Words(string text) => + text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Select(one => one.Trim('\'', '"', '`', ',', ';')); + private static ToolShape ShapeOf(Shelved shelved) => new([shelved.Remedy.Kind], shelved.Remedy.What, shelved.Script); diff --git a/src/Loadout.Core/Tools/ToolRegistry.cs b/src/Loadout.Core/Tools/ToolRegistry.cs index 39a7f839..8f3c3332 100644 --- a/src/Loadout.Core/Tools/ToolRegistry.cs +++ b/src/Loadout.Core/Tools/ToolRegistry.cs @@ -89,6 +89,8 @@ public interface IToolRegistry /// Files the nominator's candidate: screened for secrets and audited, but /// not for genericity, because a nomination quotes where it was seen. /// Not on the command line or MCP; only calls it. + /// A submission naming a tool is filed as an idea about that tool, for the + /// Refiner, rather than as a candidate. /// /// What was found. /// What makes it this nomination, recorded so it is filed once. @@ -304,7 +306,11 @@ public OperationResult Nominate(ToolSubmission submission, string ArgumentNullException.ThrowIfNull(submission); ArgumentException.ThrowIfNullOrWhiteSpace(key); - return Put(submission with { Kind = "candidate", By = "nominator" }, generic: false, key); + // Naming a tool makes it a use of that tool for the Refiner to weigh, + // not a candidate for the Creator to build again. + var kind = submission.Tool is null ? "candidate" : "idea"; + + return Put(submission with { Kind = kind, By = "nominator" }, generic: false, key); } private OperationResult Put(ToolSubmission submission, bool generic, string? key) diff --git a/tests/Loadout.Tests/Unit/RunFinishedScheduleTests.cs b/tests/Loadout.Tests/Unit/RunFinishedScheduleTests.cs index 48998435..28fa2015 100644 --- a/tests/Loadout.Tests/Unit/RunFinishedScheduleTests.cs +++ b/tests/Loadout.Tests/Unit/RunFinishedScheduleTests.cs @@ -10,10 +10,12 @@ public sealed class RunFinishedScheduleTests { private static readonly DateTimeOffset Now = new(2026, 9, 23, 12, 0, 0, TimeSpan.Zero); + private const string LongAgo = "2026-09-01T00:00:00.0000000+00:00"; + [Fact] public void Run_finished_fires_once_per_finished_run_and_never_for_tool_works() { - var schedule = Watching(seen: "20260901-0000-0000"); + var schedule = Watching(seen: LongAgo); var runs = new List { Finished("20260920-1000-a001", "tool-works") }; ScheduleService.RunFinished(schedule, runs, Now).Fire.Should().BeFalse("a tool-works run never starts one"); @@ -22,7 +24,7 @@ public void Run_finished_fires_once_per_finished_run_and_never_for_tool_works() var (fire, seen) = ScheduleService.RunFinished(schedule, runs, Now); fire.Should().BeTrue(); - seen.Should().Be("20260921-1000-b001"); + seen.Should().Be("2026-09-23T08:00:00.0000000+00:00", "the watermark is when that run finished"); schedule.LastCommit = seen!; schedule.LastRun = Now; @@ -30,6 +32,26 @@ public void Run_finished_fires_once_per_finished_run_and_never_for_tool_works() ScheduleService.RunFinished(schedule, runs, Now.AddHours(3)).Fire.Should().BeFalse("that run has been seen"); } + [Fact] + public void A_run_that_finishes_after_a_later_starting_run_still_fires() + { + var schedule = Watching(seen: LongAgo); + + // The short run began after the long one and finished first. + var runs = new List { Finished("20260923-0900-b001", "beta", finished: Now.AddHours(-2)) }; + + var (fire, seen) = ScheduleService.RunFinished(schedule, runs, Now); + fire.Should().BeTrue(); + + schedule.LastCommit = seen!; + schedule.LastRun = Now; + + runs.Add(Finished("20260923-0600-a001", "alpha", finished: Now.AddHours(1))); + + ScheduleService.RunFinished(schedule, runs, Now.AddHours(2)).Fire + .Should().BeTrue("the long run finished after the watermark, whatever time it began"); + } + [Fact] public void First_look_records_a_baseline_without_firing() { @@ -37,13 +59,13 @@ public void First_look_records_a_baseline_without_firing() Watching(seen: string.Empty), [Finished("20260921-1000-b001", "alpha")], Now); fire.Should().BeFalse(); - seen.Should().Be("20260921-1000-b001"); + seen.Should().Be("2026-09-23T08:00:00.0000000+00:00"); } [Fact] public void Debounced_to_once_an_hour() { - var schedule = Watching(seen: "20260901-0000-0000"); + var schedule = Watching(seen: LongAgo); schedule.LastRun = Now.AddMinutes(-30); var runs = new[] { Finished("20260921-1000-b001", "alpha") }; @@ -66,7 +88,7 @@ public void Debounced_to_once_an_hour() LastCommit = seen, }; - private static RunSummary Finished(string runId, string team) => new( + private static RunSummary Finished(string runId, string team, DateTimeOffset? finished = null) => new( runId, string.Empty, team, "goal", "autonomous", - Now.AddHours(-5), Now.AddHours(-4), "done", 0m, 1, [], [], []); + Now.AddHours(-5), finished ?? Now.AddHours(-4), "done", 0m, 1, [], [], []); } diff --git a/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs b/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs index 7f6e2004..714af470 100644 --- a/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs +++ b/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs @@ -84,7 +84,7 @@ private ToolNominationPass Pass() Goal = "Look at what finished.", On = on, Enabled = true, - LastCommit = "20260901-0000-0000", + LastCommit = "2026-09-01T00:00:00.0000000+00:00", }; /// A finished run of a team, with one report carrying a command that passed. diff --git a/tests/Loadout.Tests/Unit/ToolNominatorTests.cs b/tests/Loadout.Tests/Unit/ToolNominatorTests.cs index 887fcade..7c2d450d 100644 --- a/tests/Loadout.Tests/Unit/ToolNominatorTests.cs +++ b/tests/Loadout.Tests/Unit/ToolNominatorTests.cs @@ -71,6 +71,67 @@ public async Task Covered_by_an_active_tool_is_not_nominated() Nominator().Find([]).Should().BeEmpty(); } + [Fact] + public async Task A_covered_hit_becomes_a_refiner_hint() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync(registry, ToolStoreFixture.Manifest("free-cache", "1.0"), Cache, ToolStoreFixture.Cases()); + + Shelve("alpha", "clear-cache", Cache); + Shelve("beta", "free-disk", Cache); + + Nominator().Scan([]).Should().ContainSingle(one => one.Filed != null && one.Filed.Succeeded); + + var filed = File.ReadAllText(Directory.EnumerateFiles(Path.Combine(_store.Paths.Paths.State, "tools", "inbox")).Single()); + filed.Should().Contain("kind: idea").And.Contain("tool: free-cache").And.Contain("by: nominator"); + } + + [Fact] + public async Task A_command_an_active_tool_wraps_is_not_nominated() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync( + registry, ToolStoreFixture.Manifest("prune-layers", "1.0"), "docker system prune --force\n", ToolStoreFixture.Cases()); + + Run("20260901-1000-a001", "alpha", Command("docker system prune --force")); + Run("20260902-1000-b001", "beta", Command("docker system prune --force")); + + Nominator().Find(["When the disk fills, docker system prune clears the build layers."]).Should().BeEmpty(); + } + + [Fact] + public void Another_teams_same_named_script_does_not_count() + { + Shelve("alpha", "clear-cache", Cache, revision: 1); + Run("20260901-1000-b001", "beta", Command("pwsh -NoProfile -File remedies/clear-cache.ps1")); + Run("20260902-1000-b002", "beta", Command("pwsh -NoProfile -File remedies/clear-cache.ps1")); + + Nominator().Find([]).Should().NotContain(one => one.Rule == 2); + } + + [Fact] + public void fix_ps1_does_not_match_prefix_ps1() + { + Shelve("alpha", "fix", Cache, revision: 1); + Run("20260901-1000-a001", "alpha", Command("pwsh -NoProfile -File remedies/prefix.ps1")); + Run("20260902-1000-a002", "alpha", Command("pwsh -NoProfile -File remedies/prefix.ps1")); + + Nominator().Find([]).Should().NotContain(one => one.Rule == 2); + } + + [Fact] + public void A_third_team_joining_a_cluster_is_not_filed_again() + { + Shelve("alpha", "clear-cache", Cache); + Shelve("beta", "free-disk", Cache); + Nominator().Scan([]).Should().ContainSingle(one => one.Filed != null && one.Filed.Succeeded); + + Shelve("gamma", "tidy-cache", Cache); + + Nominator().Scan([]).Should().OnlyContain(one => one.Filed == null, "the cluster was filed before gamma joined it"); + Directory.EnumerateFiles(Path.Combine(_store.Paths.Paths.State, "tools", "inbox")).Should().ContainSingle(); + } + [Fact] public void A_nomination_never_carries_a_secret_value() { From d080f42618476308177c4550c6d386f7bd479fb7 Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 18:02:42 +0100 Subject: [PATCH 12/15] Keep an unreadable folder from stopping the nomination pass or the run The review of dd24997 returned it with three findings. (1) The daemon awaited the nomination pass with nothing around it, so one folder the pass could not read threw out of FireAsync and the scheduled run never started. Starting a schedule's team is now TeamDaemonCommand.StartAsync, which catches IOException, UnauthorizedAccessException and RegexMatchTimeoutException from the pass, writes one line to the daemon's output ("nominated nothing before : ...") and starts the run anyway. The daemon takes the pass through a new IToolNominationPass, which is the seam the test needed. Inside the nominator, one unreadable folder now skips only itself: the enumeration of team work directories, each team's shelf, and each run's summary and documents are guarded separately. RunJournal.Summarise lists the run folder too (NodePermissions.Pending), which is where the test's first failure came from, so it is inside the per-run guard. (2) The 'passed over' branch in ToolNominationPass tested topics.Failed, which MemoryService.ListAsync never returns: it leaves out a scope it cannot read. It is replaced by a catch around each project's memory read that logs "passed over the lessons of " and carries on. BeforeAsync takes an optional log callback for these lines, which the daemon supplies. (3) Tests: - An_unreadable_run_folder_does_not_stop_the_pass denies listing on one of two finished runs (an ACL deny on Windows, mode --x on Unix) and expects the other still to be nominated. Before the change it failed with UnauthorizedAccessException from RunJournal.Summarise. - The_daemon_still_starts_the_run_when_the_nomination_pass_throws uses a pass that throws UnauthorizedAccessException and expects "team run" to be asked and the log line to be written. With UnauthorizedAccessException taken out of the catch it failed with that exception. --- src/Loadout.Cli/Commands/TeamDaemonCommand.cs | 65 +++++++++++++------ src/Loadout.Core/ServiceRegistration.cs | 2 +- src/Loadout.Core/Tools/ToolNominationPass.cs | 35 ++++++++-- src/Loadout.Core/Tools/ToolNominator.cs | 60 +++++++++++++---- .../Loadout.Tests/Unit/DaemonControlTests.cs | 53 ++++++++++++++- .../Unit/ToolNominationPassTests.cs | 58 +++++++++++++++++ 6 files changed, 230 insertions(+), 43 deletions(-) diff --git a/src/Loadout.Cli/Commands/TeamDaemonCommand.cs b/src/Loadout.Cli/Commands/TeamDaemonCommand.cs index f31991a1..000aeff0 100644 --- a/src/Loadout.Cli/Commands/TeamDaemonCommand.cs +++ b/src/Loadout.Cli/Commands/TeamDaemonCommand.cs @@ -81,7 +81,7 @@ public sealed class TeamDaemonCommand : AsyncCommand private readonly Loadout.Core.Workspace.IWorkspaceManager _workspace; private readonly Loadout.Agents.IAgentRegistry _agents; private readonly IProcessLauncher _launcher; - private readonly Loadout.Core.Tools.ToolNominationPass _nominations; + private readonly Loadout.Core.Tools.IToolNominationPass _nominations; /// /// Every command this daemon runs, counted while it runs. @@ -114,7 +114,7 @@ public TeamDaemonCommand( Loadout.Core.Workspace.IWorkspaceManager workspace, Loadout.Agents.IAgentRegistry agents, IProcessLauncher launcher, - Loadout.Core.Tools.ToolNominationPass nominations) + Loadout.Core.Tools.IToolNominationPass nominations) { _launcher = launcher; _nominations = nominations; @@ -843,25 +843,7 @@ private async Task FireAsync(CommandOutput output, CancellationToken ct) $"[dim]{now.ToLocalTime():HH:mm}[/] starting {Markup.Escape(schedule.Id)}: " + $"{Markup.Escape(schedule.Team)} on {Markup.Escape(schedule.Project)}"); - // Before the team starts, so what finished work nominated is - // in the inbox the Creator is about to read. - var nominated = await _nominations.BeforeAsync(schedule, ct).ConfigureAwait(false); - - if (nominated.Count(one => one.Filed is { Succeeded: true }) is > 0 and var filed) - { - output.WriteLine($"[dim]{now.ToLocalTime():HH:mm}[/] nominated {filed} for the tool catalogue"); - } - - var code = await _commands.RunAsync( - "team run", - [ - schedule.Team, - schedule.Goal, - "--project", schedule.Project, - "--autonomy", schedule.Autonomy, - "--non-interactive", - ], - ct).ConfigureAwait(false); + var code = await StartAsync(schedule, now, output, ct).ConfigureAwait(false); // The run names itself, and the newest one on this machine is // the one just finished. Recorded afterwards so the schedule @@ -892,6 +874,47 @@ await _schedules.StartedAsync(schedule.Id, now, runId, CancellationToken.None) } } + /// Starts one schedule's team, nominating first where it reads the catalogue. + /// The run's exit code. + /// + /// The nomination goes before the team starts, so what finished work + /// nominated is in the inbox the Creator is about to read. It is not + /// allowed to stop the run: a pass that cannot read something says so in + /// one line and the team starts anyway, because the run is what was + /// scheduled and the nominations can wait for the next one. + /// + internal async Task StartAsync(TeamSchedule schedule, DateTimeOffset now, CommandOutput output, CancellationToken ct) + { + void Log(string line) => + output.WriteLine($"[dim]{now.ToLocalTime():HH:mm}[/] {Markup.Escape(line)}"); + + try + { + var nominated = await _nominations.BeforeAsync(schedule, Log, ct).ConfigureAwait(false); + + if (nominated.Count(one => one.Filed is { Succeeded: true }) is > 0 and var filed) + { + Log($"nominated {filed} for the tool catalogue"); + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException + or System.Text.RegularExpressions.RegexMatchTimeoutException) + { + Log($"nominated nothing before {schedule.Id}: {ex.Message}"); + } + + return await _commands.RunAsync( + "team run", + [ + schedule.Team, + schedule.Goal, + "--project", schedule.Project, + "--autonomy", schedule.Autonomy, + "--non-interactive", + ], + ct).ConfigureAwait(false); + } + /// Whether the schedules were held at the last look. private bool _held; diff --git a/src/Loadout.Core/ServiceRegistration.cs b/src/Loadout.Core/ServiceRegistration.cs index 37d8cbf3..06647555 100644 --- a/src/Loadout.Core/ServiceRegistration.cs +++ b/src/Loadout.Core/ServiceRegistration.cs @@ -166,7 +166,7 @@ public static IServiceCollection AddCoreServices(this IServiceCollection service provider.GetRequiredService(), provider.GetRequiredService(), provider.GetRequiredService())); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Loadout.Core/Tools/ToolNominationPass.cs b/src/Loadout.Core/Tools/ToolNominationPass.cs index 06745870..80e8dd77 100644 --- a/src/Loadout.Core/Tools/ToolNominationPass.cs +++ b/src/Loadout.Core/Tools/ToolNominationPass.cs @@ -9,6 +9,19 @@ namespace Loadout.Core.Tools; +/// What the daemon runs before starting a schedule that reads the tool catalogue. +public interface IToolNominationPass +{ + /// Files what the nominator finds, when this schedule is one that reads it. + /// The schedule about to start. + /// Where a line goes for each thing passed over. + /// Cancellation token. + Task? Filed)>> BeforeAsync( + TeamSchedule schedule, + Action? log = null, + CancellationToken ct = default); +} + /// /// The nominator's pass over finished work, run when something is about to /// look at the tool catalogue. @@ -26,7 +39,7 @@ namespace Loadout.Core.Tools; /// failing the pass; the runs and shelves are still worth reading without it. /// /// -public sealed class ToolNominationPass +public sealed class ToolNominationPass : IToolNominationPass { private readonly IToolRegistry _registry; private readonly IRunJournal _journal; @@ -67,6 +80,7 @@ public static bool Precedes(TeamSchedule schedule) /// Each nomination found, as gives it; empty when none is due. public async Task? Filed)>> BeforeAsync( TeamSchedule schedule, + Action? log = null, CancellationToken ct = default) { if (!Precedes(schedule)) @@ -74,18 +88,19 @@ public static bool Precedes(TeamSchedule schedule) return []; } - var lessons = await LessonsAsync(ct).ConfigureAwait(false); + var lessons = await LessonsAsync(log, ct).ConfigureAwait(false); return new ToolNominator(_registry, _journal, _remedies, _paths).Scan(lessons); } /// The text of every lesson topic of every registered project. - private async Task> LessonsAsync(CancellationToken ct) + private async Task> LessonsAsync(Action? log, CancellationToken ct) { var listed = await _projects.ListAsync(ct).ConfigureAwait(false); if (listed.Failed) { + log?.Invoke("nominating without lessons: the projects could not be listed"); return []; } @@ -93,14 +108,22 @@ private async Task> LessonsAsync(CancellationToken ct) foreach (var project in listed.Value!) { - var topics = await _memory.ListAsync(_workspace.LocalPath, project.Entry.Slug, ct).ConfigureAwait(false); + IReadOnlyList topics; - if (topics.Failed) + // The memory service leaves out a scope it cannot read rather than + // failing, so what reaches here unreadable arrives as a throw. + try + { + topics = (await _memory.ListAsync(_workspace.LocalPath, project.Entry.Slug, ct).ConfigureAwait(false)) + .Value ?? []; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { + log?.Invoke($"passed over the lessons of {project.Entry.Slug}: {ex.Message}"); continue; } - lessons.AddRange(topics.Value! + lessons.AddRange(topics .Where(one => one.Kind == MemoryKind.Lesson) .Select(one => string.Join('\n', [one.Description, .. one.Facts]))); } diff --git a/src/Loadout.Core/Tools/ToolNominator.cs b/src/Loadout.Core/Tools/ToolNominator.cs index 2f6eb2de..686eb405 100644 --- a/src/Loadout.Core/Tools/ToolNominator.cs +++ b/src/Loadout.Core/Tools/ToolNominator.cs @@ -199,21 +199,39 @@ private List Shelves() return shelved; } - foreach (var team in Directory.EnumerateDirectories(work).Select(Path.GetFileName).OfType().Order(StringComparer.Ordinal)) + List teams; + + try { - if (string.Equals(team, ScheduleService.ToolWorksTeam, StringComparison.OrdinalIgnoreCase) - || _remedies.All(team) is not { Succeeded: true } all) - { - continue; - } + teams = [.. Directory.EnumerateDirectories(work).Select(Path.GetFileName).OfType().Order(StringComparer.Ordinal)]; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + return shelved; + } - foreach (var remedy in all.Value!) + foreach (var team in teams) + { + // One team's shelf that cannot be read is passed over, and only it. + try { - if (_remedies.ScriptOf(team, remedy) is { Succeeded: true } script) + if (string.Equals(team, ScheduleService.ToolWorksTeam, StringComparison.OrdinalIgnoreCase) + || _remedies.All(team) is not { Succeeded: true } all) + { + continue; + } + + foreach (var remedy in all.Value!) { - shelved.Add(new Shelved(team, remedy, script.Value!)); + if (_remedies.ScriptOf(team, remedy) is { Succeeded: true } script) + { + shelved.Add(new Shelved(team, remedy, script.Value!)); + } } } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + } } return shelved; @@ -225,14 +243,30 @@ private List Evidence() foreach (var id in _journal.List(Depth)) { - if (_journal.Summarise(id) is not { Succeeded: true } summarised - || summarised.Value! is not { Finished: not null } run - || string.Equals(run.Team, ScheduleService.ToolWorksTeam, StringComparison.OrdinalIgnoreCase)) + RunSummary run; + IReadOnlyList documents; + + // A run folder that cannot be listed is passed over, and only it: + // the other runs are still worth reading. Summarising lists the + // folder too, so both are inside the guard. + try + { + if (_journal.Summarise(id) is not { Succeeded: true } summarised + || summarised.Value! is not { Finished: not null } finished + || string.Equals(finished.Team, ScheduleService.ToolWorksTeam, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + run = finished; + documents = RunDocuments.In(run.Directory); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { continue; } - foreach (var document in RunDocuments.In(run.Directory).Where(one => one.Kind == "report")) + foreach (var document in documents.Where(one => one.Kind == "report")) { string text; diff --git a/tests/Loadout.Tests/Unit/DaemonControlTests.cs b/tests/Loadout.Tests/Unit/DaemonControlTests.cs index 56f361a4..bc5ee270 100644 --- a/tests/Loadout.Tests/Unit/DaemonControlTests.cs +++ b/tests/Loadout.Tests/Unit/DaemonControlTests.cs @@ -234,10 +234,36 @@ public void A_restart_of_one_serving_nothing_serves_nothing() .Should().Contain("--no-dashboard").And.NotContain("--port"); } + [Fact] + public async Task The_daemon_still_starts_the_run_when_the_nomination_pass_throws() + { + var commands = new Asked(); + var daemon = Daemon(commands, nominations: new ThrowingPass()); + var schedule = new Loadout.Models.Teams.TeamSchedule + { + Id = "tools-on-finish", + Team = "tool-works", + Project = "loadout-cli", + Goal = "Look at what finished.", + On = "run-finished", + Autonomy = "autonomous", + }; + + var code = await daemon.StartAsync(schedule, DateTimeOffset.UtcNow, Output(), CancellationToken.None); + + code.Should().Be(0); + commands.Paths.Should().ContainSingle().Which.Should().Be("team run", + "a pass that cannot read what finished is no reason not to start what was scheduled"); + _said.ToString().Should().Contain("nominated nothing before tools-on-finish"); + } + private static int Occurrences(string text, string word) => (text.Length - text.Replace(word, string.Empty, StringComparison.Ordinal).Length) / word.Length; - private TeamDaemonCommand Daemon(ICommandCatalogue commands, IProcessLauncher? launcher = null) => + private TeamDaemonCommand Daemon( + ICommandCatalogue commands, + IProcessLauncher? launcher = null, + Loadout.Core.Tools.IToolNominationPass? nominations = null) => new( secrets: null!, client: null!, @@ -259,7 +285,30 @@ private TeamDaemonCommand Daemon(ICommandCatalogue commands, IProcessLauncher? l workspace: null!, agents: null!, launcher ?? new StubProcessLauncher(string.Empty), - nominations: null!); + nominations: nominations!); + + /// A nomination pass whose folders cannot be read. + private sealed class ThrowingPass : Loadout.Core.Tools.IToolNominationPass + { + public Task? Filed)>> BeforeAsync( + Loadout.Models.Teams.TeamSchedule schedule, Action? log = null, CancellationToken ct = default) => + throw new UnauthorizedAccessException("Access to the path 'runs' is denied."); + } + + /// A command that finishes at once, remembering what it was asked. + private sealed class Asked : ICommandCatalogue + { + public List Paths { get; } = []; + + public IReadOnlyList Commands => []; + + public Task RunAsync(string path, IReadOnlyList arguments, CancellationToken ct = default) + { + Paths.Add(path); + return Task.FromResult(0); + } + } private CommandOutput Output() => new( diff --git a/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs b/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs index 714af470..19ddfc25 100644 --- a/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs +++ b/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs @@ -61,6 +61,64 @@ public async Task Any_other_schedule_nominates_nothing() Directory.Exists(Path.Combine(_store.Paths.Paths.State, "tools", "inbox")).Should().BeFalse(); } + [Fact] + public async Task An_unreadable_run_folder_does_not_stop_the_pass() + { + FinishedRun("20260923-1000-a001", "alpha", "docker system prune --force"); + FinishedRun("20260923-1100-b002", "beta", "git gc --aggressive"); + var locked = new RunJournal(_store.Paths).DirectoryOf("20260923-1100-b002"); + + Unlistable(locked, true); + + try + { + FluentActions.Invoking(() => Directory.EnumerateFiles(locked).ToList()) + .Should().Throw("the fixture has to be a folder that cannot be listed"); + + var nominated = await Pass().BeforeAsync(Schedule("tools-on-finish", "tidy-up", ScheduleService.RunFinishedEvent)); + + nominated.Should().ContainSingle(one => one.Filed!.Succeeded, + "the run that can be read is still read, and only the one that cannot is passed over"); + } + finally + { + Unlistable(locked, false); + } + } + + /// Takes away, or gives back, the right to list a folder's contents. + private static void Unlistable(string directory, bool locked) + { + if (OperatingSystem.IsWindows()) + { + var info = new DirectoryInfo(directory); + var security = info.GetAccessControl(); + var rule = new System.Security.AccessControl.FileSystemAccessRule( + System.Security.Principal.WindowsIdentity.GetCurrent().User!, + System.Security.AccessControl.FileSystemRights.ListDirectory, + System.Security.AccessControl.AccessControlType.Deny); + + if (locked) + { + security.AddAccessRule(rule); + } + else + { + security.RemoveAccessRule(rule); + } + + info.SetAccessControl(security); + } + else + { + // Execute alone: a file whose name is known can still be opened, + // but the folder cannot be listed. + File.SetUnixFileMode(directory, locked + ? UnixFileMode.UserExecute + : UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + } + private ToolNominationPass Pass() { var workspace = new WorkspaceManager( From 6d20bd6b1049cd02d11ac7c4d3191a71b74c38b7 Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 18:09:37 +0100 Subject: [PATCH 13/15] Test the tool catalogue end to end: one team's lesson, another team's tool GlobalToolLifecycleTests drives a lesson from team A into a global tool that team B uses, over the catalogue's own code. Scripted: team A's two runs are written as the runner writes them (a journal and a remediator's report), and its remedy is placed on its shelf directly. The Creator's turns are the calls its commands make - tools search, submit, verify, promote - not an agent session. Team B's run is not driven through TeamRunner: its node is a brief rendered by TeamRunner.Render and a permission policy from ToolOffer.For, the two calls the runner makes when briefing a remediator. A person's answers (agreeing to a harness run, trusting a version) are written into the records the CLI writes. Every harness run goes through a stub launcher, so nothing needs pwsh. Real: the daemon's nomination pass, the genericity and overlap screens, the verify hold and regression gate, write-once promotion, the trust ruling, and usage. Three tests: the lesson crossing teams with no project detail under versions/; a failed refinement leaving 1.0 active and known-good; and each way a project can ride along being refused. For the last, a case carrying an absolute path is refused by the harness (ToolHarness.RunAsync) before promote, so that draft now expects verify's gate to fail naming the path; the other three still pass verify and are refused at promote. Mutation checks, each undone before the suite ran: - ToolGenericity.Check returning nothing for any text (a plain inversion of IsNullOrWhiteSpace does not compile under warnings-as-errors) fails tests 1 and 3. - regressions.Count <= 1 in ToolPromotion.GateAsync fails test 2. Suite: 3227 passed, 0 failed, 22 skipped. --- .../Integration/GlobalToolLifecycleTests.cs | 427 ++++++++++++++++++ 1 file changed, 427 insertions(+) create mode 100644 tests/Loadout.Tests/Integration/GlobalToolLifecycleTests.cs diff --git a/tests/Loadout.Tests/Integration/GlobalToolLifecycleTests.cs b/tests/Loadout.Tests/Integration/GlobalToolLifecycleTests.cs new file mode 100644 index 00000000..2cbd245d --- /dev/null +++ b/tests/Loadout.Tests/Integration/GlobalToolLifecycleTests.cs @@ -0,0 +1,427 @@ +using System.Text; +using FluentAssertions; +using Loadout.Agents.Teams; +using Loadout.Core.Configuration; +using Loadout.Core.Instructions; +using Loadout.Core.Teams; +using Loadout.Core.Tools; +using Loadout.Core.Workspace; +using Loadout.Models.Configuration; +using Loadout.Models.Instructions; +using Loadout.Models.Platform; +using Loadout.Models.Results; +using Loadout.Models.Teams; +using Loadout.Models.Tools; +using Loadout.Platform.Abstractions; +using Loadout.Tests.Fakes; +using Loadout.Tests.Unit; +using Xunit; + +namespace Loadout.Tests.Integration; + +/// +/// A lesson from one team becoming a tool another team uses, end to end, over +/// the catalogue's own code. +/// +/// +/// +/// What is scripted, and what is real. Team A's two runs are written as the +/// runner writes them (a journal and a remediator's report), and team B's node +/// is a brief rendered by and a permission +/// policy built by , the two calls the runner makes +/// when it briefs a remediator. The Creator's turns are the calls its commands +/// make: tools search, tools submit, tools verify. Every +/// harness run goes through a stub launcher, so nothing here needs pwsh. +/// +/// +/// Real: the daemon's nomination pass, the genericity and overlap screens, the +/// verify hold and the regression gate, write-once promotion, the trust ruling, +/// and usage. A person's answers - agreeing to a harness run, trusting a +/// version - are written into the same records the CLI writes them to. +/// +/// +public sealed class GlobalToolLifecycleTests : IDisposable +{ + private const string TeamA = "system-watch"; + private const string ProjectA = "alpha"; + private const string TeamB = "beta-ops"; + private const string ProjectB = "beta"; + + private static readonly string[] RunsA = ["20260921-0900-a001", "20260922-0900-a002"]; + + /// Team A's fix, as it was kept on its shelf: the cache it clears is written into it. + private const string HardCoded = + "$CachePath = 'D:\\alpha\\build\\cache'\nGet-ChildItem $CachePath -Recurse | Remove-Item -Force\nWrite-Output 'freed'\n"; + + /// The same fix with the path made an input. + private const string Generic = + "param([Parameter(Mandatory)][string]$CachePath)\nGet-ChildItem $CachePath -Recurse | Remove-Item -Force\nWrite-Output 'freed'\n"; + + private readonly ToolStoreFixture _store = new(ProjectA, TeamA, ProjectB, TeamB); + + public void Dispose() => _store.Dispose(); + + [Fact] + public async Task A_lesson_from_one_team_becomes_a_tool_another_team_uses() + { + var (registry, _) = _store.Registry(); + + // 1. Team A's remediator fixes a full disk with a remedy that names + // its own project's cache, and the same remedy passes again. + Shelve(TeamA, "clear-alpha-cache", HardCoded, revision: 1); + + foreach (var run in RunsA) + { + FinishedRun(run, TeamA, "pwsh -NoProfile -File remedies/clear-alpha-cache.ps1"); + } + + // 2. The daemon's run-finished schedule fires on it, and the pass that + // precedes starting tool-works files the nomination. + var schedule = new TeamSchedule + { + Id = "tools-on-finish", + Team = ScheduleService.ToolWorksTeam, + Project = ProjectB, + Goal = "Look at what finished.", + On = ScheduleService.RunFinishedEvent, + Enabled = true, + LastCommit = "2026-09-01T00:00:00.0000000+00:00", + }; + + var journal = new RunJournal(_store.Paths); + ScheduleService.RunFinished(schedule, [.. journal.List(50).Select(one => journal.Summarise(one).Value!)], DateTimeOffset.UtcNow) + .Fire.Should().BeTrue("team A's runs have finished and have not been seen"); + + var nominated = await Pass(registry).BeforeAsync(schedule); + + nominated.Should().ContainSingle(one => one.Nomination.Rule == 2 && one.Filed!.Succeeded, + "a revised remedy passed in two of its team's runs"); + var nomination = nominated.Single().Filed!.Value!.Id; + + // 3. The Creator searches first and finds nothing to extend. + registry.Search("disk cache").Should().BeEmpty(); + + // Its first submission still carries team A's cache, and is refused + // for it, naming the path and the project. + var first = registry.Submit(new ToolSubmission( + "candidate", "Frees disk held by a build cache.", By: "creator", Script: HardCoded, + Capabilities: ["disk", "cache", "cleanup"], Summary: "Frees disk held by the alpha build cache.")); + + first.Failed.Should().BeTrue("the draft still names one project's path"); + first.Error.Should().Contain("absolute path").And.Contain("'alpha'"); + + var second = registry.Submit(new ToolSubmission( + "candidate", "Frees disk held by a build cache.", By: "creator", Script: Generic, + Capabilities: ["disk", "cache", "cleanup"], Summary: "Frees disk held by a build cache directory.")); + + second.Succeeded.Should().BeTrue(second.Error); + + // 4. Verify is held for a person, as this machine has no rule for + // tool-test; the person agrees to that draft, and it runs. + var manifest = Manifest("free-disk-by-cache", "1.0"); + var cases = ToolStoreFixture.Cases(); + var draft = _store.Draft(manifest, Generic, cases); + + var held = await registry.VerifyAsync(draft, new ToolTestConsent(null, [])); + held.Value!.Ruling.Should().NotBe(RemedyRuling.Run, "nobody has agreed to this harness run"); + held.Value.Gate.Should().BeNull(); + + var verified = await registry.VerifyAsync(draft, ToolStoreFixture.Agreed(manifest, Generic, cases)); + verified.Value!.Gate!.Passed.Should().BeTrue(verified.Value.Because); + + var promoted = registry.Promote(draft, new ToolPromotionRequest( + "nomination", "inbox/" + nomination, Actor: "creator", Owner: ScheduleService.ToolWorksTeam, Kind: "disk", + Summary: "Frees disk held by a build cache directory.", Capabilities: ["disk", "cache", "cleanup"])); + + promoted.Succeeded.Should().BeTrue(promoted.Error); + + var shown = registry.Show("free-disk-by-cache").Value!; + shown.Record.Active.Should().Be("1.0"); + + registry.Audit("free-disk-by-cache").Select(one => one.Action).Should().ContainInOrder("verify", "verify", "promote"); + registry.Audit().Select(one => one.Action).Should().ContainInOrder("nominate", "submit", "verify", "promote"); + + // 5. Team B, on another project, is briefed; the brief points at the + // catalogue and names no tool. + var brief = BriefFor(TeamB); + var read = TeamRunner.Render(brief); + + read.Should().Contain(TeamRunner.ToolsPointer); + read.Should().NotContain("free-disk-by-cache").And.NotContain(ProjectA).And.NotContain(TeamA); + + // Its remediator searches in its own words, and finds the tool. + registry.Search("disk full").Select(one => one.Name).Should().Equal("free-disk-by-cache"); + registry.Show("free-disk-by-cache").Value!.Active!.Inputs.Should().Contain(one => one.Name == "CachePath"); + + // Offered, but held, until a person trusts that version. + var rules = new Dictionary { ["disk"] = "trusted" }; + var file = "free-disk-by-cache.v1.0.ps1"; + + ToolOffer.For(registry, ToolOffer.Remediator, rules, []).Should().ContainSingle() + .Which.Ruling.Should().NotBe("run", "nobody has trusted it on this machine"); + + // What `loadout tools trust free-disk-by-cache@1.0` writes. + var trusted = new List + { + new() + { + Tool = "free-disk-by-cache", + Version = "1.0", + Fingerprint = RemedyCeiling.Fingerprint(registry.ScriptOf("free-disk-by-cache", "1.0")!), + By = "person", + At = DateTimeOffset.UtcNow, + }, + }; + registry.RecordTrust("free-disk-by-cache", "1.0", revoked: false, "person"); + + var offered = ToolOffer.For(registry, ToolOffer.Remediator, rules, trusted); + offered.Should().ContainSingle().Which.Ruling.Should().Be("run"); + + var policy = new NodePolicy("20260923-1400-b001", "remediator/1", ToolOffer.Remediator, ["Bash"], [], Ask: true, Remedies: offered); + var decided = NodePermissions.Decide(policy, "Bash", Calling($"pwsh -NoProfile -File ./{file} -CachePath ./beta-cache")); + + decided.Allowed.Should().BeTrue(); + decided.Reason.Should().Contain("tool:free-disk-by-cache@1.0"); + + registry.RecordUsage(new ToolUsage + { + Tool = "free-disk-by-cache", + Version = "1.0", + Outcome = ToolOutcome.Ok, + Team = TeamB, + Run = "20260923-1400-b001", + }).Succeeded.Should().BeTrue(); + + // 6. What crossed from team A to team B, and what did not. + var head = registry.Show("free-disk-by-cache").Value!.Record; + head.UsageSummary.Teams.Should().Be(1); + head.UsageSummary.Ok.Should().Be(1); + + head.Lineage[0].Source.Should().Be("inbox/" + nomination); + var source = Path.Combine(registry.Root(), head.Lineage[0].Source + ".yaml"); + File.Exists(source).Should().BeTrue("lineage[0] resolves to the nomination"); + File.ReadAllText(source).Should().Contain("by: nominator").And.Contain("clear-alpha-cache"); + + foreach (var written in Directory.EnumerateFiles(Path.Combine(registry.Root(), "free-disk-by-cache", "versions"), "*", SearchOption.AllDirectories)) + { + var text = File.ReadAllText(written); + + foreach (var forbidden in new[] { ProjectA, TeamA, "D:\\", "D:/" }.Concat(RunsA)) + { + text.Should().NotContain(forbidden, $"{Path.GetFileName(written)} is shared by every team"); + } + } + + // A second tool changes nothing in team B's brief. + var before = Encoding.UTF8.GetBytes(TeamRunner.Render(BriefFor(TeamB))); + await _store.PromoteAsync(registry, Manifest("prune-old-logs", "1.0"), "param([string]$LogPath)\nRemove-Item (Join-Path $LogPath '*.log')\n", ToolStoreFixture.Cases()); + registry.Search(string.Empty).Should().HaveCount(2); + + Encoding.UTF8.GetBytes(TeamRunner.Render(BriefFor(TeamB))).Should().Equal(before); + } + + [Fact] + public async Task A_failed_refinement_leaves_the_known_good_version_active() + { + // 1.1 keeps its own cases but breaks one 1.0 was trusted for. + var launcher = new ExitByCase(request => + request.Arguments.Any(one => one.Contains(Path.DirectorySeparatorChar + "1.1-", StringComparison.Ordinal)) + && request.Arguments.Contains("7") ? 1 : 0); + var registry = new ToolRegistry(_store.Paths, new ToolHarness(launcher), TimeProvider.System, () => _store.Known); + + var known = ToolStoreFixture.Cases(); + known.Add(new ToolCase + { + Name = "keeps-newer-files", + Class = ToolCaseClass.All[0], + Args = new Dictionary { ["CachePath"] = "{tmp}/cache", ["OlderThanDays"] = "7" }, + Expect = new ToolCaseExpect { Exit = 0 }, + }); + + await _store.PromoteAsync(registry, Manifest("free-disk-by-cache", "1.0"), Generic, known); + + var refined = Manifest("free-disk-by-cache", "1.1"); + var script = Generic + "# refined\n"; + var cases = ToolStoreFixture.Cases(prefix: "v11-"); + var draft = _store.Draft(refined, script, cases); + + var verified = await registry.VerifyAsync(draft, ToolStoreFixture.Agreed(refined, script, cases)); + + verified.Value!.Gate!.Passed.Should().BeFalse("1.1 fails a case 1.0 passed"); + verified.Value.Gate.Regressions.Should().Equal("keeps-newer-files"); + + registry.Promote(draft, new ToolPromotionRequest("idea", "inbox/test")).Failed.Should().BeTrue(); + + var head = registry.Show("free-disk-by-cache").Value!.Record; + head.Active.Should().Be("1.0"); + head.KnownGood.Should().Equal("1.0"); + Directory.Exists(Path.Combine(registry.Root(), "free-disk-by-cache", "versions", "1.1")).Should().BeFalse(); + registry.Audit("free-disk-by-cache").Should().Contain(one => one.Action == "reject" && one.Version == "1.1"); + } + + [Fact] + public async Task Nothing_project_specific_reaches_the_registry() + { + var (registry, _) = _store.Registry(); + + // Each way a project can ride along: in the script, the origin, a + // case's argument, and an example. + var drafts = new List<(ToolVersion Manifest, string Script, List Cases)> + { + (Manifest("from-script", "1.0"), HardCoded, ToolStoreFixture.Cases()), + (Manifest("from-origin", "1.0"),Generic, ToolStoreFixture.Cases()), + (Manifest("from-case", "1.0"), Generic, ToolStoreFixture.Cases()), + (Manifest("from-example", "1.0"), Generic, ToolStoreFixture.Cases()), + }; + + drafts[1].Manifest.Origin = "The system-watch team's cache kept filling the disk."; + drafts[2].Cases[0].Args["CachePath"] = "/home/nigel/alpha/cache"; + drafts[3].Manifest.Examples[0].Command = "pwsh -File tool.ps1 -CachePath C:\\work\\beta\\cache"; + + foreach (var (manifest, script, cases) in drafts) + { + var draft = _store.Draft(manifest, script, cases); + var verified = await registry.VerifyAsync(draft, ToolStoreFixture.Agreed(manifest, script, cases)); + var promoted = registry.Promote(draft, new ToolPromotionRequest("lesson", "inbox/test")); + + if (manifest.Name == "from-case") + { + // The harness refuses a case's absolute path before it runs, so + // the gate fails at verify and promote has nothing to take. + verified.Value!.Gate!.Passed.Should().BeFalse("a case may only use {tmp}"); + verified.Value.Because.Should().Contain("'CachePath' is an absolute path"); + promoted.Failed.Should().BeTrue(manifest.Name + " did not pass verify"); + } + else + { + verified.Value!.Gate!.Passed.Should().BeTrue(manifest.Name + " runs; what it carries is the question"); + promoted.Failed.Should().BeTrue(manifest.Name + " carries a project"); + promoted.Error.Should().Contain("carries a project"); + } + + Directory.Exists(Path.Combine(registry.Root(), manifest.Name)).Should().BeFalse(manifest.Name + " left nothing behind"); + } + + registry.Search(string.Empty, all: true).Should().BeEmpty(); + } + + // ------------------------------------------------------------ helpers + + private static ToolVersion Manifest(string name, string version) + { + var manifest = ToolStoreFixture.Manifest(name, version); + manifest.Inputs = [new ToolInput { Name = "CachePath", Type = "path", Required = true, Describe = "The cache directory to clear." }]; + + return manifest; + } + + private static Brief BriefFor(string team) => new( + "20260923-1400-b001", + "remediator", + "lead", + ToolOffer.Remediator, + "The build disk is full; free space without touching sources.", + DeliverableKind.Answer, + [], + new BriefConstraints("implement", null, 40, null, []), + ["the disk has room again"], + TeamDirectory: "/teams/work/" + team); + + private static string Calling(string command) => + System.Text.Json.JsonSerializer.Serialize(new { command }); + + private ToolNominationPass Pass(IToolRegistry registry) => + new( + registry, + new RunJournal(_store.Paths), + new RemedyBook(_store.Paths), + _store.Paths, + new NoLessons(), + new FakeProjects(ProjectB, _store.Paths.Paths.State), + new WorkspaceManager( + _store.Paths, new FakeGit(_store.Paths.Paths.State), new YamlStore(new NoOpFilePermissions()), TimeProvider.System)); + + /// A remedy on a team's shelf, as a node registers one. + private void Shelve(string team, string name, string script, int revision) + { + var shelf = Path.Combine(_store.Paths.Paths.State, "teams", "work", team, "remedies"); + Directory.CreateDirectory(shelf); + + File.WriteAllText(Path.Combine(shelf, name + ".ps1"), script); + File.WriteAllText( + Path.Combine(shelf, name + ".yaml"), + $"name: {name}\nkind: disk\nwhat: Clears the build cache when the disk fills.\nassumes: pwsh\nproves: the disk has room\n" + + $"script: {name}.ps1\nrevision: {revision}\n"); + } + + /// A finished run, as the runner leaves one: its journal and its remediator's report. + private void FinishedRun(string runId, string team, string command) + { + var journal = new RunJournal(_store.Paths); + var directory = journal.DirectoryOf(runId); + Directory.CreateDirectory(directory); + + File.WriteAllLines(Path.Combine(directory, "journal.jsonl"), + [ + """{"at":"2026-09-21T10:00:00+00:00","kind":"run.started","data":{"team":""" + "\"" + team + "\"" + + ""","goal":"keep the build disk clear","autonomy":"autonomous"}}""", + """{"at":"2026-09-21T10:30:00+00:00","kind":"run.finished","data":{"ended":"done","outcome":"done"}}""", + ]); + + var report = new Report("remediator", ReportStatus.Done, "Cleared the cache; the disk has room.", [], + [new ReportEvidence(EvidenceKind.Command, command, EvidenceResult.Pass, "exit 0, freed")], []); + File.WriteAllText(Path.Combine(directory, "report-remediator-1.json"), ReportReader.Write(report)); + } + + /// A launcher whose exit code depends on what it is asked to run. + private sealed class ExitByCase(Func exit) : IProcessLauncher + { + private readonly StubProcessLauncher _rest = new(string.Empty); + + public Task> RunAsync( + ProcessRequest request, TimeSpan? timeout = null, CancellationToken ct = default) => + Task.FromResult(OperationResult.Ok(new ProcessOutcome(exit(request), "freed", string.Empty))); + + public Task> RunInteractiveAsync(ProcessRequest request, CancellationToken ct = default) => + _rest.RunInteractiveAsync(request, ct); + + public Task> StartPipedAsync(ProcessRequest request, CancellationToken ct = default) => + _rest.StartPipedAsync(request, ct); + + public OperationResult StartDetached(ProcessRequest request) => _rest.StartDetached(request); + } + + /// Memory with no lessons in it, so only runs and shelves are read. + private sealed class NoLessons : IMemoryService + { + public Task>> ListAsync( + string workspaceRoot, string slug, CancellationToken ct = default) => + Task.FromResult(OperationResult>.Ok([])); + + public Task> AuditAsync( + string workspaceRoot, string slug, int staleMonths = 6, CancellationToken ct = default) => + throw new NotSupportedException(); + + public Task> WriteAsync( + string workspaceRoot, string slug, string name, string description, MemoryKind kind, + IReadOnlyList facts, bool acknowledgedSimilar = false, + MemoryScope scope = MemoryScope.Project, CancellationToken ct = default) => + throw new NotSupportedException(); + + public OperationResult ValidateWrite(string name, string description, IReadOnlyList facts) => + throw new NotSupportedException(); + + public Task RebuildIndexAsync(string workspaceRoot, string slug, CancellationToken ct = default) => + throw new NotSupportedException(); + + public Task> CleanAsync( + string workspaceRoot, string slug, bool apply, CancellationToken ct = default) => + throw new NotSupportedException(); + + public IReadOnlyList CleanupPaths(string workspaceRoot, string slug) => + throw new NotSupportedException(); + + public Task> ReadIndexAsync(string workspaceRoot, string slug, CancellationToken ct = default) => + throw new NotSupportedException(); + } +} From d0d5a9f8e18a00e7d6ce0d8a5a267ff14a2f2d06 Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 18:17:43 +0100 Subject: [PATCH 14/15] Fix the review of d080f42: filed keys, whole-command covers, root skip Four findings from the reviewer's return of d080f42. Rule 1 now records every member fingerprint of a cluster it files, not only the lowest. A cluster {A,B} filed under A, which then loses A and gains C, was filed again as {B,C}; the audit note now carries every key and the filed check reads them all. A_cluster_that_loses_its_filed_key_is_not_filed_again failed before this and passes after. A command with no script is covered by a tool only where the tool's script runs it: its words, whole, at the start of a line or after a pipe, separator or bracket. The substring check read "# Make sure the cache path exists" as running make and hid a rule 3 nomination. A_one_word_command_is_not_covered_by_a_script_mentioning_it_in_prose failed before this and passes after. A_third_team_joining_a_cluster_is_not_filed_again now gives gamma a near copy whose fingerprint sorts lowest, so the cluster's key moves and only the Also check can recognise it. Replacing that check with the primary key alone fails the test; reverted. An_unreadable_run_folder_does_not_stop_the_pass takes a permission away from itself, which uid 0 cannot lose on Unix, so it now skips there through a DeniableFact attribute beside the other platform attributes. Not exercised as root on this Windows machine. --- src/Loadout.Core/Tools/ToolNominator.cs | 36 +++++++++--- .../Platform/PlatformFactAttributes.cs | 19 +++++++ .../Unit/ToolNominationPassTests.cs | 3 +- .../Loadout.Tests/Unit/ToolNominatorTests.cs | 55 ++++++++++++++++++- 4 files changed, 103 insertions(+), 10 deletions(-) diff --git a/src/Loadout.Core/Tools/ToolNominator.cs b/src/Loadout.Core/Tools/ToolNominator.cs index 686eb405..657f0dd1 100644 --- a/src/Loadout.Core/Tools/ToolNominator.cs +++ b/src/Loadout.Core/Tools/ToolNominator.cs @@ -68,12 +68,15 @@ public ToolNominator(IToolRegistry registry, IRunJournal journal, IRemedyBook re /// Each nomination found, with what filing it came to; null where it was filed before. public IReadOnlyList<(ToolNomination Nomination, OperationResult? Filed)> Scan(IReadOnlyList lessons) { + // A note ends "key " and every key the nomination was filed under. var filed = _registry.Audit() .Where(one => one.Action == "nominate") .Select(one => one.Note ?? string.Empty) - .ToList(); + .Where(note => note.Contains(" key ", StringComparison.Ordinal)) + .SelectMany(note => note[(note.IndexOf(" key ", StringComparison.Ordinal) + 5)..].Split(' ', StringSplitOptions.RemoveEmptyEntries)) + .ToHashSet(StringComparer.Ordinal); - bool Filed(string key) => filed.Any(note => note.EndsWith(" key " + KeyOf(key), StringComparison.Ordinal)); + bool Filed(string key) => filed.Contains(KeyOf(key)); return [ @@ -82,7 +85,9 @@ .. Sorted(lessons).Select(sorted => var (one, tool) = sorted; var prefix = tool is null ? string.Empty : "hint " + tool + " "; - if (new[] { one.Key }.Concat(one.Also ?? []).Any(key => Filed(prefix + key))) + var keys = new[] { one.Key }.Concat(one.Also ?? []).ToList(); + + if (keys.Any(key => Filed(prefix + key))) { return (one, (OperationResult?)null); } @@ -93,7 +98,9 @@ .. Sorted(lessons).Select(sorted => return (one, _registry.Nominate( new ToolSubmission("candidate", text, Tool: tool, By: "nominator", Script: one.Script, Summary: one.Summary), - KeyOf(prefix + one.Key))); + // Every member's key, so the cluster is still recognised + // after the script it was keyed by leaves it. + string.Join(' ', keys.Select(key => KeyOf(prefix + key))))); }), ]; } @@ -126,8 +133,8 @@ public IReadOnlyList Find(IReadOnlyList lessons) => /// /// A remedy is compared script to script. A command has no script, and a /// sentence about it shares too little with a tool's script to overlap, - /// so it is covered where the tool's examples run the same shape, or its - /// script or capabilities name the command. + /// so it is covered where the tool's examples run the same shape, its + /// script runs the command, or its capabilities name it. /// private static bool Covers(ToolOffered tool, ToolNomination nomination) { @@ -142,10 +149,25 @@ private static bool Covers(ToolOffered tool, ToolNomination nomination) var named = Named(shape) ?? shape; return tool.Version.Examples.Any(one => string.Equals(Shape(one.Command), shape, StringComparison.Ordinal)) - || tool.Script.Contains(named, StringComparison.OrdinalIgnoreCase) + || Runs(tool.Script, named) || tool.Record.Capabilities.Any(one => string.Equals(one, named, StringComparison.OrdinalIgnoreCase)); } + /// Whether a script runs a command: its words, whole, where a command starts. + /// + /// Where a command starts is the start of a line or after a pipe, a + /// separator or an opening bracket, so "make sure" in a comment or a + /// quoted sentence is not running make, and "remake" is not either. + /// + private static bool Runs(string script, string command) + { + var words = command.Split(' ', StringSplitOptions.RemoveEmptyEntries).Select(Regex.Escape); + var pattern = @"(?:^|[|;&({])[ \t]*(?:[^\s|;&(){}'""#]*[/\\])?" + string.Join(@"[ \t]+", words) + @"(?![\w.-])"; + + return Regex.IsMatch( + script, pattern, RegexOptions.Multiline | RegexOptions.IgnoreCase | RegexOptions.CultureInvariant, TimeSpan.FromSeconds(1)); + } + /// A command with its arguments replaced, so two runs of it compare equal. public static string Shape(string command) { diff --git a/tests/Loadout.Tests/Platform/PlatformFactAttributes.cs b/tests/Loadout.Tests/Platform/PlatformFactAttributes.cs index 0ad9d4df..503c3626 100644 --- a/tests/Loadout.Tests/Platform/PlatformFactAttributes.cs +++ b/tests/Loadout.Tests/Platform/PlatformFactAttributes.cs @@ -46,6 +46,25 @@ public UnixFactAttribute() } } +/// +/// A test that takes a permission away from itself, which root on Unix cannot +/// lose: mode bits do not apply to uid 0, so the fixture would not hold. +/// +/// +/// Not skipped for an elevated Windows process, because a deny entry naming +/// the user's own SID applies to administrators too. +/// +public sealed class DeniableFactAttribute : FactAttribute +{ + public DeniableFactAttribute() + { + if (!OperatingSystem.IsWindows() && Environment.IsPrivilegedProcess) + { + Skip = "Running as root: file mode bits cannot deny uid 0 anything."; + } + } +} + /// /// A Unix test that additionally cannot pass on macOS, where setting a pty /// window size does not work. diff --git a/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs b/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs index 19ddfc25..0c07eabd 100644 --- a/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs +++ b/tests/Loadout.Tests/Unit/ToolNominationPassTests.cs @@ -8,6 +8,7 @@ using Loadout.Models.Results; using Loadout.Models.Teams; using Loadout.Tests.Fakes; +using Loadout.Tests.Platform; using Xunit; namespace Loadout.Tests.Unit; @@ -61,7 +62,7 @@ public async Task Any_other_schedule_nominates_nothing() Directory.Exists(Path.Combine(_store.Paths.Paths.State, "tools", "inbox")).Should().BeFalse(); } - [Fact] + [DeniableFact] public async Task An_unreadable_run_folder_does_not_stop_the_pass() { FinishedRun("20260923-1000-a001", "alpha", "docker system prune --force"); diff --git a/tests/Loadout.Tests/Unit/ToolNominatorTests.cs b/tests/Loadout.Tests/Unit/ToolNominatorTests.cs index 7c2d450d..c6a41a53 100644 --- a/tests/Loadout.Tests/Unit/ToolNominatorTests.cs +++ b/tests/Loadout.Tests/Unit/ToolNominatorTests.cs @@ -126,12 +126,63 @@ public void A_third_team_joining_a_cluster_is_not_filed_again() Shelve("beta", "free-disk", Cache); Nominator().Scan([]).Should().ContainSingle(one => one.Filed != null && one.Filed.Succeeded); - Shelve("gamma", "tidy-cache", Cache); + // A near copy that sorts first, so the cluster's key moves to it and + // only the keys it was filed under before can recognise it. + var lowest = NearCopies(8).First(one => + string.CompareOrdinal(RemedyCeiling.Fingerprint(one), RemedyCeiling.Fingerprint(Cache)) < 0); + Shelve("gamma", "tidy-cache", lowest); - Nominator().Scan([]).Should().OnlyContain(one => one.Filed == null, "the cluster was filed before gamma joined it"); + var again = Nominator().Scan([]); + + again.Should().ContainSingle(one => one.Nomination.Rule == 1, "gamma's near copy joins the same cluster"); + again.Single().Nomination.Key.Should().Be("1 " + RemedyCeiling.Fingerprint(lowest)); + again.Should().OnlyContain(one => one.Filed == null, "the cluster was filed before gamma joined it"); + Directory.EnumerateFiles(Path.Combine(_store.Paths.Paths.State, "tools", "inbox")).Should().ContainSingle(); + } + + [Fact] + public void A_cluster_that_loses_its_filed_key_is_not_filed_again() + { + var copies = NearCopies(3) + .OrderBy(RemedyCeiling.Fingerprint, StringComparer.Ordinal) + .ToList(); + + Shelve("alpha", "clear-cache", copies[0]); + Shelve("beta", "free-disk", copies[1]); + Nominator().Scan([]).Should().ContainSingle(one => one.Filed != null && one.Filed.Succeeded); + + // The script the cluster was keyed by goes, and another joins. + Directory.Delete(Path.Combine(_store.Paths.Paths.State, "teams", "work", "alpha"), recursive: true); + Shelve("gamma", "tidy-cache", copies[2]); + + var again = Nominator().Scan([]); + + again.Should().ContainSingle(one => one.Nomination.Rule == 1, "beta and gamma still keep the same remedy"); + again.Should().OnlyContain(one => one.Filed == null, "beta's script was one of the cluster filed before"); Directory.EnumerateFiles(Path.Combine(_store.Paths.Paths.State, "tools", "inbox")).Should().ContainSingle(); } + [Fact] + public async Task A_one_word_command_is_not_covered_by_a_script_mentioning_it_in_prose() + { + var (registry, _) = _store.Registry(); + await _store.PromoteAsync( + registry, + ToolStoreFixture.Manifest("free-cache", "1.0"), + "# Make sure the cache path exists before clearing it.\n" + Cache, + ToolStoreFixture.Cases()); + + Run("20260901-1000-a001", "alpha", Command("make")); + Run("20260902-1000-b001", "beta", Command("make")); + + Nominator().Find([]).Should().ContainSingle(one => one.Rule == 3 && one.Key == "3 make", + "the tool's script says 'make sure', which is not running make"); + } + + /// Copies of the cache script differing only in what they print, so each has its own fingerprint. + private static List NearCopies(int count) => + [.. Enumerable.Range(0, count).Select(one => Cache + $"Write-Output 'pass {one}'\n")]; + [Fact] public void A_nomination_never_carries_a_secret_value() { From d6b247f9bd508cc1e4921c520d657d140b0f609b Mon Sep 17 00:00:00 2001 From: Nigel Date: Wed, 23 Sep 2026 18:23:56 +0100 Subject: [PATCH 15/15] Cover a project name carried inside a case's {tmp} argument The from-case draft in Nothing_project_specific_reaches_the_registry uses an absolute path, which the harness refuses at verify, so the genericity check over case arguments (ToolGenericity.cs:96) was never what stopped it. A fifth draft, from-case-name, passes the harness with {tmp}/alpha/cache and must be refused at promote with "carries a project". Mutation-checked: deleting ToolGenericity.cs:96 fails the test with "Expected promoted.Failed to be True because from-case-name carries a project, but found False." --- .../Loadout.Tests/Integration/GlobalToolLifecycleTests.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/Loadout.Tests/Integration/GlobalToolLifecycleTests.cs b/tests/Loadout.Tests/Integration/GlobalToolLifecycleTests.cs index 2cbd245d..02d74fce 100644 --- a/tests/Loadout.Tests/Integration/GlobalToolLifecycleTests.cs +++ b/tests/Loadout.Tests/Integration/GlobalToolLifecycleTests.cs @@ -265,19 +265,24 @@ public async Task Nothing_project_specific_reaches_the_registry() var (registry, _) = _store.Registry(); // Each way a project can ride along: in the script, the origin, a - // case's argument, and an example. + // case's argument (as an absolute path, and by name inside {tmp}), + // and an example. var drafts = new List<(ToolVersion Manifest, string Script, List Cases)> { (Manifest("from-script", "1.0"), HardCoded, ToolStoreFixture.Cases()), (Manifest("from-origin", "1.0"),Generic, ToolStoreFixture.Cases()), (Manifest("from-case", "1.0"), Generic, ToolStoreFixture.Cases()), (Manifest("from-example", "1.0"), Generic, ToolStoreFixture.Cases()), + (Manifest("from-case-name", "1.0"), Generic, ToolStoreFixture.Cases()), }; drafts[1].Manifest.Origin = "The system-watch team's cache kept filling the disk."; drafts[2].Cases[0].Args["CachePath"] = "/home/nigel/alpha/cache"; drafts[3].Manifest.Examples[0].Command = "pwsh -File tool.ps1 -CachePath C:\\work\\beta\\cache"; + // Only the genericity check sees this one: the harness takes {tmp}. + drafts[4].Cases[0].Args["CachePath"] = "{tmp}/alpha/cache"; + foreach (var (manifest, script, cases) in drafts) { var draft = _store.Draft(manifest, script, cases);