diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43f00ff7..dce36a57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,8 +37,9 @@ jobs: # red path unreachable if it ran after Test. - name: SEAMS.md drift check run: ./tools/check-seam-index.sh - # Integration tests need fixture-managed side containers (Postgres, Kokoro) — they run - # on-demand on a dev box, not in CI (same policy as the pre-migration pipeline). + # Test tiers (SPEC F177.1): tier 1 = this filter, every fact without Category=Integration, + # including the fixture Postgres facts; tier 2 (nightly) = Category=Integration: Kokoro, a + # stream capture or a full compose stack (nightly.yml); tier 3 = Skip="manual: …", checked by ear. # ffmpeg: the F88 artwork specs (Story222) drive the REAL binary — both fixture generation # and the extraction under test — matching the api image, which ships ffmpeg (gh-#105). # gh-#588: the retry loop itself lives in .github/actions/install-ffmpeg (was diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cdd83eed..0d24a3dc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,8 +53,8 @@ jobs: run: dotnet restore GenWave.sln - name: Build (warnings are errors) run: dotnet build GenWave.sln --no-restore --configuration Release - # Same policy as ci.yml: integration tests need fixture-managed side containers and run - # on a dev box, not here. + # Test tiers (SPEC F177.1): same tier-1 filter as ci.yml's build-test job — see that job's + # comment for the full tier 1/2/3 breakdown. # ffmpeg: the F88 artwork specs (Story222) drive the REAL binary (gh-#105) — same step # as ci.yml's build-test; this gate runs its own test pass and needs it too. # gh-#588: the retry loop itself lives in .github/actions/install-ffmpeg (was diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index def4cdba..485a4be7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,6 +54,14 @@ cd admin-ui && npx tsc --noEmit && npm run lint && npm run typecheck:specs && np See the [README](README.md) for prerequisites and how to run the full stack. +## 🧪 Test tiers + +| Tier | What | Filter | Where it runs | +|---|---|---|---| +| 1 | Every fact without the trait, including the fixture Postgres facts | `Category!=Integration` | Every PR (`ci.yml`) and every release tag (`release.yml`) | +| 2 | Needs more than Postgres: a Kokoro container, an ffmpeg stream capture, or a full compose stack | `Category=Integration` | `nightly.yml`, and by hand on a dev box | +| 3 | `Skip="manual: …"` facts | `manual:` | By hand, checked by listening to the stream | + ## ✅ Pull requests - One concern per PR; conventional-commit style messages (`feat:`, `fix:`, `docs:`, `chore:`). diff --git a/tests/GenWave.Host.Tests/HostTierTraitConventionGuard.cs b/tests/GenWave.Host.Tests/HostTierTraitConventionGuard.cs new file mode 100644 index 00000000..bdfa2550 --- /dev/null +++ b/tests/GenWave.Host.Tests/HostTierTraitConventionGuard.cs @@ -0,0 +1,34 @@ +namespace GenWave.Host.Tests; + +/// +/// Host-side twin of GenWave.MediaLibrary.Tests.IntegrationTraitConventionGuard (SPEC F177.3, +/// STORY-439, PLAN T504): asserts reports zero +/// violations over this assembly's exported types, using its default +/// list. Pure reflection, no +/// container, no database — it runs in tier 1 and fails fast at the source instead of as an opaque +/// container error when a class silently tries to bring up Kokoro or the live compose stack from +/// inside dotnet test --filter "Category!=Integration". +/// +/// +/// Named HostTierTraitConventionGuard rather than the bare TierTraitConventionGuard the +/// reflection helper carries: this type lives directly +/// in the GenWave.Host.Tests namespace, which encloses GenWave.Host.Tests.Specs — a +/// same-named type here would out-rank Story439_TestTiers.cs's own +/// using GenWave.Host.Tests.Support; for every unqualified TierTraitConventionGuard +/// call in that file (C# resolves enclosing-namespace declarations before sibling-level usings). +/// +public sealed class HostTierTraitConventionGuard +{ + [Fact] + public void EveryFixtureConsumingTestClassIsCategorisedByTier() + { + var offenders = Support.TierTraitConventionGuard.FindViolations(typeof(KokoroFixture).Assembly.GetExportedTypes()); + + Assert.True( + offenders.Count == 0, + "Test classes consuming KokoroFixture or a stack fixture (SPEC F177.3) without " + + "[Trait(\"Category\", \"Integration\")] at class level or on every fact — CI's tier " + + "1 filter would try to start a container it cannot reach:\n " + + string.Join("\n ", offenders)); + } +} diff --git a/tests/GenWave.Host.Tests/Specs/Story230_RequestsArtworkGate.cs b/tests/GenWave.Host.Tests/Specs/Story230_RequestsArtworkGate.cs index 55f2b843..22a0bd0a 100644 --- a/tests/GenWave.Host.Tests/Specs/Story230_RequestsArtworkGate.cs +++ b/tests/GenWave.Host.Tests/Specs/Story230_RequestsArtworkGate.cs @@ -9,7 +9,10 @@ // from every run forever), but a runtime reachability/precondition check that writes a clear // skip line and returns cleanly when the live stack isn't up or isn't configured for this gate, // and drives the REAL surfaces (an anonymous wish POST, the admin booth-log/settings/media APIs, -// a single ICY metadata handshake against the live Icecast stream) when it is. Both facts share +// a single ICY metadata handshake against the live Icecast stream) when it is. Per SPEC F177.1 +// (STORY-439, PLAN T504), ScenarioFlywheel carries [Trait("Category", "Integration")] — it needs +// more than Postgres (the live compose stack), so it is tier 2 and runs nightly, never in PR CI's +// tier 1 filter. Both facts share // ONE flywheel run via an xUnit collection fixture (RequestsArtworkFlywheelFixture) — mirroring // KokoroFixture's shared-container idiom — because POSTing the wish twice would trip // Requests:PerIpCooldownMinutes (default 5) on the second fact. @@ -90,6 +93,7 @@ static IReadOnlyList SpectatorWireFieldCensus() => public sealed class FlywheelCollection : ICollectionFixture; [Collection(FlywheelCollectionName)] + [Trait("Category", "Integration")] public sealed class ScenarioFlywheel(RequestsArtworkFlywheelFixture fixture, ITestOutputHelper output) { [Fact] diff --git a/tests/GenWave.Host.Tests/Specs/Story439_TestTiers.cs b/tests/GenWave.Host.Tests/Specs/Story439_TestTiers.cs index a44e9d88..2568d1f7 100644 --- a/tests/GenWave.Host.Tests/Specs/Story439_TestTiers.cs +++ b/tests/GenWave.Host.Tests/Specs/Story439_TestTiers.cs @@ -5,8 +5,8 @@ // nothing here ever starts a Kokoro. AC4–AC7 are text pins over ci.yml, CONTRIBUTING.md and the // MediaLibrary guard (Story107's repo-root grep-assert idiom). // -// RED at plan time: the guard is a throwing skeleton; ci.yml still says integration runs -// "on-demand on a dev box"; CONTRIBUTING has no "Test tiers" table. +// Was red at plan time (the guard was a throwing skeleton, ci.yml said integration runs +// "on-demand on a dev box", CONTRIBUTING had no "Test tiers" table); T504 turned it green. using GenWave.Host.Tests.Support; @@ -14,8 +14,6 @@ namespace GenWave.Host.Tests.Specs; public static class FeatureTestTiersNamedPinnedAndGuarded { - const string Pending = "pending: T504 — the Host tier guard, the ci.yml comment and the CONTRIBUTING tier table (STORY-439)"; - static string RepoRoot => RepoRootLocator.Find(AppContext.BaseDirectory); static string ReadRepoFile(params string[] parts) => File.ReadAllText(Path.Combine([RepoRoot, .. parts])); @@ -47,7 +45,7 @@ public void SomeFact() { } public sealed class ScenarioTheShippedAssemblyPasses { - [Fact(Skip = Pending)] + [Fact] public void ReportsZeroViolations() { var violations = TierTraitConventionGuard.FindViolations(typeof(KokoroFixture).Assembly.GetExportedTypes()); @@ -64,11 +62,11 @@ public sealed class ScenarioCiPinsTheTierOneFilter public void TheTestStepFiltersOutIntegration() => Assert.Contains("--filter \"Category!=Integration\"", ci, StringComparison.Ordinal); - [Fact(Skip = Pending)] + [Fact] public void NoLineSaysIntegrationRunsOnADevBox() => Assert.DoesNotContain("on-demand on a dev box", ci, StringComparison.Ordinal); - [Fact(Skip = Pending)] + [Fact] public void OneLineNamesTierTwoNightly() => Assert.Single(ci.Split('\n'), line => line.Contains("tier 2 (nightly)", StringComparison.Ordinal)); } @@ -84,15 +82,15 @@ public ScenarioContributingStatesTheThreeTiers() table = start < 0 ? "" : text[start..]; } - [Fact(Skip = Pending)] + [Fact] public void NamesTheTierOneFilter() => Assert.Contains("`Category!=Integration`", table, StringComparison.Ordinal); - [Fact(Skip = Pending)] + [Fact] public void NamesTheTierTwoFilter() => Assert.Contains("`Category=Integration`", table, StringComparison.Ordinal); - [Fact(Skip = Pending)] + [Fact] public void NamesTheManualPrefix() => Assert.Contains("`manual:`", table, StringComparison.Ordinal); } @@ -101,9 +99,9 @@ public sealed class ScenarioTheMediaLibraryGuardIsUntouched { // The committed baseline hash of IntegrationTraitConventionGuard.cs (2026-09-15) — T504 adds // a Host twin, it never widens the MediaLibrary guard's scope. - const string BaselineSha256 = "pending: T504 records the baseline hash here"; + const string BaselineSha256 = "b8382267f983fd617a9b9be13c708a92fa8715beb5b0fc8c9eff734666cc7552"; - [Fact(Skip = Pending)] + [Fact] public void IsByteIdenticalToTheBaseline() { var bytes = File.ReadAllBytes(Path.Combine(RepoRoot, "tests", "GenWave.MediaLibrary.Tests", "IntegrationTraitConventionGuard.cs")); @@ -119,7 +117,7 @@ public void IsByteIdenticalToTheBaseline() public sealed class ScenarioAKokoroConsumerWithoutTheTraitIsRejected { - [Fact(Skip = Pending)] + [Fact] public void ReportsThatClassByName() { var violations = TierTraitConventionGuard.FindViolations([typeof(KokoroConsumerWithoutTrait)]); @@ -130,7 +128,7 @@ public void ReportsThatClassByName() public sealed class ScenarioAStackFixtureConsumerWithoutTheTraitIsRejected { - [Fact(Skip = Pending)] + [Fact] public void ReportsThatClassByName() { var violations = TierTraitConventionGuard.FindViolations( diff --git a/tests/GenWave.Host.Tests/Support/TierTraitConventionGuard.cs b/tests/GenWave.Host.Tests/Support/TierTraitConventionGuard.cs index e2b60475..3067eeb8 100644 --- a/tests/GenWave.Host.Tests/Support/TierTraitConventionGuard.cs +++ b/tests/GenWave.Host.Tests/Support/TierTraitConventionGuard.cs @@ -1,21 +1,142 @@ +using System.Reflection; + namespace GenWave.Host.Tests.Support; /// /// The Host tier guard (SPEC F177.3, STORY-439, PLAN T504) — the reflection twin of /// GenWave.MediaLibrary.Tests.IntegrationTraitConventionGuard: any test class that takes a -/// KokoroFixture (constructor, IClassFixture<> or ICollectionFixture<>) -/// or any type in , and lacks [Trait("Category", "Integration")] -/// at class level or on every fact, is a violation — it would try to start a container in the PR lane. +/// KokoroFixture (constructor, IClassFixture<> or ICollectionFixture<>, +/// or a [Collection(name)] whose [CollectionDefinition(name)] class implements +/// ICollectionFixture<> for it) or any type in , and lacks +/// [Trait("Category", "Integration")] at class level or on every fact, is a violation — it +/// would try to start a container or a live compose stack in the PR lane. Only facts declared on +/// the class itself count (DeclaredOnly): facts inherited from a base test class are not +/// checked, and no Host test class inherits facts today. /// -/// Skeleton at plan time — throws until T504 lands. +/// +/// Today the only stack fixture is Story230's RequestsArtworkFlywheelFixture — its flywheel +/// facts need the LIVE compose stack (requests enabled, PublicBaseUrl set, a real catalog), so +/// F177.1 puts them in tier 2. F178's stream-capture gate runs as a shell script, not a fixture, so +/// it never appears here; there are no other stack/capture fixtures in Host.Tests today. +/// internal static class TierTraitConventionGuard { - /// Fixture types that need a full stack or an audio capture — tier 2 by definition. - /// Empty until a stack fixture exists; the guard's own list, extended in place. - public static IReadOnlyList StackFixtureTypes { get; } = []; + /// Fixture types that need a full stack or an audio capture — tier 2 by definition. + public static IReadOnlyList StackFixtureTypes { get; } = + [typeof(GenWave.Host.Tests.Specs.RequestsArtworkFlywheelFixture)]; /// Full names of the offending classes (one entry per class, never per fact). public static IReadOnlyList FindViolations( - IEnumerable types, IReadOnlyCollection? stackFixtureTypes = null) => - throw new NotImplementedException("pending: T504 — TierTraitConventionGuard (STORY-439)"); + IEnumerable types, IReadOnlyCollection? stackFixtureTypes = null) + { + var watchedTypes = new HashSet(stackFixtureTypes ?? StackFixtureTypes) { typeof(KokoroFixture) }; + var typeList = types as IReadOnlyList ?? types.ToList(); + + var collectionDefinitionsByAssembly = typeList + .Select(type => type.Assembly) + .Distinct() + .ToDictionary(assembly => assembly, CollectionDefinitionsByName); + + var offenders = new List(); + + foreach (var type in typeList) + { + if (!ConsumesWatchedFixture(type, watchedTypes, collectionDefinitionsByAssembly[type.Assembly])) + continue; + if (HasIntegrationTrait(type.GetCustomAttributesData())) + continue; + + var testMethods = type + .GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static | BindingFlags.DeclaredOnly) + .Where(IsTestMethod) + .ToList(); + if (testMethods.Count == 0) + continue; // e.g. a bare [CollectionDefinition] marker class — nothing to guard. + + var untraited = testMethods + .Where(method => !HasIntegrationTrait(method.GetCustomAttributesData())) + .Select(method => method.Name) + .ToList(); + if (untraited.Count == 0) + continue; // every fact already carries the trait. + + offenders.Add($"{type.FullName} (missing [Trait(\"Category\", \"Integration\")] on: {string.Join(", ", untraited)})"); + } + + return offenders; + } + + static bool ConsumesWatchedFixture(Type type, HashSet watchedTypes, IReadOnlyDictionary collectionDefinitionsByName) + { + if (ImplementsFixtureInterfaceFor(type, watchedTypes)) + return true; + + if (TryGetCollectionName(type, out var collectionName) + && collectionDefinitionsByName.TryGetValue(collectionName, out var definitionType) + && ImplementsFixtureInterfaceFor(definitionType, watchedTypes)) + return true; + + return type + .GetConstructors(BindingFlags.Public | BindingFlags.Instance) + .Any(ctor => ctor.GetParameters().Any(parameter => watchedTypes.Contains(parameter.ParameterType))); + } + + static bool ImplementsFixtureInterfaceFor(Type type, HashSet watchedTypes) => + type.GetInterfaces().Any(implemented => + implemented.IsGenericType + && (implemented.GetGenericTypeDefinition() == typeof(IClassFixture<>) + || implemented.GetGenericTypeDefinition() == typeof(ICollectionFixture<>)) + && watchedTypes.Contains(implemented.GetGenericArguments()[0])); + + static bool TryGetCollectionName(Type type, out string name) + { + var attribute = type.GetCustomAttributesData().FirstOrDefault(a => a.AttributeType == typeof(CollectionAttribute)); + if (attribute is { ConstructorArguments.Count: 1 } && attribute.ConstructorArguments[0].Value is string collectionName) + { + name = collectionName; + return true; + } + + name = ""; + return false; + } + + /// Maps every [CollectionDefinition(name)] class in + /// to its name, so a [Collection(name)] consumer can be traced back to the fixture(s) its + /// collection actually shares. + static IReadOnlyDictionary CollectionDefinitionsByName(Assembly assembly) + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (var type in assembly.GetTypes()) + { + if (TryGetCollectionDefinitionName(type, out var name)) + map[name] = type; + } + + return map; + } + + static bool TryGetCollectionDefinitionName(Type type, out string name) + { + var attribute = type.GetCustomAttributesData().FirstOrDefault(a => a.AttributeType == typeof(CollectionDefinitionAttribute)); + if (attribute is { ConstructorArguments.Count: 1 } && attribute.ConstructorArguments[0].Value is string definitionName) + { + name = definitionName; + return true; + } + + name = ""; + return false; + } + + static bool HasIntegrationTrait(IEnumerable attributes) => + attributes.Any(a => + a.AttributeType == typeof(TraitAttribute) + && a.ConstructorArguments.Count == 2 + && a.ConstructorArguments[0].Value as string == "Category" + && a.ConstructorArguments[1].Value as string == "Integration"); + + static bool IsTestMethod(MethodInfo method) => + method.GetCustomAttributesData().Any(a => + a.AttributeType == typeof(FactAttribute) || a.AttributeType.IsSubclassOf(typeof(FactAttribute))); }