Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@ jobs:
# red path unreachable if it ran after Test.
- name: SEAMS.md drift check
run: ./tools/check-seam-index.sh
# Integration tests need fixture-managed side containers (Postgres, Kokoro) — they run
# on-demand on a dev box, not in CI (same policy as the pre-migration pipeline).
# Test tiers (SPEC F177.1): tier 1 = this filter, every fact without Category=Integration,
# including the fixture Postgres facts; tier 2 (nightly) = Category=Integration: Kokoro, a
# stream capture or a full compose stack (nightly.yml); tier 3 = Skip="manual: …", checked by ear.
# ffmpeg: the F88 artwork specs (Story222) drive the REAL binary — both fixture generation
# and the extraction under test — matching the api image, which ships ffmpeg (gh-#105).
# gh-#588: the retry loop itself lives in .github/actions/install-ffmpeg (was
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ jobs:
run: dotnet restore GenWave.sln
- name: Build (warnings are errors)
run: dotnet build GenWave.sln --no-restore --configuration Release
# Same policy as ci.yml: integration tests need fixture-managed side containers and run
# on a dev box, not here.
# Test tiers (SPEC F177.1): same tier-1 filter as ci.yml's build-test job — see that job's
# comment for the full tier 1/2/3 breakdown.
# ffmpeg: the F88 artwork specs (Story222) drive the REAL binary (gh-#105) — same step
# as ci.yml's build-test; this gate runs its own test pass and needs it too.
# gh-#588: the retry loop itself lives in .github/actions/install-ffmpeg (was
Expand Down
8 changes: 8 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,14 @@ cd admin-ui && npx tsc --noEmit && npm run lint && npm run typecheck:specs && np

See the [README](README.md) for prerequisites and how to run the full stack.

## 🧪 Test tiers

| Tier | What | Filter | Where it runs |
|---|---|---|---|
| 1 | Every fact without the trait, including the fixture Postgres facts | `Category!=Integration` | Every PR (`ci.yml`) and every release tag (`release.yml`) |
| 2 | Needs more than Postgres: a Kokoro container, an ffmpeg stream capture, or a full compose stack | `Category=Integration` | `nightly.yml`, and by hand on a dev box |
| 3 | `Skip="manual: …"` facts | `manual:` | By hand, checked by listening to the stream |

## ✅ Pull requests

- One concern per PR; conventional-commit style messages (`feat:`, `fix:`, `docs:`, `chore:`).
Expand Down
34 changes: 34 additions & 0 deletions tests/GenWave.Host.Tests/HostTierTraitConventionGuard.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
namespace GenWave.Host.Tests;

/// <summary>
/// Host-side twin of <c>GenWave.MediaLibrary.Tests.IntegrationTraitConventionGuard</c> (SPEC F177.3,
/// STORY-439, PLAN T504): asserts <see cref="Support.TierTraitConventionGuard"/> reports zero
/// violations over this assembly's exported types, using its default
/// <see cref="Support.TierTraitConventionGuard.StackFixtureTypes"/> list. Pure reflection, no
/// container, no database — it runs in tier 1 and fails fast at the source instead of as an opaque
/// container error when a class silently tries to bring up Kokoro or the live compose stack from
/// inside <c>dotnet test --filter "Category!=Integration"</c>.
/// </summary>
/// <remarks>
/// Named <c>HostTierTraitConventionGuard</c> rather than the bare <c>TierTraitConventionGuard</c> the
/// <see cref="Support.TierTraitConventionGuard"/> reflection helper carries: this type lives directly
/// in the <c>GenWave.Host.Tests</c> namespace, which encloses <c>GenWave.Host.Tests.Specs</c> — a
/// same-named type here would out-rank Story439_TestTiers.cs's own
/// <c>using GenWave.Host.Tests.Support;</c> for every unqualified <c>TierTraitConventionGuard</c>
/// call in that file (C# resolves enclosing-namespace declarations before sibling-level usings).
/// </remarks>
public sealed class HostTierTraitConventionGuard
{
[Fact]
public void EveryFixtureConsumingTestClassIsCategorisedByTier()
{
var offenders = Support.TierTraitConventionGuard.FindViolations(typeof(KokoroFixture).Assembly.GetExportedTypes());

Assert.True(
offenders.Count == 0,
"Test classes consuming KokoroFixture or a stack fixture (SPEC F177.3) without "
+ "[Trait(\"Category\", \"Integration\")] at class level or on every fact — CI's tier "
+ "1 filter would try to start a container it cannot reach:\n "
+ string.Join("\n ", offenders));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,10 @@
// from every run forever), but a runtime reachability/precondition check that writes a clear
// skip line and returns cleanly when the live stack isn't up or isn't configured for this gate,
// and drives the REAL surfaces (an anonymous wish POST, the admin booth-log/settings/media APIs,
// a single ICY metadata handshake against the live Icecast stream) when it is. Both facts share
// a single ICY metadata handshake against the live Icecast stream) when it is. Per SPEC F177.1
// (STORY-439, PLAN T504), ScenarioFlywheel carries [Trait("Category", "Integration")] — it needs
// more than Postgres (the live compose stack), so it is tier 2 and runs nightly, never in PR CI's
// tier 1 filter. Both facts share
// ONE flywheel run via an xUnit collection fixture (RequestsArtworkFlywheelFixture) — mirroring
// KokoroFixture's shared-container idiom — because POSTing the wish twice would trip
// Requests:PerIpCooldownMinutes (default 5) on the second fact.
Expand Down Expand Up @@ -90,6 +93,7 @@ static IReadOnlyList<string> SpectatorWireFieldCensus() =>
public sealed class FlywheelCollection : ICollectionFixture<RequestsArtworkFlywheelFixture>;

[Collection(FlywheelCollectionName)]
[Trait("Category", "Integration")]
public sealed class ScenarioFlywheel(RequestsArtworkFlywheelFixture fixture, ITestOutputHelper output)
{
[Fact]
Expand Down
26 changes: 12 additions & 14 deletions tests/GenWave.Host.Tests/Specs/Story439_TestTiers.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,15 @@
// nothing here ever starts a Kokoro. AC4–AC7 are text pins over ci.yml, CONTRIBUTING.md and the
// MediaLibrary guard (Story107's repo-root grep-assert idiom).
//
// RED at plan time: the guard is a throwing skeleton; ci.yml still says integration runs
// "on-demand on a dev box"; CONTRIBUTING has no "Test tiers" table.
// Was red at plan time (the guard was a throwing skeleton, ci.yml said integration runs
// "on-demand on a dev box", CONTRIBUTING had no "Test tiers" table); T504 turned it green.

using GenWave.Host.Tests.Support;

namespace GenWave.Host.Tests.Specs;

public static class FeatureTestTiersNamedPinnedAndGuarded
{
const string Pending = "pending: T504 — the Host tier guard, the ci.yml comment and the CONTRIBUTING tier table (STORY-439)";

static string RepoRoot => RepoRootLocator.Find(AppContext.BaseDirectory);
static string ReadRepoFile(params string[] parts) => File.ReadAllText(Path.Combine([RepoRoot, .. parts]));

Expand Down Expand Up @@ -47,7 +45,7 @@ public void SomeFact() { }

public sealed class ScenarioTheShippedAssemblyPasses
{
[Fact(Skip = Pending)]
[Fact]
public void ReportsZeroViolations()
{
var violations = TierTraitConventionGuard.FindViolations(typeof(KokoroFixture).Assembly.GetExportedTypes());
Expand All @@ -64,11 +62,11 @@ public sealed class ScenarioCiPinsTheTierOneFilter
public void TheTestStepFiltersOutIntegration() =>
Assert.Contains("--filter \"Category!=Integration\"", ci, StringComparison.Ordinal);

[Fact(Skip = Pending)]
[Fact]
public void NoLineSaysIntegrationRunsOnADevBox() =>
Assert.DoesNotContain("on-demand on a dev box", ci, StringComparison.Ordinal);

[Fact(Skip = Pending)]
[Fact]
public void OneLineNamesTierTwoNightly() =>
Assert.Single(ci.Split('\n'), line => line.Contains("tier 2 (nightly)", StringComparison.Ordinal));
}
Expand All @@ -84,15 +82,15 @@ public ScenarioContributingStatesTheThreeTiers()
table = start < 0 ? "" : text[start..];
}

[Fact(Skip = Pending)]
[Fact]
public void NamesTheTierOneFilter() =>
Assert.Contains("`Category!=Integration`", table, StringComparison.Ordinal);

[Fact(Skip = Pending)]
[Fact]
public void NamesTheTierTwoFilter() =>
Assert.Contains("`Category=Integration`", table, StringComparison.Ordinal);

[Fact(Skip = Pending)]
[Fact]
public void NamesTheManualPrefix() =>
Assert.Contains("`manual:`", table, StringComparison.Ordinal);
}
Expand All @@ -101,9 +99,9 @@ public sealed class ScenarioTheMediaLibraryGuardIsUntouched
{
// The committed baseline hash of IntegrationTraitConventionGuard.cs (2026-09-15) — T504 adds
// a Host twin, it never widens the MediaLibrary guard's scope.
const string BaselineSha256 = "pending: T504 records the baseline hash here";
const string BaselineSha256 = "b8382267f983fd617a9b9be13c708a92fa8715beb5b0fc8c9eff734666cc7552";

[Fact(Skip = Pending)]
[Fact]
public void IsByteIdenticalToTheBaseline()
{
var bytes = File.ReadAllBytes(Path.Combine(RepoRoot, "tests", "GenWave.MediaLibrary.Tests", "IntegrationTraitConventionGuard.cs"));
Expand All @@ -119,7 +117,7 @@ public void IsByteIdenticalToTheBaseline()

public sealed class ScenarioAKokoroConsumerWithoutTheTraitIsRejected
{
[Fact(Skip = Pending)]
[Fact]
public void ReportsThatClassByName()
{
var violations = TierTraitConventionGuard.FindViolations([typeof(KokoroConsumerWithoutTrait)]);
Expand All @@ -130,7 +128,7 @@ public void ReportsThatClassByName()

public sealed class ScenarioAStackFixtureConsumerWithoutTheTraitIsRejected
{
[Fact(Skip = Pending)]
[Fact]
public void ReportsThatClassByName()
{
var violations = TierTraitConventionGuard.FindViolations(
Expand Down
139 changes: 130 additions & 9 deletions tests/GenWave.Host.Tests/Support/TierTraitConventionGuard.cs
Original file line number Diff line number Diff line change
@@ -1,21 +1,142 @@
using System.Reflection;

namespace GenWave.Host.Tests.Support;

/// <summary>
/// The Host tier guard (SPEC F177.3, STORY-439, PLAN T504) — the reflection twin of
/// <c>GenWave.MediaLibrary.Tests.IntegrationTraitConventionGuard</c>: any test class that takes a
/// <c>KokoroFixture</c> (constructor, <c>IClassFixture&lt;&gt;</c> or <c>ICollectionFixture&lt;&gt;</c>)
/// or any type in <see cref="StackFixtureTypes"/>, and lacks <c>[Trait("Category", "Integration")]</c>
/// at class level or on every fact, is a violation — it would try to start a container in the PR lane.
/// <c>KokoroFixture</c> (constructor, <c>IClassFixture&lt;&gt;</c> or <c>ICollectionFixture&lt;&gt;</c>,
/// or a <c>[Collection(name)]</c> whose <c>[CollectionDefinition(name)]</c> class implements
/// <c>ICollectionFixture&lt;&gt;</c> for it) or any type in <see cref="StackFixtureTypes"/>, and lacks
/// <c>[Trait("Category", "Integration")]</c> at class level or on every fact, is a violation — it
/// would try to start a container or a live compose stack in the PR lane. Only facts declared on
/// the class itself count (<c>DeclaredOnly</c>): facts inherited from a base test class are not
/// checked, and no Host test class inherits facts today.
/// </summary>
/// <remarks>Skeleton at plan time — throws until T504 lands.</remarks>
/// <remarks>
/// Today the only stack fixture is Story230's <c>RequestsArtworkFlywheelFixture</c> — its flywheel
/// facts need the LIVE compose stack (requests enabled, PublicBaseUrl set, a real catalog), so
/// F177.1 puts them in tier 2. F178's stream-capture gate runs as a shell script, not a fixture, so
/// it never appears here; there are no other stack/capture fixtures in Host.Tests today.
/// </remarks>
internal static class TierTraitConventionGuard
{
/// <summary>Fixture types that need a full stack or an audio capture — tier 2 by definition.
/// Empty until a stack fixture exists; the guard's own list, extended in place.</summary>
public static IReadOnlyList<Type> StackFixtureTypes { get; } = [];
/// <summary>Fixture types that need a full stack or an audio capture — tier 2 by definition.</summary>
public static IReadOnlyList<Type> StackFixtureTypes { get; } =
[typeof(GenWave.Host.Tests.Specs.RequestsArtworkFlywheelFixture)];

/// <summary>Full names of the offending classes (one entry per class, never per fact).</summary>
public static IReadOnlyList<string> FindViolations(
IEnumerable<Type> types, IReadOnlyCollection<Type>? stackFixtureTypes = null) =>
throw new NotImplementedException("pending: T504 — TierTraitConventionGuard (STORY-439)");
IEnumerable<Type> types, IReadOnlyCollection<Type>? stackFixtureTypes = null)
{
var watchedTypes = new HashSet<Type>(stackFixtureTypes ?? StackFixtureTypes) { typeof(KokoroFixture) };
var typeList = types as IReadOnlyList<Type> ?? types.ToList();

var collectionDefinitionsByAssembly = typeList
.Select(type => type.Assembly)
.Distinct()
.ToDictionary(assembly => assembly, CollectionDefinitionsByName);

var offenders = new List<string>();

foreach (var type in typeList)
{
if (!ConsumesWatchedFixture(type, watchedTypes, collectionDefinitionsByAssembly[type.Assembly]))
continue;
if (HasIntegrationTrait(type.GetCustomAttributesData()))
continue;

var testMethods = type
.GetMethods(BindingFlags.Public | BindingFlags.Instance | BindingFlags.Static | BindingFlags.DeclaredOnly)
.Where(IsTestMethod)
.ToList();
if (testMethods.Count == 0)
continue; // e.g. a bare [CollectionDefinition] marker class — nothing to guard.

var untraited = testMethods
.Where(method => !HasIntegrationTrait(method.GetCustomAttributesData()))
.Select(method => method.Name)
.ToList();
if (untraited.Count == 0)
continue; // every fact already carries the trait.

offenders.Add($"{type.FullName} (missing [Trait(\"Category\", \"Integration\")] on: {string.Join(", ", untraited)})");
}

return offenders;
}

static bool ConsumesWatchedFixture(Type type, HashSet<Type> watchedTypes, IReadOnlyDictionary<string, Type> collectionDefinitionsByName)
{
if (ImplementsFixtureInterfaceFor(type, watchedTypes))
return true;

if (TryGetCollectionName(type, out var collectionName)
&& collectionDefinitionsByName.TryGetValue(collectionName, out var definitionType)
&& ImplementsFixtureInterfaceFor(definitionType, watchedTypes))
return true;

return type
.GetConstructors(BindingFlags.Public | BindingFlags.Instance)
.Any(ctor => ctor.GetParameters().Any(parameter => watchedTypes.Contains(parameter.ParameterType)));
}

static bool ImplementsFixtureInterfaceFor(Type type, HashSet<Type> watchedTypes) =>
type.GetInterfaces().Any(implemented =>
implemented.IsGenericType
&& (implemented.GetGenericTypeDefinition() == typeof(IClassFixture<>)
|| implemented.GetGenericTypeDefinition() == typeof(ICollectionFixture<>))
&& watchedTypes.Contains(implemented.GetGenericArguments()[0]));

static bool TryGetCollectionName(Type type, out string name)
{
var attribute = type.GetCustomAttributesData().FirstOrDefault(a => a.AttributeType == typeof(CollectionAttribute));
if (attribute is { ConstructorArguments.Count: 1 } && attribute.ConstructorArguments[0].Value is string collectionName)
{
name = collectionName;
return true;
}

name = "";
return false;
}

/// <summary>Maps every <c>[CollectionDefinition(name)]</c> class in <paramref name="assembly"/>
/// to its name, so a <c>[Collection(name)]</c> consumer can be traced back to the fixture(s) its
/// collection actually shares.</summary>
static IReadOnlyDictionary<string, Type> CollectionDefinitionsByName(Assembly assembly)
{
var map = new Dictionary<string, Type>(StringComparer.Ordinal);
foreach (var type in assembly.GetTypes())
{
if (TryGetCollectionDefinitionName(type, out var name))
map[name] = type;
}

return map;
}

static bool TryGetCollectionDefinitionName(Type type, out string name)
{
var attribute = type.GetCustomAttributesData().FirstOrDefault(a => a.AttributeType == typeof(CollectionDefinitionAttribute));
if (attribute is { ConstructorArguments.Count: 1 } && attribute.ConstructorArguments[0].Value is string definitionName)
{
name = definitionName;
return true;
}

name = "";
return false;
}

static bool HasIntegrationTrait(IEnumerable<CustomAttributeData> attributes) =>
attributes.Any(a =>
a.AttributeType == typeof(TraitAttribute)
&& a.ConstructorArguments.Count == 2
&& a.ConstructorArguments[0].Value as string == "Category"
&& a.ConstructorArguments[1].Value as string == "Integration");

static bool IsTestMethod(MethodInfo method) =>
method.GetCustomAttributesData().Any(a =>
a.AttributeType == typeof(FactAttribute) || a.AttributeType.IsSubclassOf(typeof(FactAttribute)));
}
Loading