diff --git a/.github/workflows/skill-acceptance.yml b/.github/workflows/skill-acceptance.yml new file mode 100644 index 0000000..1b06587 --- /dev/null +++ b/.github/workflows/skill-acceptance.yml @@ -0,0 +1,22 @@ +name: Skill Acceptance + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + skill-acceptance: + name: Public skill acceptance + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Verify skill test discovery + run: tests/skill-test-runner.sh + - name: Run every skill acceptance test + run: scripts/run-skill-tests.sh diff --git a/scripts/run-skill-tests.sh b/scripts/run-skill-tests.sh new file mode 100755 index 0000000..c1cdd72 --- /dev/null +++ b/scripts/run-skill-tests.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -gt 1 ]]; then + printf 'Usage: %s [repository-root]\n' "${0##*/}" >&2 + exit 2 +fi + +if [[ $# -eq 1 ]]; then + repo="$(cd -P "$1" && pwd)" +else + repo="$(cd -P "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +fi + +if [[ ! -d "$repo/skills" ]]; then + printf 'No skills found under %s/skills.\n' "$repo" + exit 0 +fi + +skill_files=() +while IFS= read -r -d '' skill_file; do + skill_files+=("$skill_file") +done < <(find "$repo/skills" -mindepth 2 -maxdepth 2 -type f -name SKILL.md -print0 | sort -z) + +if [[ ${#skill_files[@]} -eq 0 ]]; then + printf 'No skills found under %s/skills.\n' "$repo" + exit 0 +fi + +for skill_file in "${skill_files[@]}"; do + skill_name="$(basename "$(dirname "$skill_file")")" + test_file="$repo/tests/$skill_name-skill.sh" + if [[ ! -f "$test_file" ]]; then + printf 'Skill %s is missing its acceptance test: tests/%s-skill.sh\n' \ + "$skill_name" "$skill_name" >&2 + exit 1 + fi + if [[ ! -x "$test_file" ]]; then + printf 'Skill acceptance test is not executable: tests/%s-skill.sh\n' \ + "$skill_name" >&2 + exit 1 + fi + printf 'Running %s skill acceptance test...\n' "$skill_name" + "$test_file" +done diff --git a/tests/skill-test-runner.sh b/tests/skill-test-runner.sh new file mode 100755 index 0000000..10d13b1 --- /dev/null +++ b/tests/skill-test-runner.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo="$(cd -P "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +runner="$repo/scripts/run-skill-tests.sh" +root="$(mktemp -d)" +trap 'rm -rf "$root"' EXIT +fixture="$root/repository with spaces" +mkdir -p "$fixture/skills/example" "$fixture/tests" +printf '%s\n' '---' 'description: Fixture skill' '---' >"$fixture/skills/example/SKILL.md" +cat >"$fixture/tests/example-skill.sh" <<'TEST' +#!/usr/bin/env bash +set -euo pipefail +printf 'example executed\n' >>"$SKILL_RUNNER_TEST_LOG" +TEST +chmod 0755 "$fixture/tests/example-skill.sh" + +# Discovery proof: a skill package causes its matching executable acceptance +# script to run, including when the repository path contains spaces. +log="$root/executed.log" +SKILL_RUNNER_TEST_LOG="$log" "$runner" "$fixture" >/dev/null +test "$(cat "$log")" = "example executed" + +# Coverage proof: a skill without a conventionally named acceptance test makes +# the runner fail instead of allowing a green check with missing coverage. +mkdir -p "$fixture/skills/uncovered" +printf '%s\n' '---' 'description: Uncovered fixture skill' '---' >"$fixture/skills/uncovered/SKILL.md" +if SKILL_RUNNER_TEST_LOG="$log" "$runner" "$fixture" >"$root/stdout" 2>"$root/stderr"; then + printf 'Runner accepted a skill without an acceptance test.\n' >&2 + exit 1 +fi +grep -F 'tests/uncovered-skill.sh' "$root/stderr" >/dev/null + +# Bootstrap proof: the workflow can land before the first skill and begins +# enforcing coverage as soon as a skills directory appears. +empty="$root/empty repository" +mkdir -p "$empty" +"$runner" "$empty" >"$root/empty-output" +grep -F 'No skills found' "$root/empty-output" >/dev/null + +printf 'Skill test discovery executes every matching test and fails closed on missing coverage.\n'