ci: stop the plan job fetching the kernel's history; split the host tests

.github/workflows/kernel-ci-jobs.yml

@@ -218,17 +218,17 @@ jobs:
           retention-days: 14
           if-no-files-found: ignore
 
+  # Needs nothing from plan, so it starts with it.
   host-tests:
     name: host tests (${{ matrix.group }})
-    needs: plan
     runs-on: ubuntu-24.04
     timeout-minutes: 45
     permissions:
       contents: read # check out the repository
     strategy:
       fail-fast: false
       matrix:
-        group: [tools, aurora-sep]
+        group: [tools, aurora-sep, aurora-sep-slow]
     env:
       GROUP: ${{ matrix.group }}
     steps:

tools/aurora-ci/host-tests.json

@@ -7,7 +7,8 @@
   "commits: history the suite reads; host-tests.py --fetch-commits fetches it into a shallow checkout.",
   "Every test-like file under the discovery roots is listed in suites or excluded (tests/test_host_tests.py checks).",
   "helpers: files that look like suites but are run by a listed suite.",
-  "serial: the suite measures time; it runs alone, after the parallel batch."
+  "serial: the suite measures time; it runs alone, after the parallel batch.",
+  "group: the CI job that runs the suite; aurora-sep-slow holds the longest aurora-sep suites so they run beside the rest."
  ],
  "apt": [
   "bison",
@@ -1286,7 +1287,7 @@
   },
   {
    "name": "tools/aurora-sep/tests/test_m3_air_default.py",
-   "group": "aurora-sep",
+   "group": "aurora-sep-slow",
    "argv": [
     "python3",
     "-m",
@@ -1447,7 +1448,7 @@
   },
   {
    "name": "tools/aurora-sep/tests/test_m3_pro_mesa.py",
-   "group": "aurora-sep",
+   "group": "aurora-sep-slow",
    "argv": [
     "python3",
     "-m",
@@ -1512,7 +1513,7 @@
   },
   {
    "name": "tools/aurora-sep/tests/test_m3max_kit.py",
-   "group": "aurora-sep",
+   "group": "aurora-sep-slow",
    "argv": [
     "python3",
     "-m",

tools/aurora-ci/lookup-build.py

@@ -68,8 +68,11 @@ def kernel_id(repo, rev):
 
 
 def ensure_commit(repo, sha, remote="origin"):
-    """Fetch a commit's trees (no file contents) if the checkout lacks it."""
-    if subprocess.run(["git", "-C", str(repo), "cat-file", "-e", f"{sha}^{{commit}}"],
+    """Fetch a commit's trees (no file contents) if the checkout lacks it.
+
+    The check must not fetch: a partial clone fetches a missing object on
+    demand with all of its history, millions of kernel commits and trees."""
+    if subprocess.run(["git", "-C", str(repo), "--no-lazy-fetch", "cat-file", "-e", f"{sha}^{{commit}}"],
                       stderr=subprocess.DEVNULL).returncode:
         git(repo, "fetch", "--quiet", "--no-tags", "--depth=1", "--filter=blob:none", remote, sha)
 

tools/aurora-ci/tests/test_host_tests.py

@@ -136,6 +136,16 @@ def setUp(self):
         super().setUp()
         self.manifest = host.load_manifest()
 
+    def workflow_groups(self):
+        """The groups the host-tests job's matrix runs."""
+        workflow = (TREE / ".github/workflows/kernel-ci-jobs.yml").read_text()
+        job = workflow.split("\n  host-tests:\n", 1)[1].split("\n  self-test:\n", 1)[0]
+        return re.search(r"^ +group: \[(.*)\]$", job, re.M).group(1).replace(" ", "").split(",")
+
+    def test_the_workflow_runs_every_group(self):
+        groups = {suite.get("group", "tools") for suite in self.manifest["suites"]}
+        self.assertEqual(sorted(groups), sorted(self.workflow_groups()))
+
     def test_every_test_like_file_is_listed_or_excluded(self):
         listed = set(self.manifest["excluded"]) | set(self.manifest.get("helpers", {}))
         for suite in self.manifest["suites"]:
@@ -157,7 +167,7 @@ def test_entries_are_well_formed(self):
         for suite in self.manifest["suites"]:
             with self.subTest(suite=suite["name"]):
                 self.assertTrue(suite["argv"])
-                self.assertIn(suite.get("group", "tools"), ("tools", "aurora-sep"))
+                self.assertIn(suite.get("group", "tools"), self.workflow_groups())
                 for commit in suite.get("commits", []):
                     self.assertRegex(commit, r"^[0-9a-f]{40}$")
                 if "known_failure" in suite:

tools/aurora-ci/tests/test_lookup_build.py

@@ -1,4 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0
+import subprocess
+
 from support import TempDirTestCase, commit_file, git, load
 
 lookup = load("lookup-build")
@@ -72,6 +74,28 @@ def test_ci_changes_do_not_change_the_key(self):
         self.assertEqual(self.key(), before)
 
 
+class EnsureCommitTest(TempDirTestCase):
+    def present(self, repo, sha):
+        return subprocess.run(["git", "-C", str(repo), "--no-lazy-fetch", "cat-file", "-e", f"{sha}^{{commit}}"],
+                              stderr=subprocess.DEVNULL).returncode == 0
+
+    def test_fetches_only_the_missing_commit_into_a_partial_clone(self):
+        # The plan job's checkout: depth 1, blob:none. Asking a partial clone
+        # for a missing object fetches it with all of its history.
+        origin = self.init_repo("origin")
+        first = commit_file(origin, "Makefile", "1\n", "first")
+        base = commit_file(origin, "Makefile", "2\n", "base")
+        commit_file(origin, "Makefile", "3\n", "head")
+        git(origin, "config", "uploadpack.allowFilter", "true")
+        git(origin, "config", "uploadpack.allowAnySHA1InWant", "true")
+        clone = self.tmp / "clone"
+        git(self.tmp, "clone", "--quiet", "--depth=1", "--filter=blob:none", "--no-checkout",
+            origin.as_uri(), str(clone))
+        lookup.ensure_commit(clone, base)
+        self.assertTrue(self.present(clone, base))
+        self.assertFalse(self.present(clone, first))
+
+
 REPO = 1000