diff --git a/.rat-excludes b/.rat-excludes index dc4e0ffcd6a16..01e51426e8da0 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -328,8 +328,9 @@ www-hash.txt # Vendored-in code /src/airflow/providers/google/_vendor/* -# TypeScript bundle golden fixture: its bytes are digest-pinned, so no license header can be added +# TypeScript bundle golden fixtures: their bytes are digest-pinned, so no license header can be added /ts-sdk/tests/cli/fixtures/bundle-v1.min.mjs +/ts-sdk/tests/cli/fixtures/bundle-v1-with-task-handlers.min.mjs # Java SDK build outputs /java-sdk/bin/* diff --git a/airflow-core/adr/lang-sdk/0014-bundle-metadata-and-cache-digest.md b/airflow-core/adr/lang-sdk/0014-bundle-metadata-and-cache-digest.md index aecdc4d9bc4f0..c626a78833a36 100644 --- a/airflow-core/adr/lang-sdk/0014-bundle-metadata-and-cache-digest.md +++ b/airflow-core/adr/lang-sdk/0014-bundle-metadata-and-cache-digest.md @@ -119,4 +119,7 @@ something; consumers compare for equality and interpret nothing. - Dynamic Dag rendering works. - The canonical schema drops the identifier mapping entirely, the coordinator task execution side will rely on persisted rel_path instead of discovering the artifact every time. -- The packer no longer needs to execute the artifact at all. `supervisor_schema_version` is a compile-time constant of the SDK. +- The Go packer runs nothing. It takes `supervisor_schema_version`, a compile-time constant of the SDK, from its own go-sdk and the SDK version from the binary's build information, and it refuses a binary built against another go-sdk than its own. A cross-built binary packs without a host build, and the bundle binary has no `--airflow-metadata` flag. +- The TypeScript packer still runs the bundle, for its schema version, the source file of each Dag declared in TypeScript ([ADR-0015](0015-per-dag-source-in-bundle-artifact.md)) and its pack-time checks, but it embeds no `task_handlers`. `dag_source_paths` is the one place a Dag id remains in TypeScript metadata. It is for display, and nothing reads it to find an artifact. +- A bundle packed before this change may still carry `dags` (Go) or `task_handlers` (TypeScript) in its metadata. Readers ignore them. +- Java never had an inventory. The Gradle plugin adds `Airflow-Cache-Digest` to the JAR manifest, next to its `Main-Class`, and for a fat JAR `Airflow-Supervisor-Schema-Version` too, which a thin JAR takes from the `airflow-sdk` JAR. It runs nothing. `Airflow-Cache-Digest` marks a handler JAR. diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst index 0503810efe2c1..8ef59dcd92d6e 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/go.rst @@ -28,10 +28,10 @@ to a compiled Go *bundle* that is launched by :class:`~airflow.sdk.coordinators.executable.ExecutableCoordinator` for each task instance. Because Go is a compiled language, every task must be compiled ahead of time and registered inside a single, -self-contained native executable called a **bundle**. The bundle also embeds its Dag source and a metadata -manifest (the ``dag_id`` and ``task_id`` map) in a footer appended to the executable, so the executable *is* -the bundle: one runnable file to ship, with no separate manifest or archive. The -:ref:`airflow-go-pack ` tool builds and packs that bundle. +self-contained native executable called a **bundle**. The bundle also embeds its Dag source and a small +metadata manifest (the SDK version and the supervisor schema version it was built against) in a footer +appended to the executable, so the executable *is* the bundle: one runnable file to ship, with no separate +manifest or archive. The :ref:`airflow-go-pack ` tool builds and packs that bundle. .. contents:: Contents :local: @@ -136,7 +136,8 @@ Go entry point Build a bundle with ``airflow.Bundle()``, register a handler for each task, and call ``Serve`` as the last statement of ``main``. The ``Register`` calls are the single source of truth for which ``dag_id`` and task -names this bundle can run, so the generated manifest can never drift from what the binary actually executes. +names this bundle can run: the Dag processor asks the bundle for them, so it always sees what the binary +actually executes. .. code-block:: go @@ -458,6 +459,10 @@ SDKs, specified in :doc:`task-sdk:executable-bundle-spec`. to your bundle module's ``go.mod`` and run it with ``go tool airflow-go-pack``. This pins the packer version per project. +The packer never runs the binary it packs. It reads the go-sdk version the binary was built against from the +binary's build information, and refuses a binary built against a different go-sdk version than the packer's, +so run it from the module that builds the bundle. + Build and pack in one step; any flags after ``--`` are forwarded verbatim to ``go build``: .. code-block:: bash @@ -484,20 +489,13 @@ architecture than your build machine (for example, deploying to a Linux host fro --output /opt/airflow/go-task-handlers/sample-dag-bundle \ ./example/bundle -Alternatively, pack a pre-built binary with ``--executable`` / ``--source``. The packer normally execs the -binary with ``--airflow-metadata`` to read its manifest, but a cross-compiled binary cannot run on the build -host. In that case, generate the manifest on a machine that *can* run the binary and feed it to the packer -with ``--airflow-metadata``: +Alternatively, pack a pre-built binary with ``--executable`` / ``--source``. The packer does not run the +binary, so one built for any platform packs on any host: .. code-block:: bash - # On a linux/amd64 machine: - go build -o my-bundle ./example/bundle - ./my-bundle --airflow-metadata > airflow-metadata.yaml - - # Back on the darwin/arm64 machine: - go tool airflow-go-pack --executable ./my-bundle --source main.go \ - --airflow-metadata airflow-metadata.yaml + GOOS=linux GOARCH=amd64 go build -o my-bundle ./example/bundle + go tool airflow-go-pack --executable ./my-bundle --source ./example/bundle/main.go (``--executable`` is mutually exclusive with ``--goos`` / ``--goarch`` and with ``go build`` flags after ``--``, since it packs an already-built binary instead of building one.) diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/index.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/index.rst index 323646a814bf2..17a73c5328017 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/index.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/index.rst @@ -264,8 +264,8 @@ coordinator; JVM languages, for example, compile to bytecode that requires a JRE To support a new such language, produce a *bundle* in the shared on-disk format the coordinator consumes and speak the coordinator IPC protocol (the ``--comm`` / ``--logs`` socket arguments). That format - the ``AFBNDL01`` footer appended to the executable, the binary integrity hash, and the ``airflow-metadata.yaml`` -manifest of ``dag_id``\ s and ``task_id``\ s - is specified, together with the reader algorithm and the -compatibility/versioning rules, in :doc:`task-sdk:executable-bundle-spec`. That page also publishes a -machine-readable JSON Schema for the manifest, for use by build tooling and validators. Follow the spec to -make a new language's bundles discoverable by Airflow with no change to the scheduler, worker, or UI; the -:doc:`Go SDK ` is a worked reference implementation. +manifest - is specified, together with the reader algorithm and the compatibility/versioning rules, in +:doc:`task-sdk:executable-bundle-spec`. That page also publishes a machine-readable JSON Schema for the +manifest, for use by build tooling and validators. Follow the spec to make a new language's bundles +discoverable by Airflow with no change to the scheduler, worker, or UI; the :doc:`Go SDK ` is a worked +reference implementation. diff --git a/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst b/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst index 838be4ce0b316..580f9f2d0c8cd 100644 --- a/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst +++ b/airflow-core/docs/authoring-and-scheduling/language-sdks/typescript.rst @@ -493,10 +493,11 @@ Building and packaging ---------------------- ``airflow-ts-pack`` (shipped with the SDK) bundles the entry module and all of its imports with esbuild into -a single self-contained, minified ESM file, ``bundle.min.mjs``, and embeds the manifest (the ``dag_id`` and -``task_id`` map plus the supervisor schema version) after a leading compact JSON ``//# airflowBundle=...`` -layout header. The layout records the byte ranges and SHA-256 digests of the manifest and executable code, -so there is one file to deploy, with no separate manifest or ``node_modules``. +a single self-contained, minified ESM file, ``bundle.min.mjs``, and embeds the manifest (the SDK version, the +supervisor schema version, the entry file and, for each Dag declared in TypeScript, the file that declares it) +after a leading compact JSON ``//# airflowBundle=...`` layout header. The layout records the byte ranges and +SHA-256 digests of the manifest and executable code, so there is one file to deploy, with no separate manifest or +``node_modules``. The code is minified because an integrity digest is only worth taking over an artifact nobody is expected to read or edit in place. Function names are kept through minification, since a task id defaults to its @@ -517,9 +518,8 @@ that only supply utilities or types are not embedded. npm install --save-dev esbuild npx airflow-ts-pack src/main.ts --outdir dist -Use ``--outdir `` to choose the output directory (default ``dist``), ``--outfile `` to name the -artifact exactly, which helps when one Dag bundle holds several bundles, and ``--source `` to set -the source name displayed in the Airflow UI (default: the entry file's basename). ``--outdir`` and +Use ``--outdir `` to choose the output directory (default ``dist``), or ``--outfile `` to name the +artifact exactly, which helps when one Dag bundle holds several bundles. ``--outdir`` and ``--outfile`` are mutually exclusive, and an ``--outfile`` name must end in ``.min.mjs`` so the coordinator can find it. diff --git a/airflow-core/tests/unit/dag_processing/test_task_handler_processor_go.py b/airflow-core/tests/unit/dag_processing/test_task_handler_processor_go.py index 8cf2f485fb71f..dd44847d52849 100644 --- a/airflow-core/tests/unit/dag_processing/test_task_handler_processor_go.py +++ b/airflow-core/tests/unit/dag_processing/test_task_handler_processor_go.py @@ -25,8 +25,10 @@ from typing import TYPE_CHECKING from unittest import mock +import jsonschema import pytest import structlog +import yaml from airflow.dag_processing.processor import TaskHandlerDeclaration, TaskHandlerParam from airflow.dag_processing.task_handler_processor import LangSDKTaskHandlerProcessorProcess @@ -103,6 +105,28 @@ def _go_coordinator(monkeypatch, tmp_path): reset_coordinator_manager() +def test_a_packed_go_bundle_records_no_dag_inventory(go_bundle): + completed = subprocess.run( + ["go", "tool", "airflow-go-pack", "inspect", os.fspath(go_bundle)], + cwd=GO_SDK_PATH, + env={**os.environ, "CGO_ENABLED": "0"}, + capture_output=True, + text=True, + check=False, + ) + assert completed.returncode == 0, completed.stderr + + manifest = yaml.safe_load(completed.stdout) + + assert manifest["sdk"]["language"] == "go" + assert manifest["sdk"]["supervisor_schema_version"] + assert "dags" not in manifest + schema = json.loads( + (AIRFLOW_ROOT_PATH / "task-sdk" / "docs" / "airflow-metadata.schema.json").read_text() + ) + jsonschema.Draft202012Validator(schema).validate(manifest) + + def test_probes_the_task_handlers_of_a_packed_go_bundle(go_bundle): result = LangSDKTaskHandlerProcessorProcess.run( coordinator="go", diff --git a/airflow-core/tests/unit/dag_processing/test_task_handler_processor_ts.py b/airflow-core/tests/unit/dag_processing/test_task_handler_processor_ts.py index 54211b64992e9..5e25324af2960 100644 --- a/airflow-core/tests/unit/dag_processing/test_task_handler_processor_ts.py +++ b/airflow-core/tests/unit/dag_processing/test_task_handler_processor_ts.py @@ -114,6 +114,18 @@ def _declare(task_id: str) -> TaskHandlerDeclaration: return TaskHandlerDeclaration(task_id=task_id, binding="named", params=None) +def test_a_packed_bundle_embeds_no_task_handlers(example_bundle): + metadata_line = example_bundle.read_bytes().split(b"\n")[1] + prefix = b"//# airflowMetadata=" + assert metadata_line.startswith(prefix) + + metadata = json.loads(metadata_line[len(prefix) :]) + + assert metadata["sdk"]["language"] == "typescript" + assert "dag_source_paths" in metadata + assert "task_handlers" not in metadata + + @pytest.mark.usefixtures("fresh_coordinator_manager") @conf_vars({("sdk", "coordinators"): json.dumps(COORDINATORS)}) @mock.patch.object(supervisor, "_should_use_exec", autospec=True, return_value=False) diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index d02057be259bb..db0032f6499d1 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -543,9 +543,10 @@ def _setup_java_sdk_integration(dot_env_file, tmp_dir): def _run_go_sdk_pack(output_path, *, capture_output=False, native=False): """Run ``go tool airflow-go-pack`` natively or inside the pinned Go toolchain container. - ``go tool airflow-go-pack`` builds the bundle package, reads its - --airflow-metadata, and appends the source + airflow-metadata.yaml + the - AFBNDL01 trailer, writing a single self-contained executable bundle. + ``go tool airflow-go-pack`` builds the bundle package, reads the go-sdk + version from the binary's build information (it never runs the binary), and + appends the source + airflow-metadata.yaml + the AFBNDL01 trailer, writing a + single self-contained executable bundle. CGO_ENABLED=0 yields a fully static binary that runs on the stock worker. In ``native`` mode (used in CI, where the host already has a Go toolchain plus @@ -559,10 +560,6 @@ def _run_go_sdk_pack(output_path, *, capture_output=False, native=False): * HOME points at a writable, gitignored dir under go-sdk/bin so the Go build and module caches persist between runs (first run downloads modules once; subsequent runs skip straight to compilation). - * USER/HOME must be set because the SDK calls user.Current() at init; with - cgo disabled Go's pure-Go resolver reads those env vars instead of libc, - and panics if either is empty (the same vars are set on the worker and - the Dag processor in go.yml so the packed binary runs the same way there). """ if native: cwd = GO_SDK_ROOT_PATH diff --git a/contributing-docs/30_new_language_sdk.rst b/contributing-docs/30_new_language_sdk.rst index def83a02f3724..d94e51bc69d7c 100644 --- a/contributing-docs/30_new_language_sdk.rst +++ b/contributing-docs/30_new_language_sdk.rst @@ -228,7 +228,10 @@ If the target runtime compiles to a self-contained native executable, the :class:`~airflow.sdk.coordinators.executable.ExecutableCoordinator` can discover and launch it automatically. For the coordinator to understand the bundle correctly, extra metadata should be appended to the executable by a -custom bundling step at build-time. +custom bundling step at build-time. The metadata names the SDK and the +supervisor schema version and lists no Dags or tasks: the Dag processor asks the +bundle which task handlers it registers, so the bundling step does not need to +run the executable. See :ref:`Executable Bundle Spec` in Task SDK documentation for details. @@ -536,9 +539,10 @@ authored in the target language. An SDK declares each one independently. The task can read, write, and delete Airflow Variables. ``self-contained-bundle`` (MUST) - The SDK's build artifact embeds its own Airflow metadata (``dag_id``, ``task_id`` and - the rest of the task descriptor) inside the *same* artifact as the task code, rather - than shipping it in a separate sidecar file, so the deployable unit is self-describing. + The SDK's build artifact embeds its own Airflow metadata (what Airflow needs to run it, + such as the supervisor schema version it was built against) inside the *same* artifact + as the task code, rather than shipping it in a separate sidecar file, so the deployable + unit is self-describing. Each runtime satisfies this its own way — a Go binary carries an ``AFBNDL01`` metadata trailer (see `Native Executable Bundle Format`_), a JVM artifact embeds it in the jar, a Node bundle embeds it in the package. diff --git a/go-sdk/README.md b/go-sdk/README.md index fcb3c95602211..b0d4527ba2219 100644 --- a/go-sdk/README.md +++ b/go-sdk/README.md @@ -37,10 +37,10 @@ another. Python tasks are imported and run in-process. Go is compiled, so the model is different. A single binary that bundles one or more Dags' task functions is called a **bundle**. You build one with -the SDK's packer, `airflow-go-pack`, which compiles your code and appends a metadata footer (the manifest -of `dag_id`s and `task_id`s, plus the Dag source) to the executable. The result is a **self-contained -executable bundle**: a single runnable file that *is* the bundle, with no separate manifest or archive to -ship alongside it. +the SDK's packer, `airflow-go-pack`, which compiles your code and appends a metadata footer (a small +manifest with the SDK and supervisor schema versions, plus the Dag source) to the executable. The result +is a **self-contained executable bundle**: a single runnable file that *is* the bundle, with no separate +manifest or archive to ship alongside it. ## You still need a Python stub Dag (for now) @@ -265,17 +265,14 @@ the full range of task states, and alternate XCom backends without implementing ./example/bundle ``` - Alternatively, use `--executable`/`--source`. The packer normally execs the binary to read - its metadata; a cross-compiled binary cannot run on the host, so generate the metadata on a machine that - can run it and pass the file with `--airflow-metadata`: + Alternatively, use `--executable`/`--source`. The packer never runs the binary, so one built for any + platform packs on any host. It reads the go-sdk version from the binary's build information and refuses a + binary built against a different go-sdk version than the packer's, so run it from the module that builds + the bundle: ```bash - # on linux/amd64 machine: - go build -o my-bundle ./example/bundle - ./my-bundle --airflow-metadata > airflow-metadata.yaml - - # on darwin/arm64 machine: - go tool airflow-go-pack --executable ./my-bundle --source main.go --airflow-metadata airflow-metadata.yaml + GOOS=linux GOARCH=amd64 go build -o my-bundle ./example/bundle + go tool airflow-go-pack --executable ./my-bundle --source ./example/bundle/main.go ``` > [!NOTE] diff --git a/go-sdk/adr/0001-bundle-packing-options.md b/go-sdk/adr/0001-bundle-packing-options.md index c5a31bbaaaf7d..bb87a51fc254c 100644 --- a/go-sdk/adr/0001-bundle-packing-options.md +++ b/go-sdk/adr/0001-bundle-packing-options.md @@ -44,6 +44,11 @@ options below still describe valid *packer mechanisms*; only the artefact each one writes has changed from a ZIP to a footer-augmented executable. +Option D is retired: the bundle binary no longer accepts `--airflow-metadata`, +and the packer no longer runs the binary but reads the go-sdk version from +its build information (see +[ADR-0014](../../airflow-core/adr/lang-sdk/0014-bundle-metadata-and-cache-digest.md)). + ## Context The executable provider's bundle spec diff --git a/go-sdk/adr/0002-use-go-tool-directive-for-bundle-packer.md b/go-sdk/adr/0002-use-go-tool-directive-for-bundle-packer.md index 3f5efde805c2c..366ba0d04a41e 100644 --- a/go-sdk/adr/0002-use-go-tool-directive-for-bundle-packer.md +++ b/go-sdk/adr/0002-use-go-tool-directive-for-bundle-packer.md @@ -36,6 +36,12 @@ A standalone binary + Option D introspection contract + Option H ZIP output; read them with the ADR 0004 substitution in mind, and treat ADR 0004 as authoritative wherever the two disagree. +The `--airflow-metadata` introspection this ADR has the packer run is retired: +the packer no longer runs the binary but reads the go-sdk version from its +build information, and neither the bundle binary nor the packer accepts +`--airflow-metadata` any more (see +[ADR-0014](../../airflow-core/adr/lang-sdk/0014-bundle-metadata-and-cache-digest.md)). + ## Context [ADR 0001](0001-bundle-packing-options.md) enumerated nine candidate diff --git a/go-sdk/adr/0003-coordinator-protocol-msgpack-ipc.md b/go-sdk/adr/0003-coordinator-protocol-msgpack-ipc.md index 241c0839e9544..5d707062af6f7 100644 --- a/go-sdk/adr/0003-coordinator-protocol-msgpack-ipc.md +++ b/go-sdk/adr/0003-coordinator-protocol-msgpack-ipc.md @@ -40,6 +40,10 @@ decision in this ADR is unaffected: the binary still honours the container format it ships inside. Read the ZIP mentions below with the ADR 0004 substitution in mind. +The `--airflow-metadata` mode described below is removed, so the bundle binary +speaks only the coordinator protocol (see +[ADR-0014](../../airflow-core/adr/lang-sdk/0014-bundle-metadata-and-cache-digest.md)). + ## Context A Go SDK bundle binary today (the artefact built from diff --git a/go-sdk/adr/0004-self-contained-executable-bundle.md b/go-sdk/adr/0004-self-contained-executable-bundle.md index 9ee9dd70624dd..0875a2581f4e0 100644 --- a/go-sdk/adr/0004-self-contained-executable-bundle.md +++ b/go-sdk/adr/0004-self-contained-executable-bundle.md @@ -30,6 +30,11 @@ packer mechanism (Option A standalone packer + Option D introspection contract + Option H `tool` directive) is unchanged; only the artefact the packer writes is changed. +The `--airflow-metadata` introspection this ADR has the packer run is retired: +the packer no longer runs the binary but reads the go-sdk version from its +build information, and the manifest no longer lists Dags (see +[ADR-0014](../../airflow-core/adr/lang-sdk/0014-bundle-metadata-and-cache-digest.md)). + ## Context ADR 0001 / ADR 0002 picked a ZIP archive as the bundle container, diff --git a/go-sdk/adr/0005-retire-go-edge-worker.md b/go-sdk/adr/0005-retire-go-edge-worker.md index 9c5d924de60fe..64082f09409bd 100644 --- a/go-sdk/adr/0005-retire-go-edge-worker.md +++ b/go-sdk/adr/0005-retire-go-edge-worker.md @@ -26,6 +26,10 @@ Date: 2026-08-20 Accepted. Supersedes the dual-runtime portion of [ADR 0003](0003-coordinator-protocol-msgpack-ipc.md). +`--airflow-metadata` has since been removed, so it is no longer available to the +bundle packer (see +[ADR-0014](../../airflow-core/adr/lang-sdk/0014-bundle-metadata-and-cache-digest.md)). + ## Context The Go SDK supported two task-execution architectures. The standalone Go Edge diff --git a/go-sdk/airflow/bundle.go b/go-sdk/airflow/bundle.go index 58f04354ea358..72bac1d12671f 100644 --- a/go-sdk/airflow/bundle.go +++ b/go-sdk/airflow/bundle.go @@ -125,8 +125,8 @@ func (b *BundleRef) Register(items ...Registerable) { } // taskHandlerMap holds the registered task handlers by dag_id and task_id. -// It also keeps registration order. The --airflow-metadata manifest, and the reply to the Dag -// processor's task handler parse, list the tasks of each Dag in that order. +// It also keeps registration order. The reply to the Dag processor's task handler parse lists +// the tasks of each Dag in that order. type taskHandlerMap struct { mu sync.RWMutex handlers map[string]map[string]bundle.Task diff --git a/go-sdk/airflow/bundle_test.go b/go-sdk/airflow/bundle_test.go index 31a654c453f1e..a5ce3048bd6e7 100644 --- a/go-sdk/airflow/bundle_test.go +++ b/go-sdk/airflow/bundle_test.go @@ -224,10 +224,17 @@ func TestRegisterRejectsNilDag(t *testing.T) { ) } +// closeRegistration calls serve without the coordinator flags, so it returns an error at once. +// Serve closes registration whatever it goes on to do. +func closeRegistration(t *testing.T, b *BundleRef) { + t.Helper() + require.ErrorIs(t, b.serve(nil, io.Discard), errCoordinatorFlagsRequired) +} + func TestRegisterAfterServePanics(t *testing.T) { b := Bundle() b.Register(TaskHandler("py_etl", "transform", noop)) - require.NoError(t, b.serve([]string{"--airflow-metadata"}, io.Discard)) + closeRegistration(t, b) want := "airflow.BundleRef.Register: Serve has already been called; " + "register everything before Serve" @@ -239,7 +246,7 @@ func TestRegisterAfterServePanics(t *testing.T) { // Register later is covered without a flag of its own. func TestRegisterAfterServeRejectsEveryKindOfItem(t *testing.T) { b := Bundle() - require.NoError(t, b.serve([]string{"--airflow-metadata"}, io.Discard)) + closeRegistration(t, b) var nilItem Registerable assert.Panics(t, func() { b.Register(nilItem) }, diff --git a/go-sdk/airflow/dag.go b/go-sdk/airflow/dag.go index e0b48277909e5..a46a40584f635 100644 --- a/go-sdk/airflow/dag.go +++ b/go-sdk/airflow/dag.go @@ -53,7 +53,7 @@ type DagRef struct { // Add every task before Register. [DagRef.Task] panics once the Dag is registered. // // [BundleRef.Serve] does not yet serve the Dags that Dag returns. It leaves them out of the -// --airflow-metadata manifest and cannot run their tasks. +// reply to the Dag processor's task handler parse and cannot run their tasks. func Dag(dagID string, spec ...DagSpec) *DagRef { if len(spec) > 1 { panic(fmt.Sprintf( diff --git a/go-sdk/airflow/serve.go b/go-sdk/airflow/serve.go index 910cd5a1398f2..0d8d8df190ea8 100644 --- a/go-sdk/airflow/serve.go +++ b/go-sdk/airflow/serve.go @@ -34,18 +34,11 @@ var errCoordinatorFlagsRequired = errors.New( "--comm and --logs are required for bundle execution", ) -// errFormatRequiresMetadata is returned by Serve when --format is supplied -// without --airflow-metadata, the only mode whose encoding it selects. -var errFormatRequiresMetadata = errors.New( - "--format is only valid together with --airflow-metadata", -) - // serveMode tags the protocol the binary will speak this run. type serveMode int const ( - modeAirflowMetadata serveMode = iota // --airflow-metadata: print the manifest JSON (ADR 0002/0004) - modeCoordinator // --comm/--logs: msgpack-over-IPC (ADR 0003) + modeCoordinator serveMode = iota // --comm/--logs: msgpack-over-IPC (ADR 0003) modeCoordinatorUsageError // missing coordinator flags ) @@ -54,12 +47,9 @@ const ( // Serve closes registration: a [BundleRef.Register] call that reaches the bundle afterwards // panics rather than changing what the running bundle answers for. // -// The command-line flags of the executable decide what Serve does. -// With --airflow-metadata it prints the bundle's manifest and returns, which is how -// airflow-go-pack reads the Dag and task ids of the registered task handlers. -// With --comm and --logs, which Airflow passes, it speaks the coordinator protocol: it either -// runs one task, or tells the Dag processor which task handlers it registers for the Dags the -// Dag processor asks about. +// The binary speaks only the coordinator protocol, on the --comm and --logs addresses Airflow +// passes. It either runs one task, or tells the Dag processor which task handlers it registers +// for the Dags the Dag processor asks about. Without both flags Serve returns an error. // // main must exit with a non-zero status when Serve returns an error, because the exit status // is how the supervisor learns that the task failed: @@ -77,24 +67,12 @@ func (b *BundleRef) serve(args []string, stdout io.Writer) error { b.closed.Store(true) // The flags go on their own FlagSet. On pflag.CommandLine, every program that imports this - // package would get them, and one that defines its own --format there would panic. + // package would get them, and one that defines its own --comm there would panic. flags := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) - // --help is output the caller asked for, so it goes to stdout. Anything else pflag prints, - // such as a deprecation warning for a flag the bundle defines, stays on stderr where it - // cannot land in the middle of the --airflow-metadata manifest. + // --help is output the caller asked for, so it goes to stdout. flags.Usage = func() { fmt.Fprintf(stdout, "Usage of %s:\n%s", flags.Name(), flags.FlagUsages()) } - printMetadata := flags.Bool( - "airflow-metadata", - false, - "print the bundle's airflow-metadata manifest and exit", - ) - metadataFormat := flags.String( - "format", - string(execution.MetadataFormatYAML), - "encoding for --airflow-metadata: yaml (default) or json; only valid with --airflow-metadata", - ) commAddr := flags.String( "comm", "", @@ -126,21 +104,7 @@ func (b *BundleRef) serve(args []string, stdout io.Writer) error { return err } - mode := decideMode(*printMetadata, *commAddr, *logsAddr) - - // --format applies only to --airflow-metadata; reject it elsewhere instead - // of silently ignoring it. - if mode != modeAirflowMetadata && flags.Changed("format") { - return errFormatRequiresMetadata - } - - switch mode { - case modeAirflowMetadata: - format, err := execution.ParseMetadataFormat(*metadataFormat) - if err != nil { - return err - } - return execution.DumpAirflowMetadata(stdout, &b.taskHandlers, format) + switch decideMode(*commAddr, *logsAddr) { case modeCoordinator: return execution.Serve(&b.taskHandlers, *commAddr, *logsAddr) case modeCoordinatorUsageError: @@ -149,10 +113,7 @@ func (b *BundleRef) serve(args []string, stdout io.Writer) error { return nil } -func decideMode(metadata bool, comm, logs string) serveMode { - if metadata { - return modeAirflowMetadata - } +func decideMode(comm, logs string) serveMode { commSet := comm != "" logsSet := logs != "" if commSet && logsSet { diff --git a/go-sdk/airflow/serve_test.go b/go-sdk/airflow/serve_test.go index 93de70af0231c..f7f4a5f107c10 100644 --- a/go-sdk/airflow/serve_test.go +++ b/go-sdk/airflow/serve_test.go @@ -19,8 +19,6 @@ package airflow import ( "bytes" - "encoding/json" - "errors" "io" "net" "testing" @@ -30,20 +28,17 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vmihailenco/msgpack/v5" - "gopkg.in/yaml.v3" "github.com/apache/airflow/go-sdk/pkg/execution" ) func TestDecideMode(t *testing.T) { tests := []struct { - name string - metadata bool - comm string - logs string - want serveMode + name string + comm string + logs string + want serveMode }{ - {name: "metadata", metadata: true, want: modeAirflowMetadata}, {name: "coordinator", comm: "127.0.0.1:1", logs: "127.0.0.1:2", want: modeCoordinator}, {name: "no flags", want: modeCoordinatorUsageError}, {name: "comm only", comm: "127.0.0.1:1", want: modeCoordinatorUsageError}, @@ -52,7 +47,7 @@ func TestDecideMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, decideMode(tt.metadata, tt.comm, tt.logs)) + assert.Equal(t, tt.want, decideMode(tt.comm, tt.logs)) }) } } @@ -66,52 +61,6 @@ func etlBundle() *BundleRef { return b } -type manifest struct { - SDK struct { - Language string `json:"language" yaml:"language"` - } `json:"sdk" yaml:"sdk"` - Dags map[string]struct { - Tasks []string `json:"tasks" yaml:"tasks"` - } `json:"dags" yaml:"dags"` -} - -func TestServePrintsAirflowMetadata(t *testing.T) { - tests := []struct { - name string - args []string - unmarshal func([]byte, any) error - }{ - {name: "yaml by default", args: []string{"--airflow-metadata"}, unmarshal: yaml.Unmarshal}, - { - name: "json", - args: []string{"--airflow-metadata", "--format", "json"}, - unmarshal: json.Unmarshal, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var stdout bytes.Buffer - require.NoError(t, etlBundle().serve(tt.args, &stdout)) - - var got manifest - require.NoError(t, tt.unmarshal(stdout.Bytes(), &got)) - assert.Equal(t, "go", got.SDK.Language) - require.Contains(t, got.Dags, "py_etl") - assert.Equal(t, []string{"transform", "extract"}, got.Dags["py_etl"].Tasks) - }) - } -} - -type failingWriter struct{ err error } - -func (w failingWriter) Write([]byte) (int, error) { return 0, w.err } - -func TestServeReportsManifestWriteError(t *testing.T) { - wantErr := errors.New("stdout is closed") - err := etlBundle().serve([]string{"--airflow-metadata"}, failingWriter{wantErr}) - require.ErrorIs(t, err, wantErr) -} - func TestServeAcceptsFlagsTheBundleDefines(t *testing.T) { saved := flag.CommandLine t.Cleanup(func() { flag.CommandLine = saved }) @@ -119,15 +68,30 @@ func TestServeAcceptsFlagsTheBundleDefines(t *testing.T) { region := flag.String("region", "", "a flag the bundle author defined") var stdout bytes.Buffer - args := []string{"--region", "us", "--airflow-metadata"} - require.NoError(t, etlBundle().serve(args, &stdout)) + err := etlBundle().serve([]string{"--region", "us"}, &stdout) + // The flag parsed, so Serve got as far as asking for the coordinator flags. + require.ErrorIs(t, err, errCoordinatorFlagsRequired) assert.Equal(t, "us", *region) - assert.Contains(t, stdout.String(), "py_etl") +} + +// The introspection flags are gone, so a bundle may use their names for flags of its own. +func TestServeAcceptsBundleFlagsNamedLikeTheRemovedOnes(t *testing.T) { + saved := flag.CommandLine + t.Cleanup(func() { flag.CommandLine = saved }) + flag.CommandLine = flag.NewFlagSet("bundle", flag.ContinueOnError) + metadata := flag.Bool("airflow-metadata", false, "a flag the bundle author defined") + format := flag.String("format", "", "a flag the bundle author defined") + + err := etlBundle().serve([]string{"--airflow-metadata", "--format", "json"}, io.Discard) + + require.ErrorIs(t, err, errCoordinatorFlagsRequired) + assert.True(t, *metadata) + assert.Equal(t, "json", *format) } func TestServeRejectsBundleFlagWithReservedName(t *testing.T) { - for _, name := range []string{"airflow-metadata", "format", "comm", "logs"} { + for _, name := range []string{"comm", "logs"} { t.Run(name, func(t *testing.T) { saved := flag.CommandLine t.Cleanup(func() { flag.CommandLine = saved }) @@ -135,7 +99,7 @@ func TestServeRejectsBundleFlagWithReservedName(t *testing.T) { flag.String(name, "", "a flag the bundle author defined") var stdout bytes.Buffer - err := etlBundle().serve([]string{"--airflow-metadata"}, &stdout) + err := etlBundle().serve(nil, &stdout) require.EqualError(t, err, "the bundle defines a --"+name+" flag, but Serve reserves that name") @@ -144,24 +108,6 @@ func TestServeRejectsBundleFlagWithReservedName(t *testing.T) { } } -// pflag prints a deprecation warning while it parses, so it must not reach the writer that -// carries the --airflow-metadata manifest. -func TestServeKeepsDeprecationWarningOutOfTheManifest(t *testing.T) { - saved := flag.CommandLine - t.Cleanup(func() { flag.CommandLine = saved }) - flag.CommandLine = flag.NewFlagSet("bundle", flag.ContinueOnError) - flag.String("region", "", "a flag the bundle author defined") - require.NoError(t, flag.CommandLine.MarkDeprecated("region", "use --zone")) - - var stdout bytes.Buffer - args := []string{"--region", "us", "--airflow-metadata", "--format", "json"} - require.NoError(t, etlBundle().serve(args, &stdout)) - - assert.NotContains(t, stdout.String(), "deprecated") - var got manifest - require.NoError(t, json.Unmarshal(stdout.Bytes(), &got)) -} - func TestServeRejectsBadFlags(t *testing.T) { tests := []struct { name string @@ -176,19 +122,14 @@ func TestServeRejectsBadFlags(t *testing.T) { wantIs: errCoordinatorFlagsRequired, }, { - name: "format without metadata", - args: []string{"--comm", "127.0.0.1:1", "--logs", "127.0.0.1:2", "--format", "json"}, - wantIs: errFormatRequiresMetadata, - }, - { - name: "default format spelled out without metadata", - args: []string{"--comm", "127.0.0.1:1", "--logs", "127.0.0.1:2", "--format", "yaml"}, - wantIs: errFormatRequiresMetadata, + name: "the removed --airflow-metadata flag", + args: []string{"--airflow-metadata"}, + wantMsg: "unknown flag: --airflow-metadata", }, { - name: "unknown metadata format", - args: []string{"--airflow-metadata", "--format", "xml"}, - wantMsg: `unsupported --airflow-metadata format "xml"`, + name: "the removed --format flag", + args: []string{"--comm", "127.0.0.1:1", "--logs", "127.0.0.1:2", "--format", "json"}, + wantMsg: "unknown flag: --format", }, { name: "unknown flag", @@ -216,7 +157,10 @@ func TestServeHelpIsNotAnError(t *testing.T) { var stdout bytes.Buffer require.NoError(t, etlBundle().serve([]string{"--help"}, &stdout)) - assert.Contains(t, stdout.String(), "--airflow-metadata") + assert.Contains(t, stdout.String(), "--comm") + assert.Contains(t, stdout.String(), "--logs") + assert.NotContains(t, stdout.String(), "--airflow-metadata") + assert.NotContains(t, stdout.String(), "--format") } // A fake supervisor sends StartupDetails over the comm socket, as the Python diff --git a/go-sdk/cmd/airflow-go-pack/helpers_test.go b/go-sdk/cmd/airflow-go-pack/helpers_test.go new file mode 100644 index 0000000000000..f393312a73e30 --- /dev/null +++ b/go-sdk/cmd/airflow-go-pack/helpers_test.go @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +// testSDKModule is the module path of go-sdk, spelled out so tests do not rely on the packer's. +const testSDKModule = "github.com/apache/airflow/go-sdk" + +var ( + fixtureOnce sync.Once + fixtureDir string + fixturePath string + fixtureErr error +) + +func TestMain(m *testing.M) { + code := m.Run() + if fixtureDir != "" { + _ = os.RemoveAll(fixtureDir) + } + os.Exit(code) +} + +func requireGo(t *testing.T) { + t.Helper() + if testing.Short() { + t.Skip("shells out to the go toolchain") + } + if _, err := exec.LookPath("go"); err != nil { + t.Skip("go toolchain not on PATH") + } +} + +// bundleBinary returns a host binary built once per test run from testdata/failingbundle. It +// links go-sdk and exits with status 3 when it runs, so a test that packs it shows that the +// packer does not run it. +func bundleBinary(t *testing.T) string { + t.Helper() + requireGo(t) + fixtureOnce.Do(func() { + fixtureDir, fixtureErr = os.MkdirTemp("", "airflow-go-pack-test-*") + if fixtureErr != nil { + return + } + fixturePath = filepath.Join(fixtureDir, "failingbundle") + cmd := exec.Command("go", "build", "-o", fixturePath, "./testdata/failingbundle") + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + if out, err := cmd.CombinedOutput(); err != nil { + fixtureErr = fmt.Errorf("building testdata/failingbundle: %w\n%s", err, out) + } + }) + require.NoError(t, fixtureErr) + return fixturePath +} + +func readFile(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + return data +} + +// crossArchFor returns an architecture different from the host that the Go +// toolchain can target, or "" if we have no safe mapping for this host. +func crossArchFor(hostArch string) string { + switch hostArch { + case "amd64": + return "arm64" + case "arm64": + return "amd64" + default: + return "" + } +} + +func requireCrossArch(t *testing.T) string { + t.Helper() + requireGo(t) + crossArch := crossArchFor(runtime.GOARCH) + if crossArch == "" { + t.Skipf("no cross-arch mapping for host arch %q", runtime.GOARCH) + } + return crossArch +} diff --git a/go-sdk/cmd/airflow-go-pack/inspect_test.go b/go-sdk/cmd/airflow-go-pack/inspect_test.go index 5841f3d360e7a..0e1ea618ce8ea 100644 --- a/go-sdk/cmd/airflow-go-pack/inspect_test.go +++ b/go-sdk/cmd/airflow-go-pack/inspect_test.go @@ -30,17 +30,41 @@ import ( // inspect reads a bundle through bundlefooter.Read and prints the embedded // manifest, prefixing the source too under --source. func TestInspectCmd(t *testing.T) { - dir := t.TempDir() - exe := filepath.Join(dir, "input-bin") - require.NoError(t, os.WriteFile(exe, []byte("binary-bytes"), 0o755)) - source := []byte("package main\n\nfunc main() {}\n") manifest := []byte( "airflow_bundle_metadata_version: \"1.0\"\n" + + "sdk:\n" + + " language: \"go\"\n" + + " version: \"0.1.0\"\n" + + " supervisor_schema_version: \"2026-10-30\"\n" + + "source: \"main.go\"\n", + ) + inspectBundle(t, manifest) +} + +// A bundle packed before the manifest dropped its Dag inventory still prints, with the inventory +// as it was written. +func TestInspectCmd_BundleWithADagInventory(t *testing.T) { + manifest := []byte( + "airflow_bundle_metadata_version: \"1.0\"\n" + + "sdk:\n" + + " language: \"go\"\n" + + " version: \"0.1.0\"\n" + + " supervisor_schema_version: \"2026-06-16\"\n" + + "source: \"main.go\"\n" + "dags:\n" + " my_dag:\n" + " tasks:\n" + " - \"t1\"\n", ) + inspectBundle(t, manifest) +} + +func inspectBundle(t *testing.T, manifest []byte) { + t.Helper() + dir := t.TempDir() + exe := filepath.Join(dir, "input-bin") + require.NoError(t, os.WriteFile(exe, []byte("binary-bytes"), 0o755)) + source := []byte("package main\n\nfunc main() {}\n") bundle := filepath.Join(dir, "bundle") require.NoError(t, writeBundle(exe, bundle, source, fixedManifest(manifest))) diff --git a/go-sdk/cmd/airflow-go-pack/main.go b/go-sdk/cmd/airflow-go-pack/main.go index cc04b315ac42e..feede2588b076 100644 --- a/go-sdk/cmd/airflow-go-pack/main.go +++ b/go-sdk/cmd/airflow-go-pack/main.go @@ -16,9 +16,9 @@ // under the License. // Command airflow-go-pack builds a self-contained Airflow bundle from a Go -// package. It runs `go build`, exec's the freshly built binary with -// `--airflow-metadata` to obtain the manifest, and appends the source plus -// manifest plus AFBNDL01 trailer to the executable as specified by ADR 0004. +// package. It runs `go build`, reads the go-sdk version from the binary's build +// information, and appends the source plus manifest plus AFBNDL01 trailer to the +// executable as specified by ADR 0004. It never runs the binary. // // Usage: // @@ -50,42 +50,42 @@ func newRootCmd() *cobra.Command { root := &cobra.Command{ Use: "airflow-go-pack [package]", Short: "Build a self-contained Airflow bundle from a Go package", - Long: `airflow-go-pack builds a Go bundle binary, queries it for its DAG/task -identity via --airflow-metadata, and appends the source plus an -airflow-metadata.yaml manifest plus an AFBNDL01 trailer to the -executable. The result is a single self-contained file that drops into -[executable] bundles_folder. + Long: `airflow-go-pack builds a Go bundle binary and appends the source plus an +airflow-metadata.yaml manifest plus an AFBNDL01 trailer to the executable. +The result is a single self-contained file that drops into the Dag bundle +a Go coordinator reads. + +The packer never runs the binary. It records the go-sdk version the binary +was built against, read from the binary's build information, and refuses a +binary built against a different go-sdk than the packer itself. Run it as +"go tool airflow-go-pack" from the module that builds the bundle, so both +use that module's go-sdk. By default the packer builds the package in the current directory. Pass a different package as the positional argument; pass extra go build flags after a "--" separator. ---executable expects a binary that runs on this host (same OS/arch). To -build a bundle for a different platform you have two options: run the -packer with --goos/--goarch so it cross-builds the deployable artefact -while building a host-arch binary (forwarding your -- build flags) solely -to read the manifest; or pack a pre-built cross binary with --executable -and supply its manifest via --airflow-metadata, captured by running the -binary on its native platform (mybundle --airflow-metadata > meta.yaml). +--executable packs a binary you built yourself, for any platform, as it is. +To build for a different platform, pass --goos/--goarch so the packer +cross-builds the bundle, or cross-build the binary yourself and pack it with +--executable. Use --goos/--goarch rather than the GOOS/GOARCH env vars: under "go tool airflow-go-pack" those env vars cross-build the packer itself, -which then cannot exec on the host. +which then cannot run on the host. Examples: go tool airflow-go-pack go tool airflow-go-pack ./cmd/my-bundle -- -trimpath -tags=prod go tool airflow-go-pack --executable ./build/example --source main.go - # Cross-platform via the build path: cross-build + host introspection. + # Cross-platform via the build path. go tool airflow-go-pack --goos linux --goarch amd64 ./cmd/my-bundle -- -trimpath - # Cross-platform via a pre-built binary: pack it with its captured manifest. + # Cross-platform via a pre-built binary. GOOS=linux GOARCH=arm64 go build -o ./build/example-arm64 ./cmd/my-bundle - go build -o ./build/example-on-native-host ./cmd/my-bundle - ./build/example-on-native-host --airflow-metadata > meta.yaml go tool airflow-go-pack --executable ./build/example-arm64 \ - --source ./cmd/my-bundle/main.go --airflow-metadata meta.yaml + --source ./cmd/my-bundle/main.go `, // Only count args BEFORE "--" toward the positional limit; args // after "--" are forwarded verbatim to `go build` and must not @@ -128,10 +128,6 @@ Examples: root.Flags().StringVar(&opts.output, "output", "", "output bundle path (defaults to ./)") - root.Flags().StringVar(&opts.airflowMetadata, "airflow-metadata", - "", - "path to a pre-captured --airflow-metadata manifest (JSON or YAML); skips "+ - "introspecting the binary") root.Flags().StringVar(&opts.goos, "goos", "", "target GOOS for the bundle (cross-compile); prefer this over the GOOS env "+ diff --git a/go-sdk/cmd/airflow-go-pack/pack.go b/go-sdk/cmd/airflow-go-pack/pack.go index 23d542832d5bb..941e97c22b0dd 100644 --- a/go-sdk/cmd/airflow-go-pack/pack.go +++ b/go-sdk/cmd/airflow-go-pack/pack.go @@ -21,7 +21,6 @@ import ( "bytes" "crypto/sha256" "encoding/hex" - "errors" "fmt" "go/ast" "go/parser" @@ -31,7 +30,6 @@ import ( "os/exec" "path/filepath" "runtime" - "sort" "strings" "gopkg.in/yaml.v3" @@ -42,14 +40,13 @@ import ( // packOptions are the flags accepted by the root pack command. type packOptions struct { - pkg string // target package (default ".") - source string // override the auto-detected DAG source file - executable string // pack a pre-built binary instead of building - output string // override the default output path - airflowMetadata string // path to a pre-captured --airflow-metadata manifest (JSON or YAML) - goos string // target GOOS for the deployable build (falls back to env GOOS, then host) - goarch string // target GOARCH for the deployable build (falls back to env GOARCH, then host) - buildArgs []string // forwarded verbatim to `go build` (already includes the leading "--") + pkg string // target package (default ".") + source string // override the auto-detected DAG source file + executable string // pack a pre-built binary instead of building + output string // override the default output path + goos string // target GOOS for the deployable build (falls back to env GOOS, then host) + goarch string // target GOARCH for the deployable build (falls back to env GOARCH, then host) + buildArgs []string // forwarded verbatim to `go build` (already includes the leading "--") } func runPack(stdout, stderr io.Writer, opts *packOptions) error { @@ -113,18 +110,14 @@ func runPack(stdout, stderr io.Writer, opts *packOptions) error { ) } - // execPath is the binary that receives the footer (the deployable artefact, - // which MAY be cross-compiled). introspectPath is the binary obtainMetadata - // reads --airflow-metadata from. By default that means exec'ing it on the - // host (so it must be host-runnable, hence the cross-compile sidecar below), - // but --airflow-metadata bypasses it entirely. - var execPath, introspectPath string + // execPath is the binary that receives the footer: the deployable artefact, which may be + // cross-compiled. The packer never runs it. + var execPath string cleanupExec := func() {} defer func() { cleanupExec() }() if opts.executable != "" { execPath = opts.executable - introspectPath = opts.executable } else { targetGOOS, targetGOARCH := targetPlatform(opts) artifact, cleanup, err := buildPackage(stderr, opts.pkg, opts.buildArgs, targetGOOS, targetGOARCH) @@ -133,47 +126,24 @@ func runPack(stdout, stderr io.Writer, opts *packOptions) error { } execPath = artifact cleanupExec = cleanup - introspectPath = artifact - - // Reading the manifest means exec'ing the binary, so it must be a - // host-native build. When cross-compiling, the artefact cannot run - // here; build a throwaway host binary from the same sources and the - // same forwarded `--` build flags (DAG/task identity is arch-independent) - // solely to introspect. This sidecar is unnecessary when - // --airflow-metadata supplies the manifest directly. - crossCompiling := targetGOOS != runtime.GOOS || targetGOARCH != runtime.GOARCH - if crossCompiling && opts.airflowMetadata == "" { - hostBin, cleanupHost, err := buildPackage(stderr, opts.pkg, opts.buildArgs, runtime.GOOS, runtime.GOARCH) - if err != nil { - return fmt.Errorf("building host binary for metadata introspection: %w", err) - } - prevCleanup := cleanupExec - cleanupExec = func() { cleanupHost(); prevCleanup() } - introspectPath = hostBin - } } if _, err := os.Stat(execPath); err != nil { return fmt.Errorf("executable %s: %w", execPath, err) } - if err := rejectOutputAlias(output, execPath, sourcePath, opts.airflowMetadata); err != nil { + if err := rejectOutputAlias(output, execPath, sourcePath); err != nil { return err } - meta, err := obtainMetadata(opts, introspectPath) + sdk, err := readSDK(execPath) if err != nil { return err } - if len(meta.Dags) == 0 { - return fmt.Errorf("bundle exposes no dags: nothing to pack") - } - for dagID, dag := range meta.Dags { - if len(dag.Tasks) == 0 { - fmt.Fprintf(stderr, "warning: dag %q has no tasks\n", dagID) - } + meta := airflowmetadata.Manifest{ + AirflowBundleMetadataVersion: airflowmetadata.FormatVersion, + SDK: sdk, } - warnOnSuspiciousIDs(stderr, meta) sourceBytes, err := os.ReadFile(sourcePath) if err != nil { @@ -206,8 +176,13 @@ func runPack(stdout, stderr io.Writer, opts *packOptions) error { return err } - fmt.Fprintf(stdout, "Wrote bundle %s (sdk=%s/%s, dags=%d)\n", - output, meta.SDK.Language, meta.SDK.Version, len(meta.Dags)) + fmt.Fprintf( + stdout, + "Wrote bundle %s (sdk=%s/%s)\n", + output, + meta.SDK.Language, + meta.SDK.Version, + ) return nil } @@ -337,9 +312,8 @@ func targetPlatform(opts *packOptions) (goos, goarch string) { // given GOOS/GOARCH and returns the path to the freshly built executable plus a // cleanup function. extraArgs is the slice that comes after the "--" separator // on the airflow-go-pack command line; we drop the leading "--" before -// forwarding. GOOS/GOARCH are set explicitly (overriding any ambient env) so -// the caller controls the target: the deployable build uses the resolved target -// platform, the introspection sidecar uses the host. +// forwarding. GOOS/GOARCH are set explicitly, overriding any ambient env, so +// the caller controls the target platform. func buildPackage( stderr io.Writer, pkg string, @@ -380,104 +354,6 @@ func buildPackage( return outPath, cleanup, nil } -func readAirflowMetadata(execPath string) (airflowmetadata.Manifest, error) { - out, err := runIntrospect(execPath, "--airflow-metadata") - if err != nil { - return airflowmetadata.Manifest{}, err - } - // Decode with a YAML decoder: it reads the binary's YAML default and its - // --format json output alike (JSON is a subset of YAML). - var meta airflowmetadata.Manifest - if err := yaml.Unmarshal(out, &meta); err != nil { - return airflowmetadata.Manifest{}, fmt.Errorf( - "decoding --airflow-metadata output (YAML/JSON): %w", - err, - ) - } - return meta, nil -} - -// obtainMetadata resolves the bundle manifest either from an explicit -// --airflow-metadata file or by exec'ing a host-runnable introspection binary. -// In --executable mode a binary that cannot be exec'd on the host is a hard -// error with remediation guidance: --executable expects a same-platform binary, -// and the packer never silently rebuilds a host binary, because a rebuild from -// unknown inputs (the original build tags, ldflags, and GOOS/GOARCH-specific -// files are not known here, and build flags are rejected in --executable mode) -// can advertise a different DAG/task set than the artefact actually shipped. -func obtainMetadata(opts *packOptions, introspectPath string) (airflowmetadata.Manifest, error) { - if opts.airflowMetadata != "" { - meta, err := readMetadataFile(opts.airflowMetadata) - if err != nil { - return airflowmetadata.Manifest{}, fmt.Errorf( - "--airflow-metadata %s: %w", - opts.airflowMetadata, - err, - ) - } - return meta, nil - } - - meta, err := readAirflowMetadata(introspectPath) - if err == nil { - return meta, nil - } - if opts.executable != "" && errors.Is(err, errExecNotStartable) { - return airflowmetadata.Manifest{}, fmt.Errorf( - "cannot exec --executable %q on %s/%s to read its --airflow-metadata: %w\n"+ - "--executable expects a binary that runs on this host. To pack a binary for a\n"+ - "different platform, drop --executable and let the packer cross-build instead:\n"+ - " airflow-go-pack --goos --goarch ./path/to/pkg [-- ]\n"+ - "(the packer builds a host-arch binary, forwarding your -- build flags, solely to\n"+ - "read the manifest). Alternatively pass --airflow-metadata with the manifest captured\n"+ - "from the binary on its native platform: %s --airflow-metadata > airflow-metadata.yaml", - opts.executable, runtime.GOOS, runtime.GOARCH, err, opts.executable, - ) - } - return airflowmetadata.Manifest{}, fmt.Errorf("--airflow-metadata: %w", err) -} - -// readMetadataFile parses a manifest from a pre-captured --airflow-metadata -// file. It accepts both the JSON a bundle binary prints (via -// `mybundle --airflow-metadata`) and the airflow-metadata.yaml embedded in an -// existing bundle: YAML is a superset of JSON, so a YAML decoder reads either. -func readMetadataFile(path string) (airflowmetadata.Manifest, error) { - data, err := os.ReadFile(path) - if err != nil { - return airflowmetadata.Manifest{}, err - } - var meta airflowmetadata.Manifest - if err := yaml.Unmarshal(data, &meta); err != nil { - return airflowmetadata.Manifest{}, fmt.Errorf("decoding metadata (YAML/JSON): %w", err) - } - return meta, nil -} - -// errExecNotStartable marks an introspection failure where the process never -// ran — typically the binary was built for a different CPU arch / OS, so the -// OS rejected the exec (e.g. "exec format error", "bad CPU type"). It is -// distinct from the binary running and exiting non-zero (an *exec.ExitError), -// which signals a genuine --airflow-metadata failure rather than an -// unrunnable binary. -var errExecNotStartable = errors.New("introspection binary could not be exec'd") - -func runIntrospect(execPath string, flag string) ([]byte, error) { - cmd := exec.Command(execPath, flag) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - var exitErr *exec.ExitError - if !errors.As(err, &exitErr) { - // The process did not start (no exit status). Wrap with the - // sentinel so callers can decide whether to fall back to a build. - return nil, fmt.Errorf("%w: %s %s: %v", errExecNotStartable, execPath, flag, err) - } - return nil, fmt.Errorf("%s %s: %w: %s", execPath, flag, err, stderr.String()) - } - return stdout.Bytes(), nil -} - // cacheDigestDomain starts the cache digest's input, so the digest cannot equal // a SHA-256 taken of the same bytes for another purpose, and a later definition // can change the version. @@ -514,12 +390,10 @@ func computeDigests(execPath string, source, baseManifest []byte) (bundleDigests }, nil } -// renderManifest serialises the airflow-metadata manifest as deterministic, -// sorted-key YAML matching airflow-metadata.schema.json. It injects the schema's -// source field (the filename the manifest is built from) and, when digests is -// not nil, the digests mapping, both of which the producer's Manifest omits -// because only the packer knows them; every other field is copied from the -// introspected manifest verbatim. +// renderManifest serialises the airflow-metadata manifest as deterministic YAML matching +// airflow-metadata.schema.json. It injects the schema's source field (the filename the manifest is +// built from) and, when digests is not nil, the digests mapping, both of which only the packer +// knows. func renderManifest( meta airflowmetadata.Manifest, sourceName string, @@ -530,31 +404,6 @@ func renderManifest( version = airflowmetadata.FormatVersion } - dagIDs := make([]string, 0, len(meta.Dags)) - for id := range meta.Dags { - dagIDs = append(dagIDs, id) - } - sort.Strings(dagIDs) - - dagsNode := &yaml.Node{Kind: yaml.MappingNode} - for _, id := range dagIDs { - tasks := meta.Dags[id].Tasks - taskItems := make([]*yaml.Node, 0, len(tasks)) - for _, t := range tasks { - taskItems = append(taskItems, quotedScalar(t)) - } - dagsNode.Content = append(dagsNode.Content, - scalar(id), - &yaml.Node{ - Kind: yaml.MappingNode, - Content: []*yaml.Node{ - scalar("tasks"), - {Kind: yaml.SequenceNode, Content: taskItems}, - }, - }, - ) - } - root := &yaml.Node{Kind: yaml.DocumentNode} manifest := &yaml.Node{ Kind: yaml.MappingNode, @@ -585,7 +434,6 @@ func renderManifest( }, ) } - manifest.Content = append(manifest.Content, scalar("dags"), dagsNode) root.Content = []*yaml.Node{manifest} var buf bytes.Buffer @@ -600,37 +448,32 @@ func renderManifest( return buf.Bytes(), nil } -// scalar emits a plain (unquoted) node. It is used for structural keys -// (e.g. "sdk", "tasks") and for the Dag ID mapping keys. +// scalar emits a plain (unquoted) node. It is used for the structural keys, +// such as "sdk" and "digests". func scalar(value string) *yaml.Node { return &yaml.Node{Kind: yaml.ScalarNode, Value: value} } -// quotedScalar emits a double-quoted node. Data-bearing string *values* — task -// IDs, the source filename, and the SDK fields — go through this so a value -// that looks like a number, bool, or date (e.g. a task named "123" or "true") -// round-trips as a string rather than being retyped by the YAML parser. +// quotedScalar emits a double-quoted node. Data-bearing string values, such as +// the source filename and the SDK fields, go through this so a value that looks +// like a number, bool, or date (e.g. a version of "1.0") round-trips as a string +// rather than being retyped by the YAML parser. func quotedScalar(value string) *yaml.Node { return &yaml.Node{Kind: yaml.ScalarNode, Value: value, Style: yaml.DoubleQuotedStyle} } // rejectOutputAlias fails if output resolves to the same file as any pack -// input: the executable, the source, or a supplied --airflow-metadata file. -// Packing copies the executable to output with O_TRUNC and renames it into -// place, so an aliased output would clobber the input. metadataPath is empty -// when --airflow-metadata is not used and is skipped in that case. -func rejectOutputAlias(output, execPath, sourcePath, metadataPath string) error { +// input: the executable or the source. Packing copies the executable to output +// with O_TRUNC and renames it into place, so an aliased output would clobber +// the input. +func rejectOutputAlias(output, execPath, sourcePath string) error { for _, in := range []struct { path string kind string }{ {execPath, "executable"}, {sourcePath, "source"}, - {metadataPath, "--airflow-metadata file"}, } { - if in.path == "" { - continue - } alias, err := sameFile(output, in.path) if err != nil { return fmt.Errorf("resolving output path %s: %w", output, err) diff --git a/go-sdk/cmd/airflow-go-pack/pack_integration_test.go b/go-sdk/cmd/airflow-go-pack/pack_integration_test.go index 1d75ab6c41936..971016a2c4028 100644 --- a/go-sdk/cmd/airflow-go-pack/pack_integration_test.go +++ b/go-sdk/cmd/airflow-go-pack/pack_integration_test.go @@ -21,239 +21,368 @@ import ( "bytes" "crypto/sha256" "encoding/hex" - "encoding/json" "io" "os" "os/exec" "path/filepath" "regexp" "runtime" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" - "github.com/apache/airflow/go-sdk/internal/airflowmetadata" "github.com/apache/airflow/go-sdk/internal/bundlefooter" "github.com/apache/airflow/go-sdk/pkg/execution" ) -// crossArchFor returns an architecture different from the host that the Go -// toolchain can target, or "" if we have no safe mapping for this host. -func crossArchFor(hostArch string) string { - switch hostArch { - case "amd64": - return "arm64" - case "arm64": - return "amd64" - default: - return "" - } -} - -// End-to-end cross-arch --executable test: a binary built for an arch the host -// cannot run is packed into a spec-conforming bundle with its binary region -// preserved byte-for-byte. The caller supplies the artefact's own -// --airflow-metadata output (captured here from a host build of the same -// sources) rather than the packer rebuilding to guess the metadata. -func TestPack_CrossArchExecutableWithMetadataFile(t *testing.T) { - if testing.Short() { - t.Skip("cross-arch pack test shells out to `go build` twice") - } - if _, err := exec.LookPath("go"); err != nil { - t.Skip("go toolchain not on PATH") - } - crossArch := crossArchFor(runtime.GOARCH) - if crossArch == "" { - t.Skipf("no cross-arch mapping for host arch %q", runtime.GOARCH) - } - - // The example bundle is a real bundle that answers --airflow-metadata, - // so it exercises the genuine metadata path. - exampleDir, err := filepath.Abs(filepath.Join("..", "..", "example", "bundle")) - require.NoError(t, err) - sourceFile := filepath.Join(exampleDir, "main.go") - if _, err := os.Stat(sourceFile); err != nil { - t.Skipf("example bundle source not found: %v", err) - } - - tmp := t.TempDir() - crossBin := filepath.Join(tmp, "prebuilt_cross") - hostBin := filepath.Join(tmp, "prebuilt_host") - - // Build the example for a foreign arch (the --executable input) and for - // the host. CGO is disabled so the cross build needs no C toolchain. - goBuild(t, exampleDir, crossBin, runtime.GOOS, crossArch) - goBuild(t, exampleDir, hostBin, runtime.GOOS, runtime.GOARCH) - - crossBytes, err := os.ReadFile(crossBin) - require.NoError(t, err) - hostBytes, err := os.ReadFile(hostBin) - require.NoError(t, err) - require.False(t, bytes.Equal(crossBytes, hostBytes), - "cross and host builds should differ; cross-compile may not have taken effect") - - // Capture the artefact's own --airflow-metadata JSON from the host build, - // standing in for the author running the binary on its native platform. - metaJSON := filepath.Join(tmp, "airflow-metadata.json") - captureMetadata(t, hostBin, metaJSON) - - // Pack the foreign-arch executable through the real CLI command, feeding - // the captured metadata so no host rebuild is needed. - outPath := filepath.Join(tmp, "bundle") +// packBundleFixture packs the fixture bundle package through the real CLI command, as a user would, and +// returns the bundle's source, manifest and bytes. It reports the packer's output on failure. +func packBundleFixture(t *testing.T, args ...string) (source, manifest, bundleBytes []byte) { + t.Helper() + outPath := filepath.Join(t.TempDir(), "bundle") + var stderr bytes.Buffer cmd := newRootCmd() - cmd.SetArgs([]string{ - "--executable", crossBin, - "--source", sourceFile, - "--airflow-metadata", metaJSON, - "--output", outPath, - }) - cmd.SetOut(&bytes.Buffer{}) - cmd.SetErr(&bytes.Buffer{}) - require.NoError(t, cmd.Execute()) - - bundleBytes, err := os.ReadFile(outPath) - require.NoError(t, err) - - // Read parses the trailer and verifies binary_sha256 over the binary - // region; success means the bundle is spec-valid. - source, metadata, err := bundlefooter.Read(outPath) + cmd.SetArgs(append([]string{"--output", outPath}, args...)) + cmd.SetOut(&stderr) + cmd.SetErr(&stderr) + require.NoError(t, cmd.Execute(), stderr.String()) + + // Read parses the trailer and verifies binary_sha256 over the binary region; success means + // the bundle is spec-valid. + source, manifest, err := bundlefooter.Read(outPath) require.NoError(t, err) + return source, manifest, readFile(t, outPath) +} - srcBytes, err := os.ReadFile(sourceFile) - require.NoError(t, err) - assert.Equal(t, srcBytes, source, "embedded source must match --source bytes") - - // sdk.version is environment-dependent and not asserted verbatim: a plain - // `go build` from a local module tree leaves Main.Version unset and yields - // "(devel)", while Go 1.24's VCS stamping (e.g. in CI, building from the - // git checkout) yields a pseudo-version like v0.0.0--, - // and a tagged-release build yields a semver tag. Assert the version line - // matches an accepted form, then fold the observed value into the expected - // manifest so the remaining fields and ordering are checked exactly. - // supervisor_schema_version and the format version come from SDK constants - // and change only when those constants do. +// The manifest the packer writes for a binary it did not run: the SDK block comes from the +// binary's build information and the packer's own go-sdk, and there is no Dag inventory. +// +// sdk.version is environment-dependent and not asserted verbatim: a plain `go build` from a local +// module tree leaves Main.Version unset and yields "(devel)", while Go 1.24's VCS stamping (e.g. in +// CI, building from the git checkout) yields a pseudo-version like v0.0.0--, and +// a tagged-release build yields a semver tag. The test asserts the version matches an accepted +// form, then folds the observed value into the expected manifest so the remaining fields and +// ordering are checked exactly. +func assertManifestWithoutDags(t *testing.T, manifest, binaryRegion []byte, source string) { + t.Helper() versionLine := regexp.MustCompile(`(?m)^ version: "([^"]*)"$`) - m := versionLine.FindStringSubmatch(string(metadata)) - require.NotNil(t, m, "manifest must contain an sdk.version line:\n%s", metadata) - sdkVersion := m[1] - assert.Regexp(t, `^(\(devel\)|v[0-9].*)$`, sdkVersion, + m := versionLine.FindStringSubmatch(string(manifest)) + require.NotNil(t, m, "manifest must contain an sdk.version line:\n%s", manifest) + assert.Regexp(t, `^(\(devel\)|v[0-9].*)$`, m[1], `sdk.version must be "(devel)" or a v-prefixed module version`) - // The cache digest covers the environment-dependent binary and sdk.version - // too, so it is folded in the same way. + // The cache digest covers the environment-dependent binary and sdk.version too. cacheLine := regexp.MustCompile(`(?m)^ cache: "([0-9a-f]{64})"$`) - c := cacheLine.FindStringSubmatch(string(metadata)) - require.NotNil(t, c, "manifest must contain a digests.cache line:\n%s", metadata) - crossHash := sha256.Sum256(crossBytes) + c := cacheLine.FindStringSubmatch(string(manifest)) + require.NotNil(t, c, "manifest must contain a digests.cache line:\n%s", manifest) + binaryHash := sha256.Sum256(binaryRegion) - expectedManifest := `airflow_bundle_metadata_version: "1.0" + assert.Equal(t, `airflow_bundle_metadata_version: "1.0" sdk: language: "go" - version: "` + sdkVersion + `" - supervisor_schema_version: "` + execution.SupervisorSchemaVersion + `" -source: "main.go" + version: "`+m[1]+`" + supervisor_schema_version: "`+execution.SupervisorSchemaVersion+`" +source: "`+source+`" digests: - integrity: "` + hex.EncodeToString(crossHash[:]) + `" - cache: "` + c[1] + `" -dags: - concurrent_xcom_dag: - tasks: - - "pull_xcoms_concurrently" - simple_dag: - tasks: - - "extract" - - "transform" - - "load" - taskflow_binding_dag: - tasks: - - "make_config" - - "make_numbers" - - "make_region" - - "via_flat_args" - - "via_struct_no_tags" - - "via_struct_arg_tag" - - "via_struct_default_arg" - - "via_struct_more_args" - - "via_struct_fewer_args" - - "via_flat_map" - - "via_struct_map" - - "via_plain_map" - variable_write_dag: - tasks: - - "write_and_delete_variable" -` - assert.Equal(t, expectedManifest, string(metadata)) - - // The packed binary region must be exactly the foreign-arch executable: - // the captured metadata describes it, and the binary is never rebuilt. - binaryRegion := bundleBytes[:len(bundleBytes)-len(source)-len(metadata)-bundlefooter.TrailerSize] - assert.Equal(t, crossBytes, binaryRegion, - "packed binary region must be the foreign-arch --executable, not a host rebuild") + integrity: "`+hex.EncodeToString(binaryHash[:])+`" + cache: "`+c[1]+`" +`, string(manifest)) } -// End-to-end build-mode cross-compile (no --executable): with GOOS/GOARCH set, -// the packer builds the target-arch artefact and a host-arch binary (solely to -// read the manifest), forwarding the `--` go build flags. The packed binary -// must be the target-arch artefact built with the forwarded flags. -func TestPack_CrossCompileBuildModeForwardsFlags(t *testing.T) { - if testing.Short() { - t.Skip("cross-arch pack test shells out to `go build` twice") - } - if _, err := exec.LookPath("go"); err != nil { - t.Skip("go toolchain not on PATH") - } - crossArch := crossArchFor(runtime.GOARCH) - if crossArch == "" { - t.Skipf("no cross-arch mapping for host arch %q", runtime.GOARCH) - } +func binaryRegionOf(bundleBytes, source, manifest []byte) []byte { + return bundleBytes[:len(bundleBytes)-len(source)-len(manifest)-bundlefooter.TrailerSize] +} - exampleDir, err := filepath.Abs(filepath.Join("..", "..", "example", "bundle")) - require.NoError(t, err) - if _, err := os.Stat(filepath.Join(exampleDir, "main.go")); err != nil { - t.Skipf("example bundle source not found: %v", err) - } +// Packing a package whose binary exits non-zero succeeds, so the packer does not run it. +func TestPack_BuildsWithoutRunningTheBundle(t *testing.T) { + requireGo(t) + + source, manifest, bundleBytes := packBundleFixture(t, "./testdata/failingbundle") + + srcBytes := readFile(t, filepath.Join("testdata", "failingbundle", "main.go")) + assert.Equal(t, srcBytes, source, "embedded source must be the file with func main") + assertManifestWithoutDags(t, manifest, binaryRegionOf(bundleBytes, source, manifest), "main.go") +} - // Cross-compile via the environment, exactly as a user would. CGO is - // disabled so the cross build needs no C toolchain. +// A cross-arch build-mode pack needs no host build of the bundle: the artefact is the target-arch +// build with the forwarded flags, and the host-arch binary the packer used to run is gone. +func TestPack_CrossCompileBuildModeForwardsFlags(t *testing.T) { + crossArch := requireCrossArch(t) + // Cross-compile via the environment, exactly as a user would. CGO is disabled so the cross + // build needs no C toolchain. t.Setenv("GOOS", runtime.GOOS) t.Setenv("GOARCH", crossArch) t.Setenv("CGO_ENABLED", "0") - tmp := t.TempDir() - outPath := filepath.Join(tmp, "bundle") - - // Pack the example package (no --executable, no --source), forwarding - // -trimpath after the "--" separator to the internal go build. - cmd := newRootCmd() - cmd.SetArgs([]string{exampleDir, "--output", outPath, "--", "-trimpath"}) - cmd.SetOut(&bytes.Buffer{}) - cmd.SetErr(&bytes.Buffer{}) - require.NoError(t, cmd.Execute()) - - source, metadata, err := bundlefooter.Read(outPath) - require.NoError(t, err) - assert.Contains(t, string(metadata), "simple_dag:", - "manifest must be read from the host introspection build") - - // Independently build the target-arch artefact with the same forwarded - // flag; the packed binary region must match it byte-for-byte, proving the - // deployable artefact is the cross build (not the host introspection one) - // and that -trimpath was forwarded. - wantBin := filepath.Join(tmp, "want_cross") - build := exec.Command("go", "build", "-trimpath", "-o", wantBin, exampleDir) + source, manifest, bundleBytes := packBundleFixture( + t, + "./testdata/failingbundle", + "--", + "-trimpath", + ) + + // Independently build the target-arch artefact with the same forwarded flag; the packed binary + // region must match it byte-for-byte, proving the deployable artefact is the cross build and + // that -trimpath was forwarded. + wantBin := filepath.Join(t.TempDir(), "want_cross") + build := exec.Command("go", "build", "-trimpath", "-o", wantBin, "./testdata/failingbundle") build.Env = append(os.Environ(), "GOOS="+runtime.GOOS, "GOARCH="+crossArch, "CGO_ENABLED=0") if combined, berr := build.CombinedOutput(); berr != nil { t.Fatalf("reference cross build failed: %v\n%s", berr, combined) } - wantBytes, err := os.ReadFile(wantBin) - require.NoError(t, err) + assert.Equal(t, readFile(t, wantBin), binaryRegionOf(bundleBytes, source, manifest), + "packed binary must be the cross-built artefact with -trimpath forwarded") + assertManifestWithoutDags(t, manifest, readFile(t, wantBin), "main.go") +} + +// A cross-arch --executable pack needs no manifest and no host build: a binary built for an +// architecture the host cannot run is packed with its binary region preserved byte-for-byte, and +// the SDK version is read from the binary. +func TestPack_CrossArchExecutable(t *testing.T) { + crossArch := requireCrossArch(t) + crossBin := filepath.Join(t.TempDir(), "prebuilt_cross") + goBuild(t, "./testdata/failingbundle", crossBin, runtime.GOOS, crossArch) + require.NotEqual(t, readFile(t, bundleBinary(t)), readFile(t, crossBin), + "cross and host builds should differ; cross-compile may not have taken effect") + sourceFile := filepath.Join("testdata", "failingbundle", "main.go") + + source, manifest, bundleBytes := packBundleFixture( + t, + "--executable", + crossBin, + "--source", + sourceFile, + ) + + assert.Equal(t, readFile(t, sourceFile), source, "embedded source must match --source bytes") + binaryRegion := binaryRegionOf(bundleBytes, source, manifest) + assert.Equal(t, readFile(t, crossBin), binaryRegion, + "packed binary region must be the foreign-arch --executable, not a rebuild") + assertManifestWithoutDags(t, manifest, binaryRegion, "main.go") +} + +// Build information survives a stripped binary, so the version can still be read. +func TestPack_StrippedBinary(t *testing.T) { + requireGo(t) + + source, manifest, bundleBytes := packBundleFixture( + t, "./testdata/failingbundle", "--", "-ldflags=-s -w", + ) + + // The packed binary must be the stripped build, so the flag reached `go build` and the + // version was read from a binary without a symbol table. + wantBin := filepath.Join(t.TempDir(), "want_stripped") + build := exec.Command( + "go", "build", "-ldflags=-s -w", "-o", wantBin, "./testdata/failingbundle", + ) + if combined, berr := build.CombinedOutput(); berr != nil { + t.Fatalf("reference stripped build failed: %v\n%s", berr, combined) + } + binaryRegion := binaryRegionOf(bundleBytes, source, manifest) + assert.Equal(t, readFile(t, wantBin), binaryRegion, + "packed binary must be the stripped build with -ldflags forwarded") + assert.NotEqual(t, readFile(t, bundleBinary(t)), binaryRegion, + "packed binary must differ from the unstripped build") + assertManifestWithoutDags(t, manifest, binaryRegion, "main.go") +} + +// A Go binary that does not link go-sdk is not a bundle. +func TestRunPack_RejectsBinaryWithoutSDK(t *testing.T) { + requireGo(t) + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "go.mod"), + []byte("module nosdk\n\ngo 1.24\n"), 0o644)) + source := filepath.Join(dir, "main.go") + require.NoError(t, os.WriteFile(source, []byte("package main\n\nfunc main() {}\n"), 0o644)) + exe := filepath.Join(dir, "nosdk") + build := exec.Command("go", "build", "-o", exe, ".") + build.Dir = dir + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("building the binary failed: %v\n%s", err, out) + } + out := filepath.Join(dir, "bundle") + + err := runPack( + io.Discard, + io.Discard, + &packOptions{executable: exe, source: source, output: out}, + ) + + require.Error(t, err) + assert.Contains(t, err.Error(), "does not link github.com/apache/airflow/go-sdk") + _, statErr := os.Stat(out) + assert.True(t, os.IsNotExist(statErr), "no bundle should be written") +} - bundleBytes, err := os.ReadFile(outPath) +// writeBundleModule writes a module that requires go-sdk at sdkVersion, resolved to this checkout +// of go-sdk through a replace directive, with airflow-go-pack as a tool and a bundle package that +// links go-sdk. The module copies the SDK's requirements and checksums, so it builds offline. +func writeBundleModule(t *testing.T, dir, name, sdkVersion string) { + t.Helper() + sdkDir, err := filepath.Abs(filepath.Join("..", "..")) require.NoError(t, err) - binaryRegion := bundleBytes[:len(bundleBytes)-len(source)-len(metadata)-bundlefooter.TrailerSize] - assert.Equal(t, wantBytes, binaryRegion, - "packed binary must be the cross-built artefact with -trimpath forwarded") + goMod := strings.Replace( + string(readFile(t, filepath.Join(sdkDir, "go.mod"))), + "module "+testSDKModule, "module example.com/"+name, 1, + ) + goMod += "\nrequire " + testSDKModule + " " + sdkVersion + "\n\nreplace " + testSDKModule + " => " + + filepath.ToSlash( + sdkDir, + ) + "\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "go.mod"), []byte(goMod), 0o644)) + goSum := readFile(t, filepath.Join(sdkDir, "go.sum")) + require.NoError(t, os.WriteFile(filepath.Join(dir, "go.sum"), goSum, 0o644)) + mainGo := readFile(t, filepath.Join("testdata", "failingbundle", "main.go")) + require.NoError(t, os.WriteFile(filepath.Join(dir, "main.go"), mainGo, 0o644)) +} + +// goInModule runs go in dir without network access and returns its combined output. +func goInModule(t *testing.T, dir string, args ...string) (string, error) { + t.Helper() + cmd := exec.Command("go", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GOPROXY=off", "GOFLAGS=-mod=readonly", "CGO_ENABLED=0") + out, err := cmd.CombinedOutput() + return string(out), err +} + +// The packer and the bundle binary must be built against the same go-sdk. This runs the packer as +// a user does, with `go tool` from modules that resolve go-sdk through a replace directive. +func TestPack_TwoModules(t *testing.T) { + requireGo(t) + packerModule := t.TempDir() + writeBundleModule(t, packerModule, "packer", "v0.0.0") + + t.Run( + "a module that replaces go-sdk with a local path packs with its own go tool", + func(t *testing.T) { + out := filepath.Join(t.TempDir(), "bundle") + + output, err := goInModule( + t, + packerModule, + "tool", + "airflow-go-pack", + "--output", + out, + ".", + ) + + require.NoError(t, err, output) + assert.Contains(t, output, "Wrote bundle "+out+" (sdk=go/(devel))\n") + _, manifest, err := bundlefooter.Read(out) + require.NoError(t, err) + // A replacement by a local path has the version "(devel)", which is what the manifest records. + assert.Contains(t, string(manifest), ` version: "(devel)"`) + assert.Contains( + t, + string(manifest), + ` supervisor_schema_version: "`+execution.SupervisorSchemaVersion+`"`, + ) + }, + ) + + t.Run("a binary built against another go-sdk is refused", func(t *testing.T) { + otherModule := t.TempDir() + writeBundleModule(t, otherModule, "other", "v0.0.1") + binary := filepath.Join(otherModule, "other") + output, err := goInModule(t, otherModule, "build", "-o", binary, ".") + require.NoError(t, err, output) + out := filepath.Join(t.TempDir(), "bundle") + + output, err = goInModule( + t, + packerModule, + "tool", + "airflow-go-pack", + "--executable", + binary, + "--source", + filepath.Join(otherModule, "main.go"), + "--output", + out, + ) + + require.Error(t, err) + assert.Contains(t, output, "built against go-sdk v0.0.1 replaced by ") + assert.Contains(t, output, "airflow-go-pack was built against go-sdk v0.0.0 replaced by ") + assert.Contains(t, output, "go tool airflow-go-pack") + _, statErr := os.Stat(out) + assert.True(t, os.IsNotExist(statErr), "no bundle should be written") + }) + + t.Run("a binary built in go-sdk is refused by a replacing packer", func(t *testing.T) { + // go-sdk is the main module of the binary and a replaced dependency of the packer, so + // the comparison runs. -buildvcs=false keeps the main module's version "(devel)", + // whatever the checkout's version control says. + sdkDir, err := filepath.Abs(filepath.Join("..", "..")) + require.NoError(t, err) + fixtureDir := filepath.Join(sdkDir, "cmd", "airflow-go-pack", "testdata", "failingbundle") + binary := filepath.Join(t.TempDir(), "inside") + output, err := goInModule(t, sdkDir, "build", "-buildvcs=false", "-o", binary, fixtureDir) + require.NoError(t, err, output) + out := filepath.Join(t.TempDir(), "bundle") + + output, err = goInModule( + t, + packerModule, + "tool", + "airflow-go-pack", + "--executable", + binary, + "--source", + filepath.Join(fixtureDir, "main.go"), + "--output", + out, + ) + + require.Error(t, err) + assert.Contains(t, output, "built against go-sdk (devel) (main module)") + assert.Contains(t, output, "airflow-go-pack was built against go-sdk v0.0.0 replaced by ") + _, statErr := os.Stat(out) + assert.True(t, os.IsNotExist(statErr), "no bundle should be written") + }) + + t.Run("a build flag that changes the module graph is refused", func(t *testing.T) { + // -modfile makes the build resolve go-sdk through another go.mod, as a user's flag after + // "--" could. + other := strings.Replace( + string(readFile(t, filepath.Join(packerModule, "go.mod"))), + "require "+testSDKModule+" v0.0.0", "require "+testSDKModule+" v0.0.1", 1, + ) + require.NoError( + t, + os.WriteFile(filepath.Join(packerModule, "other.mod"), []byte(other), 0o644), + ) + require.NoError(t, os.WriteFile( + filepath.Join( + packerModule, + "other.sum", + ), + readFile(t, filepath.Join(packerModule, "go.sum")), + 0o644, + )) + out := filepath.Join(t.TempDir(), "bundle") + + output, err := goInModule( + t, + packerModule, + "tool", + "airflow-go-pack", + "--output", + out, + ".", + "--", + "-modfile=other.mod", + ) + + require.Error(t, err) + assert.Contains(t, output, "built against go-sdk v0.0.1 replaced by ") + assert.Contains(t, output, "airflow-go-pack was built against go-sdk v0.0.0 replaced by ") + _, statErr := os.Stat(out) + assert.True(t, os.IsNotExist(statErr), "no bundle should be written") + }) } // When --source is supplied, the packer skips source discovery, so a package @@ -296,60 +425,6 @@ func TestRunPack_SourceBypassesDiscovery(t *testing.T) { "packing should proceed to the build step when --source is given") } -// Checks the bundle binary's --airflow-metadata encodings: default is YAML, -// --format json emits JSON, both decode to the same manifest, and --format -// without --airflow-metadata is a hard error. -func TestBundleBinary_AirflowMetadataFormats(t *testing.T) { - if testing.Short() { - t.Skip("shells out to `go build`") - } - if _, err := exec.LookPath("go"); err != nil { - t.Skip("go toolchain not on PATH") - } - - exampleDir, err := filepath.Abs(filepath.Join("..", "..", "example", "bundle")) - require.NoError(t, err) - if _, err := os.Stat(filepath.Join(exampleDir, "main.go")); err != nil { - t.Skipf("example bundle source not found: %v", err) - } - - hostBin := filepath.Join(t.TempDir(), "bundle") - goBuild(t, exampleDir, hostBin, runtime.GOOS, runtime.GOARCH) - - assertManifest := func(t *testing.T, meta airflowmetadata.Manifest) { - t.Helper() - assert.Equal(t, "go", meta.SDK.Language) - require.Contains(t, meta.Dags, "simple_dag") - assert.Equal(t, []string{"extract", "transform", "load"}, meta.Dags["simple_dag"].Tasks) - } - - // Default: YAML. A JSON document would start with '{'. - yamlOut, err := exec.Command(hostBin, "--airflow-metadata").Output() - require.NoError(t, err, "running %s --airflow-metadata", hostBin) - assert.False(t, bytes.HasPrefix(bytes.TrimSpace(yamlOut), []byte("{")), - "default --airflow-metadata must emit YAML, not JSON") - assert.Contains(t, string(yamlOut), "airflow_bundle_metadata_version:") - var fromYAML airflowmetadata.Manifest - require.NoError(t, yaml.Unmarshal(yamlOut, &fromYAML)) - assertManifest(t, fromYAML) - - // --format json: JSON output. - jsonOut, err := exec.Command(hostBin, "--airflow-metadata", "--format", "json").Output() - require.NoError(t, err, "running %s --airflow-metadata --format json", hostBin) - assert.True(t, bytes.HasPrefix(bytes.TrimSpace(jsonOut), []byte("{")), - "--format json must emit JSON") - var fromJSON airflowmetadata.Manifest - require.NoError(t, json.Unmarshal(jsonOut, &fromJSON)) - assertManifest(t, fromJSON) - - assert.Equal(t, fromYAML, fromJSON, "both encodings must decode to the same manifest") - - // --format without --airflow-metadata is a usage error. - bad, err := exec.Command(hostBin, "--format", "json").CombinedOutput() - require.Error(t, err, "--format without --airflow-metadata must exit non-zero") - assert.Contains(t, string(bad), "--format is only valid together with --airflow-metadata") -} - // Running the packer from a directory that is not a bundle main package must // turn the bare `go list` failure into an actionable error pointing at a // package path or --source. @@ -377,13 +452,3 @@ func goBuild(t *testing.T, pkgDir, out, goos, goarch string) { t.Fatalf("go build %s for %s/%s failed: %v\n%s", pkgDir, goos, goarch, err, combined) } } - -// captureMetadata runs a host-runnable bundle binary with --airflow-metadata -// and writes its JSON stdout to outPath. -func captureMetadata(t *testing.T, hostBin, outPath string) { - t.Helper() - cmd := exec.Command(hostBin, "--airflow-metadata") - out, err := cmd.Output() - require.NoError(t, err, "running %s --airflow-metadata", hostBin) - require.NoError(t, os.WriteFile(outPath, out, 0o644)) -} diff --git a/go-sdk/cmd/airflow-go-pack/pack_test.go b/go-sdk/cmd/airflow-go-pack/pack_test.go index ce5f90894dc4c..530b7261a78da 100644 --- a/go-sdk/cmd/airflow-go-pack/pack_test.go +++ b/go-sdk/cmd/airflow-go-pack/pack_test.go @@ -21,7 +21,6 @@ import ( "bytes" "crypto/sha256" "encoding/hex" - "errors" "io" "os" "path/filepath" @@ -35,85 +34,55 @@ import ( "github.com/apache/airflow/go-sdk/internal/airflowmetadata" "github.com/apache/airflow/go-sdk/internal/bundlefooter" + "github.com/apache/airflow/go-sdk/pkg/execution" ) -func TestRenderManifest_DeterministicDagOrdering(t *testing.T) { - meta := airflowmetadata.Manifest{ +func testSDK() airflowmetadata.Manifest { + return airflowmetadata.Manifest{ AirflowBundleMetadataVersion: "1.0", SDK: airflowmetadata.SDK{ Language: "go", Version: "0.1.0", SupervisorSchemaVersion: "2026-06-16", }, - Dags: map[string]airflowmetadata.Dag{ - "zeta_dag": {Tasks: []string{"a", "b"}}, - "alpha_dag": {Tasks: []string{"x"}}, - }, } +} - got1, err := renderManifest(meta, "main.go", nil) +func TestRenderManifest(t *testing.T) { + got1, err := renderManifest(testSDK(), "main.go", nil) require.NoError(t, err) - got2, err := renderManifest(meta, "main.go", nil) + got2, err := renderManifest(testSDK(), "main.go", nil) require.NoError(t, err) assert.Equal(t, got1, got2, "manifest should be byte-identical for identical input") - - expected := `airflow_bundle_metadata_version: "1.0" + assert.Equal(t, `airflow_bundle_metadata_version: "1.0" sdk: language: "go" version: "0.1.0" supervisor_schema_version: "2026-06-16" source: "main.go" -dags: - alpha_dag: - tasks: - - "x" - zeta_dag: - tasks: - - "a" - - "b" -` - assert.Equal(t, expected, string(got1)) +`, string(got1)) } -// Values (task IDs, source, SDK fields) are quoted so a scalar-looking value -// stays a string; Dag ID keys stay plain scalars. +// Values (source, SDK fields) are quoted so a scalar-looking value stays a string; keys stay plain. func TestRenderManifest_QuotesValuesNotKeys(t *testing.T) { - meta := airflowmetadata.Manifest{ - AirflowBundleMetadataVersion: "1.0", - SDK: airflowmetadata.SDK{ - Language: "go", - Version: "0.1.0", - SupervisorSchemaVersion: "2026-06-16", - }, - Dags: map[string]airflowmetadata.Dag{ - "my_dag": {Tasks: []string{"123", "true"}}, - }, - } + meta := testSDK() + meta.SDK.Version = "1.0" - got, err := renderManifest(meta, "main.go", nil) + got, err := renderManifest(meta, "true", nil) require.NoError(t, err) - // Task values that look like scalars are quoted. - assert.Contains(t, string(got), `- "123"`) - assert.Contains(t, string(got), `- "true"`) - // The Dag ID key is a plain scalar, not quoted. - assert.Contains(t, string(got), "\n my_dag:\n") - assert.NotContains(t, string(got), `"my_dag"`) + assert.Contains(t, string(got), "\n version: \"1.0\"\n") + assert.Contains(t, string(got), "\nsource: \"true\"\n") + assert.NotContains(t, string(got), `"sdk"`) } func TestRenderManifest_Digests(t *testing.T) { - meta := airflowmetadata.Manifest{ - AirflowBundleMetadataVersion: "1.0", - SDK: airflowmetadata.SDK{ - Language: "go", - Version: "0.1.0", - SupervisorSchemaVersion: "2026-06-16", - }, - Dags: map[string]airflowmetadata.Dag{"my_dag": {Tasks: []string{"t1"}}}, - } - - got, err := renderManifest(meta, "main.go", &bundleDigests{Integrity: "aa11", Cache: "bb22"}) + got, err := renderManifest( + testSDK(), + "main.go", + &bundleDigests{Integrity: "aa11", Cache: "bb22"}, + ) require.NoError(t, err) assert.Equal(t, `airflow_bundle_metadata_version: "1.0" @@ -125,26 +94,18 @@ source: "main.go" digests: integrity: "aa11" cache: "bb22" -dags: - my_dag: - tasks: - - "t1" `, string(got)) } -func TestRenderManifest_EmptyDags(t *testing.T) { - meta := airflowmetadata.Manifest{ - AirflowBundleMetadataVersion: "1.0", - SDK: airflowmetadata.SDK{ - Language: "go", - Version: "0.1.0", - SupervisorSchemaVersion: "2026-06-16", - }, - Dags: map[string]airflowmetadata.Dag{}, - } - got, err := renderManifest(meta, "main.go", nil) +// The manifest is rendered without the packer running the binary: a Dag inventory is not part of it. +func TestRenderManifest_HasNoDagInventory(t *testing.T) { + got, err := renderManifest(testSDK(), "main.go", nil) require.NoError(t, err) - assert.Contains(t, string(got), "dags: {}") + + var parsed map[string]any + require.NoError(t, yaml.Unmarshal(got, &parsed)) + assert.NotContains(t, parsed, "dags") + assert.NotContains(t, parsed, "task_handlers") } // Forwarded `go build` flags after "--" must not count against MaximumNArgs(1). @@ -163,35 +124,6 @@ func TestRootArgs_AllowsBuildFlagsAfterDoubleDash(t *testing.T) { } } -// A file the OS refuses to exec is errExecNotStartable; a binary that runs and -// exits non-zero is not. -func TestRunIntrospect_ClassifiesExecFailure(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("exec-format semantics differ on Windows") - } - dir := t.TempDir() - - // A non-binary file with the exec bit set: execve rejects it with an - // exec-format error, standing in for a foreign-arch executable. - garbage := filepath.Join(dir, "garbage") - require.NoError(t, os.WriteFile(garbage, []byte("not a real executable\n"), 0o755)) - _, err := runIntrospect(garbage, "--airflow-metadata") - require.Error(t, err) - assert.ErrorIs(t, err, errExecNotStartable) - - // A script that starts and exits non-zero is a genuine run failure, not - // an unrunnable binary, so it must NOT be classified as not-startable. - failing := filepath.Join(dir, "failing") - require.NoError(t, os.WriteFile(failing, []byte("#!/bin/sh\nexit 3\n"), 0o755)) - _, err = runIntrospect(failing, "--airflow-metadata") - require.Error(t, err) - assert.False( - t, - errors.Is(err, errExecNotStartable), - "non-zero exit should not be errExecNotStartable", - ) -} - func TestRootArgs_RejectsExtraPositionalBeforeDash(t *testing.T) { cmd := newRootCmd() cmd.RunE = func(*cobra.Command, []string) error { return nil } @@ -258,37 +190,6 @@ func TestRunPack_RejectsOutputAliasingExecutable(t *testing.T) { assert.Equal(t, original, got, "executable must not be truncated when output aliases it") } -// When --output resolves to the same file as --airflow-metadata, runPack must -// refuse before the bundle is renamed onto it; the manifest file survives. -func TestRunPack_RejectsOutputAliasingMetadataFile(t *testing.T) { - dir := t.TempDir() - exe := filepath.Join(dir, "foreign") - require.NoError(t, os.WriteFile(exe, []byte("foreign-arch-binary-bytes"), 0o755)) - source := filepath.Join(dir, "main.go") - require.NoError(t, os.WriteFile(source, []byte("package main\nfunc main() {}\n"), 0o644)) - meta := filepath.Join(dir, "airflow-metadata.json") - original := []byte( - `{"airflow_bundle_metadata_version":"1.0",` + - `"sdk":{"language":"go","version":"0.1.0","supervisor_schema_version":"2026-06-16"},` + - `"dags":{"my_dag":{"tasks":["t1"]}}}`, - ) - require.NoError(t, os.WriteFile(meta, original, 0o644)) - - err := runPack(io.Discard, io.Discard, &packOptions{ - executable: exe, - source: source, - airflowMetadata: meta, - output: meta, - }) - require.Error(t, err) - assert.Contains(t, err.Error(), "same file as the --airflow-metadata file") - - // The guard must fire before any write: the manifest file is intact. - got, readErr := os.ReadFile(meta) - require.NoError(t, readErr) - assert.Equal(t, original, got, "metadata file must not be clobbered when output aliases it") -} - // When the default output path names an existing directory, the packer must // reject it with --output guidance, not a bare os.Rename "file exists". func TestRunPack_RejectsDirectoryOutput(t *testing.T) { @@ -333,122 +234,71 @@ func TestRunPack_RejectsExecutableWithCrossFlags(t *testing.T) { } } -// A foreign-arch --executable that cannot be exec'd on the host is a hard error -// with remediation guidance, never a silent host rebuild that could describe a -// different DAG/task set than the shipped binary. -func TestRunPack_FailsFastWhenExecutableUnrunnableAndNoMetadata(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("exec-format semantics differ on Windows") - } +// A file that is not a Go binary cannot say which go-sdk it was built against, so it is refused +// before anything is written. +func TestRunPack_RejectsExecutableWithoutBuildInformation(t *testing.T) { dir := t.TempDir() - // A non-binary file with the exec bit set stands in for a foreign-arch - // executable: execve rejects it with an exec-format error. - exe := filepath.Join(dir, "foreign") - require.NoError(t, os.WriteFile(exe, []byte("not a runnable binary\n"), 0o755)) + exe := filepath.Join(dir, "not-go") + require.NoError(t, os.WriteFile(exe, []byte("not a Go binary\n"), 0o755)) source := filepath.Join(dir, "main.go") require.NoError(t, os.WriteFile(source, []byte("package main\nfunc main() {}\n"), 0o644)) out := filepath.Join(dir, "bundle") - err := runPack(io.Discard, io.Discard, &packOptions{ - executable: exe, - source: source, - output: out, - }) - require.Error(t, err) - assert.Contains(t, err.Error(), "cannot exec --executable") - assert.Contains(t, err.Error(), "--goarch", "error must point at the cross-build workflow") - assert.Contains( - t, - err.Error(), - "--airflow-metadata", - "error must mention the --airflow-metadata escape hatch", + err := runPack( + io.Discard, + io.Discard, + &packOptions{executable: exe, source: source, output: out}, ) + require.Error(t, err) + assert.Contains(t, err.Error(), "reading the build information of "+exe) _, statErr := os.Stat(out) - assert.True(t, os.IsNotExist(statErr), "no bundle should be written on a fail-fast") + assert.True(t, os.IsNotExist(statErr), "no bundle should be written") } -// --airflow-metadata short-circuits introspection: a binary that cannot run on -// the host is packed using the supplied JSON manifest, and its bytes survive. -func TestRunPack_UsesMetadataFile(t *testing.T) { +// The packer never runs the binary: the fixture exits with status 3 when it runs. +func TestRunPack_DoesNotRunTheBinary(t *testing.T) { dir := t.TempDir() - exe := filepath.Join(dir, "foreign") - exeBytes := []byte("foreign-arch-binary-bytes") + exe := filepath.Join(dir, "bundle-bin") + exeBytes := readFile(t, bundleBinary(t)) require.NoError(t, os.WriteFile(exe, exeBytes, 0o755)) source := filepath.Join(dir, "main.go") require.NoError(t, os.WriteFile(source, []byte("package main\nfunc main() {}\n"), 0o644)) - meta := filepath.Join(dir, "airflow-metadata.json") - require.NoError(t, os.WriteFile(meta, []byte( - `{"airflow_bundle_metadata_version":"1.0",`+ - `"sdk":{"language":"go","version":"0.1.0","supervisor_schema_version":"2026-06-16"},`+ - `"dags":{"my_dag":{"tasks":["t1"]}}}`, - ), 0o644)) out := filepath.Join(dir, "bundle") - err := runPack(io.Discard, io.Discard, &packOptions{ - executable: exe, - source: source, - airflowMetadata: meta, - output: out, - }) - require.NoError(t, err) + require.NoError(t, runPack(io.Discard, io.Discard, &packOptions{ + executable: exe, + source: source, + output: out, + })) gotSource, gotMeta, err := bundlefooter.Read(out) require.NoError(t, err) - srcBytes, err := os.ReadFile(source) - require.NoError(t, err) - assert.Equal(t, srcBytes, gotSource) + assert.Equal(t, readFile(t, source), gotSource) assert.Contains( t, string(gotMeta), - "my_dag:", - "Dag from --airflow-metadata must appear in the manifest", + `supervisor_schema_version: "`+execution.SupervisorSchemaVersion+`"`, ) - - bundleBytes, err := os.ReadFile(out) - require.NoError(t, err) + bundleBytes := readFile(t, out) binaryRegion := bundleBytes[:len(bundleBytes)-len(gotSource)-len(gotMeta)-bundlefooter.TrailerSize] assert.Equal(t, exeBytes, binaryRegion, "the supplied --executable must be packed verbatim") } -// --airflow-metadata also accepts a YAML manifest, not only the JSON the -// binary prints. -func TestRunPack_AcceptsYAMLMetadataFile(t *testing.T) { - dir := t.TempDir() - exe := filepath.Join(dir, "foreign") - require.NoError(t, os.WriteFile(exe, []byte("foreign-arch-binary-bytes"), 0o755)) - source := filepath.Join(dir, "main.go") - require.NoError(t, os.WriteFile(source, []byte("package main\nfunc main() {}\n"), 0o644)) - meta := filepath.Join(dir, "airflow-metadata.yaml") - require.NoError(t, os.WriteFile(meta, []byte( - "airflow_bundle_metadata_version: \"1.0\"\n"+ - "sdk:\n"+ - " language: \"go\"\n"+ - " version: \"0.1.0\"\n"+ - " supervisor_schema_version: \"2026-06-16\"\n"+ - "dags:\n"+ - " yaml_dag:\n"+ - " tasks:\n"+ - " - \"t1\"\n", - ), 0o644)) - out := filepath.Join(dir, "bundle") +// --airflow-metadata is gone: the packer takes no manifest from the user. +func TestRootCmd_RejectsAirflowMetadataFlag(t *testing.T) { + cmd := newRootCmd() + cmd.RunE = func(*cobra.Command, []string) error { return nil } + cmd.SetArgs( + []string{"--executable", "bin", "--source", "main.go", "--airflow-metadata", "meta.yaml"}, + ) + cmd.SetOut(io.Discard) + cmd.SetErr(io.Discard) - err := runPack(io.Discard, io.Discard, &packOptions{ - executable: exe, - source: source, - airflowMetadata: meta, - output: out, - }) - require.NoError(t, err) + err := cmd.Execute() - _, gotMeta, err := bundlefooter.Read(out) - require.NoError(t, err) - assert.Contains( - t, - string(gotMeta), - "yaml_dag:", - "Dag from a YAML --airflow-metadata file must appear in the manifest", - ) + require.Error(t, err) + assert.Contains(t, err.Error(), "unknown flag: --airflow-metadata") } // fixedManifest is a writeBundle renderMetadata that ignores the staged executable. @@ -456,23 +306,20 @@ func fixedManifest(manifest []byte) func(string) ([]byte, error) { return func(string) ([]byte, error) { return manifest, nil } } -// packDigests packs the given executable, source and JSON manifest in dir, and -// returns the digests the packed bundle's manifest records. -func packDigests(t *testing.T, dir string, exe, source, meta []byte) (integrity, cache string) { +// packDigests packs the given executable and source in dir, and returns the digests the packed +// bundle's manifest records. +func packDigests(t *testing.T, dir string, exe, source []byte) (integrity, cache string) { t.Helper() exePath := filepath.Join(dir, "exe") require.NoError(t, os.WriteFile(exePath, exe, 0o755)) sourcePath := filepath.Join(dir, "main.go") require.NoError(t, os.WriteFile(sourcePath, source, 0o644)) - metaPath := filepath.Join(dir, "airflow-metadata.json") - require.NoError(t, os.WriteFile(metaPath, meta, 0o644)) out := filepath.Join(dir, "bundle") require.NoError(t, runPack(io.Discard, io.Discard, &packOptions{ - executable: exePath, - source: sourcePath, - airflowMetadata: metaPath, - output: out, + executable: exePath, + source: sourcePath, + output: out, })) _, gotMeta, err := bundlefooter.Read(out) @@ -488,13 +335,10 @@ func packDigests(t *testing.T, dir string, exe, source, meta []byte) (integrity, } func TestRunPack_RecordsDigests(t *testing.T) { - exe := []byte("binary-bytes") + exe := readFile(t, bundleBinary(t)) source := []byte("package main\nfunc main() {}\n") - meta := []byte(`{"airflow_bundle_metadata_version":"1.0",` + - `"sdk":{"language":"go","version":"0.1.0","supervisor_schema_version":"2026-06-16"},` + - `"dags":{"my_dag":{"tasks":["t1"]}}}`) - integrity, cache := packDigests(t, t.TempDir(), exe, source, meta) + integrity, cache := packDigests(t, t.TempDir(), exe, source) binaryHash := sha256.Sum256(exe) assert.Equal( @@ -507,7 +351,7 @@ func TestRunPack_RecordsDigests(t *testing.T) { assert.NotEqual(t, integrity, cache) t.Run("a repack of the same inputs", func(t *testing.T) { - gotIntegrity, gotCache := packDigests(t, t.TempDir(), exe, source, meta) + gotIntegrity, gotCache := packDigests(t, t.TempDir(), exe, source) assert.Equal(t, integrity, gotIntegrity) assert.Equal(t, cache, gotCache) }) @@ -518,19 +362,15 @@ func TestRunPack_RecordsDigests(t *testing.T) { return changed } for name, tc := range map[string]struct { - exe, source, meta []byte - integrityChanges bool + exe, source []byte + integrityChanges bool }{ - "one source byte": {exe: exe, source: oneByte(source, len(source)-2), meta: meta}, - "one binary byte": {exe: oneByte(exe, 0), source: source, meta: meta, integrityChanges: true}, - "the manifest": { - exe: exe, - source: source, - meta: bytes.Replace(meta, []byte(`"0.1.0"`), []byte(`"0.1.1"`), 1), - }, + "one source byte": {exe: exe, source: oneByte(source, len(source)-2)}, + // A byte after the binary leaves its build information readable. + "one binary byte": {exe: append(bytes.Clone(exe), 0), source: source, integrityChanges: true}, } { t.Run(name, func(t *testing.T) { - gotIntegrity, gotCache := packDigests(t, t.TempDir(), tc.exe, tc.source, tc.meta) + gotIntegrity, gotCache := packDigests(t, t.TempDir(), tc.exe, tc.source) assert.NotEqual(t, cache, gotCache) if tc.integrityChanges { assert.NotEqual(t, integrity, gotIntegrity) @@ -541,6 +381,19 @@ func TestRunPack_RecordsDigests(t *testing.T) { } } +func TestComputeDigests_CoverTheManifest(t *testing.T) { + exe := filepath.Join(t.TempDir(), "exe") + require.NoError(t, os.WriteFile(exe, []byte("binary-bytes"), 0o755)) + + first, err := computeDigests(exe, []byte("source"), []byte("manifest")) + require.NoError(t, err) + changed, err := computeDigests(exe, []byte("source"), []byte("manifest 2")) + require.NoError(t, err) + + assert.Equal(t, first.Integrity, changed.Integrity) + assert.NotEqual(t, first.Cache, changed.Cache) +} + // With no explicit --output, packing "./bundle" from a package dir named // "bundle" derives a default output that collides with the pre-built binary. func TestRunPack_RejectsDefaultOutputAliasingExecutable(t *testing.T) { diff --git a/go-sdk/cmd/airflow-go-pack/sdkversion.go b/go-sdk/cmd/airflow-go-pack/sdkversion.go new file mode 100644 index 0000000000000..cfacd0d594089 --- /dev/null +++ b/go-sdk/cmd/airflow-go-pack/sdkversion.go @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "debug/buildinfo" + "errors" + "fmt" + "runtime/debug" + + "github.com/apache/airflow/go-sdk/internal/airflowmetadata" + "github.com/apache/airflow/go-sdk/pkg/execution" +) + +// sdkModulePath is the import path of the Go SDK module. +const sdkModulePath = "github.com/apache/airflow/go-sdk" + +// sdkModule is the Go SDK module as one build includes it. +type sdkModule struct { + // version is the module version, or "(devel)" for a main module without one. + version string + // replacePath and replaceVersion are set when a replace directive substitutes the module. + // A replacement by a local path has the version "(devel)". + replacePath string + replaceVersion string + // main is set when the SDK is the main module of the build, as when the build runs inside + // the SDK's own module. + main bool +} + +// findSDKModule returns how info includes the Go SDK module. ok is false when the build does +// not link it. +func findSDKModule(info *debug.BuildInfo) (module sdkModule, ok bool) { + if info.Main.Path == sdkModulePath { + module = sdkModule{version: info.Main.Version} + if module.version == "" { + module.version = "(devel)" + } + module.replacePath, module.replaceVersion = replacement(info.Main.Replace) + // The packer run by `go tool` from another module also reports the SDK as its main + // module, because that is the module of its main package. It carries a checksum or a + // replacement, which the main module of a build never does. A vendored build records no + // checksum for any module, so it is the SDK's own build only if a dependency has one. + module.main = info.Main.Replace == nil && info.Main.Sum == "" && hasChecksummedDep(info) + return module, true + } + for _, dep := range info.Deps { + if dep.Path != sdkModulePath { + continue + } + module = sdkModule{version: dep.Version} + module.replacePath, module.replaceVersion = replacement(dep.Replace) + return module, true + } + return sdkModule{}, false +} + +func hasChecksummedDep(info *debug.BuildInfo) bool { + for _, dep := range info.Deps { + if dep.Sum != "" { + return true + } + } + return false +} + +func replacement(replace *debug.Module) (path, version string) { + if replace == nil { + return "", "" + } + return replace.Path, replace.Version +} + +// recordedVersion is the version written to the manifest: the replacement's version when it has +// one, else the module's, else "(devel)". +func (m sdkModule) recordedVersion() string { + switch { + case m.replaceVersion != "": + return m.replaceVersion + case m.version != "": + return m.version + default: + return "(devel)" + } +} + +// sameBuildAs reports whether both builds include the same code of the SDK: the same version +// and the same replacement. +func (m sdkModule) sameBuildAs(other sdkModule) bool { + return m.version == other.version && + m.replacePath == other.replacePath && + m.replaceVersion == other.replaceVersion +} + +func (m sdkModule) String() string { + description := m.version + if m.main { + description += " (main module)" + } + if m.replacePath != "" { + description += " replaced by " + m.replacePath + if m.replaceVersion != "" { + description += "@" + m.replaceVersion + } + } + return description +} + +// checkSDKVersion refuses a bundle binary built against another Go SDK than the packer and +// returns the SDK version to record for it. +// +// The packer writes its own supervisor schema version into the manifest, which is only right +// for a binary built against the same SDK. When the SDK is the main module of both builds, +// as when packing inside the SDK module, the two are accepted without a comparison: Go stamps +// the main module's version from version control, so two builds of the same sources can differ. +// Otherwise the builds must include the same version and the same replacement of it. +func checkSDKVersion(packer, binary *debug.BuildInfo) (string, error) { + linked, ok := findSDKModule(binary) + if !ok { + return "", fmt.Errorf( + "the bundle binary does not link %s, so it is not a Go SDK bundle", + sdkModulePath, + ) + } + own, ok := findSDKModule(packer) + if !ok { + return "", fmt.Errorf("airflow-go-pack's build information does not list %s", sdkModulePath) + } + if !(linked.main && own.main) && !own.sameBuildAs(linked) { + return "", fmt.Errorf( + "the bundle binary was built against go-sdk %s, but airflow-go-pack was built "+ + "against go-sdk %s; pack with `go tool airflow-go-pack` from the module that "+ + "built the binary, so both use the same go-sdk", + linked, own, + ) + } + return linked.recordedVersion(), nil +} + +// readSDK reads the SDK block of the manifest for the bundle binary at path, without running +// it. The version comes from the binary's build information, which Go keeps in a stripped +// binary and for any target platform. The supervisor schema version is the packer's own, which +// checkSDKVersion has shown to be the binary's too. +func readSDK(path string) (airflowmetadata.SDK, error) { + binary, err := buildinfo.ReadFile(path) + if err != nil { + return airflowmetadata.SDK{}, fmt.Errorf( + "reading the build information of %s: %w; it must be a binary built from a Go package", + path, err, + ) + } + packer, ok := debug.ReadBuildInfo() + if !ok { + return airflowmetadata.SDK{}, errors.New( + "airflow-go-pack has no build information to compare", + ) + } + version, err := checkSDKVersion(packer, binary) + if err != nil { + return airflowmetadata.SDK{}, err + } + return airflowmetadata.SDK{ + Language: "go", + Version: version, + SupervisorSchemaVersion: execution.SupervisorSchemaVersion, + }, nil +} diff --git a/go-sdk/cmd/airflow-go-pack/sdkversion_test.go b/go-sdk/cmd/airflow-go-pack/sdkversion_test.go new file mode 100644 index 0000000000000..ea7680b99e935 --- /dev/null +++ b/go-sdk/cmd/airflow-go-pack/sdkversion_test.go @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package main + +import ( + "runtime/debug" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mainModule is a build inside the SDK's own module: the main module has no checksum, and its +// dependencies do. +func mainModule(version string) *debug.BuildInfo { + return &debug.BuildInfo{ + Main: debug.Module{Path: sdkModulePath, Version: version}, + Deps: []*debug.Module{{Path: "github.com/spf13/cobra", Version: "v1.10.1", Sum: "h1:dep="}}, + } +} + +// vendoredTool is the packer run by `go tool` from a module that vendors its dependencies: a +// vendored build records no checksum for any module. +func vendoredTool(version string) *debug.BuildInfo { + return &debug.BuildInfo{ + Main: debug.Module{Path: sdkModulePath, Version: version}, + Deps: []*debug.Module{{Path: "github.com/spf13/cobra", Version: "v1.10.1"}}, + } +} + +// tool is the packer run by `go tool` from another module: it reports the SDK as its main +// module, with the module's version and its checksum or replacement. +func tool(version string, replace *debug.Module) *debug.BuildInfo { + sum := "" + if replace == nil { + sum = "h1:abc=" + } + return &debug.BuildInfo{ + Main: debug.Module{Path: sdkModulePath, Version: version, Sum: sum, Replace: replace}, + Deps: []*debug.Module{{Path: "github.com/spf13/cobra", Version: "v1.10.1", Sum: "h1:dep="}}, + } +} + +func dependency(version string, replace *debug.Module) *debug.BuildInfo { + return &debug.BuildInfo{ + Main: debug.Module{Path: "example.com/bundle", Version: "(devel)"}, + Deps: []*debug.Module{ + {Path: "github.com/spf13/cobra", Version: "v1.10.1"}, + {Path: sdkModulePath, Version: version, Replace: replace}, + }, + } +} + +func withoutSDK() *debug.BuildInfo { + return &debug.BuildInfo{ + Main: debug.Module{Path: "example.com/other", Version: "(devel)"}, + Deps: []*debug.Module{{Path: "github.com/spf13/cobra", Version: "v1.10.1"}}, + } +} + +func TestCheckSDKVersion(t *testing.T) { + local := &debug.Module{Path: "/src/go-sdk", Version: "(devel)"} + for _, tc := range []struct { + name string + packer *debug.BuildInfo + binary *debug.BuildInfo + version string + errors []string + }{ + { + name: "the SDK is the main module of both builds, whatever its versions", + packer: mainModule("v0.0.0-20261002120000-abcdef123456"), + binary: mainModule("(devel)"), + version: "(devel)", + }, + { + name: "the SDK is the main module of both builds and the binary has no version", + packer: mainModule("(devel)"), + binary: mainModule(""), + version: "(devel)", + }, + { + name: "the SDK is a dependency of both builds at one version", + packer: dependency("v0.1.0", nil), + binary: dependency("v0.1.0", nil), + version: "v0.1.0", + }, + { + name: "the SDK is a dependency of both builds at two versions", + packer: dependency("v0.2.0", nil), + binary: dependency("v0.1.0", nil), + errors: []string{"go-sdk v0.1.0", "go-sdk v0.2.0", "go tool airflow-go-pack"}, + }, + { + name: "both builds replace the SDK with the same local path", + packer: dependency("v0.0.0", local), + binary: dependency("v0.0.0", local), + version: "(devel)", + }, + { + name: "one build replaces the SDK", + packer: dependency("v0.1.0", nil), + binary: dependency("v0.1.0", local), + errors: []string{"go-sdk v0.1.0 replaced by /src/go-sdk@(devel)", "go-sdk v0.1.0"}, + }, + { + name: "the builds replace the SDK with two paths", + packer: dependency("v0.1.0", local), + binary: dependency("v0.1.0", &debug.Module{Path: "/other/go-sdk", Version: "(devel)"}), + errors: []string{"replaced by /other/go-sdk", "replaced by /src/go-sdk"}, + }, + { + name: "both builds replace the SDK with one released fork", + packer: dependency("v0.1.0", &debug.Module{Path: "example.com/fork", Version: "v1.2.3"}), + binary: dependency("v0.1.0", &debug.Module{Path: "example.com/fork", Version: "v1.2.3"}), + version: "v1.2.3", + }, + { + name: "the builds replace the SDK with two versions of a fork", + packer: dependency("v0.1.0", &debug.Module{Path: "example.com/fork", Version: "v1.2.3"}), + binary: dependency("v0.1.0", &debug.Module{Path: "example.com/fork", Version: "v1.2.4"}), + errors: []string{"replaced by example.com/fork@v1.2.4", "replaced by example.com/fork@v1.2.3"}, + }, + { + name: "the packer is run by go tool and the binary uses the same release", + packer: tool("v0.1.0", nil), + binary: dependency("v0.1.0", nil), + version: "v0.1.0", + }, + { + name: "the packer is run by go tool and the binary uses another release", + packer: tool("v0.2.0", nil), + binary: dependency("v0.1.0", nil), + errors: []string{"go-sdk v0.1.0", "go-sdk v0.2.0"}, + }, + { + name: "the packer is run by go tool and both replace the SDK with the same local path", + packer: tool("v0.0.0", local), + binary: dependency("v0.0.0", local), + version: "(devel)", + }, + { + name: "the packer is run by go tool and the binary is built inside the SDK", + packer: tool("v0.0.0", local), + binary: mainModule("(devel)"), + errors: []string{"go-sdk (devel) (main module)", "go-sdk v0.0.0 replaced by /src/go-sdk@(devel)"}, + }, + { + name: "the packer is run by go tool with a released SDK and the binary is built inside the SDK", + packer: tool("v0.1.0", nil), + binary: mainModule("(devel)"), + errors: []string{ + "go-sdk (devel) (main module)", + "airflow-go-pack was built against go-sdk v0.1.0;", + }, + }, + { + name: "the packer is run by go tool from a vendoring module and the binary uses the same release", + packer: vendoredTool("v0.1.0"), + binary: dependency("v0.1.0", nil), + version: "v0.1.0", + }, + { + name: "the packer is run by go tool from a vendoring module and the binary is built inside the SDK", + packer: vendoredTool("v0.1.0"), + binary: mainModule("(devel)"), + errors: []string{ + "go-sdk (devel) (main module)", + "airflow-go-pack was built against go-sdk v0.1.0;", + }, + }, + { + name: "the packer is built inside a tagged SDK checkout and the binary uses the same release", + packer: mainModule("v0.1.0"), + binary: dependency("v0.1.0", nil), + version: "v0.1.0", + }, + { + name: "the packer is built inside the SDK and the binary uses a release", + packer: mainModule("(devel)"), + binary: dependency("v0.1.0", nil), + errors: []string{"go-sdk v0.1.0", "go-sdk (devel) (main module)"}, + }, + { + name: "the binary is built inside the SDK and the packer uses a release", + packer: dependency("v0.1.0", nil), + binary: mainModule("(devel)"), + errors: []string{"go-sdk (devel) (main module)", "go-sdk v0.1.0"}, + }, + { + name: "the binary does not link the SDK", + packer: dependency("v0.1.0", nil), + binary: withoutSDK(), + errors: []string{"does not link github.com/apache/airflow/go-sdk"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + version, err := checkSDKVersion(tc.packer, tc.binary) + if len(tc.errors) == 0 { + require.NoError(t, err) + assert.Equal(t, tc.version, version) + return + } + require.Error(t, err) + for _, fragment := range tc.errors { + assert.Contains(t, err.Error(), fragment) + } + }) + } +} + +func TestSDKModule_RecordedVersion(t *testing.T) { + for name, tc := range map[string]struct { + module sdkModule + want string + }{ + "a version": {sdkModule{version: "v0.1.0"}, "v0.1.0"}, + "a replacement version": {sdkModule{version: "v0.1.0", replaceVersion: "v1.2.3"}, "v1.2.3"}, + "a replacement without one": {sdkModule{version: "v0.0.0", replacePath: "/src/go-sdk"}, "v0.0.0"}, + "no version": {sdkModule{}, "(devel)"}, + "the main module's version": {sdkModule{version: "v0.1.0", main: true}, "v0.1.0"}, + "a main module without one": {sdkModule{version: "(devel)", main: true}, "(devel)"}, + } { + t.Run(name, func(t *testing.T) { + assert.Equal(t, tc.want, tc.module.recordedVersion()) + }) + } +} + +// A test binary reports the SDK module as its main module, as the packer built inside the SDK does. +func TestReadSDK_OfTheFixtureBinary(t *testing.T) { + sdk, err := readSDK(bundleBinary(t)) + + require.NoError(t, err) + assert.Equal(t, "go", sdk.Language) + assert.NotEmpty(t, sdk.Version) + assert.NotEmpty(t, sdk.SupervisorSchemaVersion) +} diff --git a/go-sdk/cmd/airflow-go-pack/testdata/failingbundle/main.go b/go-sdk/cmd/airflow-go-pack/testdata/failingbundle/main.go new file mode 100644 index 0000000000000..1d7aa1df9dcaf --- /dev/null +++ b/go-sdk/cmd/airflow-go-pack/testdata/failingbundle/main.go @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Command failingbundle is a bundle binary that links go-sdk and exits non-zero when it runs. +// The packer's tests pack it to show that packing never runs the bundle binary. +package main + +import ( + "fmt" + "os" + + "github.com/apache/airflow/go-sdk/airflow" +) + +func main() { + _ = airflow.Bundle() + fmt.Fprintln(os.Stderr, "failingbundle: this binary must not run") + os.Exit(3) +} diff --git a/go-sdk/cmd/airflow-go-pack/validate.go b/go-sdk/cmd/airflow-go-pack/validate.go deleted file mode 100644 index 2c9a40a726405..0000000000000 --- a/go-sdk/cmd/airflow-go-pack/validate.go +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package main - -import ( - "fmt" - "io" - "regexp" - "sort" - "strings" - "unicode/utf8" - - "github.com/apache/airflow/go-sdk/internal/airflowmetadata" -) - -const maxIDLength = 250 - -var idRegex = regexp.MustCompile(`^[\p{L}\p{N}_.-]+$`) - -// warnOnSuspiciousIDs checks every dag and task id in the manifest against the -// rules the Airflow server enforces (airflow.utils.helpers.validate_key). It is -// best-effort and only warns: the server validates authoritatively, and checks -// like the '..' one depend on server configuration the packer cannot see. -func warnOnSuspiciousIDs(stderr io.Writer, meta airflowmetadata.Manifest) { - dagIDs := make([]string, 0, len(meta.Dags)) - for id := range meta.Dags { - dagIDs = append(dagIDs, id) - } - // Map iteration order is random; sort so warnings print in a stable order. - sort.Strings(dagIDs) - for _, dagID := range dagIDs { - warnOnSuspiciousID(stderr, fmt.Sprintf("dag id %q", dagID), dagID) - for _, taskID := range meta.Dags[dagID].Tasks { - warnOnSuspiciousID(stderr, fmt.Sprintf("task id %q in dag %q", taskID, dagID), taskID) - } - } -} - -func warnOnSuspiciousID(stderr io.Writer, label, id string) { - if length := utf8.RuneCountInString(id); length > maxIDLength { - fmt.Fprintf( - stderr, - "warning: %s is longer than %d characters (%d); the Airflow server will reject it\n", - label, maxIDLength, length, - ) - } - if !idRegex.MatchString(id) { - fmt.Fprintf( - stderr, - "warning: %s must be made of alphanumeric characters, dashes, dots, and underscores; the Airflow server will reject it\n", - label, - ) - } else if strings.Contains(id, "..") { - fmt.Fprintf( - stderr, - "warning: %s contains '..'; the Airflow server will reject it unless [core] allow_double_dot_in_ids is enabled\n", - label, - ) - } -} diff --git a/go-sdk/cmd/airflow-go-pack/validate_test.go b/go-sdk/cmd/airflow-go-pack/validate_test.go deleted file mode 100644 index 4aa785ec64298..0000000000000 --- a/go-sdk/cmd/airflow-go-pack/validate_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package main - -import ( - "bytes" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/apache/airflow/go-sdk/internal/airflowmetadata" -) - -func warningsFor(dags map[string]airflowmetadata.Dag) string { - var buf bytes.Buffer - warnOnSuspiciousIDs(&buf, airflowmetadata.Manifest{Dags: dags}) - return buf.String() -} - -func dagCharsetWarning(id string) string { - return fmt.Sprintf( - "warning: dag id %q must be made of alphanumeric characters, dashes, dots, and underscores; the Airflow server will reject it\n", - id, - ) -} - -func TestWarnOnSuspiciousIDs_ValidIdsProduceNoWarnings(t *testing.T) { - for _, id := range []string{ - "simple", "with-dash", "with.dot", "with_underscore", "0numeric", - "café_dag", "任務", strings.Repeat("a", 250), strings.Repeat("任", 250), - } { - out := warningsFor(map[string]airflowmetadata.Dag{id: {Tasks: []string{id}}}) - assert.Empty(t, out, "id %q should not warn", id) - } -} - -func TestWarnOnSuspiciousIDs_TooLongIDWarns(t *testing.T) { - for _, id := range []string{strings.Repeat("a", 251), strings.Repeat("任", 251)} { - out := warningsFor(map[string]airflowmetadata.Dag{id: {}}) - expected := fmt.Sprintf( - "warning: dag id %q is longer than 250 characters (251); the Airflow server will reject it\n", - id, - ) - assert.Equal(t, expected, out, "id %q", id) - } -} - -func TestWarnOnSuspiciousIDs_InvalidCharsWarn(t *testing.T) { - // "a..b c" also locks the else-if: a charset failure suppresses the '..' warning. - for _, id := range []string{"", "with space", "with/slash", "with:colon", "with\ttab", "a..b c"} { - out := warningsFor(map[string]airflowmetadata.Dag{id: {}}) - assert.Equal(t, dagCharsetWarning(id), out, "id %q", id) - } -} - -func TestWarnOnSuspiciousIDs_DoubleDotWarns(t *testing.T) { - out := warningsFor(map[string]airflowmetadata.Dag{"a..b": {}}) - assert.Equal( - t, - "warning: dag id \"a..b\" contains '..'; the Airflow server will reject it unless [core] allow_double_dot_in_ids is enabled\n", - out, - ) -} - -func TestWarnOnSuspiciousIDs_TooLongInvalidIDGetsBothWarnings(t *testing.T) { - id := strings.Repeat("a", 250) + " b" - out := warningsFor(map[string]airflowmetadata.Dag{id: {}}) - expected := fmt.Sprintf( - "warning: dag id %q is longer than 250 characters (252); the Airflow server will reject it\n", - id, - ) + dagCharsetWarning(id) - assert.Equal(t, expected, out) -} - -func TestWarnOnSuspiciousIDs_SortsDagIDsForStableOutput(t *testing.T) { - out := warningsFor(map[string]airflowmetadata.Dag{ - "delta d": {}, - "alpha d": {}, - "charlie d": {}, - "bravo d": {}, - }) - expected := dagCharsetWarning("alpha d") + dagCharsetWarning("bravo d") + - dagCharsetWarning("charlie d") + dagCharsetWarning("delta d") - assert.Equal(t, expected, out) -} - -func TestWarnOnSuspiciousIDs_TaskWarningNamesItsDag(t *testing.T) { - out := warningsFor(map[string]airflowmetadata.Dag{"my_dag": {Tasks: []string{"bad task"}}}) - assert.Equal( - t, - "warning: task id \"bad task\" in dag \"my_dag\" must be made of alphanumeric characters, dashes, dots, and underscores; the Airflow server will reject it\n", - out, - ) -} - -// Packing succeeds despite a suspicious dag id: the check is best-effort and -// the server-side validation stays the source of truth. -func TestRunPack_WarnsOnSuspiciousIDsButPacks(t *testing.T) { - dir := t.TempDir() - exe := filepath.Join(dir, "foreign") - require.NoError(t, os.WriteFile(exe, []byte("foreign-arch-binary-bytes"), 0o755)) - source := filepath.Join(dir, "main.go") - require.NoError(t, os.WriteFile(source, []byte("package main\nfunc main() {}\n"), 0o644)) - meta := filepath.Join(dir, "airflow-metadata.json") - require.NoError(t, os.WriteFile(meta, []byte( - `{"airflow_bundle_metadata_version":"1.0",`+ - `"sdk":{"language":"go","version":"0.1.0","supervisor_schema_version":"2026-06-16"},`+ - `"dags":{"bad dag":{"tasks":["t1"]}}}`, - ), 0o644)) - out := filepath.Join(dir, "bundle") - - var stderr bytes.Buffer - err := runPack(&bytes.Buffer{}, &stderr, &packOptions{ - executable: exe, - source: source, - airflowMetadata: meta, - output: out, - }) - require.NoError(t, err) - assert.Contains(t, stderr.String(), `warning: dag id "bad dag" must be made of`) - assert.FileExists(t, out) -} diff --git a/go-sdk/example/bundle/Justfile b/go-sdk/example/bundle/Justfile index 07df82a2280a6..7c14021f12006 100644 --- a/go-sdk/example/bundle/Justfile +++ b/go-sdk/example/bundle/Justfile @@ -25,8 +25,8 @@ default: build: pack # One-step build + pack. The single `go tool airflow-go-pack` -# invocation runs `go build` internally, queries the binary for its -# DAG/task identity via --airflow-metadata, and appends the source plus +# invocation runs `go build` internally, reads the go-sdk version from the +# binary's build information without running it, and appends the source plus # airflow-metadata.yaml plus AFBNDL01 trailer. The output is a single # self-contained executable bundle, named after the bundle's package # directory and written to the current directory. Drop it into the diff --git a/go-sdk/internal/airflowmetadata/airflowmetadata.go b/go-sdk/internal/airflowmetadata/airflowmetadata.go index 2045daebb17f5..fa84250ca3c99 100644 --- a/go-sdk/internal/airflowmetadata/airflowmetadata.go +++ b/go-sdk/internal/airflowmetadata/airflowmetadata.go @@ -15,27 +15,20 @@ // specific language governing permissions and limitations // under the License. -// Package airflowmetadata defines the airflow-metadata manifest wire shape -// shared between the producer (a bundle binary's --airflow-metadata flag, -// emitted from pkg/execution) and the consumer (airflow-go-pack, which decodes -// it and renders the embedded airflow-metadata.yaml). Keeping the definition in -// one place stops the two sides from drifting. The canonical schema is -// airflow-metadata.schema.json in the Task SDK docs. +// Package airflowmetadata defines the airflow-metadata manifest wire shape that airflow-go-pack +// renders into the airflow-metadata.yaml it embeds in a bundle. The packer is the only producer; +// the canonical schema is airflow-metadata.schema.json in the Task SDK docs. package airflowmetadata // FormatVersion is the bundle-spec version emitted manifests conform to. const FormatVersion = "1.0" -// Manifest is the shape printed by a bundle binary's --airflow-metadata flag -// (YAML by default, JSON under --format json). It mirrors -// airflow-metadata.schema.json minus the source and digests fields, which only -// the packer can resolve from the build inputs. The yaml tags let the packer's -// --airflow-metadata flag decode a captured manifest in either JSON or YAML -// (the airflow-metadata.yaml in a bundle). +// Manifest is the part of the manifest the packer builds from the bundle binary and its own SDK. +// It mirrors airflow-metadata.schema.json minus the source and digests fields, which only the +// packer can resolve from the build inputs. type Manifest struct { - AirflowBundleMetadataVersion string `json:"airflow_bundle_metadata_version" yaml:"airflow_bundle_metadata_version"` - SDK SDK `json:"sdk" yaml:"sdk"` - Dags map[string]Dag `json:"dags" yaml:"dags"` + AirflowBundleMetadataVersion string `json:"airflow_bundle_metadata_version" yaml:"airflow_bundle_metadata_version"` + SDK SDK `json:"sdk" yaml:"sdk"` } // SDK identifies the SDK that produced the bundle. @@ -44,8 +37,3 @@ type SDK struct { Version string `json:"version" yaml:"version"` SupervisorSchemaVersion string `json:"supervisor_schema_version" yaml:"supervisor_schema_version"` } - -// Dag is the static description of a single DAG declared in the bundle. -type Dag struct { - Tasks []string `json:"tasks" yaml:"tasks"` -} diff --git a/go-sdk/internal/bundle/doc.go b/go-sdk/internal/bundle/doc.go index dda8b6ed6d4c4..09d880b49b384 100644 --- a/go-sdk/internal/bundle/doc.go +++ b/go-sdk/internal/bundle/doc.go @@ -16,7 +16,7 @@ // under the License. // Package bundle defines what the coordinator runtime needs from a bundle: -// the tasks it looks up and runs, and the Dag and task ids it lists in the manifest. +// the tasks it looks up and runs, and the task handlers it lists for the Dag processor. // // Package airflow builds both from the task handlers a bundle registers. package bundle diff --git a/go-sdk/internal/bundle/task.go b/go-sdk/internal/bundle/task.go index 245f2d79308ee..c70a1ef7fc954 100644 --- a/go-sdk/internal/bundle/task.go +++ b/go-sdk/internal/bundle/task.go @@ -54,9 +54,7 @@ type TaskHandlerInfo struct { } // EnumerableBundle lists the registered task handlers in registration order. -// DumpAirflowMetadata in pkg/execution builds the --airflow-metadata manifest -// from that list, which is how airflow-go-pack reads a bundle's Dag and task ids -// without running a task. +// The task handler parse in pkg/execution answers the Dag processor from that list. type EnumerableBundle interface { ListTaskHandlers() []TaskHandlerInfo } diff --git a/go-sdk/pkg/execution/messages.go b/go-sdk/pkg/execution/messages.go index 77df9add847a6..231d127a19d85 100644 --- a/go-sdk/pkg/execution/messages.go +++ b/go-sdk/pkg/execution/messages.go @@ -28,9 +28,9 @@ import ( // SupervisorSchemaVersion is the dated AIP-72 supervisor wire-schema version // (YYYY-MM-DD) this SDK's coordinator protocol is compiled against. It must -// match the "api_version" of the schema the models are generated from, and is -// reported in a bundle's airflow-metadata manifest as -// sdk.supervisor_schema_version so the supervisor can down/upgrade messages to +// match the "api_version" of the schema the models are generated from. +// airflow-go-pack writes it into a bundle's airflow-metadata manifest as +// sdk.supervisor_schema_version, so the supervisor can down/upgrade messages to // a shape the bundle understands. const SupervisorSchemaVersion = "2026-10-30" diff --git a/go-sdk/pkg/execution/metadata.go b/go-sdk/pkg/execution/metadata.go deleted file mode 100644 index 05dbebd70a92b..0000000000000 --- a/go-sdk/pkg/execution/metadata.go +++ /dev/null @@ -1,141 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package execution - -import ( - "encoding/json" - "fmt" - "io" - "runtime/debug" - - "gopkg.in/yaml.v3" - - "github.com/apache/airflow/go-sdk/internal/airflowmetadata" - "github.com/apache/airflow/go-sdk/internal/bundle" -) - -// sdkModulePath is the import path of the SDK module. Used to identify the -// SDK version from the bundle binary's build info dependencies. -const sdkModulePath = "github.com/apache/airflow/go-sdk" - -// MetadataFormat selects the encoding DumpAirflowMetadata writes to stdout for -// the bundle binary's --airflow-metadata flag. -type MetadataFormat string - -const ( - // MetadataFormatYAML is the default; it matches the airflow-metadata.yaml a - // bundle embeds, so `mybundle --airflow-metadata > airflow-metadata.yaml` - // yields a ready-to-use file. - MetadataFormatYAML MetadataFormat = "yaml" - // MetadataFormatJSON is opt-in via --format json. - MetadataFormatJSON MetadataFormat = "json" -) - -// ParseMetadataFormat validates a --format value and returns the matching -// MetadataFormat. An empty value defaults to YAML. -func ParseMetadataFormat(s string) (MetadataFormat, error) { - switch MetadataFormat(s) { - case "", MetadataFormatYAML: - return MetadataFormatYAML, nil - case MetadataFormatJSON: - return MetadataFormatJSON, nil - default: - return "", fmt.Errorf( - "unsupported --airflow-metadata format %q: want %q or %q", - s, MetadataFormatYAML, MetadataFormatJSON, - ) - } -} - -// DumpAirflowMetadata writes the bundle's airflow-metadata manifest to w, as YAML -// by default or as JSON when format is MetadataFormatJSON. It only reads the -// registered Dag and task ids, without running a task or calling an external service. -// airflow-go-pack execs the binary with --airflow-metadata and decodes this -// output to build the embedded manifest. -func DumpAirflowMetadata( - w io.Writer, - b bundle.EnumerableBundle, - format MetadataFormat, -) error { - data, err := encodeManifest(collectManifest(b), format) - if err != nil { - return err - } - _, err = w.Write(data) - return err -} - -func collectManifest(b bundle.EnumerableBundle) airflowmetadata.Manifest { - meta := airflowmetadata.Manifest{ - AirflowBundleMetadataVersion: airflowmetadata.FormatVersion, - SDK: airflowmetadata.SDK{ - Language: "go", - Version: sdkVersion(), - SupervisorSchemaVersion: SupervisorSchemaVersion, - }, - Dags: make(map[string]airflowmetadata.Dag), - } - for _, handler := range b.ListTaskHandlers() { - dag := meta.Dags[handler.DagID] - dag.Tasks = append(dag.Tasks, handler.TaskID) - meta.Dags[handler.DagID] = dag - } - return meta -} - -// encodeManifest renders the manifest, ensuring exactly one trailing newline -// (yaml.Marshal adds one; JSON does not). -func encodeManifest(meta airflowmetadata.Manifest, format MetadataFormat) ([]byte, error) { - switch format { - case MetadataFormatYAML, "": - return yaml.Marshal(meta) - case MetadataFormatJSON: - data, err := json.MarshalIndent(meta, "", " ") - if err != nil { - return nil, err - } - return append(data, '\n'), nil - default: - return nil, fmt.Errorf("unsupported airflow-metadata format %q", format) - } -} - -// sdkVersion returns the version of the SDK module linked into this binary, -// derived from runtime/debug.ReadBuildInfo. Falls back to "(devel)" when -// build info is unavailable (e.g. tests, bundle binaries built from a local -// replace directive). -func sdkVersion() string { - info, ok := debug.ReadBuildInfo() - if !ok { - return "(devel)" - } - if info.Main.Path == sdkModulePath && info.Main.Version != "" { - return info.Main.Version - } - for _, dep := range info.Deps { - if dep.Path == sdkModulePath { - if dep.Replace != nil && dep.Replace.Version != "" { - return dep.Replace.Version - } - if dep.Version != "" { - return dep.Version - } - } - } - return "(devel)" -} diff --git a/go-sdk/pkg/execution/metadata_test.go b/go-sdk/pkg/execution/metadata_test.go deleted file mode 100644 index d105b24d898c3..0000000000000 --- a/go-sdk/pkg/execution/metadata_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package execution - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" - - "github.com/apache/airflow/go-sdk/internal/airflowmetadata" -) - -func sampleManifest() airflowmetadata.Manifest { - return airflowmetadata.Manifest{ - // "1.0" and a task named "123" must survive a YAML round-trip as strings. - AirflowBundleMetadataVersion: "1.0", - SDK: airflowmetadata.SDK{ - Language: "go", - Version: "(devel)", - SupervisorSchemaVersion: "2026-06-16", - }, - Dags: map[string]airflowmetadata.Dag{ - "simple_dag": {Tasks: []string{"extract", "transform", "load"}}, - "odd_dag": {Tasks: []string{"123"}}, - }, - } -} - -func TestParseMetadataFormat(t *testing.T) { - tests := []struct { - in string - want MetadataFormat - wantErr bool - }{ - {in: "", want: MetadataFormatYAML}, - {in: "yaml", want: MetadataFormatYAML}, - {in: "json", want: MetadataFormatJSON}, - {in: "YAML", wantErr: true}, - {in: "yml", wantErr: true}, - {in: "xml", wantErr: true}, - } - for _, tc := range tests { - t.Run(tc.in, func(t *testing.T) { - got, err := ParseMetadataFormat(tc.in) - if tc.wantErr { - require.Error(t, err) - return - } - require.NoError(t, err) - assert.Equal(t, tc.want, got) - }) - } -} - -func TestEncodeManifest_JSON(t *testing.T) { - meta := sampleManifest() - data, err := encodeManifest(meta, MetadataFormatJSON) - require.NoError(t, err) - - assert.Equal(t, byte('{'), data[0], "JSON output must start with an object") - assert.Equal(t, byte('\n'), data[len(data)-1], "output must end with a newline") - assert.NotEqual( - t, - byte('\n'), - data[len(data)-2], - "output must have exactly one trailing newline", - ) - assert.Contains(t, string(data), "\n \"sdk\"", "JSON must be indented four spaces") - - var got airflowmetadata.Manifest - require.NoError(t, json.Unmarshal(data, &got)) - assert.Equal(t, meta, got, "JSON must round-trip back to the manifest") -} - -func TestEncodeManifest_YAML(t *testing.T) { - meta := sampleManifest() - // The empty format is the default and must encode as YAML. - for _, format := range []MetadataFormat{MetadataFormatYAML, ""} { - data, err := encodeManifest(meta, format) - require.NoError(t, err) - - assert.NotEqual(t, byte('{'), data[0], "YAML output must not be JSON") - assert.Equal(t, byte('\n'), data[len(data)-1], "output must end with a newline") - assert.NotEqual( - t, - byte('\n'), - data[len(data)-2], - "output must have exactly one trailing newline", - ) - assert.Contains(t, string(data), "airflow_bundle_metadata_version:") - - var got airflowmetadata.Manifest - require.NoError(t, yaml.Unmarshal(data, &got)) - assert.Equal(t, meta, got, "YAML must round-trip back to the manifest") - } -} - -func TestEncodeManifest_UnsupportedFormat(t *testing.T) { - _, err := encodeManifest(sampleManifest(), MetadataFormat("xml")) - require.Error(t, err) -} diff --git a/task-sdk/docs/airflow-metadata.schema.json b/task-sdk/docs/airflow-metadata.schema.json index d2a8f35ba000a..84ebf1a693c59 100644 --- a/task-sdk/docs/airflow-metadata.schema.json +++ b/task-sdk/docs/airflow-metadata.schema.json @@ -2,9 +2,9 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://airflow.apache.org/schemas/sdk-executable/airflow-metadata-1.0.schema.json", "title": "Airflow Executable SDK Bundle Metadata", - "description": "Build-time manifest declaring DAG and task identifiers exposed by an Airflow native-executable SDK bundle. See the Executable Bundle Spec documentation in the Airflow Task SDK.", + "description": "Build-time manifest describing an Airflow native-executable SDK bundle: the SDK that produced it, the name of its source file and the digests the packer records. See the Executable Bundle Spec documentation in the Airflow Task SDK.", "type": "object", - "required": ["airflow_bundle_metadata_version", "sdk", "source", "dags"], + "required": ["airflow_bundle_metadata_version", "sdk", "source"], "additionalProperties": true, "properties": { "airflow_bundle_metadata_version": { @@ -56,32 +56,6 @@ "pattern": "^[0-9a-f]{64}$" } } - }, - "dags": { - "type": "object", - "description": "Mapping of dag_id to DAG entry. Every dag_id the bundle exposes must appear here.", - "minProperties": 1, - "additionalProperties": { - "$ref": "#/$defs/dagEntry" - } - } - }, - "$defs": { - "dagEntry": { - "type": "object", - "description": "Static description of a single DAG declared in the bundle.", - "required": ["tasks"], - "additionalProperties": true, - "properties": { - "tasks": { - "type": "array", - "description": "Static list of task_ids declared in the DAG.", - "items": { - "type": "string", - "minLength": 1 - } - } - } } } } diff --git a/task-sdk/docs/executable-bundle-spec.rst b/task-sdk/docs/executable-bundle-spec.rst index a5ec030d2f451..a6c0e9b8208cd 100644 --- a/task-sdk/docs/executable-bundle-spec.rst +++ b/task-sdk/docs/executable-bundle-spec.rst @@ -169,8 +169,7 @@ extending the trailer rather than inserting between existing blobs. ``airflow-metadata.yaml`` schema -------------------------------- -The metadata region carries the same YAML manifest documented previously, -produced at build time from a static scan of the DAG source. A +The metadata region carries a YAML manifest the packer writes at build time. A machine-readable JSON Schema is published at :download:`airflow-metadata.schema.json` for use by build tooling, validators, and editors. @@ -183,15 +182,6 @@ and editors. version: "0.1.0" supervisor_schema_version: "2026-06-16" source: example.go - dags: - example_dag: - tasks: - - extract - - transform - - load - another_dag: - tasks: - - run Top-level keys: @@ -241,21 +231,11 @@ Top-level keys: SHA-256 digests of the binary region, the source region, and the manifest as rendered without the ``digests`` key, in that order. -``dags`` (mapping, required) - Mapping of ``dag_id`` to a *DAG entry*. Every ``dag_id`` the bundle - exposes MUST appear here. No coordinator reads these keys to find a - bundle: the Dag processor lists bundles by their trailer, and a task - runs the bundle it is bound to. - -DAG entry fields: - -``tasks`` (list of strings, required) - Static list of ``task_id``\ s declared in the DAG. Empty lists are - permitted but discouraged. - -Unrecognized top-level or DAG-entry keys MUST be ignored by the consumer so -that future SDK versions can extend the manifest without breaking older -runtimes. +Unrecognized top-level keys MUST be ignored by the consumer so that future SDK +versions can extend the manifest without breaking older runtimes. A bundle +packed before the manifest dropped its Dag inventory still carries a ``dags`` +mapping of each ``dag_id`` to the ``tasks`` of that Dag. No coordinator reads +it, and consumers MUST ignore it. Examples -------- diff --git a/task-sdk/docs/ts-bundle-spec.rst b/task-sdk/docs/ts-bundle-spec.rst index fd2402cf6f097..f86b078d796f3 100644 --- a/task-sdk/docs/ts-bundle-spec.rst +++ b/task-sdk/docs/ts-bundle-spec.rst @@ -137,11 +137,9 @@ The ``airflowMetadata`` payload is compact UTF-8 JSON with this logical shape: "version": "0.1.0-beta1", "supervisor_schema_version": "2026-06-16" }, - "source": "main.ts", - "task_handlers": { - "example": { - "tasks": ["extract", "load"] - } + "entrypoint_path": "src/main.ts", + "dag_source_paths": { + "example": "src/main.ts" } } @@ -149,14 +147,17 @@ The packer serializes this object without insignificant whitespace and escapes t separators (U+2028 and U+2029), keeping it in one newline-terminated JavaScript comment without a second encoding layer. The SHA-256 digest detects changes to the exact serialized bytes. -``task_handlers`` is keyed by Dag ID and lists the task IDs the bundle handles for each. It is named for what a -TypeScript bundle actually provides: handlers for Dags declared elsewhere, not Dag definitions of its own. No -coordinator reads it to find a bundle: the Dag processor lists bundles by suffix and layout line, and a task runs the -bundle it is bound to. - -The metadata ``source`` value is the logical authoring name displayed for the Dag, a filename rather than content. -The source region named in the layout header is what carries the content. The two are separate fields in separate -documents, and neither is used to execute the bundle. +``entrypoint_path`` is the path of the entry file the bundle was built from, relative to the directory it was packed +in. ``dag_source_paths`` maps the Dag ID of each Dag declared in TypeScript to the path of the file that declares it, +for each Dag the packer could attribute to a file. A Dag declared in Python, for which the bundle only provides task +handlers, has no entry, and neither has a Dag the packer could not attribute to a file: a reader shows the +``entrypoint_path`` file for a Dag without an entry. Both fields are for display: neither is used to find or run a +bundle. + +The metadata does not list the task handlers a bundle provides, and no coordinator reads Dag IDs or task IDs from it to +find a bundle: the Dag processor lists bundles by suffix and layout line, and a task runs the bundle it is bound to. A +bundle packed before the metadata dropped them may carry a ``task_handlers`` mapping of Dag ID to ``tasks``, which +readers ignore. Reader Algorithm ---------------- diff --git a/task-sdk/tests/task_sdk/coordinators/executable/test_coordinator.py b/task-sdk/tests/task_sdk/coordinators/executable/test_coordinator.py index 257c1e3f6408a..dd204486e2b15 100644 --- a/task-sdk/tests/task_sdk/coordinators/executable/test_coordinator.py +++ b/task-sdk/tests/task_sdk/coordinators/executable/test_coordinator.py @@ -54,8 +54,8 @@ _DEFAULT_BINARY_PAYLOAD = b"\x7fELF" + b"binary-stub-payload" -def _make_metadata(dag_ids, source_filename: str = "example.go") -> dict: - return { +def _make_metadata(dag_ids=None, source_filename: str = "example.go") -> dict: + metadata = { "airflow_bundle_metadata_version": "1.0", "sdk": { "language": "go", @@ -63,14 +63,17 @@ def _make_metadata(dag_ids, source_filename: str = "example.go") -> dict: "supervisor_schema_version": "2026-06-16", }, "source": source_filename, - "dags": {dag_id: {"tasks": ["task1"]} for dag_id in dag_ids}, } + if dag_ids is not None: + # The Dag inventory that packers wrote before the manifest dropped it. + metadata["dags"] = {dag_id: {"tasks": ["task1"]} for dag_id in dag_ids} + return metadata def _build_bundle( path: Path, *, - dag_ids=("tutorial_dag",), + dag_ids=None, source: str | bytes = "package main\n\nfunc main() {}\n", source_filename: str = "example.go", metadata: dict | bytes | None = None, @@ -200,7 +203,7 @@ def test_clear_drops_all_entries(self): def _make_metadata_with_digests(**digests: str) -> dict: - return {**_make_metadata(["etl"]), "digests": digests} + return {**_make_metadata(), "digests": digests} class TestReadCacheDigest: @@ -281,7 +284,7 @@ def test_rejects_a_file_that_is_not_a_valid_bundle(self, tmp_path, build): ExecutableCoordinator()._build_task_handler_command(path=bundle) def test_rejects_a_bundle_without_a_schema_version(self, tmp_path): - metadata = _make_metadata(["etl"]) + metadata = _make_metadata() del metadata["sdk"]["supervisor_schema_version"] bundle = _build_bundle(tmp_path / "etl", metadata=metadata) @@ -289,6 +292,36 @@ def test_rejects_a_bundle_without_a_schema_version(self, tmp_path): ExecutableCoordinator()._build_task_handler_command(path=bundle) +class TestManifestWithAndWithoutADagInventory: + """A manifest lists no Dags now. A bundle packed earlier still carries a ``dags`` mapping, which is ignored.""" + + @pytest.fixture(params=[None, ["etl", "other_dag"]], ids=["without-dags", "with-dags"]) + def bundle(self, request, tmp_path): + metadata = _make_metadata(request.param) + assert ("dags" in metadata) is (request.param is not None) + return _build_bundle(tmp_path / "etl", metadata=metadata) + + def test_is_listed_as_a_candidate(self, bundle): + assert _list_candidates(bundle.parent) == [ + TaskHandlerCandidate(rel_path="etl", size_bytes=bundle.stat().st_size, cache_digest=None) + ] + + def test_is_started_with_its_schema_version(self, bundle): + command, schema_version = ExecutableCoordinator()._build_task_handler_command(path=bundle) + + assert command == [str(bundle.resolve())] + assert schema_version == "2026-06-16" + + def test_runs_for_a_task_that_names_it(self, bundle, mock_client): + with register_dag_bundle("go-task-handlers", bundle.parent) as name: + reference = TaskHandlerArtifactRef(bundle_info=BundleInfo(name=name), rel_path="etl") + _, popen_calls = _execute_task( + mock_client, "other-dags", dag_rel_path="dag.py", task_handler_artifact=reference + ) + + assert popen_calls[0][0] == str(bundle.resolve()) + + def _list_candidates(bundle_path: Path) -> list[TaskHandlerCandidate]: return ExecutableCoordinator().list_task_handler_candidates(bundle_path) @@ -354,7 +387,7 @@ def test_lists_a_bundle_it_cannot_run_with_its_trailer_error(self, tmp_path): @pytest.fixture def bundles_dir(tmp_path): """A directory with one bundle that declares no Dag id, so a task cannot find it by its Dag id.""" - _build_bundle(tmp_path / "my_bundle", dag_ids=[]) + _build_bundle(tmp_path / "my_bundle") return tmp_path @@ -430,7 +463,7 @@ def test_a_referenced_bundle_runs_even_when_another_registers_the_same_dag_id( def test_a_referenced_bundle_runs_whatever_dag_ids_it_declares( self, bundles_dir, go_task_handlers, mock_client ): - _build_bundle(bundles_dir / "etl", dag_ids=[]) + _build_bundle(bundles_dir / "etl") reference = TaskHandlerArtifactRef(rel_path="etl") _, popen_calls = _execute_task( @@ -537,7 +570,7 @@ def test_a_jar_is_not_an_artifact_of_this_coordinator(self, bundles_dir, go_task def test_a_bundle_with_an_unknown_schema_version_raises_before_the_runtime_starts( self, bundles_dir, go_task_handlers, mock_client ): - metadata = _make_metadata(["tutorial_dag"]) + metadata = _make_metadata() metadata["sdk"]["supervisor_schema_version"] = "1999-01-01" _build_bundle(bundles_dir / "bogus", metadata=metadata) reference = TaskHandlerArtifactRef(rel_path="bogus") diff --git a/task-sdk/tests/task_sdk/coordinators/node/_bundle_test_utils.py b/task-sdk/tests/task_sdk/coordinators/node/_bundle_test_utils.py index 8039cbc032cdd..d20ec4e3659ee 100644 --- a/task-sdk/tests/task_sdk/coordinators/node/_bundle_test_utils.py +++ b/task-sdk/tests/task_sdk/coordinators/node/_bundle_test_utils.py @@ -41,6 +41,7 @@ def metadata_json( metadata_version: str | None = "1.0", dag_source_paths: dict[str, str] | None = None, entrypoint_path: str | None = DEFAULT_SOURCE_PATH, + task_handlers: bool = False, ) -> bytes: if dag_source_paths is None: dag_source_paths = {dag_id: DEFAULT_SOURCE_PATH for dag_id in dag_ids} @@ -54,7 +55,9 @@ def metadata_json( if entrypoint_path is not None: metadata["entrypoint_path"] = entrypoint_path metadata["dag_source_paths"] = dag_source_paths - metadata["task_handlers"] = {dag_id: {"tasks": ["test_task"]} for dag_id in dag_ids} + if task_handlers: + # Bundles packed before airflow-ts-pack stopped embedding the handlers it checks. + metadata["task_handlers"] = {dag_id: {"tasks": ["test_task"]} for dag_id in dag_ids} if metadata_version is not None: metadata = {"airflow_bundle_metadata_version": metadata_version, **metadata} return json.dumps(metadata, separators=(",", ":"), ensure_ascii=False).encode() diff --git a/task-sdk/tests/task_sdk/coordinators/node/test_bundle_reader.py b/task-sdk/tests/task_sdk/coordinators/node/test_bundle_reader.py index ceec75f8f52a5..85d60288016bd 100644 --- a/task-sdk/tests/task_sdk/coordinators/node/test_bundle_reader.py +++ b/task-sdk/tests/task_sdk/coordinators/node/test_bundle_reader.py @@ -53,7 +53,16 @@ from tests_common.test_utils.paths import AIRFLOW_ROOT_PATH -TYPESCRIPT_V1_FIXTURE = AIRFLOW_ROOT_PATH / "ts-sdk" / "tests" / "cli" / "fixtures" / "bundle-v1.min.mjs" +TYPESCRIPT_FIXTURES = AIRFLOW_ROOT_PATH / "ts-sdk" / "tests" / "cli" / "fixtures" +TYPESCRIPT_V1_FIXTURE = TYPESCRIPT_FIXTURES / "bundle-v1.min.mjs" +# The same bundle as packed before airflow-ts-pack stopped embedding task_handlers in the metadata. +TYPESCRIPT_V1_WITH_TASK_HANDLERS_FIXTURE = TYPESCRIPT_FIXTURES / "bundle-v1-with-task-handlers.min.mjs" + + +def _embedded_metadata(bundle: pathlib.Path) -> dict: + metadata_line = bundle.read_bytes().split(b"\n")[1] + assert metadata_line.startswith(METADATA_PREFIX) + return json.loads(metadata_line[len(METADATA_PREFIX) :]) @pytest.fixture(autouse=True) @@ -68,6 +77,21 @@ def test_reads_bundle_produced_by_typescript_encoder(self): assert metadata.supervisor_schema_version == SCHEMA_VERSION + def test_a_bundle_packed_today_embeds_no_task_handlers(self): + assert "task_handlers" not in _embedded_metadata(TYPESCRIPT_V1_FIXTURE) + + def test_reads_a_bundle_packed_with_task_handlers_in_its_metadata(self): + assert _embedded_metadata(TYPESCRIPT_V1_WITH_TASK_HANDLERS_FIXTURE)["task_handlers"] == { + "test_dag": {"tasks": ["test_task"]} + } + + metadata = read_bundle(TYPESCRIPT_V1_WITH_TASK_HANDLERS_FIXTURE) + + assert metadata.supervisor_schema_version == SCHEMA_VERSION + assert read_bundle_source(TYPESCRIPT_V1_WITH_TASK_HANDLERS_FIXTURE, "test_dag") == read_bundle_source( + TYPESCRIPT_V1_FIXTURE, "test_dag" + ) + def test_reads_source_embedded_by_typescript_encoder(self): # Real encoder output, whose source region needs both escape branches. Recovering it # exactly is the cross-language agreement on the escape scheme, which the Python helper @@ -470,9 +494,8 @@ def test_rejects_metadata_that_is_not_a_mapping(self, tmp_path): ) def test_reads_metadata_whether_or_not_it_lists_task_handlers(self, tmp_path, task_handlers): metadata = json.loads(_metadata_json("sales")) - if task_handlers is None: - del metadata["task_handlers"] - else: + assert "task_handlers" not in metadata + if task_handlers is not None: metadata["task_handlers"] = task_handlers write_bundle(tmp_path, metadata_payload=json.dumps(metadata).encode()) diff --git a/task-sdk/tests/task_sdk/coordinators/node/test_coordinator.py b/task-sdk/tests/task_sdk/coordinators/node/test_coordinator.py index 152f6b5bd41cf..1a0a68ec95332 100644 --- a/task-sdk/tests/task_sdk/coordinators/node/test_coordinator.py +++ b/task-sdk/tests/task_sdk/coordinators/node/test_coordinator.py @@ -38,7 +38,10 @@ from airflow.sdk.execution_time.comms import TaskHandlerArtifactRef from airflow.sdk.execution_time.coordinator import TaskHandlerArtifactError, TaskHandlerCandidate +from tests_common.test_utils.paths import AIRFLOW_ROOT_PATH + SCHEMA_VERSION = "2026-06-16" +TYPESCRIPT_FIXTURES = AIRFLOW_ROOT_PATH / "ts-sdk" / "tests" / "cli" / "fixtures" @pytest.fixture(autouse=True) @@ -267,6 +270,32 @@ def test_a_bundle_with_an_unknown_schema_version_raises_before_the_runtime_start mock_start.assert_not_called() +class TestBundlesPackedByAirflowTsPack: + """A bundle reads whether or not its metadata still carries the task_handlers older packers embedded.""" + + @pytest.fixture( + params=["bundle-v1.min.mjs", "bundle-v1-with-task-handlers.min.mjs"], + ids=["without-task-handlers", "with-task-handlers"], + ) + def packed_bundle(self, request, tmp_path): + bundle = tmp_path / "handlers.min.mjs" + bundle.write_bytes((TYPESCRIPT_FIXTURES / request.param).read_bytes()) + return bundle + + def test_is_listed_as_a_candidate(self, packed_bundle): + candidates = NodeCoordinator().list_task_handler_candidates(packed_bundle.parent) + + assert [(c.rel_path, c.error) for c in candidates] == [("handlers.min.mjs", None)] + + def test_is_started_with_its_schema_version(self, packed_bundle): + command, schema_version = NodeCoordinator(node_executable="node")._build_task_handler_command( + path=packed_bundle + ) + + assert command == ["node", str(packed_bundle)] + assert schema_version == SCHEMA_VERSION + + class TestListTaskHandlerCandidates: def test_lists_min_mjs_files_with_a_layout_line(self, tmp_path): bundle = write_bundle(tmp_path, "test_dag", name="team-a/handlers.min.mjs") diff --git a/task-sdk/tests/task_sdk/coordinators/test_airflow_metadata_schema.py b/task-sdk/tests/task_sdk/coordinators/test_airflow_metadata_schema.py new file mode 100644 index 0000000000000..91a8c2f665fff --- /dev/null +++ b/task-sdk/tests/task_sdk/coordinators/test_airflow_metadata_schema.py @@ -0,0 +1,127 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""The bundle metadata JSON Schema accepts what the packers write, and the manifests older packers wrote.""" + +from __future__ import annotations + +import copy +import json +import re +import textwrap +from typing import Any + +import jsonschema +import pytest +import yaml + +from tests_common.test_utils.paths import AIRFLOW_ROOT_PATH + +DOCS_PATH = AIRFLOW_ROOT_PATH / "task-sdk" / "docs" +SCHEMA = json.loads((DOCS_PATH / "airflow-metadata.schema.json").read_text()) + +DIGESTS = {"integrity": "a" * 64, "cache": "b" * 64} + +# What a packer writes today: no Dag inventory. +MANIFEST: dict[str, Any] = { + "airflow_bundle_metadata_version": "1.0", + "sdk": {"language": "go", "version": "0.1.0", "supervisor_schema_version": "2026-10-30"}, + "source": "main.go", + "digests": DIGESTS, +} + +# What a packer wrote before the manifest dropped the inventory. +MANIFEST_WITH_DAGS: dict[str, Any] = { + **MANIFEST, + "dags": { + "simple_dag": {"tasks": ["extract", "transform", "load"]}, + "another_dag": {"tasks": []}, + }, +} + + +def _is_valid(manifest: dict[str, Any]) -> bool: + return jsonschema.Draft202012Validator(SCHEMA).is_valid(manifest) + + +def _get_spec_example() -> dict[str, Any]: + """Return the manifest the executable bundle spec shows in its first YAML example.""" + spec = (DOCS_PATH / "executable-bundle-spec.rst").read_text() + block = re.search(r"\.\. code-block:: yaml\n\n((?: .*\n|\n)+)", spec) + assert block is not None + return yaml.safe_load(textwrap.dedent(block.group(1))) + + +def test_the_schema_is_a_valid_draft_2020_12_schema(): + jsonschema.Draft202012Validator.check_schema(SCHEMA) + + +@pytest.mark.parametrize("digests", [True, False], ids=["with-digests", "without-digests"]) +def test_accepts_a_manifest_without_a_dag_inventory(digests): + manifest = copy.deepcopy(MANIFEST) + if not digests: + del manifest["digests"] + + assert _is_valid(manifest) + + +def test_accepts_a_manifest_an_older_packer_wrote_with_its_dag_inventory(): + assert _is_valid(MANIFEST_WITH_DAGS) + + +def test_accepts_the_example_in_the_spec(): + example = _get_spec_example() + + assert "dags" not in example + assert _is_valid(example) + + +def test_does_not_describe_a_dag_inventory(): + assert "dags" not in SCHEMA["properties"] + assert "dags" not in SCHEMA["required"] + assert "$defs" not in SCHEMA + + +def test_keeps_the_bundle_spec_version(): + assert ( + SCHEMA["properties"]["airflow_bundle_metadata_version"]["pattern"] == r"^[0-9]+\.[0-9]+(\.[0-9]+)?$" + ) + assert MANIFEST["airflow_bundle_metadata_version"] == "1.0" + assert "1.0" in SCHEMA["$id"] + + +@pytest.mark.parametrize("field", ["airflow_bundle_metadata_version", "sdk", "source"]) +def test_requires_what_a_reader_needs(field): + manifest = copy.deepcopy(MANIFEST) + del manifest[field] + + assert not _is_valid(manifest) + + +@pytest.mark.parametrize("field", ["language", "version", "supervisor_schema_version"]) +def test_requires_the_sdk_fields(field): + manifest = copy.deepcopy(MANIFEST) + del manifest["sdk"][field] + + assert not _is_valid(manifest) + + +def test_rejects_a_supervisor_schema_version_that_is_not_a_date(): + manifest = copy.deepcopy(MANIFEST) + manifest["sdk"]["supervisor_schema_version"] = "latest" + + assert not _is_valid(manifest) diff --git a/ts-sdk/README.md b/ts-sdk/README.md index 2e71736ef5dab..7ad9f0a4ee05c 100644 --- a/ts-sdk/README.md +++ b/ts-sdk/README.md @@ -311,7 +311,9 @@ airflow-ts-pack src/main.ts --outdir dist ``` It bundles the entrypoint into a minified `dist/bundle.min.mjs` with esbuild, then runs that bundle with -`--airflow-metadata` so it reports its own registered Dag/task pairs and supervisor schema version. The manifest is +`--airflow-metadata` so it reports its supervisor schema version and the file each of its Dags was declared in. The +packer also checks the Dag and task pairs the bundle registers, and warns about ids the Airflow server would reject, +but does not embed them. The manifest, with the SDK version, the schema version and those source files, is embedded as a compact JSON `//# airflowMetadata=...` comment after a leading compact JSON `//# airflowBundle=...` layout descriptor, and the entry module is embedded verbatim in a `/*# airflowSource ... #*/` block comment so Airflow can show the source a bundle was authored from, which its minified code no longer is. The CLI records the @@ -326,7 +328,6 @@ Options: - `--outdir `: output directory (default `dist`) - `--outfile `: exact output path, whose name must end in `.min.mjs` -- `--source `: display name of the primary source file shown in the Airflow UI (default: entry basename) ## TaskClient diff --git a/ts-sdk/example/README.md b/ts-sdk/example/README.md index 85b5c74099f80..b06a6a8696aea 100644 --- a/ts-sdk/example/README.md +++ b/ts-sdk/example/README.md @@ -28,8 +28,8 @@ This example shows the coordinator-mode shape for TypeScript task handlers: - `dist/bundle.min.mjs` is the generated Node.js bundle that Airflow launches. The build uses the SDK's `airflow-ts-pack` tool, which bundles the entrypoint -with esbuild and embeds the Airflow metadata generated from the bundle's -registered tasks, producing a single deployable file. +with esbuild and embeds the Airflow metadata read from the bundle, producing a +single deployable file. ## Build diff --git a/ts-sdk/src/cli/bundle-encoder.ts b/ts-sdk/src/cli/bundle-encoder.ts index b5f2d063bd0c6..f35957cb10d21 100644 --- a/ts-sdk/src/cli/bundle-encoder.ts +++ b/ts-sdk/src/cli/bundle-encoder.ts @@ -30,9 +30,9 @@ * -> executable JavaScript * * The header records each region's byte range and digest, and each source - * region also carries the path it was compiled from. Metadata describes what - * the bundle serves, source regions carry the author's original files, and - * the executable JavaScript runs the task handlers. + * region also carries the path it was compiled from. Metadata names the SDK and + * the source file of each Dag declared in TypeScript, source regions carry the + * author's original files, and the executable JavaScript runs the task handlers. * * This module owns the on-disk encoding. Readers must use the header's named * byte ranges and paths rather than incidental line positions. @@ -54,6 +54,8 @@ export const EMBEDDED_SOURCE_MARKER = "/*# airflowSource:"; export const EMBEDDED_SOURCE_CLOSE = "\n#*/\n"; export interface BundleEncoderInput { + /** What the built bundle reported. The encoder embeds its schema version and `dag_source_paths`; + * the task handlers it lists are not embedded. */ bundleManifest: BundleManifest; sdkVersion: string; /** Project-relative path of the entry file `airflow-ts-pack` bundled. Always present among @@ -71,7 +73,6 @@ interface BundleMetadata { sdk: { language: string; version: string; supervisor_schema_version: string }; entrypoint_path: string; dag_source_paths: BundleManifest["dag_source_paths"]; - task_handlers: BundleManifest["task_handlers"]; } interface VerifiedByteRange { @@ -245,7 +246,7 @@ function encodeMetadata(input: BundleEncoderInput): Buffer { if (metadata.length > EMBEDDED_METADATA_MAX_BYTES) { throw new Error( `Embedded airflow metadata is ${metadata.length} bytes, ` + - `over the ${EMBEDDED_METADATA_MAX_BYTES} byte limit; reduce the number of registered tasks`, + `over the ${EMBEDDED_METADATA_MAX_BYTES} byte limit; reduce the number of Dags declared in TypeScript`, ); } return metadata; @@ -268,7 +269,6 @@ function buildBundleMetadata(input: BundleEncoderInput): BundleMetadata { }, entrypoint_path: input.entrypointPath, dag_source_paths: input.bundleManifest.dag_source_paths, - task_handlers: input.bundleManifest.task_handlers, }; } diff --git a/ts-sdk/src/cli/pack.ts b/ts-sdk/src/cli/pack.ts index f1806909e0f18..5afe259d318b8 100644 --- a/ts-sdk/src/cli/pack.ts +++ b/ts-sdk/src/cli/pack.ts @@ -21,8 +21,9 @@ // `bundle.min.mjs` carries the metadata, the entrypoint source, and an integrity layout descriptor // in JavaScript comments. // -// Build first, then run the built bundle with --airflow-metadata so the manifest comes from the -// bundle's own Dag registry and schema version, never from a hand-written sidecar. +// Build first, then run the built bundle with --airflow-metadata so the schema version and the Dag +// source paths come from the bundle's own registry, never from a hand-written sidecar. The task +// handlers the bundle reports are checked here and not embedded: nothing reads them from the artifact. import { execFileSync } from "node:child_process"; import { readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; @@ -49,8 +50,8 @@ const MANIFEST_MAX_BUFFER_BYTES = 64 * 1024 * 1024; const USAGE = `Usage: airflow-ts-pack [--outdir | --outfile ] Bundles into a minified ${BUNDLE_FILENAME} with esbuild and embeds the -airflow metadata generated from the bundle's served Dags, plus each Dag-defining -source file verbatim so Airflow has readable text to display per Dag. +airflow metadata read from the bundle, plus each Dag-defining source file +verbatim so Airflow has readable text to display per Dag. Options: --outdir Output directory, holding ${BUNDLE_FILENAME} (default: dist) @@ -330,8 +331,7 @@ export async function runPack(argv: readonly string[]): Promise { `${args.entry} served nothing; register Dags or task handlers with bundle.register(...)`, ); } - // Warn rather than fail, as airflow-go-pack does: the shared schema allows a - // Dag with no tasks. + // Warn rather than fail: a Dag with no tasks is not an error. for (const [dagId, dag] of dagEntries) { if (dag.tasks.length === 0) { process.stderr.write(`warning: dag ${JSON.stringify(dagId)} has no tasks\n`); diff --git a/ts-sdk/src/cli/validate.ts b/ts-sdk/src/cli/validate.ts index 1d8098b3fcf39..962621c890499 100644 --- a/ts-sdk/src/cli/validate.ts +++ b/ts-sdk/src/cli/validate.ts @@ -25,7 +25,7 @@ const ID_REGEX = /^[\p{L}\p{N}_.-]+$/u; type WarnFn = (message: string) => void; /** - * Check every dag and task id in the manifest against the rules the Airflow + * Check every dag and task id the bundle reports against the rules the Airflow * server enforces (`airflow.utils.helpers.validate_key`). Best-effort and * warn-only: the server validates authoritatively, and checks like the `..` * one depend on server configuration the packer cannot see. diff --git a/ts-sdk/src/coordinator/manifest.ts b/ts-sdk/src/coordinator/manifest.ts index d706f984c2cb9..a7f3cbb10b8db 100644 --- a/ts-sdk/src/coordinator/manifest.ts +++ b/ts-sdk/src/coordinator/manifest.ts @@ -26,12 +26,16 @@ export const AIRFLOW_METADATA_FLAG = "--airflow-metadata"; /** Marks the manifest line on stdout, which import-time logging may also reach. */ export const AIRFLOW_METADATA_SENTINEL = "__AIRFLOW_METADATA__ "; -/** Bundle manifest fields only the built bundle itself knows: the schema version it was compiled - * against, and the task handlers it registered grouped by Dag. Named `task_handlers` because a - * TypeScript bundle provides handlers for Dags declared elsewhere, not Dag definitions. A Dag with - * no handlers keeps an empty `tasks` list so `airflow-ts-pack` can warn instead of dropping it. +/** What the built bundle reports about itself when `airflow-ts-pack` runs it with + * `--airflow-metadata`: the schema version it was compiled against, the task handlers it + * registered grouped by Dag, and the source file of each Dag declared in TypeScript. The packer + * embeds the schema version and `dag_source_paths`, and only checks `task_handlers` (a bundle + * that serves nothing, a Dag with no tasks, ids the server would reject). Named `task_handlers` + * because a TypeScript bundle provides handlers for Dags declared elsewhere, not Dag definitions. + * A Dag with no handlers keeps an empty `tasks` list so `airflow-ts-pack` can warn instead of + * dropping it. * - * `dag_source_paths` names the source file each *native* Dag was declared in — captured at + * `dag_source_paths` names the source file each *native* Dag was declared in, captured at * construction time from `airflow-ts-pack`'s module-source tag. Mixed-lang Dags (owned by * Python) are absent here. */ export interface BundleManifest { diff --git a/ts-sdk/src/sdk/bundle.ts b/ts-sdk/src/sdk/bundle.ts index 4ae41cfe16ed4..40e66e2ee0781 100644 --- a/ts-sdk/src/sdk/bundle.ts +++ b/ts-sdk/src/sdk/bundle.ts @@ -133,8 +133,9 @@ export class Bundle { * makes. * * Everything this bundle provides must be registered before `serve()` is - * awaited: what is left out is not part of the bundle, and its tasks are - * marked removed at runtime. + * awaited: a stub task whose handler is left out fails the import of its + * Python Dag file, and a task that still reaches the bundle without its + * handler is marked removed. */ async serve(): Promise { // `const { serve } = bundle` detaches the method, which would otherwise diff --git a/ts-sdk/tests/cli/fixtures/bundle-v1-with-task-handlers.min.mjs b/ts-sdk/tests/cli/fixtures/bundle-v1-with-task-handlers.min.mjs new file mode 100644 index 0000000000000..2433161574736 --- /dev/null +++ b/ts-sdk/tests/cli/fixtures/bundle-v1-with-task-handlers.min.mjs @@ -0,0 +1,18 @@ +//# airflowBundle={"code":{"start":"00000000000003ec","end":"00000000000004fb","sha256":"fa1afadd7cb7147d54e4870b849dd2390e5ad75f6153d040e7381ae8a76a5a29"},"metadata":{"start":"00000000000001de","end":"00000000000002df","sha256":"0f522f6b96c0813da84ac9ef1e4c944b93da409b502c850386964d4117b8d4b0"},"sources":[{"path":"entry.ts","start":"00000000000002fb","end":"00000000000003e7","sha256":"3e7a601893ae2cc92dd72dcc596d3fbfa19cafbaa0d3f5768973c49c3daee1c9"}]} +//# airflowMetadata={"airflow_bundle_metadata_version":"1.0","sdk":{"language":"typescript","version":"0.1.0","supervisor_schema_version":"2026-06-16"},"entrypoint_path":"entry.ts","dag_source_paths":{"test_dag":"entry.ts"},"task_handlers":{"test_dag":{"tasks":["test_task"]}}} +/*# airflowSource:entry.ts +/** Handlers for the test Dag. *\/ +import { Bundle, Dag } from "apache-airflow-ts-sdk"; + +const TERMINATOR = /\*\\//; +const dag = new Dag("test_dag"); +dag.task("test_task", async () => TERMINATOR.source); + +await new Bundle(dag).serve(); + +#*/ +var e=async function(){return"extracted"};await e(); +/*! Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + */ diff --git a/ts-sdk/tests/cli/fixtures/bundle-v1.min.mjs b/ts-sdk/tests/cli/fixtures/bundle-v1.min.mjs index 2433161574736..c6d408b9cac82 100644 --- a/ts-sdk/tests/cli/fixtures/bundle-v1.min.mjs +++ b/ts-sdk/tests/cli/fixtures/bundle-v1.min.mjs @@ -1,5 +1,5 @@ -//# airflowBundle={"code":{"start":"00000000000003ec","end":"00000000000004fb","sha256":"fa1afadd7cb7147d54e4870b849dd2390e5ad75f6153d040e7381ae8a76a5a29"},"metadata":{"start":"00000000000001de","end":"00000000000002df","sha256":"0f522f6b96c0813da84ac9ef1e4c944b93da409b502c850386964d4117b8d4b0"},"sources":[{"path":"entry.ts","start":"00000000000002fb","end":"00000000000003e7","sha256":"3e7a601893ae2cc92dd72dcc596d3fbfa19cafbaa0d3f5768973c49c3daee1c9"}]} -//# airflowMetadata={"airflow_bundle_metadata_version":"1.0","sdk":{"language":"typescript","version":"0.1.0","supervisor_schema_version":"2026-06-16"},"entrypoint_path":"entry.ts","dag_source_paths":{"test_dag":"entry.ts"},"task_handlers":{"test_dag":{"tasks":["test_task"]}}} +//# airflowBundle={"code":{"start":"00000000000003b7","end":"00000000000004c6","sha256":"fa1afadd7cb7147d54e4870b849dd2390e5ad75f6153d040e7381ae8a76a5a29"},"metadata":{"start":"00000000000001de","end":"00000000000002aa","sha256":"98d0324443dff6a3fa1d43f7b97128ad569955ef2fec7c5d2623400f75e35bd2"},"sources":[{"path":"entry.ts","start":"00000000000002c6","end":"00000000000003b2","sha256":"3e7a601893ae2cc92dd72dcc596d3fbfa19cafbaa0d3f5768973c49c3daee1c9"}]} +//# airflowMetadata={"airflow_bundle_metadata_version":"1.0","sdk":{"language":"typescript","version":"0.1.0","supervisor_schema_version":"2026-06-16"},"entrypoint_path":"entry.ts","dag_source_paths":{"test_dag":"entry.ts"}} /*# airflowSource:entry.ts /** Handlers for the test Dag. *\/ import { Bundle, Dag } from "apache-airflow-ts-sdk"; diff --git a/ts-sdk/tests/cli/pack.test.ts b/ts-sdk/tests/cli/pack.test.ts index 7b428acaa44e1..d0adf345c50f4 100644 --- a/ts-sdk/tests/cli/pack.test.ts +++ b/ts-sdk/tests/cli/pack.test.ts @@ -34,7 +34,11 @@ import { } from "../../src/cli/bundle-encoder.js"; import { parsePackArgs, runPack } from "../../src/cli/pack.js"; import { SUPERVISOR_API_VERSION } from "../../src/coordinator/protocol.js"; -import { AIRFLOW_METADATA_SENTINEL } from "../../src/coordinator/manifest.js"; +import { + AIRFLOW_METADATA_FLAG, + AIRFLOW_METADATA_SENTINEL, + type BundleManifest, +} from "../../src/coordinator/manifest.js"; const FIXTURE_ENTRY = fileURLToPath(new URL("fixtures/entry.ts", import.meta.url)); const GOLDEN_BUNDLE = fileURLToPath(new URL("fixtures/bundle-v1.min.mjs", import.meta.url)); @@ -139,7 +143,7 @@ describe("encodeBundle", () => { const bundle = encodeBundle({ bundleManifest: { supervisor_schema_version: "2026-06-16", - task_handlers: { my_dag: { tasks: ["a", 'b"c'] } }, + task_handlers: { my_dag: { tasks: ["a"] } }, dag_source_paths: { my_dag: "src/my_dag.ts" }, }, sdkVersion: "0.1.0", @@ -170,13 +174,41 @@ describe("encodeBundle", () => { const metadata = bundle.subarray(metadataStart, metadataEnd).toString("utf-8"); expect(metadata).toBe( - '{"airflow_bundle_metadata_version":"1.0","sdk":{"language":"typescript","version":"0.1.0","supervisor_schema_version":"2026-06-16"},"entrypoint_path":"src/my_dag.ts","dag_source_paths":{"my_dag":"src/my_dag.ts"},"task_handlers":{"my_dag":{"tasks":["a","b\\"c"]}}}', + '{"airflow_bundle_metadata_version":"1.0","sdk":{"language":"typescript","version":"0.1.0","supervisor_schema_version":"2026-06-16"},"entrypoint_path":"src/my_dag.ts","dag_source_paths":{"my_dag":"src/my_dag.ts"}}', ); expect(header).not.toHaveProperty("version"); expect(header).not.toHaveProperty("source"); }); + it("embeds the schema version and the Dag source paths, and no task handlers", () => { + const bundle = encodeBundle({ + bundleManifest: { + supervisor_schema_version: "2026-06-16", + task_handlers: { my_dag: { tasks: ["a"] }, py_dag: { tasks: ["stub"] } }, + dag_source_paths: { my_dag: "src/my_dag.ts" }, + }, + sdkVersion: "0.1.0", + entrypointPath: "src/my_dag.ts", + sourceFiles: { "src/my_dag.ts": "export {};\n" }, + executable: Buffer.from("export {};\n"), + }); + const header = parseHeader(bundle.subarray(0, bundle.indexOf("\n")).toString("utf-8")); + + const metadata = JSON.parse( + bundle + .subarray( + Number.parseInt(header.metadata.start, 16), + Number.parseInt(header.metadata.end, 16), + ) + .toString("utf-8"), + ); + + expect(metadata.sdk.supervisor_schema_version).toBe("2026-06-16"); + expect(metadata.dag_source_paths).toEqual({ my_dag: "src/my_dag.ts" }); + expect(metadata).not.toHaveProperty("task_handlers"); + }); + it("embeds one source region per author-owned Dag file, each with its path", () => { const bundle = encodeBundle({ bundleManifest: { @@ -348,8 +380,8 @@ describe("encodeBundle", () => { const bundle = encodeBundle({ bundleManifest: { supervisor_schema_version: "2026-06-16", - task_handlers: { "line\u2028separator": { tasks: ["paragraph\u2029separator"] } }, - dag_source_paths: {}, + task_handlers: {}, + dag_source_paths: { "line\u2028separator": "paragraph\u2029separator.ts" }, }, sdkVersion: "0.1.0", entrypointPath: "entry.ts", @@ -363,8 +395,8 @@ describe("encodeBundle", () => { expect(metadataLine).toContain("\\u2028"); expect(metadataLine).toContain("\\u2029"); expect(JSON.parse(metadataLine.slice(EMBEDDED_METADATA_PREFIX.length))).toHaveProperty( - "task_handlers.line\u2028separator.tasks", - ["paragraph\u2029separator"], + ["dag_source_paths", "line\u2028separator"], + "paragraph\u2029separator.ts", ); }); }); @@ -378,6 +410,19 @@ function readEmbeddedMetadata(bundlePath: string): string { return bundle.subarray(start, end).toString("utf-8"); } +/** What the packed bundle reports about itself when run with `--airflow-metadata`: the task + * handlers the packer checks. The embedded metadata does not carry them. */ +function readRuntimeReport(bundlePath: string): BundleManifest { + const dumped = execFileSync(process.execPath, [bundlePath, AIRFLOW_METADATA_FLAG], { + encoding: "utf-8", + }); + const line = dumped + .split("\n") + .reverse() + .find((candidate) => candidate.startsWith(AIRFLOW_METADATA_SENTINEL))!; + return JSON.parse(line.slice(AIRFLOW_METADATA_SENTINEL.length)) as BundleManifest; +} + /** Collect what runPack writes to stderr; returns a reader for the text so far. */ function captureStderr(): () => string { const chunks: string[] = []; @@ -423,19 +468,16 @@ describe("runPack", () => { dag_source_paths: { other_dag: expect.stringMatching(/entry\.ts$/) as unknown as string, }, - task_handlers: { - fixture_dag: { tasks: ["extract", "transform"] }, - other_dag: { tasks: ["solo"] }, - }, }); - - const dumped = execFileSync(process.execPath, [bundlePath, "--airflow-metadata"], { - encoding: "utf-8", + // The packer ran the bundle and checked the handlers it reported, but embeds none of them. + expect(metadata).not.toHaveProperty("task_handlers"); + + const report = readRuntimeReport(bundlePath); + expect(report.supervisor_schema_version).toBe(SUPERVISOR_API_VERSION); + expect(report.task_handlers).toEqual({ + fixture_dag: { tasks: ["extract", "transform"] }, + other_dag: { tasks: ["solo"] }, }); - expect(dumped.startsWith(AIRFLOW_METADATA_SENTINEL)).toBe(true); - expect( - JSON.parse(dumped.slice(AIRFLOW_METADATA_SENTINEL.length)).supervisor_schema_version, - ).toBe(SUPERVISOR_API_VERSION); }); it("embeds verifiable metadata and code regions", async () => { @@ -459,9 +501,7 @@ describe("runPack", () => { offset(layout.metadata.end), ); expect(createHash("sha256").update(metadataPayload).digest("hex")).toBe(layout.metadata.sha256); - expect(JSON.parse(metadataPayload.toString("utf-8"))).toHaveProperty( - "task_handlers.fixture_dag", - ); + expect(JSON.parse(metadataPayload.toString("utf-8"))).not.toHaveProperty("task_handlers"); // One source region per author-owned Dag file. The fixture declares one // native Dag (`other_dag`) in the entry, so exactly one region ships and @@ -514,7 +554,7 @@ describe("runPack", () => { expect(readFileSync(target).subarray(0, EMBEDDED_LAYOUT_PREFIX.length).toString()).toBe( EMBEDDED_LAYOUT_PREFIX, ); - expect(JSON.parse(readEmbeddedMetadata(target))).toHaveProperty("task_handlers.fixture_dag"); + expect(JSON.parse(readEmbeddedMetadata(target))).toHaveProperty("dag_source_paths.other_dag"); }); it("keeps a shebang entry runnable and reads the manifest past import-time logging", async () => { @@ -533,23 +573,31 @@ describe("runPack", () => { const metadataLine = bundle.split("\n")[1]!; const metadata = JSON.parse(metadataLine.slice(EMBEDDED_METADATA_PREFIX.length)); - expect(metadata).toHaveProperty("task_handlers.noisy_dag"); + expect(metadata).toHaveProperty("dag_source_paths.noisy_dag"); + expect(metadata).not.toHaveProperty("task_handlers"); - execFileSync(process.execPath, [bundlePath, "--airflow-metadata"], { encoding: "utf-8" }); + expect(readRuntimeReport(bundlePath).task_handlers).toHaveProperty("noisy_dag"); }); it("leaves no bundle behind when the metadata exceeds the embedded size limit", async () => { outdir = mkdtempSync(path.join(tmpdir(), "ts-pack-")); const entry = path.join(outdir, "huge-entry.ts"); + // The metadata names each Dag declared in TypeScript with its source file, so it takes many + // long Dag ids to reach the limit. The task handlers are not embedded. writeFileSync( entry, [ `import { Bundle, Dag } from ${JSON.stringify(SDK_INDEX)};`, - 'const bigDag = new Dag("big_dag");', - 'for (let i = 0; i < 5000; i += 1) bigDag.task(String(i).padStart(240, "t"), async () => undefined)();', - "await new Bundle(bigDag).serve();", + "const dags: Dag[] = [];", + "for (let i = 0; i < 5000; i += 1) {", + ' const dag = new Dag(String(i).padStart(240, "d"));', + ' dag.task("work", async () => undefined)();', + " dags.push(dag);", + "}", + "await new Bundle(...dags).serve();", ].join("\n"), ); + captureStderr(); await expect(runPack([entry, "--outdir", outdir])).rejects.toThrow( "over the 1048576 byte limit", @@ -674,7 +722,7 @@ describe("runPack", () => { expect(existsSync(path.join(outdir, "bundle.pack-staging.mjs"))).toBe(false); }); - it("warns but still packs a registered Dag with no tasks, as airflow-go-pack does", async () => { + it("warns but still packs a registered Dag with no tasks", async () => { outdir = mkdtempSync(path.join(tmpdir(), "ts-pack-")); const entry = path.join(outdir, "mixed-entry.ts"); writeFileSync( @@ -691,10 +739,9 @@ describe("runPack", () => { await runPack([entry, "--outdir", outdir]); expect(stderr()).toContain('warning: dag "empty_dag" has no tasks\n'); - expect(JSON.parse(readEmbeddedMetadata(path.join(outdir, "bundle.min.mjs")))).toHaveProperty( - "task_handlers.empty_dag.tasks", - [], - ); + const bundlePath = path.join(outdir, "bundle.min.mjs"); + expect(readRuntimeReport(bundlePath).task_handlers.empty_dag).toEqual({ tasks: [] }); + expect(JSON.parse(readEmbeddedMetadata(bundlePath))).not.toHaveProperty("task_handlers"); }); it("takes an omitted task id from the handler name, through minification", async () => { @@ -717,10 +764,9 @@ describe("runPack", () => { // Read back off the packed artifact, so this asserts what minification // left behind rather than what the source said. - expect(JSON.parse(readEmbeddedMetadata(path.join(outdir, "bundle.min.mjs")))).toHaveProperty( - "task_handlers.sales_dag.tasks", - ["extractRows", "loadRows"], - ); + expect( + readRuntimeReport(path.join(outdir, "bundle.min.mjs")).task_handlers.sales_dag!.tasks, + ).toEqual(["extractRows", "loadRows"]); expect(stderr()).toBe(""); }); @@ -759,10 +805,9 @@ describe("runPack", () => { await runPack([entry, "--outdir", outdir]); - expect(JSON.parse(readEmbeddedMetadata(path.join(outdir, "bundle.min.mjs")))).toHaveProperty( - "task_handlers.sales_dag.tasks", - ["extract_north", "load_north"], - ); + expect( + readRuntimeReport(path.join(outdir, "bundle.min.mjs")).task_handlers.sales_dag!.tasks, + ).toEqual(["extract_north", "load_north"]); }); it("packs only the Dags the served bundle holds", async () => { @@ -782,9 +827,11 @@ describe("runPack", () => { await runPack([entry, "--outdir", outdir]); - const metadata = JSON.parse(readEmbeddedMetadata(path.join(outdir, "bundle.min.mjs"))); - expect(metadata).toHaveProperty("task_handlers.sales_dag"); - expect(metadata).not.toHaveProperty("task_handlers.billing_dag"); + const bundlePath = path.join(outdir, "bundle.min.mjs"); + const metadata = JSON.parse(readEmbeddedMetadata(bundlePath)); + expect(metadata).toHaveProperty("dag_source_paths.sales_dag"); + expect(metadata).not.toHaveProperty("dag_source_paths.billing_dag"); + expect(Object.keys(readRuntimeReport(bundlePath).task_handlers)).toEqual(["sales_dag"]); }); it("embeds the entrypoint source for a mixed-language bundle with no native Dag", async () => {