Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
244 changes: 183 additions & 61 deletions airflow-core/docs/authoring-and-scheduling/language-sdks/java.rst

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,15 @@
4. A Java task that throws with retries left returns ``RetryTask`` rather than a
terminal ``FAILED``, so the supervisor marks it UP_FOR_RETRY and re-runs it;
``load`` therefore ends ``success`` on its second attempt (try_number 2).

5. Keyword arguments at the stub call site bind to a ``TaskInput``'s fields by
name, matched ignoring case and underscores; ``report`` asserts the value it
received, so it only succeeds if ``run_label`` reached its ``runLabel``
field.

The ``java_interface_example`` and ``scala_spark_example`` Dags are covered too:
the former for tasks written against ``InputTask``, whose arguments the SDK
resolves from the stub call site into a ``TaskInput`` and injects.
"""

from __future__ import annotations
Expand Down Expand Up @@ -81,6 +90,7 @@
_LOG_FETCH_TIMEOUT = 60

_ANNOTATION_DAG_ID = "java_annotation_example"
_INTERFACE_DAG_ID = "java_interface_example"
_XCOM_CASTING_DAG_ID = "java_xcom_casting_example"
_SCALA_SPARK_DAG_ID = "scala_spark_example"
_VARIABLE_WRITE_DAG_ID = "java_variable_write"
Expand Down Expand Up @@ -171,6 +181,12 @@ def annotation_example_run() -> _CompletedRun:
return _trigger_and_wait_for_dag(_ANNOTATION_DAG_ID, _JAVA_TASK_TIMEOUT)


@pytest.fixture(scope="module")
def interface_example_run() -> _CompletedRun:
"""Trigger the interface example once for all of its assertions."""
return _trigger_and_wait_for_dag(_INTERFACE_DAG_ID, _JAVA_TASK_TIMEOUT)


@pytest.fixture(scope="module")
def xcom_casting_example_run() -> _CompletedRun:
"""Trigger the XCom casting example once for all of its assertions."""
Expand Down Expand Up @@ -213,6 +229,19 @@ def test_transform_xcom_is_numeric_timestamp(self, annotation_example_run: _Comp
f"Expected 'transform' XCom to be a positive integer (millisecond timestamp), got {value!r}"
)

def test_report_binds_keyword_arguments_by_folded_name(self, annotation_example_run: _CompletedRun):
"""``report`` declares a ``TaskInput`` with no ``@ArgName``, so reaching ``success``
proves ``run_label`` folded onto its ``runLabel`` field: the task throws unless that
field holds the ``run_label="nightly"`` literal from the keyword call site."""
report_ti = annotation_example_run.get_task_instance("report")

assert report_ti.get("state") == "success", (
f"Java 'report' task did not succeed.\n"
f" task state : {report_ti.get('state')!r}\n"
f" dag state : {annotation_example_run.state!r}\n"
f" all tasks : {annotation_example_run.ti_states}"
)

def test_concurrent_client_calls_succeed(self, annotation_example_run: _CompletedRun):
"""A Java task calling the client from many threads must succeed."""
concurrent_ti = annotation_example_run.get_task_instance("concurrent")
Expand Down Expand Up @@ -377,13 +406,32 @@ def test_scratch_variable_deleted_by_java_task_is_gone(self, variable_write_run:
)


class TestJavaSDKInterfaceExample:
"""Verify tasks written against ``InputTask`` receive their bound arguments."""

def test_input_tasks_receive_their_bound_arguments(self, interface_example_run: _CompletedRun):
"""``transform`` and ``summarize`` implement ``InputTask``, so reaching ``success``
proves the supervisor's bindings arrived: ``transform`` binds the ``extract`` XCom
onto its bundle's ``extracted`` field, and ``summarize`` throws unless its bundle
holds the ``region_code="emea"`` literal from the keyword call site."""
for task_id in ("transform", "summarize"):
task_instance = interface_example_run.get_task_instance(task_id)
assert task_instance.get("state") == "success", (
f"Java {task_id!r} task did not succeed.\n"
f" task state : {task_instance.get('state')!r}\n"
f" dag state : {interface_example_run.state!r}\n"
f" all tasks : {interface_example_run.ti_states}"
)


class TestJavaSDKXComCastingExample:
"""Verify numeric XCom values are cast across declared Java types."""

def test_numeric_xcom_casting(self, xcom_casting_example_run: _CompletedRun):
"""Numeric XComs are read across declared types (int -> long -> double, and a wire
double back as a float), and a boxed param stays null when its XCom is absent."""
for task_id in ("widen_to_double", "consume_nullable", "consume_float"):
double back as a float), a boxed param stays null when its XCom is absent, and a
``List<Double>`` param keeps its element type over a literal list of integers."""
for task_id in ("widen_to_double", "consume_nullable", "consume_float", "consume_double_list"):
task_instance = xcom_casting_example_run.get_task_instance(task_id)
assert task_instance.get("state") == "success", (
f"Java {task_id!r} task did not succeed.\n"
Expand Down
2 changes: 1 addition & 1 deletion java-sdk/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -590,7 +590,7 @@ prek hook regenerate it.

<!-- BEGIN AUTO-GENERATED LANG-SDK COMPAT MATRIX -->

*Min. Airflow version: 3.3 · supervisor schema: 2026-06-16*
*Min. Airflow version: 3.3 · supervisor schema: 2026-10-30*
Comment thread
jason810496 marked this conversation as resolved.

| Dimension | Tier | Supported | Since | Notes |
|---|---|---|---|---|
Expand Down
33 changes: 28 additions & 5 deletions java-sdk/adr/0001-mixed-lang-dag-interface.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ The processor compiles this into a `Task`:
public static final class Score implements Task {
@Override
public void execute(Context context, Client client) throws Exception {
TaskArgs args = TaskArgs.of(context);
TaskArgs args = TaskArgs.of(context, client, 3);
long rows = args.require(0, Long.class);
double threshold = args.require(1, Double.class);
List<String> regions = args.require(2, new TypeReference<List<String>>() {});
Expand Down Expand Up @@ -103,7 +103,7 @@ public static class SummarizeInput implements TaskInput {
}

// summarize(region_code=..., transformed=...) is called with keyword
// arguments, which bind to the bundle's fields by name.
// arguments, which bind to the `TaskInput`'s fields by name.
public static class Summarize implements InputTask<SummarizeInput> {
public void execute(@NotNull Context context, Client client, SummarizeInput input) {
log.log(
Expand Down Expand Up @@ -155,11 +155,17 @@ One verb, overloaded, and everything registers as a class: nothing is constructe
- Annotated task data parameters bind by position; injected `Client` and `Context` do not consume
positions. Positional binding also needs no parameter names in the class file, so it does not
depend on compiling with `-parameters`.
- Keyword arguments bind through one `TaskInput` bundle, with `@ArgName` only when Python and Java
- Keyword arguments bind through one `TaskInput`, with `@ArgName` only when Python and Java
names differ.
- Interface tasks use `InputTask<MyInput>` for named fields. There is no public positional form.
- `TaskArgs` is internal. It is what generated code reads, and what every syntax above resolves to;
nothing in the public API returns or accepts it. It exposes `require(index, type)`, which fails
nothing in the public API returns or accepts it. It is opened with
`of(context, client, declared)` — the client because resolving a binding may have to pull an
upstream's XCom, and the declared count because positions carry the whole meaning of a flat
binding, so a call site that bound a different number of arguments than the method takes has
already shifted them. A parameter the call site omitted is exempt: its default still
arrives, but a method that does not declare it is not reading shifted arguments, so those are
dropped before the counts are compared. It exposes `require(index, type)`, which fails
when the position resolves to nothing, and `get(index, type)`, which yields `null` — each with a
`Class<T>` and a `TypeReference<T>` form.
- Annotation processing generates code, it never rewrites it, so the method a user writes stays
Expand All @@ -169,7 +175,24 @@ One verb, overloaded, and everything registers as a class: nothing is constructe
the generated body calls `new Parent().method(...)`.
- A parameter whose type has type arguments binds through the `TypeReference<T>` overload, so the
element type survives and the generated code never contains an unchecked cast.
- Missing reference or boxed values become `null`; primitive inputs fail clearly.
- **Neither direction of a `TaskInput` name mismatch fails the task.** A field binds by name, so a
field nothing supplies keeps its Java default and an argument no field claims changes nothing the
task reads. Both are logged, since either one means the Java signature and the Python stub
disagree. Captured defaults are not logged, since the Dag author never passed them. Flat
positional binding still fails either way, because a dropped or added argument shifts every later
one.
- **Two fields whose argument names fold alike fail when the bundle is built**, because the fold
cannot tell them apart and there is no value either could safely take. Two *arguments* that fold
alike are the other way round: they reach a field naming one of them exactly, and a field that
would match both is left unfilled and reported.
- **An argument that resolves to nothing is a value, not a mistake.** Once a parameter has claimed
its argument, a null literal or an upstream that pushed no XCom gives `null` to a reference or
boxed type, and fails for a primitive, which cannot hold it. Declaring a boxed type is how a
task says the value is optional.
- This matches the Go SDK, which logs both directions and zero-values an unmatched struct field
([go-sdk ADR-0006](../../go-sdk/adr/0006-cross-language-argument-binding.md)), and it follows the
cross-language rule in
[lang-SDK ADR-0007](../../airflow-core/adr/lang-sdk/0007-taskflow-across-language-boundary.md).
- `@Builder.XCom` is removed, keeping the Python call site as the single source of data-flow
wiring.
- **The generated `Task` is one more branch of an existing classifier.** `BuilderProcessor` already
Expand Down
2 changes: 1 addition & 1 deletion java-sdk/adr/0002-native-dag-interface.md
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ public final class EtlPipeline_Dag {
public static final class Transform implements Task {
@Override
public void execute(Context context, Client client) throws Exception {
TaskArgs args = TaskArgs.of(context);
TaskArgs args = TaskArgs.of(context, client, 1);
long extracted = args.require(0, Long.class);
double threshold = 0.9; // baked from lit(0.9) at Dag-build time
client.setXCom(new EtlPipeline().transform(client, context, extracted, threshold));
Expand Down
2 changes: 1 addition & 1 deletion java-sdk/capabilities.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ min_airflow_version: "3.3"

# Keep in sync with airflowSupervisorSchemaVersion in gradle.properties, which is what stamps the
# JAR manifest. The render hook fails if the two disagree.
supervisor_schema_version: "2026-06-16"
supervisor_schema_version: "2026-10-30"

# The runtime terminates a task with SucceedTask, RetryTask, or TaskState (failed/removed); it does
# not yet emit skipped, DeferTask, RescheduleTask, or AwaitInputTask.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,14 @@
import java.util.concurrent.Executors;
import org.apache.airflow.sdk.*;

// The Python Dag file (src/resources/dags/java_examples.py) owns the graph: each
// data parameter below receives whatever the @task.stub call site bound at its
// position, so this class registers task implementations only.
@SuppressWarnings("DuplicatedCode")
@Builder.Dag(id = "java_annotation_example")
public class AnnotationExample {
private static final System.Logger log = System.getLogger(AnnotationExample.class.getName());

@Builder.Task(id = "extract")
@Builder.TaskHandler(dag = "java_annotation_example", task = "extract")
public long extractValue(Client client) throws InterruptedException {
log.log(INFO, "Hello from task");

Expand All @@ -51,8 +53,8 @@ public long extractValue(Client client) throws InterruptedException {
return new Date().getTime();
}

@Builder.Task(id = "transform")
public long transformValue(Client client, @Builder.XCom(task = "extract") long extracted) {
@Builder.TaskHandler(dag = "java_annotation_example", task = "transform")
public long transformValue(Client client, long extracted) {
log.log(INFO, "Got XCom from extract: {0}", extracted);

var variable = client.getVariable("my_variable");
Expand All @@ -67,17 +69,33 @@ public long transformValue(Client client, @Builder.XCom(task = "extract") long e
// the task UP_FOR_RETRY -- which only works because the Java SDK now returns
// RetryTask (instead of a terminal FAILED) when ti_context.should_retry is
// set. The retry then runs this task again and it returns normally.
@Builder.Task
public void load(Context context, @Builder.XCom(task = "transform") long transformed) {
@Builder.TaskHandler(dag = "java_annotation_example")
public void load(Context context, long transformed) {
log.log(INFO, "Got XCom from transform: {0}", transformed);
if (context.ti.tryNumber == 1) {
throw new RuntimeException("I failed");
}
log.log(INFO, "Recovered on retry, try number {0}", context.ti.tryNumber);
}

// Keyword arguments bind by name instead of by position, through a TaskInput.
// Field names match ignoring case and underscores, so the stub's snake_case
// arguments reach camelCase Java fields on their own.
public static class ReportInput implements TaskInput {
public String runLabel; // binds run_label through the fold
public long transformed;
}

@Builder.TaskHandler(dag = "java_annotation_example", task = "report")
public void report(ReportInput input) {
log.log(INFO, "Report {0} for transformed value {1}", input.runLabel, input.transformed);
if (!"nightly".equals(input.runLabel)) {
throw new RuntimeException("expected run label 'nightly' but got " + input.runLabel);
}
}

// Verify one supervisor channel can handle client calls across threads.
@Builder.Task(id = "concurrent")
@Builder.TaskHandler(dag = "java_annotation_example", task = "concurrent")
public void concurrentClientCalls(Client client) throws Exception {
var pool = Executors.newFixedThreadPool(8);
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,17 @@

package org.apache.airflow.example;

import java.util.List;
import org.apache.airflow.sdk.*;
import org.jetbrains.annotations.NotNull;

public class ExampleBundleBuilder implements BundleBuilder {
@NotNull
@Override
public Iterable<DagDef> getDags() {
return List.of(
InterfaceExampleBuilder.build(),
AnnotationExampleBuilder.build(),
XComCastingExampleBuilder.build());
public class ExampleBundleBuilder {
public static Bundle build() {
return new Bundle()
.register(InterfaceExampleBuilder.build())
.register(AnnotationExample.class)
.register(XComCastingExample.class);
}

public static void main(String[] args) {
var bundle = new ExampleBundleBuilder().build();
Server.create(args).serve(bundle);
Server.create(args).serve(build());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,15 @@ public void execute(@NotNull Context context, Client client) throws Exception {
}
}

public static class Transform implements Task {
public void execute(@NotNull Context context, Client client) {
var extracted = client.getXCom("extract");
log.log(INFO, "Got XCom from extract: {0}", extracted);
public static class TransformInput implements TaskInput {
public long extracted;
}

// The Python Dag file calls transform(extracted), so the field of that name
// receives the extract task's XCom.
public static class Transform implements InputTask<TransformInput> {
public void execute(@NotNull Context context, Client client, TransformInput input) {
log.log(INFO, "Got extracted value from the bound argument: {0}", input.extracted);

var variable = client.getVariable("my_variable");
log.log(INFO, "Got variable: {0}", variable);
Expand All @@ -63,18 +68,31 @@ public void execute(@NotNull Context context, Client client) {
}
}

public static class Load implements Task {
public void execute(@NotNull Context context, Client client) {
var transformed = client.getXCom("transform");
log.log(INFO, "Got XCom from transform: {0}", transformed);
throw new RuntimeException("I failed");
public static class SummarizeInput implements TaskInput {
// Pinned so the field can be called region rather than regionCode. Or drop
// it: public String regionCode; binds region_code with nothing declared.
@ArgName("region_code")
public String region;

public long transformed;
}

// summarize(region_code=..., transformed=...) is called with keyword
// arguments, which bind to the fields by name.
public static class Summarize implements InputTask<SummarizeInput> {
public void execute(@NotNull Context context, Client client, SummarizeInput input) {
log.log(
INFO, "Summarize region {0} for transformed value {1}", input.region, input.transformed);
if (!"emea".equals(input.region)) {
throw new RuntimeException("expected region 'emea' but got " + input.region);
}
}
}

public static DagDef build() {
return new DagDef("java_interface_example")
.addTask("extract", Extract.class)
.addTask("transform", Transform.class)
.addTask("load", Load.class);
.addTask("summarize", Summarize.class);
}
}
Loading
Loading