From 66e5c9671febdae20fffbfc0147c9146ccf7a3aa Mon Sep 17 00:00:00 2001 From: Dilnaz Amanzholova Date: Tue, 1 Sep 2026 14:04:11 +0000 Subject: [PATCH 1/8] Adjust warnings, import errors and dag source models --- .../src/airflow/api/common/delete_dag.py | 3 +- .../api_fastapi/common/parameters/misc.py | 6 +-- .../core_api/datamodels/dag_sources.py | 1 + .../core_api/datamodels/dag_warning.py | 5 +- .../core_api/datamodels/import_error.py | 2 +- .../core_api/routes/public/dag_warning.py | 6 +-- .../core_api/routes/public/import_error.py | 14 +++--- .../src/airflow/dag_processing/collection.py | 18 ++++--- .../src/airflow/dag_processing/manager.py | 4 +- ...ame_to_source_reference_in_import_error.py | 50 +++++++++++++++++++ airflow-core/src/airflow/models/dagwarning.py | 5 +- airflow-core/src/airflow/models/errors.py | 18 +++++-- .../ui/openapi-gen/requests/types.gen.ts | 6 ++- .../airflow/ui/src/pages/Dag/Code/Code.tsx | 8 ++- .../routes/public/test_dag_sources.py | 4 ++ .../routes/public/test_dag_warning.py | 13 +++-- .../routes/public/test_import_error.py | 12 ++--- .../unit/dag_processing/test_collection.py | 18 +++++-- 18 files changed, 138 insertions(+), 55 deletions(-) create mode 100644 airflow-core/src/airflow/migrations/versions/0135_3_4_0_rename_filename_to_source_reference_in_import_error.py diff --git a/airflow-core/src/airflow/api/common/delete_dag.py b/airflow-core/src/airflow/api/common/delete_dag.py index e32e8bb90f3b2..e9d54563efdf6 100644 --- a/airflow-core/src/airflow/api/common/delete_dag.py +++ b/airflow-core/src/airflow/api/common/delete_dag.py @@ -80,9 +80,10 @@ def delete_dag(dag_id: str, keep_records_in_log: bool = True, *, session: Sessio # Delete entries in Import Errors table for a deleted Dag # This handles the case when the dag_id is changed in the file + targets = [ref for ref in (dag.relative_fileloc, dag.fileloc) if ref] session.execute( delete(ParseImportError).where( - ParseImportError.filename == dag.relative_fileloc, + ParseImportError.source_reference.in_(targets), ParseImportError.bundle_name == dag.bundle_name, ) ) diff --git a/airflow-core/src/airflow/api_fastapi/common/parameters/misc.py b/airflow-core/src/airflow/api_fastapi/common/parameters/misc.py index d718d2e132ee9..104e0bd94d4f0 100644 --- a/airflow-core/src/airflow/api_fastapi/common/parameters/misc.py +++ b/airflow-core/src/airflow/api_fastapi/common/parameters/misc.py @@ -269,19 +269,19 @@ def _optional_boolean(value: bool | None) -> bool | None: # Parse Import Errors QueryParseImportErrorFilenamePatternSearch = Annotated[ - _SearchParam, Depends(search_param_factory(ParseImportError.filename, "filename_pattern")) + _SearchParam, Depends(search_param_factory(ParseImportError.source_reference, "filename_pattern")) ] QueryParseImportErrorFilenamePrefixPatternSearch = Annotated[ _PrefixSearchParam, - Depends(prefix_search_param_factory(ParseImportError.filename, "filename_prefix_pattern")), + Depends(prefix_search_param_factory(ParseImportError.source_reference, "filename_prefix_pattern")), ] QueryParseImportErrorFilenameFilter = Annotated[ FilterParam, Depends( filter_param_factory( - ParseImportError.filename, + ParseImportError.source_reference, str | None, filter_name="filename", description="Exact filename match. Returns only the import error for this specific file path.", diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_sources.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_sources.py index fdc081a0ab2b7..d6dc5a3fc91a3 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_sources.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_sources.py @@ -28,3 +28,4 @@ class DAGSourceResponse(BaseModel): dag_id: str version_number: int | None dag_display_name: str = Field(validation_alias=AliasPath("dag_model", "dag_display_name")) + language: str | None = None diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py index 470a0b14fc64f..8d4a8373eef1a 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py @@ -21,18 +21,17 @@ from datetime import datetime from pydantic import AliasPath, Field - from airflow.api_fastapi.core_api.base import BaseModel -from airflow.models.dagwarning import DagWarningType class DAGWarningResponse(BaseModel): """Dag Warning serializer for responses.""" dag_id: str - warning_type: DagWarningType + warning_type: str message: str timestamp: datetime + context: dict | None = None dag_display_name: str = Field(validation_alias=AliasPath("dag_model", "dag_display_name")) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/import_error.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/import_error.py index d070df98fdb26..8709372c9348f 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/import_error.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/import_error.py @@ -30,7 +30,7 @@ class ImportErrorResponse(BaseModel): id: int = Field(alias="import_error_id") timestamp: datetime - filename: str + source_reference: str = Field(alias="filename") bundle_name: str | None stacktrace: str = Field(alias="stack_trace") diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py index c1a652c370f74..e6bb4837e17fc 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py @@ -40,7 +40,7 @@ DAGWarningCollectionResponse, ) from airflow.api_fastapi.core_api.security import ReadableDagWarningsFilterDep, requires_access_dag -from airflow.models.dagwarning import DagWarning, DagWarningType +from airflow.models.dagwarning import DagWarning dag_warning_router = AirflowRouter(tags=["DagWarning"]) @@ -52,8 +52,8 @@ def list_dag_warnings( dag_id: Annotated[FilterParam[str | None], Depends(filter_param_factory(DagWarning.dag_id, str | None))], warning_type: Annotated[ - FilterParam[DagWarningType | None], - Depends(filter_param_factory(DagWarning.warning_type, DagWarningType | None)), + FilterParam[str | None], + Depends(filter_param_factory(DagWarning.warning_type, str | None)), ], limit: QueryLimit, offset: QueryOffset, diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/import_error.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/import_error.py index d5801e6b7e4c3..9766a08692e78 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/import_error.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/import_error.py @@ -94,7 +94,7 @@ def get_import_error( file_dag_ids = set( session.scalars( select(DagModel.dag_id).where( - DagModel.relative_fileloc == error.filename, + DagModel.relative_fileloc == error.source_reference, DagModel.bundle_name == error.bundle_name, ) ).all() @@ -146,12 +146,12 @@ def get_import_errors( [ "id", "timestamp", - "filename", + "source_reference", "bundle_name", "stacktrace", ], ParseImportError, - {"import_error_id": "id"}, + {"import_error_id": "id", "filename": "source_reference"}, ).dynamic_depends() ), ], @@ -177,7 +177,7 @@ def get_import_errors( select(ParseImportError.id, ParseImportError.bundle_name) .outerjoin( files_with_any_dags, - (ParseImportError.filename == files_with_any_dags.c.relative_fileloc) + (ParseImportError.source_reference == files_with_any_dags.c.relative_fileloc) & (ParseImportError.bundle_name == files_with_any_dags.c.bundle_name), ) .where(files_with_any_dags.c.relative_fileloc.is_(None)) @@ -232,14 +232,14 @@ def get_import_errors( .outerjoin( files_with_any_dags, and_( - ParseImportError.filename == files_with_any_dags.c.relative_fileloc, + ParseImportError.source_reference == files_with_any_dags.c.relative_fileloc, ParseImportError.bundle_name == files_with_any_dags.c.bundle_name, ), ) .outerjoin( file_dags_cte, and_( - ParseImportError.filename == file_dags_cte.c.relative_fileloc, + ParseImportError.source_reference == file_dags_cte.c.relative_fileloc, ParseImportError.bundle_name == file_dags_cte.c.bundle_name, ), ) @@ -262,7 +262,7 @@ def get_import_errors( filtered_import_errors_stmt.with_only_columns( ParseImportError.id, ParseImportError.timestamp, - ParseImportError.filename, + ParseImportError.source_reference, ParseImportError.bundle_name, ParseImportError.stacktrace, ) diff --git a/airflow-core/src/airflow/dag_processing/collection.py b/airflow-core/src/airflow/dag_processing/collection.py index 36b948abad6c2..99b494021fdc2 100644 --- a/airflow-core/src/airflow/dag_processing/collection.py +++ b/airflow-core/src/airflow/dag_processing/collection.py @@ -388,15 +388,17 @@ def _update_import_errors( from airflow.listeners.listener import get_listener_manager # Read before the delete below, so an error still present is updated rather than recreated. - # Scoped to the keys being written: the table is not indexed on (bundle_name, filename) so this + # Scoped to the keys being written: the table is not indexed on (bundle_name, source_reference) so this # still scans, but it no longer builds a row per import error the deployment is carrying. existing_import_error_files: set[tuple[str | None, str | None]] = set() if import_errors: existing_import_error_files = { (bundle, filename) for bundle, filename in session.execute( - select(ParseImportError.bundle_name, ParseImportError.filename).where( - tuple_(ParseImportError.bundle_name, ParseImportError.filename).in_(list(import_errors)) + select(ParseImportError.bundle_name, ParseImportError.source_reference).where( + tuple_(ParseImportError.bundle_name, ParseImportError.source_reference).in_( + list(import_errors) + ) ) ) } @@ -407,7 +409,7 @@ def _update_import_errors( if files_to_clear: session.execute( delete(ParseImportError).where( - tuple_(ParseImportError.bundle_name, ParseImportError.filename).in_(files_to_clear) + tuple_(ParseImportError.bundle_name, ParseImportError.source_reference).in_(files_to_clear) ) ) @@ -419,11 +421,11 @@ def _update_import_errors( session.execute( update(ParseImportError) .where( - ParseImportError.filename == relative_fileloc, + ParseImportError.source_reference == relative_fileloc, ParseImportError.bundle_name == bundle_name_, ) .values( - filename=relative_fileloc, + source_reference=relative_fileloc, bundle_name=bundle_name_, timestamp=utcnow(), stacktrace=stacktrace, @@ -435,7 +437,7 @@ def _update_import_errors( import_error = session.scalar( select(ParseImportError).where( ParseImportError.bundle_name == bundle_name_, - ParseImportError.filename == relative_fileloc, + ParseImportError.source_reference == relative_fileloc, ) ) if import_error is not None: @@ -446,7 +448,7 @@ def _update_import_errors( log.exception("error calling listener") else: import_error = ParseImportError( - filename=relative_fileloc, + source_reference=relative_fileloc, bundle_name=bundle_name_, timestamp=utcnow(), stacktrace=stacktrace, diff --git a/airflow-core/src/airflow/dag_processing/manager.py b/airflow-core/src/airflow/dag_processing/manager.py index 16edef4560305..0b533796fd574 100644 --- a/airflow-core/src/airflow/dag_processing/manager.py +++ b/airflow-core/src/airflow/dag_processing/manager.py @@ -1069,10 +1069,10 @@ def clear_orphaned_import_errors( errors = session.scalars( select(ParseImportError) .where(ParseImportError.bundle_name == bundle_name) - .options(load_only(ParseImportError.filename)) + .options(load_only(ParseImportError.source_reference)) ) for error in errors: - if error.filename not in observed_filelocs: + if error.source_reference not in observed_filelocs: session.delete(error) except Exception: self.log.exception("Error removing old import errors") diff --git a/airflow-core/src/airflow/migrations/versions/0135_3_4_0_rename_filename_to_source_reference_in_import_error.py b/airflow-core/src/airflow/migrations/versions/0135_3_4_0_rename_filename_to_source_reference_in_import_error.py new file mode 100644 index 0000000000000..828376b1e2df1 --- /dev/null +++ b/airflow-core/src/airflow/migrations/versions/0135_3_4_0_rename_filename_to_source_reference_in_import_error.py @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Rename filename to source_reference in import_error. + +Revision ID: ca8499dc1004 +Revises: b6a9c2e7d410 +Create Date: 2026-08-27 12:45:02.276898 + +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "ca8499dc1004" +down_revision = "b6a9c2e7d410" +branch_labels = None +depends_on = None +airflow_version = "3.4.0" + + +def upgrade(): + """Apply rename filename to source_reference in import_error.""" + with op.batch_alter_table("import_error", schema=None) as batch_op: + batch_op.alter_column("filename", new_column_name="source_reference", type_=sa.String(length=1024)) + + +def downgrade(): + """Unapply rename filename to source_reference in import_error.""" + with op.batch_alter_table("import_error", schema=None) as batch_op: + batch_op.alter_column("source_reference", new_column_name="filename", type_=sa.String(length=1024)) diff --git a/airflow-core/src/airflow/models/dagwarning.py b/airflow-core/src/airflow/models/dagwarning.py index b0a5a3c93e1ef..79032a4d50f94 100644 --- a/airflow-core/src/airflow/models/dagwarning.py +++ b/airflow-core/src/airflow/models/dagwarning.py @@ -65,7 +65,10 @@ class DagWarning(Base): def __init__(self, dag_id: str, warning_type: str, message: str, **kwargs): super().__init__(**kwargs) self.dag_id = dag_id - self.warning_type = DagWarningType(warning_type).value # make sure valid type + try: + self.warning_type = DagWarningType(warning_type).value + except ValueError: + self.warning_type = str(warning_type) self.message = message def __eq__(self, other) -> bool: diff --git a/airflow-core/src/airflow/models/errors.py b/airflow-core/src/airflow/models/errors.py index 7d967386f79ab..dfd211c545e62 100644 --- a/airflow-core/src/airflow/models/errors.py +++ b/airflow-core/src/airflow/models/errors.py @@ -17,10 +17,11 @@ # under the License. from __future__ import annotations +import os from datetime import datetime from sqlalchemy import Integer, String, Text -from sqlalchemy.orm import Mapped, mapped_column +from sqlalchemy.orm import Mapped, mapped_column, synonym from airflow.dag_processing.bundles.manager import DagBundlesManager from airflow.models.base import Base, StringID @@ -33,13 +34,20 @@ class ParseImportError(Base): __tablename__ = "import_error" id: Mapped[int] = mapped_column(Integer, primary_key=True) timestamp: Mapped[datetime | None] = mapped_column(UtcDateTime, nullable=True) - filename: Mapped[str | None] = mapped_column(String(1024), nullable=True) + source_reference: Mapped[str | None] = mapped_column(String(1024), nullable=True) bundle_name: Mapped[str | None] = mapped_column(StringID(), nullable=True) stacktrace: Mapped[str | None] = mapped_column(Text, nullable=True) + filename = synonym("source_reference") + def full_file_path(self) -> str: """Return the full file path of the dag.""" - if self.bundle_name is None or self.filename is None: - raise ValueError("bundle_name and filename must not be None") + if self.bundle_name is None or self.source_reference is None: + raise ValueError("bundle_name and source_reference must not be None") bundle = DagBundlesManager().get_bundle(self.bundle_name) - return "/".join([str(bundle.path), self.filename]) + ref = self.source_reference + if ref.startswith(str(bundle.path)): + return ref + if os.path.isabs(ref.split(":")[0]): + return ref + return "/".join([str(bundle.path), ref]) diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index 8f70583bf018f..d57a3ea855440 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1163,6 +1163,7 @@ export type DAGSourceResponse = { dag_id: string; version_number: number | null; dag_display_name: string; + language?: string | null; }; /** @@ -1194,9 +1195,12 @@ export type DAGWarningCollectionResponse = { */ export type DAGWarningResponse = { dag_id: string; - warning_type: DagWarningType; + warning_type: string; message: string; timestamp: string; + context?: { + [key: string]: unknown; + } | null; dag_display_name: string; }; diff --git a/airflow-core/src/airflow/ui/src/pages/Dag/Code/Code.tsx b/airflow-core/src/airflow/ui/src/pages/Dag/Code/Code.tsx index ca274462779b7..e255099f3d4e1 100644 --- a/airflow-core/src/airflow/ui/src/pages/Dag/Code/Code.tsx +++ b/airflow-core/src/airflow/ui/src/pages/Dag/Code/Code.tsx @@ -183,7 +183,11 @@ export const Code = () => { )} - + ) : ( @@ -206,7 +210,7 @@ export const Code = () => { list[ParseImportError]: _import_errors = [ ParseImportError( bundle_name=bundle, - filename=filename, + source_reference=filename, stacktrace=stacktrace, timestamp=timestamp, ) @@ -565,7 +565,7 @@ def test_get_import_errors_bundle_name_and_filename_filter( session.add( ParseImportError( bundle_name=other_bundle, - filename=shared_filename, + source_reference=shared_filename, stacktrace="wrong bundle error", timestamp=TIMESTAMP1, ) @@ -618,7 +618,7 @@ def test_get_import_errors__file_registered_only_in_other_bundle_is_unregistered # has no Dag for it. error = ParseImportError( bundle_name=BUNDLE_NAME, - filename=FILENAME1, + source_reference=FILENAME1, stacktrace=STACKTRACE1, timestamp=TIMESTAMP1, ) @@ -968,7 +968,7 @@ def lonely_file_import_error( ) -> ParseImportError: error = ParseImportError( bundle_name=BUNDLE_NAME, - filename=self.LONELY_FILE_RELATIVE, + source_reference=self.LONELY_FILE_RELATIVE, stacktrace=self.LONELY_STACKTRACE, timestamp=TIMESTAMP1, ) @@ -985,7 +985,7 @@ def mixed_file_import_error( ) -> ParseImportError: error = ParseImportError( bundle_name=BUNDLE_NAME, - filename=self.MIXED_FILE_RELATIVE, + source_reference=self.MIXED_FILE_RELATIVE, stacktrace=self.MIXED_STACKTRACE, timestamp=TIMESTAMP2, ) @@ -1134,7 +1134,7 @@ def team_scoped_unregistered_import_error(self) -> Generator[ParseImportError, N session.add_all([team, bundle]) error = ParseImportError( bundle_name="team_b_bundle", - filename="team_b_unregistered.py", + source_reference="team_b_unregistered.py", stacktrace="team b stack trace", timestamp=TIMESTAMP1, ) diff --git a/airflow-core/tests/unit/dag_processing/test_collection.py b/airflow-core/tests/unit/dag_processing/test_collection.py index be36290abcece..98bf3dca0cd50 100644 --- a/airflow-core/tests/unit/dag_processing/test_collection.py +++ b/airflow-core/tests/unit/dag_processing/test_collection.py @@ -1081,7 +1081,7 @@ def test_new_import_error_replaces_old( import_error = session.scalar( select(ParseImportError).where( - ParseImportError.filename == filename, ParseImportError.bundle_name == bundle_name + ParseImportError.source_reference == filename, ParseImportError.bundle_name == bundle_name ) ) @@ -1119,7 +1119,9 @@ def test_remove_error_clears_import_error(self, testing_dag_bundle, session): session.flush() # Sanity check of pre-condition - import_errors = set(session.execute(select(ParseImportError.filename, ParseImportError.bundle_name))) + import_errors = set( + session.execute(select(ParseImportError.source_reference, ParseImportError.bundle_name)) + ) assert import_errors == {("abc.py", bundle_name), ("def.py", bundle_name)} dag = DAG(dag_id="test") @@ -1140,7 +1142,9 @@ def test_remove_error_clears_import_error(self, testing_dag_bundle, session): dag_model: DagModel = session.get(DagModel, (dag.dag_id,)) assert dag_model.has_import_errors is False - import_errors = set(session.execute(select(ParseImportError.filename, ParseImportError.bundle_name))) + import_errors = set( + session.execute(select(ParseImportError.source_reference, ParseImportError.bundle_name)) + ) assert import_errors == {("def.py", bundle_name)} @@ -1223,7 +1227,9 @@ def test_clear_import_error_for_file_without_dags(self, testing_dag_bundle, sess session.add(other_file_error) session.flush() - import_errors = set(session.execute(select(ParseImportError.filename, ParseImportError.bundle_name))) + import_errors = set( + session.execute(select(ParseImportError.source_reference, ParseImportError.bundle_name)) + ) assert import_errors == {("no_dags.py", bundle_name), ("other.py", bundle_name)} # Simulate parsing the file: it was parsed successfully (no import errors), @@ -1241,7 +1247,9 @@ def test_clear_import_error_for_file_without_dags(self, testing_dag_bundle, sess files_parsed=files_parsed, ) - import_errors = set(session.execute(select(ParseImportError.filename, ParseImportError.bundle_name))) + import_errors = set( + session.execute(select(ParseImportError.source_reference, ParseImportError.bundle_name)) + ) assert import_errors == {("other.py", bundle_name)}, "Import error for parsed file should be cleared" @pytest.mark.usefixtures("clean_db") From a1458509b813355e1f36c90258ef6fbaf056f47f Mon Sep 17 00:00:00 2001 From: Dilnaz Amanzholova Date: Tue, 8 Sep 2026 15:00:52 +0000 Subject: [PATCH 2/8] Fix Code view ESLint errors and XComs E2E test filter locator --- .../core_api/datamodels/dag_warning.py | 2 +- .../openapi/v2-rest-api-generated.yaml | 30 ++--- .../airflow/ui/openapi-gen/queries/common.ts | 4 +- .../ui/openapi-gen/queries/ensureQueryData.ts | 6 +- .../ui/openapi-gen/queries/prefetch.ts | 6 +- .../airflow/ui/openapi-gen/queries/queries.ts | 6 +- .../ui/openapi-gen/queries/suspense.ts | 6 +- .../ui/openapi-gen/requests/schemas.gen.ts | 24 ++-- .../ui/openapi-gen/requests/services.gen.ts | 2 +- .../ui/openapi-gen/requests/types.gen.ts | 127 ++++++++---------- .../airflow/ui/src/pages/Dag/Code/Code.tsx | 6 +- .../routes/public/test_dag_warning.py | 1 - .../airflowctl/api/datamodels/generated.py | 47 +++---- .../tests/airflow_ctl/api/test_operations.py | 3 +- 14 files changed, 120 insertions(+), 150 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py index 8d4a8373eef1a..541c5955a6edf 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py @@ -21,6 +21,7 @@ from datetime import datetime from pydantic import AliasPath, Field + from airflow.api_fastapi.core_api.base import BaseModel @@ -31,7 +32,6 @@ class DAGWarningResponse(BaseModel): warning_type: str message: str timestamp: datetime - context: dict | None = None dag_display_name: str = Field(validation_alias=AliasPath("dag_model", "dag_display_name")) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml index d57c05b681210..7863571ada7ac 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml @@ -3804,7 +3804,7 @@ paths: required: false schema: anyOf: - - $ref: '#/components/schemas/DagWarningType' + - type: string - type: 'null' title: Warning Type - name: limit @@ -5256,13 +5256,13 @@ paths: type: string description: 'Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, - filename, bundle_name, stacktrace, import_error_id`' + source_reference, bundle_name, stacktrace, import_error_id, filename`' default: - id title: Order By description: 'Attributes to order by, multi criteria sort is supported. Prefix - with `-` for descending order. Supported attributes: `id, timestamp, filename, - bundle_name, stacktrace, import_error_id`' + with `-` for descending order. Supported attributes: `id, timestamp, source_reference, + bundle_name, stacktrace, import_error_id, filename`' - name: filename_pattern in: query required: false @@ -14283,6 +14283,11 @@ components: dag_display_name: type: string title: Dag Display Name + language: + anyOf: + - type: string + - type: 'null' + title: Language type: object required: - content @@ -14345,7 +14350,8 @@ components: type: string title: Dag Id warning_type: - $ref: '#/components/schemas/DagWarningType' + type: string + title: Warning Type message: type: string title: Message @@ -14880,20 +14886,6 @@ components: - bundle_url title: DagVersionResponse description: Dag Version serializer for responses. - DagWarningType: - type: string - enum: - - asset conflict - - duplicate dag id - - non-existent pool - - runtime varying value - title: DagWarningType - description: 'Enum for DAG warning types. - - - This is the set of allowable values for the ``warning_type`` field - - in the DagWarning model.' DetailedHealthStatus: type: string enum: diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts index c6c5d28aff6f9..574871897ab27 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts @@ -2,7 +2,7 @@ import { UseQueryResult } from "@tanstack/react-query"; import { AssetService, AssetStateStoreService, AuthLinksService, BackfillService, CalendarService, ConfigService, ConnectionService, DagBundleService, DagParsingService, DagRunService, DagService, DagSourceService, DagStatsService, DagVersionService, DagWarningService, DashboardService, DeadlinesService, DependenciesService, EventLogService, ExperimentalService, ExtraLinksService, GanttService, GridService, ImportErrorService, JobService, LoginService, MonitorService, PartitionedDagRunService, PluginService, PoolService, ProviderService, StructureService, TaskInstanceService, TaskService, TaskStateStoreService, TeamsService, VariableService, VersionService, XcomService } from "../requests/services.gen"; -import { DagRunState, DagSchedulingState, DagWarningType, ReprocessBehavior } from "../requests/types.gen"; +import { DagRunState, DagSchedulingState, ReprocessBehavior } from "../requests/types.gen"; export type AssetServiceGetAssetsDefaultResponse = Awaited>; export type AssetServiceGetAssetsQueryResult = UseQueryResult; export const useAssetServiceGetAssetsKey = "AssetServiceGetAssets"; @@ -345,7 +345,7 @@ export const UseDagWarningServiceListDagWarningsKeyFn = ({ dagId, limit, offset, limit?: number; offset?: number; orderBy?: string[]; - warningType?: DagWarningType; + warningType?: string; } = {}, queryKey?: Array) => [useDagWarningServiceListDagWarningsKey, ...(queryKey ?? [{ dagId, limit, offset, orderBy, warningType }])]; export type DagServiceGetDagsDefaultResponse = Awaited>; export type DagServiceGetDagsQueryResult = UseQueryResult; diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts index e1c55f73cf52a..fc7148ab9562e 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts @@ -2,7 +2,7 @@ import { type QueryClient } from "@tanstack/react-query"; import { AssetService, AssetStateStoreService, AuthLinksService, BackfillService, CalendarService, ConfigService, ConnectionService, DagBundleService, DagRunService, DagService, DagSourceService, DagStatsService, DagVersionService, DagWarningService, DashboardService, DeadlinesService, DependenciesService, EventLogService, ExperimentalService, ExtraLinksService, GanttService, GridService, ImportErrorService, JobService, LoginService, MonitorService, PartitionedDagRunService, PluginService, PoolService, ProviderService, StructureService, TaskInstanceService, TaskService, TaskStateStoreService, TeamsService, VariableService, VersionService, XcomService } from "../requests/services.gen"; -import { DagRunState, DagSchedulingState, DagWarningType, ReprocessBehavior } from "../requests/types.gen"; +import { DagRunState, DagSchedulingState, ReprocessBehavior } from "../requests/types.gen"; import * as Common from "./common"; /** * Get Assets @@ -685,7 +685,7 @@ export const ensureUseDagWarningServiceListDagWarningsData = (queryClient: Query limit?: number; offset?: number; orderBy?: string[]; - warningType?: DagWarningType; + warningType?: string; } = {}) => queryClient.ensureQueryData({ queryKey: Common.UseDagWarningServiceListDagWarningsKeyFn({ dagId, limit, offset, orderBy, warningType }), queryFn: () => DagWarningService.listDagWarnings({ dagId, limit, offset, orderBy, warningType }) }); /** * Get Dags @@ -1523,7 +1523,7 @@ export const ensureUseImportErrorServiceGetImportErrorData = (queryClient: Query * @param data The data for the request. * @param data.limit * @param data.offset -* @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, filename, bundle_name, stacktrace, import_error_id` +* @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, source_reference, bundle_name, stacktrace, import_error_id, filename` * @param data.filenamePattern Case-insensitive substring match (SQL `ILIKE`). Slower than `filename_prefix_pattern` on large tables — see "Filtering with pattern parameters". * @param data.filenamePrefixPattern Case-sensitive, index-friendly prefix match. See "Filtering with pattern parameters". * @param data.filename Exact filename match. Returns only the import error for this specific file path. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts index 613336b861bad..73496ce6079cb 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts @@ -2,7 +2,7 @@ import { type QueryClient } from "@tanstack/react-query"; import { AssetService, AssetStateStoreService, AuthLinksService, BackfillService, CalendarService, ConfigService, ConnectionService, DagBundleService, DagRunService, DagService, DagSourceService, DagStatsService, DagVersionService, DagWarningService, DashboardService, DeadlinesService, DependenciesService, EventLogService, ExperimentalService, ExtraLinksService, GanttService, GridService, ImportErrorService, JobService, LoginService, MonitorService, PartitionedDagRunService, PluginService, PoolService, ProviderService, StructureService, TaskInstanceService, TaskService, TaskStateStoreService, TeamsService, VariableService, VersionService, XcomService } from "../requests/services.gen"; -import { DagRunState, DagSchedulingState, DagWarningType, ReprocessBehavior } from "../requests/types.gen"; +import { DagRunState, DagSchedulingState, ReprocessBehavior } from "../requests/types.gen"; import * as Common from "./common"; /** * Get Assets @@ -685,7 +685,7 @@ export const prefetchUseDagWarningServiceListDagWarnings = (queryClient: QueryCl limit?: number; offset?: number; orderBy?: string[]; - warningType?: DagWarningType; + warningType?: string; } = {}) => queryClient.prefetchQuery({ queryKey: Common.UseDagWarningServiceListDagWarningsKeyFn({ dagId, limit, offset, orderBy, warningType }), queryFn: () => DagWarningService.listDagWarnings({ dagId, limit, offset, orderBy, warningType }) }); /** * Get Dags @@ -1523,7 +1523,7 @@ export const prefetchUseImportErrorServiceGetImportError = (queryClient: QueryCl * @param data The data for the request. * @param data.limit * @param data.offset -* @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, filename, bundle_name, stacktrace, import_error_id` +* @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, source_reference, bundle_name, stacktrace, import_error_id, filename` * @param data.filenamePattern Case-insensitive substring match (SQL `ILIKE`). Slower than `filename_prefix_pattern` on large tables — see "Filtering with pattern parameters". * @param data.filenamePrefixPattern Case-sensitive, index-friendly prefix match. See "Filtering with pattern parameters". * @param data.filename Exact filename match. Returns only the import error for this specific file path. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts index aaa024a638c8e..e353cf6df668f 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts @@ -2,7 +2,7 @@ import { UseMutationOptions, UseQueryOptions, useMutation, useQuery } from "@tanstack/react-query"; import { AssetService, AssetStateStoreService, AuthLinksService, BackfillService, CalendarService, ConfigService, ConnectionService, DagBundleService, DagParsingService, DagRunService, DagService, DagSourceService, DagStatsService, DagVersionService, DagWarningService, DashboardService, DeadlinesService, DependenciesService, EventLogService, ExperimentalService, ExtraLinksService, GanttService, GridService, ImportErrorService, JobService, LoginService, MonitorService, PartitionedDagRunService, PluginService, PoolService, ProviderService, StructureService, TaskInstanceService, TaskService, TaskStateStoreService, TeamsService, VariableService, VersionService, XcomService } from "../requests/services.gen"; -import { AssetStateStoreBody, BackfillPostBody, BulkBody_BulkDAGBody_, BulkBody_BulkDAGRunBody_, BulkBody_BulkTaskInstanceBody_, BulkBody_ConnectionBody_, BulkBody_PoolBody_, BulkBody_VariableBody_, BulkDAGRunClearBody, ClearPartitionsBody, ClearTaskInstancesBody, ConnectionBody, ConnectionTestRequestBody, CreateAssetEventsBody, DAGPatchBody, DAGRunClearBody, DAGRunPatchBody, DAGRunsBatchBody, DagRunState, DagSchedulingState, DagWarningType, GenerateTokenBody, MaterializeAssetBody, PatchTaskInstanceBody, PoolBody, PoolPatchBody, ReprocessBehavior, TaskInstancesBatchBody, TaskStateStoreBody, TaskStateStorePatchBody, TriggerDAGRunPostBody, UpdateHITLDetailPayload, VariableBody, XComCreateBody, XComUpdateBody } from "../requests/types.gen"; +import { AssetStateStoreBody, BackfillPostBody, BulkBody_BulkDAGBody_, BulkBody_BulkDAGRunBody_, BulkBody_BulkTaskInstanceBody_, BulkBody_ConnectionBody_, BulkBody_PoolBody_, BulkBody_VariableBody_, BulkDAGRunClearBody, ClearPartitionsBody, ClearTaskInstancesBody, ConnectionBody, ConnectionTestRequestBody, CreateAssetEventsBody, DAGPatchBody, DAGRunClearBody, DAGRunPatchBody, DAGRunsBatchBody, DagRunState, DagSchedulingState, GenerateTokenBody, MaterializeAssetBody, PatchTaskInstanceBody, PoolBody, PoolPatchBody, ReprocessBehavior, TaskInstancesBatchBody, TaskStateStoreBody, TaskStateStorePatchBody, TriggerDAGRunPostBody, UpdateHITLDetailPayload, VariableBody, XComCreateBody, XComUpdateBody } from "../requests/types.gen"; import * as Common from "./common"; /** * Get Assets @@ -685,7 +685,7 @@ export const useDagWarningServiceListDagWarnings = , "queryKey" | "queryFn">) => useQuery({ queryKey: Common.UseDagWarningServiceListDagWarningsKeyFn({ dagId, limit, offset, orderBy, warningType }, queryKey), queryFn: () => DagWarningService.listDagWarnings({ dagId, limit, offset, orderBy, warningType }) as TData, ...options }); /** * Get Dags @@ -1523,7 +1523,7 @@ export const useImportErrorServiceGetImportError = , "queryKey" | "queryFn">) => useSuspenseQuery({ queryKey: Common.UseDagWarningServiceListDagWarningsKeyFn({ dagId, limit, offset, orderBy, warningType }, queryKey), queryFn: () => DagWarningService.listDagWarnings({ dagId, limit, offset, orderBy, warningType }) as TData, ...options }); /** * Get Dags @@ -1523,7 +1523,7 @@ export const useImportErrorServiceGetImportErrorSuspense = ; @@ -274,8 +274,8 @@ export type BackfillPostBody = { to_date: string; run_backwards?: boolean; dag_run_conf?: { - [key: string]: unknown; -} | null; + [key: string]: unknown; + } | null; reprocess_behavior?: ReprocessBehavior; max_active_runs?: number; /** @@ -293,8 +293,8 @@ export type BackfillResponse = { from_date: string; to_date: string; dag_run_conf: { - [key: string]: unknown; -} | null; + [key: string]: unknown; + } | null; is_paused: boolean; reprocess_behavior: ReprocessBehavior; max_active_runs: number; @@ -753,9 +753,9 @@ export type ClearTaskInstancesBody = { * A list of `task_id` or [`task_id`, `map_index`]. If only the `task_id` is provided for a mapped task, all of its map indices will be targeted. */ task_ids?: Array<(string | [ - string, - number -])> | null; + string, + number + ])> | null; /** * Clear every task in this task group. Mutually exclusive with `task_ids`. The group's tasks are resolved on the server from the dag structure, so all of them are targeted regardless of how many there are. */ @@ -786,9 +786,9 @@ export type Config = { export type ConfigOption = { key: string; value: string | [ - string, - string -]; + string, + string + ]; }; /** @@ -949,19 +949,19 @@ export type DAGDetailsResponse = { end_date: string | null; is_paused_upon_creation: boolean | null; params: { - [key: string]: unknown; -} | null; + [key: string]: unknown; + } | null; render_template_as_native_obj: boolean; template_search_path: Array<(string)> | null; timezone: string | null; last_parsed: string | null; default_args: { - [key: string]: unknown; -} | null; + [key: string]: unknown; + } | null; rerun_with_latest_version?: boolean | null; owner_links?: { - [key: string]: (string); -} | null; + [key: string]: (string); + } | null; is_favorite?: boolean; active_runs_count?: number; team_name?: string | null; @@ -1112,8 +1112,8 @@ export type DAGRunResponse = { triggered_by: DagRunTriggeredByType | null; triggering_user_name: string | null; conf: { - [key: string]: unknown; -} | null; + [key: string]: unknown; + } | null; note: string | null; dag_versions: Array; bundle_version: string | null; @@ -1198,9 +1198,6 @@ export type DAGWarningResponse = { warning_type: string; message: string; timestamp: string; - context?: { - [key: string]: unknown; - } | null; dag_display_name: string; }; @@ -1434,14 +1431,6 @@ export type DagVersionResponse = { bundle_url: string | null; }; -/** - * Enum for DAG warning types. - * - * This is the set of allowable values for the ``warning_type`` field - * in the DagWarning model. - */ -export type DagWarningType = 'asset conflict' | 'duplicate dag id' | 'non-existent pool' | 'runtime varying value'; - /** * How much of a component's work has a live instance covering it. */ @@ -1628,8 +1617,8 @@ export type HITLUser = { */ export type HTTPExceptionResponse = { detail: string | { - [key: string]: unknown; -}; + [key: string]: unknown; + }; }; export type HTTPValidationError = { @@ -1721,8 +1710,8 @@ export type MaterializeAssetBody = { logical_date?: string | null; run_after?: string | null; conf?: { - [key: string]: unknown; -} | null; + [key: string]: unknown; + } | null; note?: string | null; partition_key?: string | null; bundle_version?: string | null; @@ -2187,11 +2176,11 @@ export type TaskResponse = { doc_md: string | null; operator_name: string | null; params: { - [key: string]: unknown; -} | null; + [key: string]: unknown; + } | null; class_ref: { - [key: string]: unknown; -} | null; + [key: string]: unknown; + } | null; is_mapped: boolean | null; /** * Extract and return extra_links. @@ -2258,8 +2247,8 @@ export type TriggerDAGRunPostBody = { logical_date: string | null; run_after?: string | null; conf?: { - [key: string]: unknown; -} | null; + [key: string]: unknown; + } | null; note?: string | null; partition_key?: string | null; bundle_version?: string | null; @@ -2562,8 +2551,8 @@ export type ConnectionHookMetaData = { hook_name: string; standard_fields: StandardHookFields | null; extra_fields: { - [key: string]: unknown; -} | null; + [key: string]: unknown; + } | null; }; /** @@ -2864,8 +2853,8 @@ export type LightGridTaskInstanceSummary = { task_display_name: string; state: TaskInstanceState | null; child_states: { - [key: string]: (number); -} | null; + [key: string]: (number); + } | null; min_start_date: string | null; max_end_date: string | null; dag_version_number?: number | null; @@ -2955,8 +2944,8 @@ export type PartitionedDagRunCollectionResponse = { partitioned_dag_runs: Array; total: number; asset_expressions?: { - [key: string]: (AssetExpressionAsset | AssetExpressionAlias | AssetExpressionRef | AssetExpressionAny | AssetExpressionAll | null); -} | null; + [key: string]: (AssetExpressionAsset | AssetExpressionAlias | AssetExpressionRef | AssetExpressionAny | AssetExpressionAll | null); + } | null; }; /** @@ -3049,13 +3038,13 @@ export type TeamResponse = { */ export type Theme = { tokens?: { - [key: string]: ThemeColors; -} | null; + [key: string]: ThemeColors; + } | null; globalCss?: { - [key: string]: { - [key: string]: unknown; - }; -} | null; + [key: string]: { + [key: string]: unknown; + }; + } | null; icon?: string | null; icon_dark_mode?: string | null; }; @@ -3065,21 +3054,21 @@ export type Theme = { */ export type ThemeColors = { brand?: { - [key: string]: { - [key: string]: OklchColor; - }; -} | null; + [key: string]: { + [key: string]: OklchColor; + }; + } | null; gray?: { - [key: string]: { - [key: string]: OklchColor; - }; -} | null; + [key: string]: { + [key: string]: OklchColor; + }; + } | null; black?: { - [key: string]: OklchColor; -} | null; + [key: string]: OklchColor; + } | null; white?: { - [key: string]: OklchColor; -} | null; + [key: string]: OklchColor; + } | null; }; /** @@ -3722,7 +3711,7 @@ export type ListDagWarningsData = { * Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `dag_id, warning_type, message, timestamp` */ orderBy?: Array<(string)>; - warningType?: DagWarningType | null; + warningType?: string | null; }; export type ListDagWarningsResponse = DAGWarningCollectionResponse; @@ -4524,7 +4513,7 @@ export type GetImportErrorsData = { limit?: number; offset?: number; /** - * Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, filename, bundle_name, stacktrace, import_error_id` + * Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, source_reference, bundle_name, stacktrace, import_error_id, filename` */ orderBy?: Array<(string)>; }; diff --git a/airflow-core/src/airflow/ui/src/pages/Dag/Code/Code.tsx b/airflow-core/src/airflow/ui/src/pages/Dag/Code/Code.tsx index e255099f3d4e1..b8874d6b262bd 100644 --- a/airflow-core/src/airflow/ui/src/pages/Dag/Code/Code.tsx +++ b/airflow-core/src/airflow/ui/src/pages/Dag/Code/Code.tsx @@ -161,6 +161,8 @@ export const Code = () => { ? translate("code.noCode") : (compareCode?.content ?? ""); + const language: string = code?.language ?? "python"; + const codeStatus = ( <> { )} @@ -210,7 +212,7 @@ export const Code = () => { Date: Wed, 16 Sep 2026 12:27:07 +0000 Subject: [PATCH 3/8] Keep DagWarningType along with string type and fix import error paths --- .../core_api/datamodels/dag_warning.py | 3 +- .../openapi/v2-rest-api-generated.yaml | 19 ++++- .../core_api/routes/public/dag_warning.py | 6 +- airflow-core/src/airflow/models/dagwarning.py | 10 +-- airflow-core/src/airflow/models/errors.py | 10 +-- .../ui/openapi-gen/requests/schemas.gen.ts | 19 ++++- .../ui/openapi-gen/requests/types.gen.ts | 12 +++- airflow-core/tests/unit/models/test_errors.py | 71 +++++++++++++++++++ .../airflowctl/api/datamodels/generated.py | 46 +++++++----- .../tests/airflow_ctl/api/test_operations.py | 3 +- 10 files changed, 164 insertions(+), 35 deletions(-) create mode 100644 airflow-core/tests/unit/models/test_errors.py diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py index 541c5955a6edf..796336bdb92e6 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py @@ -23,13 +23,14 @@ from pydantic import AliasPath, Field from airflow.api_fastapi.core_api.base import BaseModel +from airflow.models.dagwarning import DagWarningType class DAGWarningResponse(BaseModel): """Dag Warning serializer for responses.""" dag_id: str - warning_type: str + warning_type: DagWarningType | str message: str timestamp: datetime dag_display_name: str = Field(validation_alias=AliasPath("dag_model", "dag_display_name")) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml index 7863571ada7ac..5d5f8842d2b99 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml @@ -3804,6 +3804,7 @@ paths: required: false schema: anyOf: + - $ref: '#/components/schemas/DagWarningType' - type: string - type: 'null' title: Warning Type @@ -14350,7 +14351,9 @@ components: type: string title: Dag Id warning_type: - type: string + anyOf: + - $ref: '#/components/schemas/DagWarningType' + - type: string title: Warning Type message: type: string @@ -14894,6 +14897,20 @@ components: - down title: DetailedHealthStatus description: How much of a component's work has a live instance covering it. + DagWarningType: + type: string + enum: + - asset conflict + - duplicate dag id + - non-existent pool + - runtime varying value + title: DagWarningType + description: 'Enum for DAG warning types. + + + This is the set of allowable values for the ``warning_type`` field + + in the DagWarning model.' DryRunBackfillCollectionResponse: properties: backfills: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py index e6bb4837e17fc..db82be115fda8 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py @@ -40,7 +40,7 @@ DAGWarningCollectionResponse, ) from airflow.api_fastapi.core_api.security import ReadableDagWarningsFilterDep, requires_access_dag -from airflow.models.dagwarning import DagWarning +from airflow.models.dagwarning import DagWarning, DagWarningType dag_warning_router = AirflowRouter(tags=["DagWarning"]) @@ -52,8 +52,8 @@ def list_dag_warnings( dag_id: Annotated[FilterParam[str | None], Depends(filter_param_factory(DagWarning.dag_id, str | None))], warning_type: Annotated[ - FilterParam[str | None], - Depends(filter_param_factory(DagWarning.warning_type, str | None)), + FilterParam[DagWarningType | str | None], + Depends(filter_param_factory(DagWarning.warning_type, DagWarningType | str | None)), ], limit: QueryLimit, offset: QueryOffset, diff --git a/airflow-core/src/airflow/models/dagwarning.py b/airflow-core/src/airflow/models/dagwarning.py index 79032a4d50f94..2145e7e6028a3 100644 --- a/airflow-core/src/airflow/models/dagwarning.py +++ b/airflow-core/src/airflow/models/dagwarning.py @@ -62,13 +62,13 @@ class DagWarning(Base): Index("idx_dag_warning_dag_id", dag_id), ) - def __init__(self, dag_id: str, warning_type: str, message: str, **kwargs): + def __init__(self, dag_id: str, warning_type: DagWarningType | str, message: str, **kwargs): super().__init__(**kwargs) self.dag_id = dag_id - try: - self.warning_type = DagWarningType(warning_type).value - except ValueError: - self.warning_type = str(warning_type) + # Dag importers supply their own warning types, so anything outside the enum is kept verbatim. + self.warning_type = ( + warning_type.value if isinstance(warning_type, DagWarningType) else str(warning_type) + ) self.message = message def __eq__(self, other) -> bool: diff --git a/airflow-core/src/airflow/models/errors.py b/airflow-core/src/airflow/models/errors.py index dfd211c545e62..8654713c35761 100644 --- a/airflow-core/src/airflow/models/errors.py +++ b/airflow-core/src/airflow/models/errors.py @@ -17,8 +17,8 @@ # under the License. from __future__ import annotations -import os from datetime import datetime +from pathlib import Path from sqlalchemy import Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column, synonym @@ -46,8 +46,8 @@ def full_file_path(self) -> str: raise ValueError("bundle_name and source_reference must not be None") bundle = DagBundlesManager().get_bundle(self.bundle_name) ref = self.source_reference - if ref.startswith(str(bundle.path)): + # A reference may address a member inside a container (``archive.zip:dags/my_dag.py``); the + # anchor alone decides whether it is already rooted, so the separator is never parsed here. + if ref.startswith(str(bundle.path)) or Path(ref).is_absolute(): return ref - if os.path.isabs(ref.split(":")[0]): - return ref - return "/".join([str(bundle.path), ref]) + return str(bundle.path / ref) diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts index 25ac9ac7dcdea..dd5b9366bf678 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -4599,7 +4599,14 @@ export const $DAGWarningResponse = { title: 'Dag Id' }, warning_type: { - type: 'string', + anyOf: [ + { + '$ref': '#/components/schemas/DagWarningType' + }, + { + type: 'string' + } + ], title: 'Warning Type' }, message: { @@ -5317,6 +5324,16 @@ export const $DetailedHealthStatus = { description: "How much of a component's work has a live instance covering it." } as const; +export const $DagWarningType = { + type: 'string', + enum: ['asset conflict', 'duplicate dag id', 'non-existent pool', 'runtime varying value'], + title: 'DagWarningType', + description: `Enum for DAG warning types. + +This is the set of allowable values for the \`\`warning_type\`\` field +in the DagWarning model.` +} as const; + export const $DryRunBackfillCollectionResponse = { properties: { backfills: { diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index 6b733ff040fc4..1996ad7cc533b 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1195,7 +1195,7 @@ export type DAGWarningCollectionResponse = { */ export type DAGWarningResponse = { dag_id: string; - warning_type: string; + warning_type: DagWarningType | string; message: string; timestamp: string; dag_display_name: string; @@ -1436,6 +1436,14 @@ export type DagVersionResponse = { */ export type DetailedHealthStatus = 'healthy' | 'degraded' | 'down'; +/** + * Enum for DAG warning types. + * + * This is the set of allowable values for the ``warning_type`` field + * in the DagWarning model. + */ +export type DagWarningType = 'asset conflict' | 'duplicate dag id' | 'non-existent pool' | 'runtime varying value'; + /** * Backfill collection serializer for responses in dry-run mode. */ @@ -3711,7 +3719,7 @@ export type ListDagWarningsData = { * Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `dag_id, warning_type, message, timestamp` */ orderBy?: Array<(string)>; - warningType?: string | null; + warningType?: DagWarningType | string | null; }; export type ListDagWarningsResponse = DAGWarningCollectionResponse; diff --git a/airflow-core/tests/unit/models/test_errors.py b/airflow-core/tests/unit/models/test_errors.py new file mode 100644 index 0000000000000..780401b0e1be1 --- /dev/null +++ b/airflow-core/tests/unit/models/test_errors.py @@ -0,0 +1,71 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from pathlib import Path +from unittest import mock + +import pytest + +from airflow.models.errors import ParseImportError + +BUNDLE_PATH = Path("/bundle") + + +@pytest.fixture +def bundle_path(): + with mock.patch("airflow.models.errors.DagBundlesManager", autospec=True) as manager: + manager.return_value.get_bundle.return_value.path = BUNDLE_PATH + yield + + +class TestFullFilePath: + @pytest.mark.usefixtures("bundle_path") + @pytest.mark.parametrize( + ("source_reference", "expected"), + [ + pytest.param("dags/my_dag.py", "/bundle/dags/my_dag.py", id="relative-file"), + pytest.param( + "archive.zip:dags/my_dag.py", + "/bundle/archive.zip:dags/my_dag.py", + id="relative-archive-member", + ), + pytest.param( + "/elsewhere/archive.zip:dags/my_dag.py", + "/elsewhere/archive.zip:dags/my_dag.py", + id="absolute-archive-member", + ), + pytest.param("/bundle/dags/my_dag.py", "/bundle/dags/my_dag.py", id="already-under-bundle"), + ], + ) + def test_resolves_reference_against_bundle(self, source_reference, expected): + error = ParseImportError(bundle_name="my-bundle", source_reference=source_reference) + + assert error.full_file_path() == expected + + @pytest.mark.parametrize( + ("bundle_name", "source_reference"), + [ + pytest.param(None, "dags/my_dag.py", id="missing-bundle"), + pytest.param("my-bundle", None, id="missing-reference"), + ], + ) + def test_raises_when_reference_is_incomplete(self, bundle_name, source_reference): + error = ParseImportError(bundle_name=bundle_name, source_reference=source_reference) + + with pytest.raises(ValueError, match="bundle_name and source_reference must not be None"): + error.full_file_path() diff --git a/airflow-ctl/src/airflowctl/api/datamodels/generated.py b/airflow-ctl/src/airflowctl/api/datamodels/generated.py index 85c78ba1f3fa3..ec9739a4ce629 100644 --- a/airflow-ctl/src/airflowctl/api/datamodels/generated.py +++ b/airflow-ctl/src/airflowctl/api/datamodels/generated.py @@ -637,18 +637,6 @@ class DagBundleResponse(BaseModel): ] -class DAGWarningResponse(BaseModel): - """ - Dag Warning serializer for responses. - """ - - dag_id: Annotated[str, Field(title="Dag Id")] - warning_type: Annotated[str, Field(title="Warning Type")] - message: Annotated[str, Field(title="Message")] - timestamp: Annotated[datetime, Field(title="Timestamp")] - dag_display_name: Annotated[str, Field(title="Dag Display Name")] - - class DagProcessorInstanceInfoResponse(BaseModel): """ Dag processor instance info serializer for responses. @@ -806,6 +794,20 @@ class DetailedHealthStatus(str, Enum): DOWN = "down" +class DagWarningType(str, Enum): + """ + Enum for DAG warning types. + + This is the set of allowable values for the ``warning_type`` field + in the DagWarning model. + """ + + ASSET_CONFLICT = "asset conflict" + DUPLICATE_DAG_ID = "duplicate dag id" + NON_EXISTENT_POOL = "non-existent pool" + RUNTIME_VARYING_VALUE = "runtime varying value" + + class DryRunBackfillResponse(BaseModel): """ Backfill serializer for responses in dry-run mode. @@ -2150,13 +2152,16 @@ class DAGVersionCollectionResponse(BaseModel): total_entries: Annotated[int, Field(title="Total Entries")] -class DAGWarningCollectionResponse(BaseModel): +class DAGWarningResponse(BaseModel): """ - Dag warning collection serializer for responses. + Dag Warning serializer for responses. """ - dag_warnings: Annotated[list[DAGWarningResponse], Field(title="Dag Warnings")] - total_entries: Annotated[int, Field(title="Total Entries")] + dag_id: Annotated[str, Field(title="Dag Id")] + warning_type: Annotated[DagWarningType | str, Field(title="Warning Type")] + message: Annotated[str, Field(title="Message")] + timestamp: Annotated[datetime, Field(title="Timestamp")] + dag_display_name: Annotated[str, Field(title="Dag Display Name")] class DagBundleCollectionResponse(BaseModel): @@ -2711,6 +2716,15 @@ class DAGRunCollectionResponse(BaseModel): ] = None +class DAGWarningCollectionResponse(BaseModel): + """ + Dag warning collection serializer for responses. + """ + + dag_warnings: Annotated[list[DAGWarningResponse], Field(title="Dag Warnings")] + total_entries: Annotated[int, Field(title="Total Entries")] + + class DagStatsCollectionResponse(BaseModel): """ Dag Stats Collection serializer for responses. diff --git a/airflow-ctl/tests/airflow_ctl/api/test_operations.py b/airflow-ctl/tests/airflow_ctl/api/test_operations.py index 6af647fbe68d2..f1eaf2e89efe1 100644 --- a/airflow-ctl/tests/airflow_ctl/api/test_operations.py +++ b/airflow-ctl/tests/airflow_ctl/api/test_operations.py @@ -77,6 +77,7 @@ DagVersionResponse, DAGWarningCollectionResponse, DAGWarningResponse, + DagWarningType, ImportErrorCollectionResponse, ImportErrorResponse, JobCollectionResponse, @@ -1191,7 +1192,7 @@ class TestDagOperations: dag_warnings=[ DAGWarningResponse( dag_id=dag_id, - warning_type="non-existent pool", + warning_type=DagWarningType.NON_EXISTENT_POOL, message="message", timestamp=datetime.datetime(2025, 1, 1, 0, 0, 0), dag_display_name=dag_display_name, From ab86cb4b816b32f526189b45e0d056f940525be1 Mon Sep 17 00:00:00 2001 From: Dilnaz Amanzholova Date: Wed, 16 Sep 2026 14:29:28 +0000 Subject: [PATCH 4/8] Populate the language field of the Dag source API The API response and the Code view's syntax highlighting already expose a language for each Dag's source, but nothing persisted it, so the field was always null and the UI silently fell back to Python for every Dag. --- airflow-core/docs/migrations-ref.rst | 4 +- .../core_api/routes/public/dag_sources.py | 1 + .../0136_3_4_0_add_language_to_dag_code.py | 52 +++++++++++++++++++ airflow-core/src/airflow/models/dagcode.py | 3 ++ airflow-core/src/airflow/utils/db.py | 2 +- .../routes/public/test_dag_sources.py | 8 +-- .../tests/unit/models/test_dagcode.py | 8 +++ 7 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 airflow-core/src/airflow/migrations/versions/0136_3_4_0_add_language_to_dag_code.py diff --git a/airflow-core/docs/migrations-ref.rst b/airflow-core/docs/migrations-ref.rst index c0b456561fd65..f6c56c6eecb75 100644 --- a/airflow-core/docs/migrations-ref.rst +++ b/airflow-core/docs/migrations-ref.rst @@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=========================+==================+===================+==============================================================+ -| ``a61f0c9d2b47`` (head) | ``c9f4b3e7a218`` | ``3.4.0`` | Allocate attempt numbers for pending retries and cleared | +| ``e5a91c7f42b3`` (head) | ``a61f0c9d2b47`` | ``3.4.0`` | Add language column to dag_code. | ++-------------------------+------------------+-------------------+--------------------------------------------------------------+ +| ``a61f0c9d2b47`` | ``c9f4b3e7a218`` | ``3.4.0`` | Allocate attempt numbers for pending retries and cleared | | | | | tasks. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``c9f4b3e7a218`` | ``3b7a91c5df20`` | ``3.4.0`` | Replace the ``job.team_name`` column with a ``job_team`` | diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_sources.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_sources.py index f74ce76c7b7da..212e63c55f201 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_sources.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_sources.py @@ -103,6 +103,7 @@ def get_dag_source( content=content, version_number=dag_version.version_number, dag_display_name=dag_version.dag_model.dag_display_name, + language=dag_version.dag_code.language, ) if accept == Mimetype.TEXT: diff --git a/airflow-core/src/airflow/migrations/versions/0136_3_4_0_add_language_to_dag_code.py b/airflow-core/src/airflow/migrations/versions/0136_3_4_0_add_language_to_dag_code.py new file mode 100644 index 0000000000000..9672fdec2a647 --- /dev/null +++ b/airflow-core/src/airflow/migrations/versions/0136_3_4_0_add_language_to_dag_code.py @@ -0,0 +1,52 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Add language column to dag_code. + +Revision ID: e5a91c7f42b3 +Revises: ca8499dc1004 +Create Date: 2026-09-10 12:00:00.000000 + +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e5a91c7f42b3" +down_revision = "ca8499dc1004" +branch_labels = None +depends_on = None +airflow_version = "3.4.0" + + +def upgrade(): + """Apply add language column to dag_code.""" + with op.batch_alter_table("dag_code", schema=None) as batch_op: + batch_op.add_column( + sa.Column("language", sa.String(length=64), nullable=False, server_default="python") + ) + + +def downgrade(): + """Unapply add language column to dag_code.""" + with op.batch_alter_table("dag_code", schema=None) as batch_op: + batch_op.drop_column("language") diff --git a/airflow-core/src/airflow/models/dagcode.py b/airflow-core/src/airflow/models/dagcode.py index b0306b15ff130..b21af49d57f71 100644 --- a/airflow-core/src/airflow/models/dagcode.py +++ b/airflow-core/src/airflow/models/dagcode.py @@ -66,6 +66,9 @@ class DagCode(Base): ) source_code: Mapped[str] = mapped_column(Text().with_variant(MEDIUMTEXT(), "mysql"), nullable=False) source_code_hash: Mapped[str] = mapped_column(String(32), nullable=False) + language: Mapped[str] = mapped_column( + String(64), nullable=False, default="python", server_default="python" + ) dag_version_id: Mapped[UUID] = mapped_column( sa.Uuid(), ForeignKey("dag_version.id", ondelete="CASCADE"), nullable=False, unique=True ) diff --git a/airflow-core/src/airflow/utils/db.py b/airflow-core/src/airflow/utils/db.py index 950af812a7faa..9639594f7210b 100644 --- a/airflow-core/src/airflow/utils/db.py +++ b/airflow-core/src/airflow/utils/db.py @@ -117,7 +117,7 @@ class MappedClassProtocol(Protocol): "3.1.8": "509b94a1042d", "3.2.0": "1d6611b6ab7c", "3.3.0": "d2f4e1b3c5a7", - "3.4.0": "a61f0c9d2b47", + "3.4.0": "e5a91c7f42b3", } # Prefix used to identify tables holding data moved during migration. diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py index b7ee7574a8877..6c07e4d38928c 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_sources.py @@ -122,7 +122,7 @@ def test_should_respond_200_json(self, test_client, test_dag, headers): "dag_id": TEST_DAG_ID, "version_number": 1, "dag_display_name": TEST_DAG_DISPLAY_NAME, - "language": None, + "language": "python", } assert response.headers["Content-Type"].startswith("application/json") @@ -168,7 +168,7 @@ def test_should_respond_200_version(self, test_client, accept, session, test_dag "dag_id": TEST_DAG_ID, "version_number": 2, "dag_display_name": TEST_DAG_DISPLAY_NAME, - "language": None, + "language": "python", } def test_should_respond_406_unsupport_mime_type(self, test_client, test_dag): @@ -227,7 +227,7 @@ def test_source_is_redacted_when_caller_cannot_read_all_dags_in_file( "dag_id": TEST_DAG_ID, "version_number": 1, "dag_display_name": TEST_DAG_DISPLAY_NAME, - "language": None, + "language": "python", } mock_get_auth_manager.return_value.get_authorized_dag_ids.assert_called_once_with(user=mock.ANY) @@ -251,5 +251,5 @@ def test_source_is_returned_when_caller_can_read_all_dags_in_file( "dag_id": TEST_DAG_ID, "version_number": 1, "dag_display_name": TEST_DAG_DISPLAY_NAME, - "language": None, + "language": "python", } diff --git a/airflow-core/tests/unit/models/test_dagcode.py b/airflow-core/tests/unit/models/test_dagcode.py index 1a0e16d95e542..419a8f0cbd5e0 100644 --- a/airflow-core/tests/unit/models/test_dagcode.py +++ b/airflow-core/tests/unit/models/test_dagcode.py @@ -262,3 +262,11 @@ def mytask(): refreshed = DagCode.get_latest_dagcode(dag.dag_id) assert refreshed.fileloc == dag.fileloc assert refreshed.source_code_hash == original_hash + + def test_language_defaults_to_python(self, dag_maker): + """Code written without an explicit language is recorded as Python.""" + with dag_maker("dag_language_default") as dag: + pass + sync_dag_to_db(dag) + + assert DagCode.get_latest_dagcode(dag.dag_id).language == "python" From 1ac3cd199e701c6cc63db87a1474152cf428392f Mon Sep 17 00:00:00 2001 From: Dilnaz Amanzholova Date: Fri, 18 Sep 2026 10:05:55 +0000 Subject: [PATCH 5/8] Add source_reference to ParseImportError model and keep filename Renaming filename to source_reference might break compatibility across the database schema, model attributes, and public REST API endpoints that rely on filename to refer to concrete files. --- airflow-core/docs/migrations-ref.rst | 8 +- .../src/airflow/api/common/delete_dag.py | 8 +- .../api_fastapi/common/parameters/misc.py | 6 +- .../core_api/datamodels/import_error.py | 3 +- .../openapi/v2-rest-api-generated.yaml | 27 ++-- .../core_api/routes/public/import_error.py | 12 +- .../src/airflow/dag_processing/collection.py | 16 +-- .../src/airflow/dag_processing/manager.py | 4 +- ...0_add_source_reference_to_import_error.py} | 10 +- airflow-core/src/airflow/models/errors.py | 20 ++- .../ui/openapi-gen/queries/ensureQueryData.ts | 2 +- .../ui/openapi-gen/queries/prefetch.ts | 2 +- .../airflow/ui/openapi-gen/queries/queries.ts | 2 +- .../ui/openapi-gen/queries/suspense.ts | 2 +- .../ui/openapi-gen/requests/schemas.gen.ts | 25 ++-- .../ui/openapi-gen/requests/services.gen.ts | 2 +- .../ui/openapi-gen/requests/types.gen.ts | 119 +++++++++--------- .../routes/public/test_import_error.py | 44 ++++++- .../unit/dag_processing/test_collection.py | 18 +-- airflow-core/tests/unit/models/test_errors.py | 35 ++++-- .../airflowctl/api/datamodels/generated.py | 21 ++-- 21 files changed, 224 insertions(+), 162 deletions(-) rename airflow-core/src/airflow/migrations/versions/{0135_3_4_0_rename_filename_to_source_reference_in_import_error.py => 0135_3_4_0_add_source_reference_to_import_error.py} (76%) diff --git a/airflow-core/docs/migrations-ref.rst b/airflow-core/docs/migrations-ref.rst index f6c56c6eecb75..e86c4e240236a 100644 --- a/airflow-core/docs/migrations-ref.rst +++ b/airflow-core/docs/migrations-ref.rst @@ -39,12 +39,14 @@ Here's the list of all the Database Migrations that are executed via when you ru +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=========================+==================+===================+==============================================================+ -| ``e5a91c7f42b3`` (head) | ``a61f0c9d2b47`` | ``3.4.0`` | Add language column to dag_code. | +| ``e5a91c7f42b3`` (head) | ``c9f4b3e7a218`` | ``3.4.0`` | Add language column to dag_code. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ -| ``a61f0c9d2b47`` | ``c9f4b3e7a218`` | ``3.4.0`` | Allocate attempt numbers for pending retries and cleared | +| ``c9f4b3e7a218`` | ``a61f0c9d2b47`` | ``3.4.0`` | Add source_reference to import_error. | ++-------------------------+------------------+-------------------+--------------------------------------------------------------+ +| ``a61f0c9d2b47`` | ``ca8499dc1004`` | ``3.4.0`` | Allocate attempt numbers for pending retries and cleared | | | | | tasks. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ -| ``c9f4b3e7a218`` | ``3b7a91c5df20`` | ``3.4.0`` | Replace the ``job.team_name`` column with a ``job_team`` | +| ``ca8499dc1004`` | ``3b7a91c5df20`` | ``3.4.0`` | Replace the ``job.team_name`` column with a ``job_team`` | | | | | association table. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``3b7a91c5df20`` | ``5182d0596ee2`` | ``3.4.0`` | Fold task_map into xcom.mapped_length. | diff --git a/airflow-core/src/airflow/api/common/delete_dag.py b/airflow-core/src/airflow/api/common/delete_dag.py index e9d54563efdf6..83a7cd508d4a7 100644 --- a/airflow-core/src/airflow/api/common/delete_dag.py +++ b/airflow-core/src/airflow/api/common/delete_dag.py @@ -22,7 +22,7 @@ import logging from typing import TYPE_CHECKING, cast -from sqlalchemy import delete, select +from sqlalchemy import delete, or_, select from airflow import models from airflow.exceptions import AirflowException, DagNotFound @@ -80,10 +80,12 @@ def delete_dag(dag_id: str, keep_records_in_log: bool = True, *, session: Sessio # Delete entries in Import Errors table for a deleted Dag # This handles the case when the dag_id is changed in the file - targets = [ref for ref in (dag.relative_fileloc, dag.fileloc) if ref] session.execute( delete(ParseImportError).where( - ParseImportError.source_reference.in_(targets), + or_( + ParseImportError.source_reference == dag.relative_fileloc, + ParseImportError.filename == dag.fileloc, + ), ParseImportError.bundle_name == dag.bundle_name, ) ) diff --git a/airflow-core/src/airflow/api_fastapi/common/parameters/misc.py b/airflow-core/src/airflow/api_fastapi/common/parameters/misc.py index 104e0bd94d4f0..d718d2e132ee9 100644 --- a/airflow-core/src/airflow/api_fastapi/common/parameters/misc.py +++ b/airflow-core/src/airflow/api_fastapi/common/parameters/misc.py @@ -269,19 +269,19 @@ def _optional_boolean(value: bool | None) -> bool | None: # Parse Import Errors QueryParseImportErrorFilenamePatternSearch = Annotated[ - _SearchParam, Depends(search_param_factory(ParseImportError.source_reference, "filename_pattern")) + _SearchParam, Depends(search_param_factory(ParseImportError.filename, "filename_pattern")) ] QueryParseImportErrorFilenamePrefixPatternSearch = Annotated[ _PrefixSearchParam, - Depends(prefix_search_param_factory(ParseImportError.source_reference, "filename_prefix_pattern")), + Depends(prefix_search_param_factory(ParseImportError.filename, "filename_prefix_pattern")), ] QueryParseImportErrorFilenameFilter = Annotated[ FilterParam, Depends( filter_param_factory( - ParseImportError.source_reference, + ParseImportError.filename, str | None, filter_name="filename", description="Exact filename match. Returns only the import error for this specific file path.", diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/import_error.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/import_error.py index 8709372c9348f..1747168b880bd 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/import_error.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/import_error.py @@ -30,7 +30,8 @@ class ImportErrorResponse(BaseModel): id: int = Field(alias="import_error_id") timestamp: datetime - source_reference: str = Field(alias="filename") + filename: str + source_reference: str | None = None bundle_name: str | None stacktrace: str = Field(alias="stack_trace") diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml index 5d5f8842d2b99..ad0e845f7f4c9 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml @@ -5257,13 +5257,13 @@ paths: type: string description: 'Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, - source_reference, bundle_name, stacktrace, import_error_id, filename`' + filename, source_reference, bundle_name, stacktrace, import_error_id`' default: - id title: Order By description: 'Attributes to order by, multi criteria sort is supported. Prefix - with `-` for descending order. Supported attributes: `id, timestamp, source_reference, - bundle_name, stacktrace, import_error_id, filename`' + with `-` for descending order. Supported attributes: `id, timestamp, filename, + source_reference, bundle_name, stacktrace, import_error_id`' - name: filename_pattern in: query required: false @@ -14889,14 +14889,6 @@ components: - bundle_url title: DagVersionResponse description: Dag Version serializer for responses. - DetailedHealthStatus: - type: string - enum: - - healthy - - degraded - - down - title: DetailedHealthStatus - description: How much of a component's work has a live instance covering it. DagWarningType: type: string enum: @@ -14911,6 +14903,14 @@ components: This is the set of allowable values for the ``warning_type`` field in the DagWarning model.' + DetailedHealthStatus: + type: string + enum: + - healthy + - degraded + - down + title: DetailedHealthStatus + description: How much of a component's work has a live instance covering it. DryRunBackfillCollectionResponse: properties: backfills: @@ -15459,6 +15459,11 @@ components: filename: type: string title: Filename + source_reference: + anyOf: + - type: string + - type: 'null' + title: Source Reference bundle_name: anyOf: - type: string diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/import_error.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/import_error.py index 9766a08692e78..fe8cb80a69247 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/import_error.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/import_error.py @@ -94,7 +94,7 @@ def get_import_error( file_dag_ids = set( session.scalars( select(DagModel.dag_id).where( - DagModel.relative_fileloc == error.source_reference, + DagModel.relative_fileloc == error.filename, DagModel.bundle_name == error.bundle_name, ) ).all() @@ -146,12 +146,13 @@ def get_import_errors( [ "id", "timestamp", + "filename", "source_reference", "bundle_name", "stacktrace", ], ParseImportError, - {"import_error_id": "id", "filename": "source_reference"}, + {"import_error_id": "id"}, ).dynamic_depends() ), ], @@ -177,7 +178,7 @@ def get_import_errors( select(ParseImportError.id, ParseImportError.bundle_name) .outerjoin( files_with_any_dags, - (ParseImportError.source_reference == files_with_any_dags.c.relative_fileloc) + (ParseImportError.filename == files_with_any_dags.c.relative_fileloc) & (ParseImportError.bundle_name == files_with_any_dags.c.bundle_name), ) .where(files_with_any_dags.c.relative_fileloc.is_(None)) @@ -232,14 +233,14 @@ def get_import_errors( .outerjoin( files_with_any_dags, and_( - ParseImportError.source_reference == files_with_any_dags.c.relative_fileloc, + ParseImportError.filename == files_with_any_dags.c.relative_fileloc, ParseImportError.bundle_name == files_with_any_dags.c.bundle_name, ), ) .outerjoin( file_dags_cte, and_( - ParseImportError.source_reference == file_dags_cte.c.relative_fileloc, + ParseImportError.filename == file_dags_cte.c.relative_fileloc, ParseImportError.bundle_name == file_dags_cte.c.bundle_name, ), ) @@ -262,6 +263,7 @@ def get_import_errors( filtered_import_errors_stmt.with_only_columns( ParseImportError.id, ParseImportError.timestamp, + ParseImportError.filename, ParseImportError.source_reference, ParseImportError.bundle_name, ParseImportError.stacktrace, diff --git a/airflow-core/src/airflow/dag_processing/collection.py b/airflow-core/src/airflow/dag_processing/collection.py index 99b494021fdc2..70c95f8eedf79 100644 --- a/airflow-core/src/airflow/dag_processing/collection.py +++ b/airflow-core/src/airflow/dag_processing/collection.py @@ -388,17 +388,15 @@ def _update_import_errors( from airflow.listeners.listener import get_listener_manager # Read before the delete below, so an error still present is updated rather than recreated. - # Scoped to the keys being written: the table is not indexed on (bundle_name, source_reference) so this + # Scoped to the keys being written: the table is not indexed on (bundle_name, filename) so this # still scans, but it no longer builds a row per import error the deployment is carrying. existing_import_error_files: set[tuple[str | None, str | None]] = set() if import_errors: existing_import_error_files = { (bundle, filename) for bundle, filename in session.execute( - select(ParseImportError.bundle_name, ParseImportError.source_reference).where( - tuple_(ParseImportError.bundle_name, ParseImportError.source_reference).in_( - list(import_errors) - ) + select(ParseImportError.bundle_name, ParseImportError.filename).where( + tuple_(ParseImportError.bundle_name, ParseImportError.filename).in_(list(import_errors)) ) ) } @@ -409,7 +407,7 @@ def _update_import_errors( if files_to_clear: session.execute( delete(ParseImportError).where( - tuple_(ParseImportError.bundle_name, ParseImportError.source_reference).in_(files_to_clear) + tuple_(ParseImportError.bundle_name, ParseImportError.filename).in_(files_to_clear) ) ) @@ -421,10 +419,11 @@ def _update_import_errors( session.execute( update(ParseImportError) .where( - ParseImportError.source_reference == relative_fileloc, + ParseImportError.filename == relative_fileloc, ParseImportError.bundle_name == bundle_name_, ) .values( + filename=relative_fileloc, source_reference=relative_fileloc, bundle_name=bundle_name_, timestamp=utcnow(), @@ -437,7 +436,7 @@ def _update_import_errors( import_error = session.scalar( select(ParseImportError).where( ParseImportError.bundle_name == bundle_name_, - ParseImportError.source_reference == relative_fileloc, + ParseImportError.filename == relative_fileloc, ) ) if import_error is not None: @@ -448,6 +447,7 @@ def _update_import_errors( log.exception("error calling listener") else: import_error = ParseImportError( + filename=relative_fileloc, source_reference=relative_fileloc, bundle_name=bundle_name_, timestamp=utcnow(), diff --git a/airflow-core/src/airflow/dag_processing/manager.py b/airflow-core/src/airflow/dag_processing/manager.py index 0b533796fd574..16edef4560305 100644 --- a/airflow-core/src/airflow/dag_processing/manager.py +++ b/airflow-core/src/airflow/dag_processing/manager.py @@ -1069,10 +1069,10 @@ def clear_orphaned_import_errors( errors = session.scalars( select(ParseImportError) .where(ParseImportError.bundle_name == bundle_name) - .options(load_only(ParseImportError.source_reference)) + .options(load_only(ParseImportError.filename)) ) for error in errors: - if error.source_reference not in observed_filelocs: + if error.filename not in observed_filelocs: session.delete(error) except Exception: self.log.exception("Error removing old import errors") diff --git a/airflow-core/src/airflow/migrations/versions/0135_3_4_0_rename_filename_to_source_reference_in_import_error.py b/airflow-core/src/airflow/migrations/versions/0135_3_4_0_add_source_reference_to_import_error.py similarity index 76% rename from airflow-core/src/airflow/migrations/versions/0135_3_4_0_rename_filename_to_source_reference_in_import_error.py rename to airflow-core/src/airflow/migrations/versions/0135_3_4_0_add_source_reference_to_import_error.py index 828376b1e2df1..c5f122219b346 100644 --- a/airflow-core/src/airflow/migrations/versions/0135_3_4_0_rename_filename_to_source_reference_in_import_error.py +++ b/airflow-core/src/airflow/migrations/versions/0135_3_4_0_add_source_reference_to_import_error.py @@ -17,7 +17,7 @@ # under the License. """ -Rename filename to source_reference in import_error. +Add source_reference to import_error. Revision ID: ca8499dc1004 Revises: b6a9c2e7d410 @@ -39,12 +39,12 @@ def upgrade(): - """Apply rename filename to source_reference in import_error.""" + """Apply add source_reference to import_error.""" with op.batch_alter_table("import_error", schema=None) as batch_op: - batch_op.alter_column("filename", new_column_name="source_reference", type_=sa.String(length=1024)) + batch_op.add_column(sa.Column("source_reference", sa.String(length=1024), nullable=True)) def downgrade(): - """Unapply rename filename to source_reference in import_error.""" + """Unapply add source_reference to import_error.""" with op.batch_alter_table("import_error", schema=None) as batch_op: - batch_op.alter_column("source_reference", new_column_name="filename", type_=sa.String(length=1024)) + batch_op.drop_column("source_reference") diff --git a/airflow-core/src/airflow/models/errors.py b/airflow-core/src/airflow/models/errors.py index 8654713c35761..b17aa36500ca2 100644 --- a/airflow-core/src/airflow/models/errors.py +++ b/airflow-core/src/airflow/models/errors.py @@ -21,7 +21,7 @@ from pathlib import Path from sqlalchemy import Integer, String, Text -from sqlalchemy.orm import Mapped, mapped_column, synonym +from sqlalchemy.orm import Mapped, mapped_column from airflow.dag_processing.bundles.manager import DagBundlesManager from airflow.models.base import Base, StringID @@ -34,20 +34,18 @@ class ParseImportError(Base): __tablename__ = "import_error" id: Mapped[int] = mapped_column(Integer, primary_key=True) timestamp: Mapped[datetime | None] = mapped_column(UtcDateTime, nullable=True) + filename: Mapped[str | None] = mapped_column(String(1024), nullable=True) source_reference: Mapped[str | None] = mapped_column(String(1024), nullable=True) bundle_name: Mapped[str | None] = mapped_column(StringID(), nullable=True) stacktrace: Mapped[str | None] = mapped_column(Text, nullable=True) - filename = synonym("source_reference") - def full_file_path(self) -> str: """Return the full file path of the dag.""" - if self.bundle_name is None or self.source_reference is None: - raise ValueError("bundle_name and source_reference must not be None") - bundle = DagBundlesManager().get_bundle(self.bundle_name) - ref = self.source_reference - # A reference may address a member inside a container (``archive.zip:dags/my_dag.py``); the - # anchor alone decides whether it is already rooted, so the separator is never parsed here. - if ref.startswith(str(bundle.path)) or Path(ref).is_absolute(): + ref = self.source_reference or self.filename + if self.bundle_name is None or ref is None: + raise ValueError("bundle_name and (source_reference or filename) must not be None") + ref_path = Path(ref) + if ref_path.is_absolute(): return ref - return str(bundle.path / ref) + bundle = DagBundlesManager().get_bundle(self.bundle_name) + return str(bundle.path / ref_path) diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts index fc7148ab9562e..01983ce4e9958 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts @@ -1523,7 +1523,7 @@ export const ensureUseImportErrorServiceGetImportErrorData = (queryClient: Query * @param data The data for the request. * @param data.limit * @param data.offset -* @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, source_reference, bundle_name, stacktrace, import_error_id, filename` +* @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, filename, source_reference, bundle_name, stacktrace, import_error_id` * @param data.filenamePattern Case-insensitive substring match (SQL `ILIKE`). Slower than `filename_prefix_pattern` on large tables — see "Filtering with pattern parameters". * @param data.filenamePrefixPattern Case-sensitive, index-friendly prefix match. See "Filtering with pattern parameters". * @param data.filename Exact filename match. Returns only the import error for this specific file path. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts index 73496ce6079cb..501689dc55dd0 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts @@ -1523,7 +1523,7 @@ export const prefetchUseImportErrorServiceGetImportError = (queryClient: QueryCl * @param data The data for the request. * @param data.limit * @param data.offset -* @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, source_reference, bundle_name, stacktrace, import_error_id, filename` +* @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, filename, source_reference, bundle_name, stacktrace, import_error_id` * @param data.filenamePattern Case-insensitive substring match (SQL `ILIKE`). Slower than `filename_prefix_pattern` on large tables — see "Filtering with pattern parameters". * @param data.filenamePrefixPattern Case-sensitive, index-friendly prefix match. See "Filtering with pattern parameters". * @param data.filename Exact filename match. Returns only the import error for this specific file path. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts index e353cf6df668f..4951fc96970e8 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts @@ -1523,7 +1523,7 @@ export const useImportErrorServiceGetImportError = ; @@ -274,8 +274,8 @@ export type BackfillPostBody = { to_date: string; run_backwards?: boolean; dag_run_conf?: { - [key: string]: unknown; - } | null; + [key: string]: unknown; +} | null; reprocess_behavior?: ReprocessBehavior; max_active_runs?: number; /** @@ -293,8 +293,8 @@ export type BackfillResponse = { from_date: string; to_date: string; dag_run_conf: { - [key: string]: unknown; - } | null; + [key: string]: unknown; +} | null; is_paused: boolean; reprocess_behavior: ReprocessBehavior; max_active_runs: number; @@ -786,9 +786,9 @@ export type Config = { export type ConfigOption = { key: string; value: string | [ - string, - string - ]; + string, + string +]; }; /** @@ -949,19 +949,19 @@ export type DAGDetailsResponse = { end_date: string | null; is_paused_upon_creation: boolean | null; params: { - [key: string]: unknown; - } | null; + [key: string]: unknown; +} | null; render_template_as_native_obj: boolean; template_search_path: Array<(string)> | null; timezone: string | null; last_parsed: string | null; default_args: { - [key: string]: unknown; - } | null; + [key: string]: unknown; +} | null; rerun_with_latest_version?: boolean | null; owner_links?: { - [key: string]: (string); - } | null; + [key: string]: (string); +} | null; is_favorite?: boolean; active_runs_count?: number; team_name?: string | null; @@ -1112,8 +1112,8 @@ export type DAGRunResponse = { triggered_by: DagRunTriggeredByType | null; triggering_user_name: string | null; conf: { - [key: string]: unknown; - } | null; + [key: string]: unknown; +} | null; note: string | null; dag_versions: Array; bundle_version: string | null; @@ -1431,11 +1431,6 @@ export type DagVersionResponse = { bundle_url: string | null; }; -/** - * How much of a component's work has a live instance covering it. - */ -export type DetailedHealthStatus = 'healthy' | 'degraded' | 'down'; - /** * Enum for DAG warning types. * @@ -1444,6 +1439,11 @@ export type DetailedHealthStatus = 'healthy' | 'degraded' | 'down'; */ export type DagWarningType = 'asset conflict' | 'duplicate dag id' | 'non-existent pool' | 'runtime varying value'; +/** + * How much of a component's work has a live instance covering it. + */ +export type DetailedHealthStatus = 'healthy' | 'degraded' | 'down'; + /** * Backfill collection serializer for responses in dry-run mode. */ @@ -1625,8 +1625,8 @@ export type HITLUser = { */ export type HTTPExceptionResponse = { detail: string | { - [key: string]: unknown; - }; + [key: string]: unknown; +}; }; export type HTTPValidationError = { @@ -1663,6 +1663,7 @@ export type ImportErrorResponse = { import_error_id: number; timestamp: string; filename: string; + source_reference?: string | null; bundle_name: string | null; stack_trace: string; /** @@ -1718,8 +1719,8 @@ export type MaterializeAssetBody = { logical_date?: string | null; run_after?: string | null; conf?: { - [key: string]: unknown; - } | null; + [key: string]: unknown; +} | null; note?: string | null; partition_key?: string | null; bundle_version?: string | null; @@ -2184,11 +2185,11 @@ export type TaskResponse = { doc_md: string | null; operator_name: string | null; params: { - [key: string]: unknown; - } | null; + [key: string]: unknown; +} | null; class_ref: { - [key: string]: unknown; - } | null; + [key: string]: unknown; +} | null; is_mapped: boolean | null; /** * Extract and return extra_links. @@ -2255,8 +2256,8 @@ export type TriggerDAGRunPostBody = { logical_date: string | null; run_after?: string | null; conf?: { - [key: string]: unknown; - } | null; + [key: string]: unknown; +} | null; note?: string | null; partition_key?: string | null; bundle_version?: string | null; @@ -2559,8 +2560,8 @@ export type ConnectionHookMetaData = { hook_name: string; standard_fields: StandardHookFields | null; extra_fields: { - [key: string]: unknown; - } | null; + [key: string]: unknown; +} | null; }; /** @@ -2861,8 +2862,8 @@ export type LightGridTaskInstanceSummary = { task_display_name: string; state: TaskInstanceState | null; child_states: { - [key: string]: (number); - } | null; + [key: string]: (number); +} | null; min_start_date: string | null; max_end_date: string | null; dag_version_number?: number | null; @@ -2952,8 +2953,8 @@ export type PartitionedDagRunCollectionResponse = { partitioned_dag_runs: Array; total: number; asset_expressions?: { - [key: string]: (AssetExpressionAsset | AssetExpressionAlias | AssetExpressionRef | AssetExpressionAny | AssetExpressionAll | null); - } | null; + [key: string]: (AssetExpressionAsset | AssetExpressionAlias | AssetExpressionRef | AssetExpressionAny | AssetExpressionAll | null); +} | null; }; /** @@ -3046,13 +3047,13 @@ export type TeamResponse = { */ export type Theme = { tokens?: { - [key: string]: ThemeColors; - } | null; + [key: string]: ThemeColors; +} | null; globalCss?: { - [key: string]: { - [key: string]: unknown; - }; - } | null; + [key: string]: { + [key: string]: unknown; + }; +} | null; icon?: string | null; icon_dark_mode?: string | null; }; @@ -3062,21 +3063,21 @@ export type Theme = { */ export type ThemeColors = { brand?: { - [key: string]: { - [key: string]: OklchColor; - }; - } | null; + [key: string]: { + [key: string]: OklchColor; + }; +} | null; gray?: { - [key: string]: { - [key: string]: OklchColor; - }; - } | null; - black?: { + [key: string]: { [key: string]: OklchColor; - } | null; + }; +} | null; + black?: { + [key: string]: OklchColor; +} | null; white?: { - [key: string]: OklchColor; - } | null; + [key: string]: OklchColor; +} | null; }; /** @@ -4521,7 +4522,7 @@ export type GetImportErrorsData = { limit?: number; offset?: number; /** - * Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, source_reference, bundle_name, stacktrace, import_error_id, filename` + * Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `id, timestamp, filename, source_reference, bundle_name, stacktrace, import_error_id` */ orderBy?: Array<(string)>; }; diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py index 49c72835909ed..f8a0f1e12a80f 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py @@ -119,7 +119,7 @@ def import_errors(*, session: Session = NEW_SESSION) -> list[ParseImportError]: _import_errors = [ ParseImportError( bundle_name=bundle, - source_reference=filename, + filename=filename, stacktrace=stacktrace, timestamp=timestamp, ) @@ -171,6 +171,7 @@ class TestGetImportError: { "timestamp": from_datetime_to_zulu_without_ms(TIMESTAMP1), "filename": FILENAME1, + "source_reference": None, "stack_trace": STACKTRACE1, "bundle_name": BUNDLE_NAME, }, @@ -181,6 +182,7 @@ class TestGetImportError: { "timestamp": from_datetime_to_zulu_without_ms(TIMESTAMP2), "filename": FILENAME2, + "source_reference": None, "stack_trace": STACKTRACE2, "bundle_name": BUNDLE_NAME, }, @@ -191,6 +193,7 @@ class TestGetImportError: { "timestamp": from_datetime_to_zulu_without_ms(TIMESTAMP3), "filename": FILENAME3, + "source_reference": None, "stack_trace": STACKTRACE3, "bundle_name": BUNDLE_NAME, }, @@ -230,6 +233,32 @@ def test_get_import_error( ) assert response.json() == expected_body + @mock.patch("airflow.api_fastapi.core_api.routes.public.import_error.get_auth_manager") + def test_get_import_error_with_source_reference( + self, mock_get_auth_manager, test_client, session, permitted_dag_model_all + ): + error = ParseImportError( + bundle_name=BUNDLE_NAME, + filename=FILENAME1, + source_reference="archive.zip/dags/my_dag.py", + stacktrace=STACKTRACE1, + timestamp=TIMESTAMP1, + ) + session.add(error) + session.commit() + + set_mock_auth_manager__get_authorized_dag_ids(mock_get_auth_manager, permitted_dag_model_all) + response = test_client.get(f"/importErrors/{error.id}") + assert response.status_code == 200 + assert response.json() == { + "import_error_id": error.id, + "timestamp": from_datetime_to_zulu_without_ms(TIMESTAMP1), + "filename": FILENAME1, + "source_reference": "archive.zip/dags/my_dag.py", + "stack_trace": STACKTRACE1, + "bundle_name": BUNDLE_NAME, + } + def test_should_raises_401_unauthenticated(self, unauthenticated_test_client, import_errors): import_error_id = import_errors[0].id response = unauthenticated_test_client.get(f"/importErrors/{import_error_id}") @@ -275,6 +304,7 @@ def test_get_import_error__user_dont_have_read_permission_to_read_all_dags_in_fi "import_error_id": import_error_id, "timestamp": from_datetime_to_zulu_without_ms(TIMESTAMP1), "filename": FILENAME1, + "source_reference": None, "stack_trace": "REDACTED - you do not have read permission on all Dags in the file", "bundle_name": BUNDLE_NAME, "file_token": url_safe_serializer.dumps( @@ -316,6 +346,7 @@ def test_get_import_error__no_dag_in_dagmodel( "import_error_id": import_error_id, "timestamp": from_datetime_to_zulu_without_ms(TIMESTAMP1), "filename": FILENAME1, + "source_reference": None, "stack_trace": STACKTRACE1, "bundle_name": BUNDLE_NAME, "file_token": url_safe_serializer.dumps( @@ -565,7 +596,7 @@ def test_get_import_errors_bundle_name_and_filename_filter( session.add( ParseImportError( bundle_name=other_bundle, - source_reference=shared_filename, + filename=shared_filename, stacktrace="wrong bundle error", timestamp=TIMESTAMP1, ) @@ -618,7 +649,7 @@ def test_get_import_errors__file_registered_only_in_other_bundle_is_unregistered # has no Dag for it. error = ParseImportError( bundle_name=BUNDLE_NAME, - source_reference=FILENAME1, + filename=FILENAME1, stacktrace=STACKTRACE1, timestamp=TIMESTAMP1, ) @@ -785,6 +816,7 @@ def test_user_can_not_read_all_dags_in_file( "import_error_id": import_errors[0].id, "timestamp": from_datetime_to_zulu_without_ms(TIMESTAMP1), "filename": FILENAME1, + "source_reference": None, "stack_trace": expected_stack_trace, "bundle_name": BUNDLE_NAME, "file_token": url_safe_serializer.dumps( @@ -968,7 +1000,7 @@ def lonely_file_import_error( ) -> ParseImportError: error = ParseImportError( bundle_name=BUNDLE_NAME, - source_reference=self.LONELY_FILE_RELATIVE, + filename=self.LONELY_FILE_RELATIVE, stacktrace=self.LONELY_STACKTRACE, timestamp=TIMESTAMP1, ) @@ -985,7 +1017,7 @@ def mixed_file_import_error( ) -> ParseImportError: error = ParseImportError( bundle_name=BUNDLE_NAME, - source_reference=self.MIXED_FILE_RELATIVE, + filename=self.MIXED_FILE_RELATIVE, stacktrace=self.MIXED_STACKTRACE, timestamp=TIMESTAMP2, ) @@ -1134,7 +1166,7 @@ def team_scoped_unregistered_import_error(self) -> Generator[ParseImportError, N session.add_all([team, bundle]) error = ParseImportError( bundle_name="team_b_bundle", - source_reference="team_b_unregistered.py", + filename="team_b_unregistered.py", stacktrace="team b stack trace", timestamp=TIMESTAMP1, ) diff --git a/airflow-core/tests/unit/dag_processing/test_collection.py b/airflow-core/tests/unit/dag_processing/test_collection.py index 98bf3dca0cd50..be36290abcece 100644 --- a/airflow-core/tests/unit/dag_processing/test_collection.py +++ b/airflow-core/tests/unit/dag_processing/test_collection.py @@ -1081,7 +1081,7 @@ def test_new_import_error_replaces_old( import_error = session.scalar( select(ParseImportError).where( - ParseImportError.source_reference == filename, ParseImportError.bundle_name == bundle_name + ParseImportError.filename == filename, ParseImportError.bundle_name == bundle_name ) ) @@ -1119,9 +1119,7 @@ def test_remove_error_clears_import_error(self, testing_dag_bundle, session): session.flush() # Sanity check of pre-condition - import_errors = set( - session.execute(select(ParseImportError.source_reference, ParseImportError.bundle_name)) - ) + import_errors = set(session.execute(select(ParseImportError.filename, ParseImportError.bundle_name))) assert import_errors == {("abc.py", bundle_name), ("def.py", bundle_name)} dag = DAG(dag_id="test") @@ -1142,9 +1140,7 @@ def test_remove_error_clears_import_error(self, testing_dag_bundle, session): dag_model: DagModel = session.get(DagModel, (dag.dag_id,)) assert dag_model.has_import_errors is False - import_errors = set( - session.execute(select(ParseImportError.source_reference, ParseImportError.bundle_name)) - ) + import_errors = set(session.execute(select(ParseImportError.filename, ParseImportError.bundle_name))) assert import_errors == {("def.py", bundle_name)} @@ -1227,9 +1223,7 @@ def test_clear_import_error_for_file_without_dags(self, testing_dag_bundle, sess session.add(other_file_error) session.flush() - import_errors = set( - session.execute(select(ParseImportError.source_reference, ParseImportError.bundle_name)) - ) + import_errors = set(session.execute(select(ParseImportError.filename, ParseImportError.bundle_name))) assert import_errors == {("no_dags.py", bundle_name), ("other.py", bundle_name)} # Simulate parsing the file: it was parsed successfully (no import errors), @@ -1247,9 +1241,7 @@ def test_clear_import_error_for_file_without_dags(self, testing_dag_bundle, sess files_parsed=files_parsed, ) - import_errors = set( - session.execute(select(ParseImportError.source_reference, ParseImportError.bundle_name)) - ) + import_errors = set(session.execute(select(ParseImportError.filename, ParseImportError.bundle_name))) assert import_errors == {("other.py", bundle_name)}, "Import error for parsed file should be cleared" @pytest.mark.usefixtures("clean_db") diff --git a/airflow-core/tests/unit/models/test_errors.py b/airflow-core/tests/unit/models/test_errors.py index 780401b0e1be1..da5d1f36c2770 100644 --- a/airflow-core/tests/unit/models/test_errors.py +++ b/airflow-core/tests/unit/models/test_errors.py @@ -40,16 +40,21 @@ class TestFullFilePath: [ pytest.param("dags/my_dag.py", "/bundle/dags/my_dag.py", id="relative-file"), pytest.param( - "archive.zip:dags/my_dag.py", - "/bundle/archive.zip:dags/my_dag.py", + "archive.zip/dags/my_dag.py", + "/bundle/archive.zip/dags/my_dag.py", id="relative-archive-member", ), pytest.param( - "/elsewhere/archive.zip:dags/my_dag.py", - "/elsewhere/archive.zip:dags/my_dag.py", + "/elsewhere/archive.zip/dags/my_dag.py", + "/elsewhere/archive.zip/dags/my_dag.py", id="absolute-archive-member", ), pytest.param("/bundle/dags/my_dag.py", "/bundle/dags/my_dag.py", id="already-under-bundle"), + pytest.param( + "/bundle-sibling/dags/my_dag.py", + "/bundle-sibling/dags/my_dag.py", + id="sibling-bundle-prefix", + ), ], ) def test_resolves_reference_against_bundle(self, source_reference, expected): @@ -57,15 +62,25 @@ def test_resolves_reference_against_bundle(self, source_reference, expected): assert error.full_file_path() == expected + @pytest.mark.usefixtures("bundle_path") + def test_resolves_filename_when_source_reference_is_none(self): + error = ParseImportError(bundle_name="my-bundle", filename="dags/my_dag.py") + + assert error.full_file_path() == "/bundle/dags/my_dag.py" + @pytest.mark.parametrize( - ("bundle_name", "source_reference"), + ("bundle_name", "source_reference", "filename"), [ - pytest.param(None, "dags/my_dag.py", id="missing-bundle"), - pytest.param("my-bundle", None, id="missing-reference"), + pytest.param(None, "dags/my_dag.py", None, id="missing-bundle"), + pytest.param("my-bundle", None, None, id="missing-reference"), ], ) - def test_raises_when_reference_is_incomplete(self, bundle_name, source_reference): - error = ParseImportError(bundle_name=bundle_name, source_reference=source_reference) + def test_raises_when_reference_is_incomplete(self, bundle_name, source_reference, filename): + error = ParseImportError( + bundle_name=bundle_name, source_reference=source_reference, filename=filename + ) - with pytest.raises(ValueError, match="bundle_name and source_reference must not be None"): + with pytest.raises( + ValueError, match=r"bundle_name and \(source_reference or filename\) must not be None" + ): error.full_file_path() diff --git a/airflow-ctl/src/airflowctl/api/datamodels/generated.py b/airflow-ctl/src/airflowctl/api/datamodels/generated.py index ec9739a4ce629..31c18107a8066 100644 --- a/airflow-ctl/src/airflowctl/api/datamodels/generated.py +++ b/airflow-ctl/src/airflowctl/api/datamodels/generated.py @@ -784,16 +784,6 @@ class DagVersionResponse(BaseModel): bundle_url: Annotated[str | None, Field(title="Bundle Url")] -class DetailedHealthStatus(str, Enum): - """ - How much of a component's work has a live instance covering it. - """ - - HEALTHY = "healthy" - DEGRADED = "degraded" - DOWN = "down" - - class DagWarningType(str, Enum): """ Enum for DAG warning types. @@ -808,6 +798,16 @@ class DagWarningType(str, Enum): RUNTIME_VARYING_VALUE = "runtime varying value" +class DetailedHealthStatus(str, Enum): + """ + How much of a component's work has a live instance covering it. + """ + + HEALTHY = "healthy" + DEGRADED = "degraded" + DOWN = "down" + + class DryRunBackfillResponse(BaseModel): """ Backfill serializer for responses in dry-run mode. @@ -926,6 +926,7 @@ class ImportErrorResponse(BaseModel): import_error_id: Annotated[int, Field(title="Import Error Id")] timestamp: Annotated[datetime, Field(title="Timestamp")] filename: Annotated[str, Field(title="Filename")] + source_reference: Annotated[str | None, Field(title="Source Reference")] = None bundle_name: Annotated[str | None, Field(title="Bundle Name")] stack_trace: Annotated[str, Field(title="Stack Trace")] file_token: Annotated[ From 6efa53e4e58b21256f4a88c4fe74b08941e61edd Mon Sep 17 00:00:00 2001 From: Dilnaz Amanzholova Date: Fri, 25 Sep 2026 09:11:12 +0000 Subject: [PATCH 6/8] Fix stale import errors after Dag deletion and allow custom warning types --- airflow-core/docs/migrations-ref.rst | 8 +-- .../src/airflow/api/common/delete_dag.py | 2 +- .../core_api/datamodels/dag_warning.py | 2 +- .../openapi/v2-rest-api-generated.yaml | 31 +++++------ .../core_api/routes/public/dag_warning.py | 4 +- ...0_add_source_reference_to_import_error.py} | 6 +- ...=> 0139_3_4_0_add_language_to_dag_code.py} | 0 airflow-core/src/airflow/models/dagwarning.py | 49 ++++++++++++++--- airflow-core/src/airflow/models/errors.py | 2 +- .../ui/openapi-gen/requests/schemas.gen.ts | 30 +++++----- .../ui/openapi-gen/requests/types.gen.ts | 18 ++---- .../tests/unit/api/common/test_delete_dag.py | 49 +++++++++++++++++ .../routes/public/test_dag_warning.py | 23 +++++++- .../routes/public/test_import_error.py | 5 +- .../tests/unit/models/test_dagwarning.py | 55 ++++++++++++++++++- .../airflowctl/api/datamodels/generated.py | 15 +++-- .../tests/airflow_ctl/api/test_operations.py | 2 +- 17 files changed, 228 insertions(+), 73 deletions(-) rename airflow-core/src/airflow/migrations/versions/{0135_3_4_0_add_source_reference_to_import_error.py => 0138_3_4_0_add_source_reference_to_import_error.py} (94%) rename airflow-core/src/airflow/migrations/versions/{0136_3_4_0_add_language_to_dag_code.py => 0139_3_4_0_add_language_to_dag_code.py} (100%) diff --git a/airflow-core/docs/migrations-ref.rst b/airflow-core/docs/migrations-ref.rst index e86c4e240236a..4f5bf0f33ad4f 100644 --- a/airflow-core/docs/migrations-ref.rst +++ b/airflow-core/docs/migrations-ref.rst @@ -39,14 +39,14 @@ Here's the list of all the Database Migrations that are executed via when you ru +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=========================+==================+===================+==============================================================+ -| ``e5a91c7f42b3`` (head) | ``c9f4b3e7a218`` | ``3.4.0`` | Add language column to dag_code. | +| ``e5a91c7f42b3`` (head) | ``ca8499dc1004`` | ``3.4.0`` | Add language column to dag_code. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ -| ``c9f4b3e7a218`` | ``a61f0c9d2b47`` | ``3.4.0`` | Add source_reference to import_error. | +| ``ca8499dc1004`` | ``a61f0c9d2b47`` | ``3.4.0`` | Add source_reference to import_error. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ -| ``a61f0c9d2b47`` | ``ca8499dc1004`` | ``3.4.0`` | Allocate attempt numbers for pending retries and cleared | +| ``a61f0c9d2b47`` | ``c9f4b3e7a218`` | ``3.4.0`` | Allocate attempt numbers for pending retries and cleared | | | | | tasks. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ -| ``ca8499dc1004`` | ``3b7a91c5df20`` | ``3.4.0`` | Replace the ``job.team_name`` column with a ``job_team`` | +| ``c9f4b3e7a218`` | ``3b7a91c5df20`` | ``3.4.0`` | Replace the ``job.team_name`` column with a ``job_team`` | | | | | association table. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``3b7a91c5df20`` | ``5182d0596ee2`` | ``3.4.0`` | Fold task_map into xcom.mapped_length. | diff --git a/airflow-core/src/airflow/api/common/delete_dag.py b/airflow-core/src/airflow/api/common/delete_dag.py index 83a7cd508d4a7..4dc753fa945a0 100644 --- a/airflow-core/src/airflow/api/common/delete_dag.py +++ b/airflow-core/src/airflow/api/common/delete_dag.py @@ -84,7 +84,7 @@ def delete_dag(dag_id: str, keep_records_in_log: bool = True, *, session: Sessio delete(ParseImportError).where( or_( ParseImportError.source_reference == dag.relative_fileloc, - ParseImportError.filename == dag.fileloc, + ParseImportError.filename == dag.relative_fileloc, ), ParseImportError.bundle_name == dag.bundle_name, ) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py index 796336bdb92e6..470a0b14fc64f 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py @@ -30,7 +30,7 @@ class DAGWarningResponse(BaseModel): """Dag Warning serializer for responses.""" dag_id: str - warning_type: DagWarningType | str + warning_type: DagWarningType message: str timestamp: datetime dag_display_name: str = Field(validation_alias=AliasPath("dag_model", "dag_display_name")) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml index ad0e845f7f4c9..b2017e62ea475 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml @@ -3805,7 +3805,6 @@ paths: schema: anyOf: - $ref: '#/components/schemas/DagWarningType' - - type: string - type: 'null' title: Warning Type - name: limit @@ -14351,10 +14350,7 @@ components: type: string title: Dag Id warning_type: - anyOf: - - $ref: '#/components/schemas/DagWarningType' - - type: string - title: Warning Type + $ref: '#/components/schemas/DagWarningType' message: type: string title: Message @@ -14890,19 +14886,22 @@ components: title: DagVersionResponse description: Dag Version serializer for responses. DagWarningType: - type: string - enum: - - asset conflict - - duplicate dag id - - non-existent pool - - runtime varying value - title: DagWarningType - description: 'Enum for DAG warning types. - + anyOf: + - type: string + enum: + - asset conflict + - duplicate dag id + - non-existent pool + - runtime varying value + title: DagWarningType + description: 'Enum for DAG warning types. - This is the set of allowable values for the ``warning_type`` field - in the DagWarning model.' + Members are the types Airflow emits; importers may add namespaced types + such as ``yaml:deprecated_field``.' + - type: string + maxLength: 50 + pattern: ^[a-z][a-z0-9_]*:[a-z0-9_.\-]+$ DetailedHealthStatus: type: string enum: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py index db82be115fda8..c1a652c370f74 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py @@ -52,8 +52,8 @@ def list_dag_warnings( dag_id: Annotated[FilterParam[str | None], Depends(filter_param_factory(DagWarning.dag_id, str | None))], warning_type: Annotated[ - FilterParam[DagWarningType | str | None], - Depends(filter_param_factory(DagWarning.warning_type, DagWarningType | str | None)), + FilterParam[DagWarningType | None], + Depends(filter_param_factory(DagWarning.warning_type, DagWarningType | None)), ], limit: QueryLimit, offset: QueryOffset, diff --git a/airflow-core/src/airflow/migrations/versions/0135_3_4_0_add_source_reference_to_import_error.py b/airflow-core/src/airflow/migrations/versions/0138_3_4_0_add_source_reference_to_import_error.py similarity index 94% rename from airflow-core/src/airflow/migrations/versions/0135_3_4_0_add_source_reference_to_import_error.py rename to airflow-core/src/airflow/migrations/versions/0138_3_4_0_add_source_reference_to_import_error.py index c5f122219b346..bebf3be419b61 100644 --- a/airflow-core/src/airflow/migrations/versions/0135_3_4_0_add_source_reference_to_import_error.py +++ b/airflow-core/src/airflow/migrations/versions/0138_3_4_0_add_source_reference_to_import_error.py @@ -20,7 +20,7 @@ Add source_reference to import_error. Revision ID: ca8499dc1004 -Revises: b6a9c2e7d410 +Revises: c9f4b3e7a218 Create Date: 2026-08-27 12:45:02.276898 """ @@ -32,7 +32,7 @@ # revision identifiers, used by Alembic. revision = "ca8499dc1004" -down_revision = "b6a9c2e7d410" +down_revision = "c9f4b3e7a218" branch_labels = None depends_on = None airflow_version = "3.4.0" @@ -41,7 +41,7 @@ def upgrade(): """Apply add source_reference to import_error.""" with op.batch_alter_table("import_error", schema=None) as batch_op: - batch_op.add_column(sa.Column("source_reference", sa.String(length=1024), nullable=True)) + batch_op.add_column(sa.Column("source_reference", sa.String(length=2000), nullable=True)) def downgrade(): diff --git a/airflow-core/src/airflow/migrations/versions/0136_3_4_0_add_language_to_dag_code.py b/airflow-core/src/airflow/migrations/versions/0139_3_4_0_add_language_to_dag_code.py similarity index 100% rename from airflow-core/src/airflow/migrations/versions/0136_3_4_0_add_language_to_dag_code.py rename to airflow-core/src/airflow/migrations/versions/0139_3_4_0_add_language_to_dag_code.py diff --git a/airflow-core/src/airflow/models/dagwarning.py b/airflow-core/src/airflow/models/dagwarning.py index 2145e7e6028a3..d588013c5e281 100644 --- a/airflow-core/src/airflow/models/dagwarning.py +++ b/airflow-core/src/airflow/models/dagwarning.py @@ -17,6 +17,7 @@ # under the License. from __future__ import annotations +import re from datetime import datetime from enum import Enum from typing import TYPE_CHECKING @@ -32,8 +33,14 @@ from airflow.utils.sqlalchemy import UtcDateTime, get_dialect_name if TYPE_CHECKING: + from pydantic import GetJsonSchemaHandler + from pydantic.json_schema import JsonSchemaValue + from pydantic_core import CoreSchema from sqlalchemy.orm import Session +WARNING_TYPE_MAX_LENGTH = 50 +IMPORTER_WARNING_TYPE_PATTERN = r"^[a-z][a-z0-9_]*:[a-z0-9_.\-]+$" + class DagWarning(Base): """ @@ -45,7 +52,7 @@ class DagWarning(Base): """ dag_id: Mapped[str] = mapped_column(StringID(), primary_key=True) - warning_type: Mapped[str] = mapped_column(String(50), primary_key=True) + warning_type: Mapped[str] = mapped_column(String(WARNING_TYPE_MAX_LENGTH), primary_key=True) message: Mapped[str] = mapped_column(Text, nullable=False) timestamp: Mapped[datetime] = mapped_column(UtcDateTime, nullable=False, default=timezone.utcnow) @@ -62,13 +69,10 @@ class DagWarning(Base): Index("idx_dag_warning_dag_id", dag_id), ) - def __init__(self, dag_id: str, warning_type: DagWarningType | str, message: str, **kwargs): + def __init__(self, dag_id: str, warning_type: str, message: str, **kwargs): super().__init__(**kwargs) self.dag_id = dag_id - # Dag importers supply their own warning types, so anything outside the enum is kept verbatim. - self.warning_type = ( - warning_type.value if isinstance(warning_type, DagWarningType) else str(warning_type) - ) + self.warning_type = DagWarningType(warning_type).value self.message = message def __eq__(self, other) -> bool: @@ -100,11 +104,40 @@ class DagWarningType(str, Enum): """ Enum for DAG warning types. - This is the set of allowable values for the ``warning_type`` field - in the DagWarning model. + Members are the types Airflow emits; importers may add namespaced types such as ``yaml:deprecated_field``. """ ASSET_CONFLICT = "asset conflict" DUPLICATE_DAG_ID = "duplicate dag id" NONEXISTENT_POOL = "non-existent pool" RUNTIME_VARYING_VALUE = "runtime varying value" + + @classmethod + def _missing_(cls, value: object) -> DagWarningType | None: + if ( + not isinstance(value, str) + or len(value) > WARNING_TYPE_MAX_LENGTH + or not re.match(IMPORTER_WARNING_TYPE_PATTERN, value) + ): + return None + # Not registered in _value2member_map_: values come from API queries too, and caching + # them would let arbitrary input grow the enum for the life of the process. + pseudo_member = str.__new__(cls, value) + pseudo_member._name_ = value + pseudo_member._value_ = value + return pseudo_member + + @classmethod + def __get_pydantic_json_schema__( + cls, core_schema: CoreSchema, handler: GetJsonSchemaHandler + ) -> JsonSchemaValue: + return { + "anyOf": [ + handler(core_schema), + { + "type": "string", + "pattern": IMPORTER_WARNING_TYPE_PATTERN, + "maxLength": WARNING_TYPE_MAX_LENGTH, + }, + ] + } diff --git a/airflow-core/src/airflow/models/errors.py b/airflow-core/src/airflow/models/errors.py index b17aa36500ca2..47fc98113236e 100644 --- a/airflow-core/src/airflow/models/errors.py +++ b/airflow-core/src/airflow/models/errors.py @@ -35,7 +35,7 @@ class ParseImportError(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True) timestamp: Mapped[datetime | None] = mapped_column(UtcDateTime, nullable=True) filename: Mapped[str | None] = mapped_column(String(1024), nullable=True) - source_reference: Mapped[str | None] = mapped_column(String(1024), nullable=True) + source_reference: Mapped[str | None] = mapped_column(String(2000), nullable=True) bundle_name: Mapped[str | None] = mapped_column(StringID(), nullable=True) stacktrace: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts index a8e88ba858003..5f0dfc998129b 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -4599,15 +4599,7 @@ export const $DAGWarningResponse = { title: 'Dag Id' }, warning_type: { - anyOf: [ - { - '$ref': '#/components/schemas/DagWarningType' - }, - { - type: 'string' - } - ], - title: 'Warning Type' + '$ref': '#/components/schemas/DagWarningType' }, message: { type: 'string', @@ -5318,13 +5310,21 @@ export const $DagVersionResponse = { } as const; export const $DagWarningType = { - type: 'string', - enum: ['asset conflict', 'duplicate dag id', 'non-existent pool', 'runtime varying value'], - title: 'DagWarningType', - description: `Enum for DAG warning types. + anyOf: [ + { + type: 'string', + enum: ['asset conflict', 'duplicate dag id', 'non-existent pool', 'runtime varying value'], + title: 'DagWarningType', + description: `Enum for DAG warning types. -This is the set of allowable values for the \`\`warning_type\`\` field -in the DagWarning model.` +Members are the types Airflow emits; importers may add namespaced types such as \`\`yaml:deprecated_field\`\`.` + }, + { + type: 'string', + maxLength: 50, + pattern: '^[a-z][a-z0-9_]*:[a-z0-9_.\\-]+$' + } + ] } as const; export const $DetailedHealthStatus = { diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index b35dfb8ea14d8..db33aba8c49c9 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -753,9 +753,9 @@ export type ClearTaskInstancesBody = { * A list of `task_id` or [`task_id`, `map_index`]. If only the `task_id` is provided for a mapped task, all of its map indices will be targeted. */ task_ids?: Array<(string | [ - string, - number - ])> | null; + string, + number +])> | null; /** * Clear every task in this task group. Mutually exclusive with `task_ids`. The group's tasks are resolved on the server from the dag structure, so all of them are targeted regardless of how many there are. */ @@ -1195,7 +1195,7 @@ export type DAGWarningCollectionResponse = { */ export type DAGWarningResponse = { dag_id: string; - warning_type: DagWarningType | string; + warning_type: DagWarningType; message: string; timestamp: string; dag_display_name: string; @@ -1431,13 +1431,7 @@ export type DagVersionResponse = { bundle_url: string | null; }; -/** - * Enum for DAG warning types. - * - * This is the set of allowable values for the ``warning_type`` field - * in the DagWarning model. - */ -export type DagWarningType = 'asset conflict' | 'duplicate dag id' | 'non-existent pool' | 'runtime varying value'; +export type DagWarningType = 'asset conflict' | 'duplicate dag id' | 'non-existent pool' | 'runtime varying value' | string; /** * How much of a component's work has a live instance covering it. @@ -3720,7 +3714,7 @@ export type ListDagWarningsData = { * Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `dag_id, warning_type, message, timestamp` */ orderBy?: Array<(string)>; - warningType?: DagWarningType | string | null; + warningType?: DagWarningType | null; }; export type ListDagWarningsResponse = DAGWarningCollectionResponse; diff --git a/airflow-core/tests/unit/api/common/test_delete_dag.py b/airflow-core/tests/unit/api/common/test_delete_dag.py index eea327bdfcdfd..14f96e3b6fe08 100644 --- a/airflow-core/tests/unit/api/common/test_delete_dag.py +++ b/airflow-core/tests/unit/api/common/test_delete_dag.py @@ -24,8 +24,11 @@ from airflow.api.common.delete_dag import delete_dag from airflow.models import DagModel +from airflow.models.errors import ParseImportError from airflow.providers.standard.operators.empty import EmptyOperator +from tests_common.test_utils.db import clear_db_import_errors + if TYPE_CHECKING: from airflow.serialization.definitions.dag import SerializedDAG @@ -36,6 +39,13 @@ DAG_ID = "dag_to_delete" +@pytest.fixture +def clean_import_errors(): + clear_db_import_errors() + yield + clear_db_import_errors() + + def test_delete_dag_does_not_read_back_deleted_row_keys(dag_maker: DagMaker[SerializedDAG], session): """ delete_dag must not ask the database for the keys of the rows it deletes. @@ -76,3 +86,42 @@ def capture(_conn, _cursor, statement, _parameters, _context, _executemany): ) assert session.scalar(select(func.count()).select_from(DagModel).where(DagModel.dag_id == DAG_ID)) == 0 + + +@pytest.mark.parametrize( + "matching_column", + [ + pytest.param("source_reference", id="by-source-reference"), + pytest.param("filename", id="by-relative-filename"), + ], +) +@pytest.mark.usefixtures("clean_import_errors") +def test_delete_dag_removes_import_errors_for_its_file( + dag_maker: DagMaker[SerializedDAG], session, matching_column +): + with dag_maker(DAG_ID, session=session): + EmptyOperator(task_id="task") + session.commit() + dag_model = session.scalar(select(DagModel).where(DagModel.dag_id == DAG_ID)) + + session.add_all( + [ + ParseImportError( + bundle_name=dag_model.bundle_name, + stacktrace="matching", + **{matching_column: dag_model.relative_fileloc}, + ), + ParseImportError( + bundle_name=dag_model.bundle_name, + filename="unrelated.py", + source_reference="unrelated.py", + stacktrace="unrelated", + ), + ] + ) + session.commit() + + delete_dag(DAG_ID, session=session) + session.commit() + + assert session.scalars(select(ParseImportError.stacktrace)).all() == ["unrelated"] diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_warning.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_warning.py index da5446b7089e7..7a15de834c9c1 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_warning.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_dag_warning.py @@ -111,9 +111,30 @@ def test_should_respond_403(self, unauthorized_test_client): response = unauthorized_test_client.get("/dagWarnings", params={}) assert response.status_code == 403 + def test_get_dag_warnings_bad_request(self, test_client): + response = test_client.get("/dagWarnings", params={"warning_type": "invalid"}) + response_json = response.json() + assert response.status_code == 422 + assert ( + response_json["detail"][0]["msg"] + == "Input should be 'asset conflict', 'duplicate dag id', 'non-existent pool' or 'runtime varying value'" + ) + def test_get_dag_warnings_non_matching_type(self, test_client): - response = test_client.get("/dagWarnings", params={"warning_type": "non-existent-type"}) + response = test_client.get("/dagWarnings", params={"warning_type": "yaml:unused_type"}) response_json = response.json() assert response.status_code == 200 assert response_json["total_entries"] == 0 assert response_json["dag_warnings"] == [] + + def test_get_dag_warnings_importer_defined_type(self, test_client, session): + session.add(DagWarning(DAG1_ID, "yaml:schema_violation", "importer message")) + session.commit() + + response = test_client.get("/dagWarnings", params={"warning_type": "yaml:schema_violation"}) + + assert response.status_code == 200 + response_json = response.json() + assert response_json["total_entries"] == 1 + assert response_json["dag_warnings"][0]["warning_type"] == "yaml:schema_violation" + assert response_json["dag_warnings"][0]["message"] == "importer message" diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py index f8a0f1e12a80f..1ec130b401fee 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py @@ -235,7 +235,7 @@ def test_get_import_error( @mock.patch("airflow.api_fastapi.core_api.routes.public.import_error.get_auth_manager") def test_get_import_error_with_source_reference( - self, mock_get_auth_manager, test_client, session, permitted_dag_model_all + self, mock_get_auth_manager, test_client, session, permitted_dag_model_all, url_safe_serializer ): error = ParseImportError( bundle_name=BUNDLE_NAME, @@ -257,6 +257,9 @@ def test_get_import_error_with_source_reference( "source_reference": "archive.zip/dags/my_dag.py", "stack_trace": STACKTRACE1, "bundle_name": BUNDLE_NAME, + "file_token": url_safe_serializer.dumps( + {"bundle_name": BUNDLE_NAME, "relative_fileloc": FILENAME1} + ), } def test_should_raises_401_unauthenticated(self, unauthenticated_test_client, import_errors): diff --git a/airflow-core/tests/unit/models/test_dagwarning.py b/airflow-core/tests/unit/models/test_dagwarning.py index 0c5a3d3f96932..4738935274b8e 100644 --- a/airflow-core/tests/unit/models/test_dagwarning.py +++ b/airflow-core/tests/unit/models/test_dagwarning.py @@ -25,13 +25,62 @@ from sqlalchemy.exc import OperationalError from airflow.models import DagModel -from airflow.models.dagwarning import DagWarning +from airflow.models.dagwarning import DagWarning, DagWarningType from tests_common.test_utils.db import clear_db_dags -pytestmark = pytest.mark.db_test - +class TestDagWarningType: + def test_known_value_resolves_to_member(self): + assert DagWarningType("non-existent pool") is DagWarningType.NONEXISTENT_POOL + + def test_namespaced_importer_value_is_accepted(self): + warning_type = DagWarningType("yaml:schema_violation") + + assert isinstance(warning_type, DagWarningType) + assert warning_type.value == "yaml:schema_violation" + assert warning_type == "yaml:schema_violation" + + def test_importer_value_is_not_registered_as_member(self): + DagWarningType("yaml:schema_violation") + + assert "yaml:schema_violation" not in DagWarningType._value2member_map_ + assert len(DagWarningType) == 4 + + @pytest.mark.parametrize( + "value", + [ + pytest.param(1, id="non-string"), + pytest.param("yaml schema violation", id="no-namespace"), + pytest.param("non existent pool", id="mistyped-core-type"), + pytest.param("YAML:Schema_Violation", id="uppercase"), + pytest.param(":schema_violation", id="empty-namespace"), + pytest.param("yaml:", id="empty-type"), + pytest.param(f"yaml:{'x' * 46}", id="longer-than-column"), + ], + ) + def test_invalid_value_is_rejected(self, value): + with pytest.raises(ValueError, match="is not a valid DagWarningType"): + DagWarningType(value) + + def test_value_at_column_length_is_accepted(self): + value = f"yaml:{'x' * 45}" + + assert DagWarningType(value).value == value + + @pytest.mark.parametrize( + ("warning_type", "expected"), + [ + pytest.param(DagWarningType.NONEXISTENT_POOL, "non-existent pool", id="enum-member"), + pytest.param("non-existent pool", "non-existent pool", id="known-string"), + pytest.param("yaml:schema_violation", "yaml:schema_violation", id="importer-defined-string"), + ], + ) + def test_dag_warning_stores_type_value(self, warning_type, expected): + assert DagWarning("dag_1", warning_type, "message").warning_type == expected + + +@pytest.mark.db_test class TestDagWarning: def setup_method(self): clear_db_dags() diff --git a/airflow-ctl/src/airflowctl/api/datamodels/generated.py b/airflow-ctl/src/airflowctl/api/datamodels/generated.py index 31c18107a8066..eab99224720a3 100644 --- a/airflow-ctl/src/airflowctl/api/datamodels/generated.py +++ b/airflow-ctl/src/airflowctl/api/datamodels/generated.py @@ -784,12 +784,11 @@ class DagVersionResponse(BaseModel): bundle_url: Annotated[str | None, Field(title="Bundle Url")] -class DagWarningType(str, Enum): +class DagWarningType1(str, Enum): """ Enum for DAG warning types. - This is the set of allowable values for the ``warning_type`` field - in the DagWarning model. + Members are the types Airflow emits; importers may add namespaced types such as ``yaml:deprecated_field``. """ ASSET_CONFLICT = "asset conflict" @@ -798,6 +797,14 @@ class DagWarningType(str, Enum): RUNTIME_VARYING_VALUE = "runtime varying value" +class DagWarningType2(RootModel[str]): + root: Annotated[str, Field(max_length=50, pattern="^[a-z][a-z0-9_]*:[a-z0-9_.\\-]+$")] + + +class DagWarningType(RootModel[DagWarningType1 | DagWarningType2]): + root: DagWarningType1 | DagWarningType2 + + class DetailedHealthStatus(str, Enum): """ How much of a component's work has a live instance covering it. @@ -2159,7 +2166,7 @@ class DAGWarningResponse(BaseModel): """ dag_id: Annotated[str, Field(title="Dag Id")] - warning_type: Annotated[DagWarningType | str, Field(title="Warning Type")] + warning_type: DagWarningType message: Annotated[str, Field(title="Message")] timestamp: Annotated[datetime, Field(title="Timestamp")] dag_display_name: Annotated[str, Field(title="Dag Display Name")] diff --git a/airflow-ctl/tests/airflow_ctl/api/test_operations.py b/airflow-ctl/tests/airflow_ctl/api/test_operations.py index f1eaf2e89efe1..17b0a4c84f69d 100644 --- a/airflow-ctl/tests/airflow_ctl/api/test_operations.py +++ b/airflow-ctl/tests/airflow_ctl/api/test_operations.py @@ -1192,7 +1192,7 @@ class TestDagOperations: dag_warnings=[ DAGWarningResponse( dag_id=dag_id, - warning_type=DagWarningType.NON_EXISTENT_POOL, + warning_type=DagWarningType("non-existent pool"), message="message", timestamp=datetime.datetime(2025, 1, 1, 0, 0, 0), dag_display_name=dag_display_name, From 5dc86b60869ce5a586d9a94a36efed451771682e Mon Sep 17 00:00:00 2001 From: Dilnaz Amanzholova Date: Fri, 25 Sep 2026 10:58:15 +0000 Subject: [PATCH 7/8] Keep DagWarningType enum stable for generated API clients --- .../core_api/datamodels/dag_warning.py | 4 +- .../openapi/v2-rest-api-generated.yaml | 35 ++++++----- .../core_api/routes/public/dag_warning.py | 6 +- airflow-core/src/airflow/models/dagwarning.py | 61 ++++++++----------- .../ui/openapi-gen/requests/schemas.gen.ts | 32 +++++----- .../ui/openapi-gen/requests/types.gen.ts | 12 +++- .../tests/unit/models/test_dagwarning.py | 57 ++++++----------- .../airflowctl/api/datamodels/generated.py | 21 +++---- .../tests/airflow_ctl/api/test_operations.py | 2 +- 9 files changed, 107 insertions(+), 123 deletions(-) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py index 470a0b14fc64f..24dd095c75a06 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dag_warning.py @@ -23,14 +23,14 @@ from pydantic import AliasPath, Field from airflow.api_fastapi.core_api.base import BaseModel -from airflow.models.dagwarning import DagWarningType +from airflow.models.dagwarning import DagWarningTypeValue class DAGWarningResponse(BaseModel): """Dag Warning serializer for responses.""" dag_id: str - warning_type: DagWarningType + warning_type: DagWarningTypeValue message: str timestamp: datetime dag_display_name: str = Field(validation_alias=AliasPath("dag_model", "dag_display_name")) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml index b2017e62ea475..1de3fbae90fdd 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml @@ -3805,6 +3805,9 @@ paths: schema: anyOf: - $ref: '#/components/schemas/DagWarningType' + - type: string + maxLength: 50 + pattern: ^[a-z][a-z0-9_]*:[a-z0-9_.\-]+$ - type: 'null' title: Warning Type - name: limit @@ -14350,7 +14353,12 @@ components: type: string title: Dag Id warning_type: - $ref: '#/components/schemas/DagWarningType' + anyOf: + - $ref: '#/components/schemas/DagWarningType' + - type: string + maxLength: 50 + pattern: ^[a-z][a-z0-9_]*:[a-z0-9_.\-]+$ + title: Warning Type message: type: string title: Message @@ -14886,22 +14894,19 @@ components: title: DagVersionResponse description: Dag Version serializer for responses. DagWarningType: - anyOf: - - type: string - enum: - - asset conflict - - duplicate dag id - - non-existent pool - - runtime varying value - title: DagWarningType - description: 'Enum for DAG warning types. + type: string + enum: + - asset conflict + - duplicate dag id + - non-existent pool + - runtime varying value + title: DagWarningType + description: 'Enum for DAG warning types. + + This is the set of allowable values for the ``warning_type`` field - Members are the types Airflow emits; importers may add namespaced types - such as ``yaml:deprecated_field``.' - - type: string - maxLength: 50 - pattern: ^[a-z][a-z0-9_]*:[a-z0-9_.\-]+$ + in the DagWarning model.' DetailedHealthStatus: type: string enum: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py index c1a652c370f74..0096b82d1ba3e 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/public/dag_warning.py @@ -40,7 +40,7 @@ DAGWarningCollectionResponse, ) from airflow.api_fastapi.core_api.security import ReadableDagWarningsFilterDep, requires_access_dag -from airflow.models.dagwarning import DagWarning, DagWarningType +from airflow.models.dagwarning import DagWarning, DagWarningTypeValue dag_warning_router = AirflowRouter(tags=["DagWarning"]) @@ -52,8 +52,8 @@ def list_dag_warnings( dag_id: Annotated[FilterParam[str | None], Depends(filter_param_factory(DagWarning.dag_id, str | None))], warning_type: Annotated[ - FilterParam[DagWarningType | None], - Depends(filter_param_factory(DagWarning.warning_type, DagWarningType | None)), + FilterParam[DagWarningTypeValue | None], + Depends(filter_param_factory(DagWarning.warning_type, DagWarningTypeValue | None)), ], limit: QueryLimit, offset: QueryOffset, diff --git a/airflow-core/src/airflow/models/dagwarning.py b/airflow-core/src/airflow/models/dagwarning.py index d588013c5e281..11ddbc3eb0733 100644 --- a/airflow-core/src/airflow/models/dagwarning.py +++ b/airflow-core/src/airflow/models/dagwarning.py @@ -17,11 +17,11 @@ # under the License. from __future__ import annotations -import re from datetime import datetime from enum import Enum -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Annotated +from pydantic import StringConstraints, TypeAdapter from sqlalchemy import ForeignKeyConstraint, Index, String, Text, delete, select, true from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -33,13 +33,9 @@ from airflow.utils.sqlalchemy import UtcDateTime, get_dialect_name if TYPE_CHECKING: - from pydantic import GetJsonSchemaHandler - from pydantic.json_schema import JsonSchemaValue - from pydantic_core import CoreSchema from sqlalchemy.orm import Session WARNING_TYPE_MAX_LENGTH = 50 -IMPORTER_WARNING_TYPE_PATTERN = r"^[a-z][a-z0-9_]*:[a-z0-9_.\-]+$" class DagWarning(Base): @@ -72,7 +68,7 @@ class DagWarning(Base): def __init__(self, dag_id: str, warning_type: str, message: str, **kwargs): super().__init__(**kwargs) self.dag_id = dag_id - self.warning_type = DagWarningType(warning_type).value + self.warning_type = get_warning_type_value(warning_type) self.message = message def __eq__(self, other) -> bool: @@ -104,7 +100,8 @@ class DagWarningType(str, Enum): """ Enum for DAG warning types. - Members are the types Airflow emits; importers may add namespaced types such as ``yaml:deprecated_field``. + This is the set of allowable values for the ``warning_type`` field + in the DagWarning model. """ ASSET_CONFLICT = "asset conflict" @@ -112,32 +109,24 @@ class DagWarningType(str, Enum): NONEXISTENT_POOL = "non-existent pool" RUNTIME_VARYING_VALUE = "runtime varying value" - @classmethod - def _missing_(cls, value: object) -> DagWarningType | None: - if ( - not isinstance(value, str) - or len(value) > WARNING_TYPE_MAX_LENGTH - or not re.match(IMPORTER_WARNING_TYPE_PATTERN, value) - ): - return None - # Not registered in _value2member_map_: values come from API queries too, and caching - # them would let arbitrary input grow the enum for the life of the process. - pseudo_member = str.__new__(cls, value) - pseudo_member._name_ = value - pseudo_member._value_ = value - return pseudo_member - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: CoreSchema, handler: GetJsonSchemaHandler - ) -> JsonSchemaValue: - return { - "anyOf": [ - handler(core_schema), - { - "type": "string", - "pattern": IMPORTER_WARNING_TYPE_PATTERN, - "maxLength": WARNING_TYPE_MAX_LENGTH, - }, - ] - } +ImporterWarningType = Annotated[ + str, + StringConstraints(pattern=r"^[a-z][a-z0-9_]*:[a-z0-9_.\-]+$", max_length=WARNING_TYPE_MAX_LENGTH), +] +"""Warning type reported by a Dag importer, prefixed with its namespace (e.g. ``yaml:deprecated_field``).""" + +DagWarningTypeValue = DagWarningType | ImporterWarningType +"""Any valid ``warning_type``: a built-in :class:`DagWarningType` or an :data:`ImporterWarningType`.""" + +_warning_type_adapter: TypeAdapter[DagWarningType | str] = TypeAdapter(DagWarningTypeValue) + + +def get_warning_type_value(warning_type: str) -> str: + """ + Return the value to store for ``warning_type``. + + :raises ValueError: if it is neither a :class:`DagWarningType` nor an :data:`ImporterWarningType`. + """ + validated = _warning_type_adapter.validate_python(warning_type) + return validated.value if isinstance(validated, DagWarningType) else validated diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts index 5f0dfc998129b..67a76df369886 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -4599,7 +4599,17 @@ export const $DAGWarningResponse = { title: 'Dag Id' }, warning_type: { - '$ref': '#/components/schemas/DagWarningType' + anyOf: [ + { + '$ref': '#/components/schemas/DagWarningType' + }, + { + type: 'string', + maxLength: 50, + pattern: '^[a-z][a-z0-9_]*:[a-z0-9_.\\-]+$' + } + ], + title: 'Warning Type' }, message: { type: 'string', @@ -5310,21 +5320,13 @@ export const $DagVersionResponse = { } as const; export const $DagWarningType = { - anyOf: [ - { - type: 'string', - enum: ['asset conflict', 'duplicate dag id', 'non-existent pool', 'runtime varying value'], - title: 'DagWarningType', - description: `Enum for DAG warning types. + type: 'string', + enum: ['asset conflict', 'duplicate dag id', 'non-existent pool', 'runtime varying value'], + title: 'DagWarningType', + description: `Enum for DAG warning types. -Members are the types Airflow emits; importers may add namespaced types such as \`\`yaml:deprecated_field\`\`.` - }, - { - type: 'string', - maxLength: 50, - pattern: '^[a-z][a-z0-9_]*:[a-z0-9_.\\-]+$' - } - ] +This is the set of allowable values for the \`\`warning_type\`\` field +in the DagWarning model.` } as const; export const $DetailedHealthStatus = { diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index db33aba8c49c9..e7482ed74d57b 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1195,7 +1195,7 @@ export type DAGWarningCollectionResponse = { */ export type DAGWarningResponse = { dag_id: string; - warning_type: DagWarningType; + warning_type: DagWarningType | string; message: string; timestamp: string; dag_display_name: string; @@ -1431,7 +1431,13 @@ export type DagVersionResponse = { bundle_url: string | null; }; -export type DagWarningType = 'asset conflict' | 'duplicate dag id' | 'non-existent pool' | 'runtime varying value' | string; +/** + * Enum for DAG warning types. + * + * This is the set of allowable values for the ``warning_type`` field + * in the DagWarning model. + */ +export type DagWarningType = 'asset conflict' | 'duplicate dag id' | 'non-existent pool' | 'runtime varying value'; /** * How much of a component's work has a live instance covering it. @@ -3714,7 +3720,7 @@ export type ListDagWarningsData = { * Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `dag_id, warning_type, message, timestamp` */ orderBy?: Array<(string)>; - warningType?: DagWarningType | null; + warningType?: DagWarningType | string | null; }; export type ListDagWarningsResponse = DAGWarningCollectionResponse; diff --git a/airflow-core/tests/unit/models/test_dagwarning.py b/airflow-core/tests/unit/models/test_dagwarning.py index 4738935274b8e..0f07d7a91924b 100644 --- a/airflow-core/tests/unit/models/test_dagwarning.py +++ b/airflow-core/tests/unit/models/test_dagwarning.py @@ -25,59 +25,42 @@ from sqlalchemy.exc import OperationalError from airflow.models import DagModel -from airflow.models.dagwarning import DagWarning, DagWarningType +from airflow.models.dagwarning import DagWarning, DagWarningType, get_warning_type_value from tests_common.test_utils.db import clear_db_dags -class TestDagWarningType: - def test_known_value_resolves_to_member(self): - assert DagWarningType("non-existent pool") is DagWarningType.NONEXISTENT_POOL - - def test_namespaced_importer_value_is_accepted(self): - warning_type = DagWarningType("yaml:schema_violation") - - assert isinstance(warning_type, DagWarningType) - assert warning_type.value == "yaml:schema_violation" - assert warning_type == "yaml:schema_violation" - - def test_importer_value_is_not_registered_as_member(self): - DagWarningType("yaml:schema_violation") - - assert "yaml:schema_violation" not in DagWarningType._value2member_map_ - assert len(DagWarningType) == 4 +class TestGetWarningTypeValue: + @pytest.mark.parametrize( + ("warning_type", "expected"), + [ + pytest.param(DagWarningType.NONEXISTENT_POOL, "non-existent pool", id="enum-member"), + pytest.param("non-existent pool", "non-existent pool", id="built-in-string"), + pytest.param("yaml:schema_violation", "yaml:schema_violation", id="importer-type"), + pytest.param(f"yaml:{'x' * 45}", f"yaml:{'x' * 45}", id="importer-type-at-column-length"), + ], + ) + def test_valid_type_returns_stored_value(self, warning_type, expected): + assert get_warning_type_value(warning_type) == expected @pytest.mark.parametrize( - "value", + "warning_type", [ pytest.param(1, id="non-string"), pytest.param("yaml schema violation", id="no-namespace"), - pytest.param("non existent pool", id="mistyped-core-type"), + pytest.param("non existent pool", id="mistyped-built-in-type"), pytest.param("YAML:Schema_Violation", id="uppercase"), pytest.param(":schema_violation", id="empty-namespace"), pytest.param("yaml:", id="empty-type"), pytest.param(f"yaml:{'x' * 46}", id="longer-than-column"), ], ) - def test_invalid_value_is_rejected(self, value): - with pytest.raises(ValueError, match="is not a valid DagWarningType"): - DagWarningType(value) - - def test_value_at_column_length_is_accepted(self): - value = f"yaml:{'x' * 45}" - - assert DagWarningType(value).value == value + def test_invalid_type_is_rejected(self, warning_type): + with pytest.raises(ValueError, match="validation error"): + get_warning_type_value(warning_type) - @pytest.mark.parametrize( - ("warning_type", "expected"), - [ - pytest.param(DagWarningType.NONEXISTENT_POOL, "non-existent pool", id="enum-member"), - pytest.param("non-existent pool", "non-existent pool", id="known-string"), - pytest.param("yaml:schema_violation", "yaml:schema_violation", id="importer-defined-string"), - ], - ) - def test_dag_warning_stores_type_value(self, warning_type, expected): - assert DagWarning("dag_1", warning_type, "message").warning_type == expected + def test_dag_warning_stores_importer_type(self): + assert DagWarning("dag_1", "yaml:schema_violation", "message").warning_type == "yaml:schema_violation" @pytest.mark.db_test diff --git a/airflow-ctl/src/airflowctl/api/datamodels/generated.py b/airflow-ctl/src/airflowctl/api/datamodels/generated.py index eab99224720a3..acfb5c2f83492 100644 --- a/airflow-ctl/src/airflowctl/api/datamodels/generated.py +++ b/airflow-ctl/src/airflowctl/api/datamodels/generated.py @@ -501,6 +501,12 @@ class DAGTagCollectionResponse(BaseModel): total_entries: Annotated[int, Field(title="Total Entries")] +class WarningType(RootModel[str]): + root: Annotated[ + str, Field(max_length=50, pattern="^[a-z][a-z0-9_]*:[a-z0-9_.\\-]+$", title="Warning Type") + ] + + class DagBundleDetailResponse(BaseModel): """ Dag bundle serializer for the single-bundle response. @@ -784,11 +790,12 @@ class DagVersionResponse(BaseModel): bundle_url: Annotated[str | None, Field(title="Bundle Url")] -class DagWarningType1(str, Enum): +class DagWarningType(str, Enum): """ Enum for DAG warning types. - Members are the types Airflow emits; importers may add namespaced types such as ``yaml:deprecated_field``. + This is the set of allowable values for the ``warning_type`` field + in the DagWarning model. """ ASSET_CONFLICT = "asset conflict" @@ -797,14 +804,6 @@ class DagWarningType1(str, Enum): RUNTIME_VARYING_VALUE = "runtime varying value" -class DagWarningType2(RootModel[str]): - root: Annotated[str, Field(max_length=50, pattern="^[a-z][a-z0-9_]*:[a-z0-9_.\\-]+$")] - - -class DagWarningType(RootModel[DagWarningType1 | DagWarningType2]): - root: DagWarningType1 | DagWarningType2 - - class DetailedHealthStatus(str, Enum): """ How much of a component's work has a live instance covering it. @@ -2166,7 +2165,7 @@ class DAGWarningResponse(BaseModel): """ dag_id: Annotated[str, Field(title="Dag Id")] - warning_type: DagWarningType + warning_type: Annotated[DagWarningType | WarningType, Field(title="Warning Type")] message: Annotated[str, Field(title="Message")] timestamp: Annotated[datetime, Field(title="Timestamp")] dag_display_name: Annotated[str, Field(title="Dag Display Name")] diff --git a/airflow-ctl/tests/airflow_ctl/api/test_operations.py b/airflow-ctl/tests/airflow_ctl/api/test_operations.py index 17b0a4c84f69d..f1eaf2e89efe1 100644 --- a/airflow-ctl/tests/airflow_ctl/api/test_operations.py +++ b/airflow-ctl/tests/airflow_ctl/api/test_operations.py @@ -1192,7 +1192,7 @@ class TestDagOperations: dag_warnings=[ DAGWarningResponse( dag_id=dag_id, - warning_type=DagWarningType("non-existent pool"), + warning_type=DagWarningType.NON_EXISTENT_POOL, message="message", timestamp=datetime.datetime(2025, 1, 1, 0, 0, 0), dag_display_name=dag_display_name, From 3f7666f734a45e9608b04939e315d2e28437684a Mon Sep 17 00:00:00 2001 From: Dilnaz Amanzholova Date: Sun, 27 Sep 2026 19:52:32 +0000 Subject: [PATCH 8/8] Make import error source_reference required and test list ordering by it --- .../core_api/datamodels/import_error.py | 2 +- .../openapi/v2-rest-api-generated.yaml | 1 + ...0_add_source_reference_to_import_error.py} | 4 +- ...=> 0140_3_4_0_add_language_to_dag_code.py} | 0 airflow-core/src/airflow/models/dagwarning.py | 2 +- .../ui/openapi-gen/requests/schemas.gen.ts | 2 +- .../ui/openapi-gen/requests/types.gen.ts | 2 +- .../routes/public/test_import_error.py | 41 ++++++++++++++++++- .../airflowctl/api/datamodels/generated.py | 2 +- .../tests/airflow_ctl/api/test_operations.py | 1 + 10 files changed, 49 insertions(+), 8 deletions(-) rename airflow-core/src/airflow/migrations/versions/{0138_3_4_0_add_source_reference_to_import_error.py => 0139_3_4_0_add_source_reference_to_import_error.py} (96%) rename airflow-core/src/airflow/migrations/versions/{0139_3_4_0_add_language_to_dag_code.py => 0140_3_4_0_add_language_to_dag_code.py} (100%) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/import_error.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/import_error.py index 1747168b880bd..17c4e2e127bcb 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/import_error.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/import_error.py @@ -31,7 +31,7 @@ class ImportErrorResponse(BaseModel): id: int = Field(alias="import_error_id") timestamp: datetime filename: str - source_reference: str | None = None + source_reference: str | None bundle_name: str | None stacktrace: str = Field(alias="stack_trace") diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml index 1de3fbae90fdd..dabcc47c9b773 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml @@ -15487,6 +15487,7 @@ components: - import_error_id - timestamp - filename + - source_reference - bundle_name - stack_trace - file_token diff --git a/airflow-core/src/airflow/migrations/versions/0138_3_4_0_add_source_reference_to_import_error.py b/airflow-core/src/airflow/migrations/versions/0139_3_4_0_add_source_reference_to_import_error.py similarity index 96% rename from airflow-core/src/airflow/migrations/versions/0138_3_4_0_add_source_reference_to_import_error.py rename to airflow-core/src/airflow/migrations/versions/0139_3_4_0_add_source_reference_to_import_error.py index bebf3be419b61..42613b825d359 100644 --- a/airflow-core/src/airflow/migrations/versions/0138_3_4_0_add_source_reference_to_import_error.py +++ b/airflow-core/src/airflow/migrations/versions/0139_3_4_0_add_source_reference_to_import_error.py @@ -20,7 +20,7 @@ Add source_reference to import_error. Revision ID: ca8499dc1004 -Revises: c9f4b3e7a218 +Revises: a61f0c9d2b47 Create Date: 2026-08-27 12:45:02.276898 """ @@ -32,7 +32,7 @@ # revision identifiers, used by Alembic. revision = "ca8499dc1004" -down_revision = "c9f4b3e7a218" +down_revision = "a61f0c9d2b47" branch_labels = None depends_on = None airflow_version = "3.4.0" diff --git a/airflow-core/src/airflow/migrations/versions/0139_3_4_0_add_language_to_dag_code.py b/airflow-core/src/airflow/migrations/versions/0140_3_4_0_add_language_to_dag_code.py similarity index 100% rename from airflow-core/src/airflow/migrations/versions/0139_3_4_0_add_language_to_dag_code.py rename to airflow-core/src/airflow/migrations/versions/0140_3_4_0_add_language_to_dag_code.py diff --git a/airflow-core/src/airflow/models/dagwarning.py b/airflow-core/src/airflow/models/dagwarning.py index 11ddbc3eb0733..6305361d2aa02 100644 --- a/airflow-core/src/airflow/models/dagwarning.py +++ b/airflow-core/src/airflow/models/dagwarning.py @@ -119,7 +119,7 @@ class DagWarningType(str, Enum): DagWarningTypeValue = DagWarningType | ImporterWarningType """Any valid ``warning_type``: a built-in :class:`DagWarningType` or an :data:`ImporterWarningType`.""" -_warning_type_adapter: TypeAdapter[DagWarningType | str] = TypeAdapter(DagWarningTypeValue) +_warning_type_adapter: TypeAdapter[DagWarningTypeValue] = TypeAdapter(DagWarningTypeValue) def get_warning_type_value(warning_type: str) -> str: diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts index 67a76df369886..b028696f3a87d 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -6185,7 +6185,7 @@ export const $ImportErrorResponse = { } }, type: 'object', - required: ['import_error_id', 'timestamp', 'filename', 'bundle_name', 'stack_trace', 'file_token'], + required: ['import_error_id', 'timestamp', 'filename', 'source_reference', 'bundle_name', 'stack_trace', 'file_token'], title: 'ImportErrorResponse', description: 'Import Error Response.' } as const; diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index e7482ed74d57b..ceb90161cee83 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -1663,7 +1663,7 @@ export type ImportErrorResponse = { import_error_id: number; timestamp: string; filename: string; - source_reference?: string | null; + source_reference: string | null; bundle_name: string | null; stack_trace: string; /** diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py index 1ec130b401fee..02f5878db2ae4 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_import_error.py @@ -21,7 +21,7 @@ from unittest import mock import pytest -from sqlalchemy import delete +from sqlalchemy import delete, update from airflow.api_fastapi.auth.managers.models.resource_details import AccessView, DagDetails from airflow.api_fastapi.core_api.routes.public.import_error import REDACTED_STACKTRACE @@ -677,6 +677,45 @@ def test_get_import_errors__file_registered_only_in_other_bundle_is_unregistered access_view=AccessView.IMPORT_ERRORS_ALL, user=mock.ANY, team_name=None ) + @pytest.mark.parametrize( + ("order_by", "expected_filenames"), + [ + ("source_reference", [FILENAME2, FILENAME3, FILENAME1]), + ("-source_reference", [FILENAME1, FILENAME3, FILENAME2]), + ], + ) + @mock.patch("airflow.api_fastapi.core_api.routes.public.import_error.get_auth_manager") + def test_get_import_errors_source_reference( + self, + mock_get_auth_manager, + test_client, + session, + order_by, + expected_filenames, + permitted_dag_model_all, + ): + source_references = { + FILENAME1: f"c.zip/{FILENAME1}", + FILENAME2: f"a.zip/{FILENAME2}", + FILENAME3: f"b.zip/{FILENAME3}", + } + for filename, source_reference in source_references.items(): + session.execute( + update(ParseImportError) + .where(ParseImportError.filename == filename) + .values(source_reference=source_reference) + ) + session.commit() + set_mock_auth_manager__get_authorized_dag_ids(mock_get_auth_manager, permitted_dag_model_all) + set_mock_auth_manager__batch_is_authorized_dag(mock_get_auth_manager, True) + + response = test_client.get("/importErrors", params={"order_by": order_by}) + + assert response.status_code == 200 + assert [(e["filename"], e["source_reference"]) for e in response.json()["import_errors"]] == [ + (filename, source_references[filename]) for filename in expected_filenames + ] + def test_should_raises_401_unauthenticated(self, unauthenticated_test_client): response = unauthenticated_test_client.get("/importErrors") assert response.status_code == 401 diff --git a/airflow-ctl/src/airflowctl/api/datamodels/generated.py b/airflow-ctl/src/airflowctl/api/datamodels/generated.py index acfb5c2f83492..924d3420928f3 100644 --- a/airflow-ctl/src/airflowctl/api/datamodels/generated.py +++ b/airflow-ctl/src/airflowctl/api/datamodels/generated.py @@ -932,7 +932,7 @@ class ImportErrorResponse(BaseModel): import_error_id: Annotated[int, Field(title="Import Error Id")] timestamp: Annotated[datetime, Field(title="Timestamp")] filename: Annotated[str, Field(title="Filename")] - source_reference: Annotated[str | None, Field(title="Source Reference")] = None + source_reference: Annotated[str | None, Field(title="Source Reference")] bundle_name: Annotated[str | None, Field(title="Bundle Name")] stack_trace: Annotated[str, Field(title="Stack Trace")] file_token: Annotated[ diff --git a/airflow-ctl/tests/airflow_ctl/api/test_operations.py b/airflow-ctl/tests/airflow_ctl/api/test_operations.py index f1eaf2e89efe1..24ed44e262090 100644 --- a/airflow-ctl/tests/airflow_ctl/api/test_operations.py +++ b/airflow-ctl/tests/airflow_ctl/api/test_operations.py @@ -1151,6 +1151,7 @@ class TestDagOperations: import_error_id=0, timestamp=datetime.datetime(2025, 1, 1, 0, 0, 0), filename="filename", + source_reference=None, bundle_name="bundle_name", stack_trace="stack_trace", file_token="file_token",