Skip to content

Remove fab from preinstalled providers #48457

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
potiuk merged 1 commit into from
Apr 7, 2025
Merged

Remove fab from preinstalled providers #48457

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 2 additions & 11 deletions .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1186,17 +1186,8 @@ repos:
\bab_register_user\b
pass_filenames: true
files: ^airflow-core/src/airflow/migrations/versions/.*\.py$
# These migrations contain FAB related changes but existed before moving FAB auth manager
# to its own provider
exclude: >
Comment thread
vincbeck marked this conversation as resolved.
(?ix)^(
airflow-core/src/airflow/migrations/versions/00.*\.py|
airflow-core/src/airflow/migrations/versions/0106.*\.py|
airflow-core/src/airflow/migrations/versions/0118.*\.py|
airflow-core/src/airflow/migrations/versions/0119.*\.py|
airflow-core/src/airflow/migrations/versions/0121.*\.py|
airflow-core/src/airflow/migrations/versions/0124.*\.py
)$
exclude:
airflow-core/src/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py
## ADD MOST PRE-COMMITS ABOVE THAT LINE
# The below pre-commits are those requiring CI image to be built
- id: mypy-dev
Expand Down
2 changes: 1 addition & 1 deletion airflow-core/docs/img/airflow_erd.sha256
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
c0f6d179e576a7c14406900cc3eaf8995246b5e7e781f1de0c0c794e4b10c6da
aa423e4d2c250ceca76fd28b7fd788fc476f7e235326c187b6804916853e6ebf
2 changes: 1 addition & 1 deletion airflow-core/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,12 +136,12 @@ dependencies = [
# Werkzug 3 breaks Flask-Login 0.6.2
# we should remove this limitation when FAB supports Flask 2.3
"werkzeug>=2.0,<4",
"wtforms>=3.0,<4",
# pre-installed providers
"apache-airflow-providers-common-compat>=1.6.0",
"apache-airflow-providers-common-io>=1.5.2",
"apache-airflow-providers-common-sql>=1.24.1",
"apache-airflow-providers-openlineage>=2.1.2",
"apache-airflow-providers-fab>=2.0.0",
"apache-airflow-providers-smtp>=2.0.1",
"apache-airflow-providers-standard>=0.2.0",
"apache-airflow-providers-sqlite>=4.0.1",
Expand Down
13 changes: 12 additions & 1 deletion airflow-core/src/airflow/cli/commands/scheduler_command.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
from airflow.executors.executor_loader import ExecutorLoader
from airflow.jobs.job import Job, run_job
from airflow.jobs.scheduler_job_runner import SchedulerJobRunner
from airflow.providers.celery.version_compat import AIRFLOW_V_3_0_PLUS
from airflow.utils import cli as cli_utils
from airflow.utils.providers_configuration_loader import providers_configuration_loaded
from airflow.utils.scheduler_health import serve_health_check
Expand Down Expand Up @@ -60,7 +61,17 @@ def scheduler(args: Namespace):
@contextmanager
def _serve_logs(skip_serve_logs: bool = False):
"""Start serve_logs sub-process."""
from airflow.utils.serve_logs import serve_logs
if AIRFLOW_V_3_0_PLUS:
try:
from airflow.providers.fab.www.serve_logs import serve_logs
except ImportError:
raise ImportError(
"Celery requires FAB provider to be installed in order to run this command. "
"Please install the FAB provider by running: "
"pip install apache-airflow-providers-celery[fab]"
)
else:
from airflow.utils.serve_logs import serve_logs # type: ignore[no-redef]

sub_proc = None
executor_class, _ = ExecutorLoader.import_default_executor_cls()
Expand Down
14 changes: 13 additions & 1 deletion airflow-core/src/airflow/cli/commands/triggerer_command.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,26 @@
from airflow.configuration import conf
from airflow.jobs.job import Job, run_job
from airflow.jobs.triggerer_job_runner import TriggererJobRunner
from airflow.providers.celery.version_compat import AIRFLOW_V_3_0_PLUS
from airflow.utils import cli as cli_utils
from airflow.utils.providers_configuration_loader import providers_configuration_loaded
from airflow.utils.serve_logs import serve_logs


@contextmanager
def _serve_logs(skip_serve_logs: bool = False) -> Generator[None, None, None]:
"""Start serve_logs sub-process."""
if AIRFLOW_V_3_0_PLUS:
try:
from airflow.providers.fab.www.serve_logs import serve_logs
except ImportError:
raise ImportError(
"Celery requires FAB provider to be installed in order to run this command. "
"Please install the FAB provider by running: "
"pip install apache-airflow-providers-celery[fab]"
)
else:
from airflow.utils.serve_logs import serve_logs # type: ignore[no-redef]

sub_proc = None
if skip_serve_logs is False:
port = conf.getint("logging", "trigger_log_server_port", fallback=8794)
Expand Down
50 changes: 38 additions & 12 deletions airflow-core/src/airflow/migrations/versions/0028_3_0_0_drop_ab_user_id_foreign_key.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
from __future__ import annotations

from alembic import op
from sqlalchemy import inspect

# revision identifiers, used by Alembic.
revision = "044f740568ec"
Expand All @@ -37,29 +38,54 @@
airflow_version = "3.0.0"


def table_exists(table_name):
"""Check if a table exists in the database."""
inspector = inspect(op.get_bind())
return table_name in inspector.get_table_names()


def constraint_exists(table_name, constraint_name):
"""Check if a foreign key constraint exists on a table."""
inspector = inspect(op.get_bind())
foreign_keys = inspector.get_foreign_keys(table_name)
return any(fk["name"] == constraint_name for fk in foreign_keys)


def index_exists(table_name, index_name):
"""Check if an index exists on a table."""
inspector = inspect(op.get_bind())
indexes = inspector.get_indexes(table_name)
return any(idx["name"] == index_name for idx in indexes)


def upgrade():
"""Apply Drop ab_user.id foreign key."""
with op.batch_alter_table("dag_run_note", schema=None) as batch_op:
batch_op.drop_constraint("dag_run_note_user_fkey", type_="foreignkey")
if constraint_exists("dag_run_note", "dag_run_note_user_fkey"):
with op.batch_alter_table("dag_run_note", schema=None) as batch_op:
batch_op.drop_constraint("dag_run_note_user_fkey", type_="foreignkey")

with op.batch_alter_table("task_instance_note", schema=None) as batch_op:
batch_op.drop_constraint("task_instance_note_user_fkey", type_="foreignkey")
if constraint_exists("task_instance_note", "task_instance_note_user_fkey"):
with op.batch_alter_table("task_instance_note", schema=None) as batch_op:
batch_op.drop_constraint("task_instance_note_user_fkey", type_="foreignkey")

if op.get_bind().dialect.name == "mysql":
with op.batch_alter_table("dag_run_note", schema=None) as batch_op:
batch_op.drop_index("dag_run_note_user_fkey")
if index_exists("dag_run_note", "dag_run_note_user_fkey"):
with op.batch_alter_table("dag_run_note", schema=None) as batch_op:
batch_op.drop_index("dag_run_note_user_fkey")

with op.batch_alter_table("task_instance_note", schema=None) as batch_op:
batch_op.drop_index("task_instance_note_user_fkey")
if index_exists("task_instance_note", "task_instance_note_user_fkey"):
with op.batch_alter_table("task_instance_note", schema=None) as batch_op:
batch_op.drop_index("task_instance_note_user_fkey")


def downgrade():
"""Unapply Drop ab_user.id foreign key."""
with op.batch_alter_table("task_instance_note", schema=None) as batch_op:
batch_op.create_foreign_key("task_instance_note_user_fkey", "ab_user", ["user_id"], ["id"])
if table_exists("ab_user"):
with op.batch_alter_table("task_instance_note", schema=None) as batch_op:
batch_op.create_foreign_key("task_instance_note_user_fkey", "ab_user", ["user_id"], ["id"])

with op.batch_alter_table("dag_run_note", schema=None) as batch_op:
batch_op.create_foreign_key("dag_run_note_user_fkey", "ab_user", ["user_id"], ["id"])
with op.batch_alter_table("dag_run_note", schema=None) as batch_op:
batch_op.create_foreign_key("dag_run_note_user_fkey", "ab_user", ["user_id"], ["id"])

if op.get_bind().dialect.name == "mysql":
with op.batch_alter_table("task_instance_note", schema=None) as batch_op:
Expand Down
2 changes: 1 addition & 1 deletion airflow-core/tests/unit/cli/commands/test_scheduler_command.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
from airflow.cli import cli_parser
from airflow.cli.commands import scheduler_command
from airflow.executors import executor_loader
from airflow.providers.fab.www.serve_logs import serve_logs
from airflow.utils.scheduler_health import serve_health_check
from airflow.utils.serve_logs import serve_logs

from tests_common.test_utils.config import conf_vars

Expand Down
1 change: 0 additions & 1 deletion dev/breeze/src/airflow_breeze/templates/pyproject_TEMPLATE.toml.jinja2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
2 changes: 1 addition & 1 deletion dev/breeze/tests/test_selective_checks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2157,7 +2157,7 @@ def test_upgrade_to_newer_dependencies(
),
pytest.param(
("providers/celery/src/airflow/providers/celery/file.py",),
{"docs-list-as-string": "celery cncf.kubernetes"},
{"docs-list-as-string": "celery cncf.kubernetes fab"},
id="Celery python files changed",
),
pytest.param(
Expand Down
4 changes: 2 additions & 2 deletions devel-common/src/tests_common/test_utils/db.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,8 +86,6 @@ def _bootstrap_dagbag():


def initial_db_init():
from flask import Flask

from airflow.configuration import conf
from airflow.utils import db

Expand All @@ -98,6 +96,8 @@ def initial_db_init():
db.downgrade(to_revision="5f2621c13b39")
db.upgradedb(to_revision="head")
else:
from flask import Flask

from airflow.www.extensions.init_appbuilder import init_appbuilder
from airflow.www.extensions.init_auth_manager import get_auth_manager

Expand Down
1 change: 0 additions & 1 deletion providers/airbyte/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/alibaba/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/amazon/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
2 changes: 2 additions & 0 deletions providers/amazon/tests/unit/amazon/aws/auth_manager/router/test_login.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from airflow.api_fastapi.app import AUTH_MANAGER_FASTAPI_APP_PREFIX, create_app

from tests_common.test_utils.config import conf_vars
from tests_common.test_utils.mock_plugins import mock_plugin_manager

SAML_METADATA_URL = "/saml/metadata"
SAML_METADATA_PARSED = {
Expand Down Expand Up @@ -73,6 +74,7 @@ def test_client():
yield TestClient(create_app())


@mock_plugin_manager(plugins=[])
class TestLoginRouter:
def test_login(self, test_client):
response = test_client.get(AUTH_MANAGER_FASTAPI_APP_PREFIX + "/login", follow_redirects=False)
Expand Down
1 change: 0 additions & 1 deletion providers/apache/beam/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/cassandra/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/drill/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/druid/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/flink/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/hdfs/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/hive/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/iceberg/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/impala/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/kafka/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/kylin/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/livy/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/pig/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/pinot/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apache/spark/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/apprise/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/arangodb/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
1 change: 0 additions & 1 deletion providers/asana/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ apache-airflow = {workspace = true}
apache-airflow-devel-common = {workspace = true}
apache-airflow-task-sdk = {workspace = true}
apache-airflow-providers-common-sql = {workspace = true}
apache-airflow-providers-fab = {workspace = true}
apache-airflow-providers-standard = {workspace = true}

[project.urls]
Expand Down
Loading