diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml index cb053ac7e7757..f55ce81d94d1b 100644 --- a/.github/actions/install-pre-commit/action.yml +++ b/.github/actions/install-pre-commit/action.yml @@ -24,7 +24,7 @@ inputs: default: "3.9" uv-version: description: 'uv version to use' - default: "0.5.24" # Keep this comment to allow automatic replacement of uv version + default: "0.6.14" # Keep this comment to allow automatic replacement of uv version pre-commit-version: description: 'pre-commit version to use' default: "3.5.0" # Keep this comment to allow automatic replacement of pre-commit version diff --git a/Dockerfile b/Dockerfile index de4a18d6a9453..0e37d1eb5bc1b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,9 +53,9 @@ ARG PYTHON_BASE_IMAGE="python:3.8-slim-bookworm" # You can swap comments between those two args to test pip from the main version # When you attempt to test if the version of `pip` from specified branch works for our builds # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` -ARG AIRFLOW_PIP_VERSION=25.0 +ARG AIRFLOW_PIP_VERSION=25.0.1 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_UV_VERSION=0.5.24 +ARG AIRFLOW_UV_VERSION=0.6.14 ARG AIRFLOW_USE_UV="false" ARG UV_HTTP_TIMEOUT="300" ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" diff --git a/Dockerfile.ci b/Dockerfile.ci index dc3276f948de3..6732f6c024d47 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -911,7 +911,7 @@ function check_boto_upgrade() { set -x # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \ - "oss2>=2.14.0" "cryptography<43.0.0" "requests!=2.32.*,<3.0.0,>=2.24.0" + "oss2>=2.14.0" "cryptography<43.0.0" "requests!=2.32.0,!=2.32.1,!=2.32.2,<3.0.0,>=2.24.0" set +x pip check } @@ -1249,9 +1249,9 @@ COPY --from=scripts common.sh install_packaging_tools.sh install_additional_depe # You can swap comments between those two args to test pip from the main version # When you attempt to test if the version of `pip` from specified branch works for our builds # Also use `force pip` label on your PR to swap all places we use `uv` to `pip` -ARG AIRFLOW_PIP_VERSION=25.0 +ARG AIRFLOW_PIP_VERSION=25.0.1 # ARG AIRFLOW_PIP_VERSION="git+https://github.com/pypa/pip.git@main" -ARG AIRFLOW_UV_VERSION=0.5.24 +ARG AIRFLOW_UV_VERSION=0.6.14 # TODO(potiuk): automate with upgrade check (possibly) ARG AIRFLOW_PRE_COMMIT_VERSION="3.5.0" diff --git a/airflow/providers/alibaba/provider.yaml b/airflow/providers/alibaba/provider.yaml index a12c766d182b1..463cfa4d971a7 100644 --- a/airflow/providers/alibaba/provider.yaml +++ b/airflow/providers/alibaba/provider.yaml @@ -53,6 +53,7 @@ dependencies: - oss2>=2.14.0 - alibabacloud_adb20211201>=1.0.0 - alibabacloud_tea_openapi>=0.3.7 + - apscheduler<3.11.0; python_version >= "3.8" integrations: - integration-name: Alibaba Cloud OSS diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index ff7ebedc72ae8..216ca0424a9e3 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -442,8 +442,8 @@ can be used for CI images: | `DEV_APT_DEPS` | | Dev APT dependencies installed in the first part of the image (default empty means default dependencies are used) | | `ADDITIONAL_DEV_APT_DEPS` | | Additional apt dev dependencies installed in the first part of the image | | `ADDITIONAL_DEV_APT_ENV` | | Additional env variables defined when installing dev deps | -| `AIRFLOW_PIP_VERSION` | `25.0` | `pip` version used. | -| `AIRFLOW_UV_VERSION` | `0.5.24` | `uv` version used. | +| `AIRFLOW_PIP_VERSION` | `25.0.1` | `pip` version used. | +| `AIRFLOW_UV_VERSION` | `0.6.14` | `uv` version used. | | `AIRFLOW_PRE_COMMIT_VERSION` | `3.5.0` | `pre-commit` version used. | | `AIRFLOW_USE_UV` | `true` | Whether to use UV for installation. | | `PIP_PROGRESS_BAR` | `on` | Progress bar for PIP installation | diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index 08b135479de56..b553c12816814 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -302,8 +302,8 @@ │(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ │[default: 3.8]                                              │ │--integrationCore Integrations to enable when running (can be more than one).                    │ -│(all | all-testable | cassandra | celery | drill | kafka | kerberos | keycloak |    │ -│mongo | mssql | openlineage | otel | pinot | qdrant | redis | statsd | trino | ydb) │ +│(all | all-testable | cassandra | drill | kafka | kerberos | keycloak | mongo |     │ +│mssql | openlineage | otel | pinot | qdrant | redis | statsd | trino | ydb)         │ │--standalone-dag-processorRun standalone dag processor for start-airflow.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Docker Compose selection and cleanup ───────────────────────────────────────────────────────────────────────────────╮ diff --git a/dev/breeze/doc/images/output_shell.svg b/dev/breeze/doc/images/output_shell.svg index 10215706d52e4..cc3f8b70a0444 100644 --- a/dev/breeze/doc/images/output_shell.svg +++ b/dev/breeze/doc/images/output_shell.svg @@ -1,4 +1,4 @@ - + │(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ │[default: 3.8]                                              │ │--integrationCore Integrations to enable when running (can be more than one).               │ -│(all | all-testable | cassandra | celery | drill | kafka | kerberos | keycloak │ -│| mongo | mssql | openlineage | otel | pinot | qdrant | redis | statsd | trino │ -│| ydb)                                                                         │ -│--standalone-dag-processorRun standalone dag processor for start-airflow.│ -│--start-webserver-with-examplesStart minimal airflow webserver with examples (for testing purposes) when      │ -│entering breeze.                                                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Docker Compose project management ──────────────────────────────────────────────────────────────────────────────────╮ -│--project-nameName of the docker-compose project to bring down. The `docker-compose` is for legacy  │ -│breeze project name and you can use `breeze down --project-name docker-compose` to    │ -│stop all containers belonging to it.                                                  │ -│(breeze | pre-commit | docker-compose)                                                │ -│[default: breeze]                                                                     │ -│--restart,--remove-orphansRestart all containers before entering shell (also removes orphan containers).│ -│--docker-hostOptional - docker host to use when running docker commands. When set, the `--builder` │ -│option is ignored when building images.                                               │ -│(TEXT)                                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Scripts execution ──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--quietSuppress initialization output when starting.│ -│--skip-image-upgrade-checkSkip checking if the CI image is up to date.│ -│--warn-image-upgrade-neededWarn when image upgrade is needed even if --skip-upgrade-check is used.│ -│--skip-environment-initializationSkip running breeze entrypoint initialization - no user output, no db checks.│ -│--ttyWhether to allocate pseudo-tty when running docker command (useful for         │ -│pre-commit and CI to force-enable it).                                         │ -│(auto | enabled | disabled)                                                    │ -│[default: auto]                                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Database ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--backend-bDatabase backend to use. If 'none' is chosen, Breeze will start with an       │ -│invalid database configuration, meaning there will be no database available,  │ -│and any attempts to connect to the Airflow database will fail.                │ -│(>sqlite< | mysql | postgres | none)                                          │ -│[default: sqlite]                                                             │ -│--postgres-version-PVersion of Postgres used.(>13< | 14 | 15 | 16 | 17)[default: 13]│ -│--mysql-version-MVersion of MySQL used.(>8.0< | 8.4)[default: 8.0]│ -│--db-reset-d/--no-db-resetReset DB when entering the container.[default: no-db-reset]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Choose executor ────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--executorSpecify the executor to use with shell command.                                                  │ -│(LocalExecutor|KubernetesExecutor|CeleryExecutor|CeleryKubernetesExecutor|EdgeExecutor|Sequential│ -│Executor)                                                                                        │ -│[default: LocalExecutor]                                                                         │ -│--celery-brokerSpecify the celery message broker(rabbitmq|redis)[default: redis]│ -│--celery-flowerStart celery flower│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Build CI image (before entering shell) ─────────────────────────────────────────────────────────────────────────────╮ -│--force-buildForce image build no matter if it is determined as needed.│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│[default: autodetect]                                         │ -│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv]│ -│--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).(INTEGER RANGE)│ -│[default: 300; x>=1]                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Mounting the sources and volumes ───────────────────────────────────────────────────────────────────────────────────╮ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =       │ -│selected).                                                                                 │ -│(selected | all | skip | remove | tests | providers-and-tests)                             │ -│[default: selected]                                                                        │ -│--include-mypy-volumeWhether to include mounting of the mypy volume (useful for debugging mypy).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Installing packages after entering shell ───────────────────────────────────────────────────────────────────────────╮ -│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or   │ -│local context file).                                    │ -│(TEXT)                                                  │ -│--airflow-constraints-modeMode of constraints for Airflow for CI image building.  │ -│(constraints-source-providers | constraints |           │ -│constraints-no-providers)                               │ -│[default: constraints-source-providers]                 │ -│--airflow-constraints-referenceConstraint reference to use for airflow installation    │ -│(used in calculated constraints URL).                   │ -│(TEXT)                                                  │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is │ -│used                                                    │ -│(TEXT)                                                  │ -│[default: ""]                                           │ -│--airflow-skip-constraintsDo not use constraints when installing airflow.│ -│--clean-airflow-installationClean the airflow installation before installing version│ -│specified by --use-airflow-version.                     │ -│--excluded-providersJSON-string of dictionary containing excluded providers │ -│per python version ({'3.12': ['provider']})             │ -│(TEXT)                                                  │ -│--force-lowest-dependenciesRun tests for the lowest direct dependencies of Airflow │ -│or selected provider if `Provider[PROVIDER_ID]` is used │ -│as test type.                                           │ -│--install-airflow-with-constraints/--no-install-airflow…Install airflow in a separate step, with constraints    │ -│determined from package or airflow version.             │ -│[default: install-airflow-with-constraints]             │ -│--install-selected-providersComma-separated list of providers selected to be        │ -│installed (implies --use-packages-from-dist).           │ -│(TEXT)                                                  │ -│--package-formatFormat of packages that should be installed from dist.│ -│(wheel | sdist)                                       │ -│[default: wheel]                                      │ -│--providers-constraints-locationLocation of providers constraints to use (remote URL or │ -│local context file).                                    │ -│(TEXT)                                                  │ -│--providers-constraints-modeMode of constraints for Providers for CI image building.│ -│(constraints-source-providers | constraints |           │ -│constraints-no-providers)                               │ -│[default: constraints-source-providers]                 │ -│--providers-constraints-referenceConstraint reference to use for providers installation  │ -│(used in calculated constraints URL). Can be 'default'  │ -│in which case the default constraints-reference is used.│ -│(TEXT)                                                  │ -│--providers-skip-constraintsDo not use constraints when installing providers.│ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It  │ -│can also be version (to install from PyPI), `none`,     │ -│`wheel`, or `sdist` to install from `dist` folder, or   │ -│VCS URL to install from                                 │ -│(https://pip.pypa.io/en/stable/topics/vcs-support/).    │ -│Implies --mount-sources `remove`.                       │ -│(none | wheel | sdist | <airflow_version>)              │ -│--use-packages-from-distInstall all found packages (--package-format determines │ -│type) from 'dist' folder when entering breeze.          │ -│--install-airflow-python-clientInstall airflow python client packages (--package-format│ -│determines type) from 'dist' folder when entering       │ -│breeze.                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Upgrading/downgrading/removing selected packages ───────────────────────────────────────────────────────────────────╮ -│--upgrade-botoRemove aiobotocore and upgrade botocore and boto to the latest version.│ -│--downgrade-sqlalchemyDowngrade SQLAlchemy to minimum supported version.│ -│--downgrade-pendulumDowngrade Pendulum to minimum supported version.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ DB test flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-db-tests-onlyOnly runs tests that require a database│ -│--skip-db-testsSkip tests that require a database│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Other options ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--forward-credentials-fForward local credentials to container when running.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--verbose-commandsShow details of commands executed.│ -│--keep-env-variablesDo not clear environment variables that might have side effect while running tests│ -│--no-db-cleanupDo not clear the database before each test module│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│(all | all-testable | cassandra | drill | kafka | kerberos | keycloak | mongo |│ +│mssql | openlineage | otel | pinot | qdrant | redis | statsd | trino | ydb)    │ +│--standalone-dag-processorRun standalone dag processor for start-airflow.│ +│--start-webserver-with-examplesStart minimal airflow webserver with examples (for testing purposes) when      │ +│entering breeze.                                                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Docker Compose project management ──────────────────────────────────────────────────────────────────────────────────╮ +│--project-nameName of the docker-compose project to bring down. The `docker-compose` is for legacy  │ +│breeze project name and you can use `breeze down --project-name docker-compose` to    │ +│stop all containers belonging to it.                                                  │ +│(breeze | pre-commit | docker-compose)                                                │ +│[default: breeze]                                                                     │ +│--restart,--remove-orphansRestart all containers before entering shell (also removes orphan containers).│ +│--docker-hostOptional - docker host to use when running docker commands. When set, the `--builder` │ +│option is ignored when building images.                                               │ +│(TEXT)                                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Scripts execution ──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--quietSuppress initialization output when starting.│ +│--skip-image-upgrade-checkSkip checking if the CI image is up to date.│ +│--warn-image-upgrade-neededWarn when image upgrade is needed even if --skip-upgrade-check is used.│ +│--skip-environment-initializationSkip running breeze entrypoint initialization - no user output, no db checks.│ +│--ttyWhether to allocate pseudo-tty when running docker command (useful for         │ +│pre-commit and CI to force-enable it).                                         │ +│(auto | enabled | disabled)                                                    │ +│[default: auto]                                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Database ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--backend-bDatabase backend to use. If 'none' is chosen, Breeze will start with an       │ +│invalid database configuration, meaning there will be no database available,  │ +│and any attempts to connect to the Airflow database will fail.                │ +│(>sqlite< | mysql | postgres | none)                                          │ +│[default: sqlite]                                                             │ +│--postgres-version-PVersion of Postgres used.(>13< | 14 | 15 | 16 | 17)[default: 13]│ +│--mysql-version-MVersion of MySQL used.(>8.0< | 8.4)[default: 8.0]│ +│--db-reset-d/--no-db-resetReset DB when entering the container.[default: no-db-reset]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Choose executor ────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--executorSpecify the executor to use with shell command.                                                  │ +│(LocalExecutor|KubernetesExecutor|CeleryExecutor|CeleryKubernetesExecutor|EdgeExecutor|Sequential│ +│Executor)                                                                                        │ +│[default: LocalExecutor]                                                                         │ +│--celery-brokerSpecify the celery message broker(rabbitmq|redis)[default: redis]│ +│--celery-flowerStart celery flower│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Build CI image (before entering shell) ─────────────────────────────────────────────────────────────────────────────╮ +│--force-buildForce image build no matter if it is determined as needed.│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│[default: autodetect]                                         │ +│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv]│ +│--uv-http-timeoutTimeout for requests that UV makes (only used in case of UV builds).(INTEGER RANGE)│ +│[default: 300; x>=1]                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Mounting the sources and volumes ───────────────────────────────────────────────────────────────────────────────────╮ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default =       │ +│selected).                                                                                 │ +│(selected | all | skip | remove | tests | providers-and-tests)                             │ +│[default: selected]                                                                        │ +│--include-mypy-volumeWhether to include mounting of the mypy volume (useful for debugging mypy).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Installing packages after entering shell ───────────────────────────────────────────────────────────────────────────╮ +│--airflow-constraints-locationLocation of airflow constraints to use (remote URL or   │ +│local context file).                                    │ +│(TEXT)                                                  │ +│--airflow-constraints-modeMode of constraints for Airflow for CI image building.  │ +│(constraints-source-providers | constraints |           │ +│constraints-no-providers)                               │ +│[default: constraints-source-providers]                 │ +│--airflow-constraints-referenceConstraint reference to use for airflow installation    │ +│(used in calculated constraints URL).                   │ +│(TEXT)                                                  │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is │ +│used                                                    │ +│(TEXT)                                                  │ +│[default: ""]                                           │ +│--airflow-skip-constraintsDo not use constraints when installing airflow.│ +│--clean-airflow-installationClean the airflow installation before installing version│ +│specified by --use-airflow-version.                     │ +│--excluded-providersJSON-string of dictionary containing excluded providers │ +│per python version ({'3.12': ['provider']})             │ +│(TEXT)                                                  │ +│--force-lowest-dependenciesRun tests for the lowest direct dependencies of Airflow │ +│or selected provider if `Provider[PROVIDER_ID]` is used │ +│as test type.                                           │ +│--install-airflow-with-constraints/--no-install-airflow-Install airflow in a separate step, with constraints    │ +│with-constraintsdetermined from package or airflow version.             │ +│[default: install-airflow-with-constraints]             │ +│--install-selected-providersComma-separated list of providers selected to be        │ +│installed (implies --use-packages-from-dist).           │ +│(TEXT)                                                  │ +│--package-formatFormat of packages that should be installed from dist.│ +│(wheel | sdist)                                       │ +│[default: wheel]                                      │ +│--providers-constraints-locationLocation of providers constraints to use (remote URL or │ +│local context file).                                    │ +│(TEXT)                                                  │ +│--providers-constraints-modeMode of constraints for Providers for CI image building.│ +│(constraints-source-providers | constraints |           │ +│constraints-no-providers)                               │ +│[default: constraints-source-providers]                 │ +│--providers-constraints-referenceConstraint reference to use for providers installation  │ +│(used in calculated constraints URL). Can be 'default'  │ +│in which case the default constraints-reference is used.│ +│(TEXT)                                                  │ +│--providers-skip-constraintsDo not use constraints when installing providers.│ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It  │ +│can also be version (to install from PyPI), `none`,     │ +│`wheel`, or `sdist` to install from `dist` folder, or   │ +│VCS URL to install from                                 │ +│(https://pip.pypa.io/en/stable/topics/vcs-support/).    │ +│Implies --mount-sources `remove`.                       │ +│(none | wheel | sdist | <airflow_version>)              │ +│--use-packages-from-distInstall all found packages (--package-format determines │ +│type) from 'dist' folder when entering breeze.          │ +│--install-airflow-python-clientInstall airflow python client packages (--package-format│ +│determines type) from 'dist' folder when entering       │ +│breeze.                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Upgrading/downgrading/removing selected packages ───────────────────────────────────────────────────────────────────╮ +│--upgrade-botoRemove aiobotocore and upgrade botocore and boto to the latest version.│ +│--downgrade-sqlalchemyDowngrade SQLAlchemy to minimum supported version.│ +│--downgrade-pendulumDowngrade Pendulum to minimum supported version.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ DB test flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-db-tests-onlyOnly runs tests that require a database│ +│--skip-db-testsSkip tests that require a database│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Other options ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--forward-credentials-fForward local credentials to container when running.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--verbose-commandsShow details of commands executed.│ +│--keep-env-variablesDo not clear environment variables that might have side effect while running tests│ +│--no-db-cleanupDo not clear the database before each test module│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_shell.txt b/dev/breeze/doc/images/output_shell.txt index 3bc716efe2639..5135bca538a2f 100644 --- a/dev/breeze/doc/images/output_shell.txt +++ b/dev/breeze/doc/images/output_shell.txt @@ -1 +1 @@ -eb424643dff1e99d1b4dc19d53e707eb +67e33734dbc8d61a54ad08dc14d94bcd diff --git a/dev/breeze/doc/images/output_start-airflow.svg b/dev/breeze/doc/images/output_start-airflow.svg index 9000e6d1201b7..8242052db9c78 100644 --- a/dev/breeze/doc/images/output_start-airflow.svg +++ b/dev/breeze/doc/images/output_start-airflow.svg @@ -402,8 +402,8 @@ │[default: 3.8]                                              │ │--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ │--integrationCore Integrations to enable when running (can be more than one).                    │ -│(all | all-testable | cassandra | celery | drill | kafka | kerberos | keycloak |    │ -│mongo | mssql | openlineage | otel | pinot | qdrant | redis | statsd | trino | ydb) │ +│(all | all-testable | cassandra | drill | kafka | kerberos | keycloak | mongo |     │ +│mssql | openlineage | otel | pinot | qdrant | redis | statsd | trino | ydb)         │ │--standalone-dag-processorRun standalone dag processor for start-airflow.│ │--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ │--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ diff --git a/dev/breeze/doc/images/output_start-airflow.txt b/dev/breeze/doc/images/output_start-airflow.txt index 5d937717bb884..4c2d092254946 100644 --- a/dev/breeze/doc/images/output_start-airflow.txt +++ b/dev/breeze/doc/images/output_start-airflow.txt @@ -1 +1 @@ -d27f68e5a4c79e3e0bf592e42e6cc60e +2870511bad4c352ff44ac723dfc1a358 diff --git a/dev/breeze/doc/images/output_testing_core-integration-tests.svg b/dev/breeze/doc/images/output_testing_core-integration-tests.svg index 7780769ad012f..ecb9d0ce2082b 100644 --- a/dev/breeze/doc/images/output_testing_core-integration-tests.svg +++ b/dev/breeze/doc/images/output_testing_core-integration-tests.svg @@ -233,8 +233,8 @@ │[default: force-sa-warnings]                                   │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Integration tests ──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--integrationCore Integrations to enable when running (can be more than one).   │ -│(all | all-testable | celery | kerberos | keycloak | otel | statsd)│ +│--integrationCore Integrations to enable when running (can be more than one).│ +│(all | all-testable | kerberos | keycloak | otel | statsd)      │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ │--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ diff --git a/dev/breeze/doc/images/output_testing_core-integration-tests.txt b/dev/breeze/doc/images/output_testing_core-integration-tests.txt index 950ef841307a3..9ad426dde78bd 100644 --- a/dev/breeze/doc/images/output_testing_core-integration-tests.txt +++ b/dev/breeze/doc/images/output_testing_core-integration-tests.txt @@ -1 +1 @@ -6526dbf8cf168bb14b38c4023f652160 +c5567e460679ec98974d9bacdf0fca44 diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index b25fd25ec675f..2162ba8819215 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -85,10 +85,8 @@ ) from airflow_breeze.global_constants import ( ALLOWED_CELERY_BROKERS, - ALLOWED_CELERY_EXECUTORS, ALLOWED_EXECUTORS, ALLOWED_TTY, - CELERY_INTEGRATION, DEFAULT_ALLOWED_EXECUTOR, DEFAULT_CELERY_BROKER, DEFAULT_PYTHON_MAJOR_MINOR_VERSION, @@ -585,12 +583,8 @@ def start_airflow( ) if not executor: - if CELERY_INTEGRATION in integration: - # Default to a celery executor if that's the integration being used - executor = ALLOWED_CELERY_EXECUTORS[0] - else: - # Otherwise default to LocalExecutor - executor = START_AIRFLOW_DEFAULT_ALLOWED_EXECUTOR + # Otherwise default to LocalExecutor + executor = START_AIRFLOW_DEFAULT_ALLOWED_EXECUTOR platform = get_normalized_platform(platform) shell_params = ShellParams( @@ -639,12 +633,6 @@ def start_airflow( rebuild_or_pull_ci_image_if_needed(command_params=shell_params) result = enter_shell(shell_params=shell_params) fix_ownership_using_docker() - if CELERY_INTEGRATION in integration and executor not in ALLOWED_CELERY_EXECUTORS: - get_console().print( - "[warning]A non-Celery executor was used with start-airflow in combination with the Celery " - "integration, this will lead to some processes failing to start (e.g. celery worker)\n" - ) - sys.exit(result.returncode) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index efeb839d3bdfb..12e5e29524a5d 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -233,8 +233,8 @@ class VersionedFile(NamedTuple): file_name: str -AIRFLOW_PIP_VERSION = "25.0" -AIRFLOW_UV_VERSION = "0.5.24" +AIRFLOW_PIP_VERSION = "25.0.1" +AIRFLOW_UV_VERSION = "0.6.14" AIRFLOW_USE_UV = False # TODO: automate these as well WHEEL_VERSION = "0.44.0" diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 6f48f6458833c..4d0acd73929f4 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -57,9 +57,7 @@ ALLOWED_BACKENDS = [SQLITE_BACKEND, MYSQL_BACKEND, POSTGRES_BACKEND, NONE_BACKEND] ALLOWED_PROD_BACKENDS = [MYSQL_BACKEND, POSTGRES_BACKEND] DEFAULT_BACKEND = ALLOWED_BACKENDS[0] -CELERY_INTEGRATION = "celery" TESTABLE_CORE_INTEGRATIONS = [ - CELERY_INTEGRATION, "kerberos", ] TESTABLE_PROVIDERS_INTEGRATIONS = [ @@ -188,8 +186,8 @@ ALLOWED_INSTALL_MYSQL_CLIENT_TYPES = ["mariadb", "mysql"] -PIP_VERSION = "25.0" -UV_VERSION = "0.5.24" +PIP_VERSION = "25.0.1" +UV_VERSION = "0.6.14" DEFAULT_UV_HTTP_TIMEOUT = 300 DEFAULT_WSL2_HTTP_TIMEOUT = 900 diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 61e237ba3937e..2e9d91cf2300c 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -381,7 +381,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "upgrade-to-newer-dependencies": "false", "core-test-types-list-as-string": ALL_CI_SELECTIVE_TEST_TYPES, "providers-test-types-list-as-string": ALL_PROVIDERS_SELECTIVE_TEST_TYPES, - "testable-core-integrations": "['celery', 'kerberos']", + "testable-core-integrations": "['kerberos']", "testable-providers-integrations": "['cassandra', 'drill', 'kafka', 'mongo', 'pinot', 'qdrant', 'redis', 'trino', 'ydb']", "needs-mypy": "true", "mypy-checks": "['mypy-airflow', 'mypy-providers', 'mypy-docs', 'mypy-dev']", diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 037cd4c5a96c8..b038b7b111f51 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -17,6 +17,7 @@ "alibabacloud_adb20211201>=1.0.0", "alibabacloud_tea_openapi>=0.3.7", "apache-airflow>=2.7.0", + "apscheduler<3.11.0; python_version >= \"3.8\"", "oss2>=2.14.0" ], "devel-deps": [], diff --git a/scripts/ci/install_breeze.sh b/scripts/ci/install_breeze.sh index 5259628a579d3..c2c0ae2e6d658 100755 --- a/scripts/ci/install_breeze.sh +++ b/scripts/ci/install_breeze.sh @@ -21,8 +21,8 @@ cd "$( dirname "${BASH_SOURCE[0]}" )/../../" PYTHON_ARG="" -PIP_VERSION="25.0" -UV_VERSION="0.5.24" +PIP_VERSION="25.0.1" +UV_VERSION="0.6.14" if [[ ${PYTHON_VERSION=} != "" ]]; then PYTHON_ARG="--python=$(which python"${PYTHON_VERSION}") " fi diff --git a/scripts/docker/entrypoint_ci.sh b/scripts/docker/entrypoint_ci.sh index 6e9b36507ba68..abd5017fe6ef0 100755 --- a/scripts/docker/entrypoint_ci.sh +++ b/scripts/docker/entrypoint_ci.sh @@ -256,7 +256,7 @@ function check_boto_upgrade() { set -x # shellcheck disable=SC2086 ${PACKAGING_TOOL_CMD} install ${EXTRA_INSTALL_FLAGS} --upgrade boto3 botocore \ - "oss2>=2.14.0" "cryptography<43.0.0" "requests!=2.32.*,<3.0.0,>=2.24.0" + "oss2>=2.14.0" "cryptography<43.0.0" "requests!=2.32.0,!=2.32.1,!=2.32.2,<3.0.0,>=2.24.0" set +x pip check } diff --git a/tests/integration/executors/test_celery_executor.py b/tests/integration/executors/test_celery_executor.py index 9c7fe96ff186d..6c627d89c25eb 100644 --- a/tests/integration/executors/test_celery_executor.py +++ b/tests/integration/executors/test_celery_executor.py @@ -21,11 +21,9 @@ import json import logging import os -import sys from ast import literal_eval from datetime import datetime from importlib import reload -from time import sleep from unittest import mock # leave this it is used by the test worker @@ -34,11 +32,10 @@ from celery import Celery from celery.backends.base import BaseBackend, BaseKeyValueStoreBackend from celery.backends.database import DatabaseBackend -from celery.contrib.testing.worker import start_worker from kombu.asynchronous import set_event_loop from airflow.configuration import conf -from airflow.exceptions import AirflowException, AirflowTaskTimeout +from airflow.exceptions import AirflowTaskTimeout from airflow.executors import base_executor from airflow.models.dag import DAG from airflow.models.taskinstance import SimpleTaskInstance, TaskInstance @@ -130,75 +127,6 @@ def _change_state(self, key: TaskInstanceKey, state: TaskInstanceState, info=Non reload(base_executor) reload(celery_executor) - @pytest.mark.flaky(reruns=3) - @pytest.mark.parametrize("broker_url", _prepare_test_bodies()) - def test_celery_integration(self, broker_url): - from airflow.providers.celery.executors import celery_executor, celery_executor_utils - - success_command = ["airflow", "tasks", "run", "true", "some_parameter"] - fail_command = ["airflow", "version"] - - def fake_execute_command(command): - if command != success_command: - raise AirflowException("fail") - - with _prepare_app(broker_url, execute=fake_execute_command) as app: - executor = celery_executor.CeleryExecutor() - assert executor.tasks == {} - executor.start() - - with start_worker(app=app, logfile=sys.stdout, loglevel="info"): - execute_date = datetime.now() - - task_tuples_to_send = [ - ( - ("success", "fake_simple_ti", execute_date, 0), - success_command, - celery_executor_utils.celery_configuration["task_default_queue"], - celery_executor_utils.execute_command, - ), - ( - ("fail", "fake_simple_ti", execute_date, 0), - fail_command, - celery_executor_utils.celery_configuration["task_default_queue"], - celery_executor_utils.execute_command, - ), - ] - - # "Enqueue" them. We don't have a real SimpleTaskInstance, so directly edit the dict - for key, command, queue, _ in task_tuples_to_send: - executor.queued_tasks[key] = (command, 1, queue, None) - executor.task_publish_retries[key] = 1 - - executor._process_tasks(task_tuples_to_send) - for _ in range(20): - num_tasks = len(executor.tasks.keys()) - if num_tasks == 2: - break - logger.info( - "Waiting 0.1 s for tasks to be processed asynchronously. Processed so far %d", - num_tasks, - ) - sleep(0.4) - assert list(executor.tasks.keys()) == [ - ("success", "fake_simple_ti", execute_date, 0), - ("fail", "fake_simple_ti", execute_date, 0), - ] - assert ( - executor.event_buffer[("success", "fake_simple_ti", execute_date, 0)][0] == State.QUEUED - ) - assert executor.event_buffer[("fail", "fake_simple_ti", execute_date, 0)][0] == State.QUEUED - - executor.end(synchronous=True) - - assert executor.event_buffer[("success", "fake_simple_ti", execute_date, 0)][0] == State.SUCCESS - assert executor.event_buffer[("fail", "fake_simple_ti", execute_date, 0)][0] == State.FAILED - - assert "success" not in executor.tasks - assert "fail" not in executor.tasks - - assert executor.queued_tasks == {} - def test_error_sending_task(self): from airflow.providers.celery.executors import celery_executor