From a85cb5abd6cc5ffcc516f5b1158a907883eeb23e Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 18 Aug 2023 23:14:18 +0200 Subject: [PATCH] Remove "eager upgrade" from PROD image completely There were still some left-overs of EAGER_UPGRADE in PROD image building. Howwever "eager upgrade" only makes sense for CI images. PROD images when being built should use eager upgrades as they are produced in the CI image step. This PR does the following: * removes eager upgrade parameters from PROD image * instead, prod image build has a new flag for installing the images: --use-constraints-for-context-packages which will automatically use constraints from "docker-context-files" if they are present there. * modifies the CI workflows to upload constraints as artifacts and download them for PROD image build when "eager upgrade" has been used and directs it to use "source" constraints * adds back support to "upgrade to newer dependencies" label that makes it easy to test "eager upgrade" As the result, when PROD image is build in CI: * when regular PR is run, it will use latest github "source" constraints * whwn "eager upgrade" PR is run, it will use the eager-upgrade constraints that were generated during CI build --- .github/actions/build-ci-images/action.yml | 6 + .github/actions/build-prod-images/action.yml | 9 +- Dockerfile | 52 ++++- .../commands/ci_image_commands.py | 2 + .../commands/ci_image_commands_config.py | 1 + .../commands/production_image_commands.py | 41 ++-- .../production_image_commands_config.py | 4 +- .../src/airflow_breeze/global_constants.py | 1 + .../airflow_breeze/params/build_ci_params.py | 5 +- .../params/build_prod_params.py | 7 +- .../params/common_build_params.py | 14 +- .../airflow_breeze/utils/common_options.py | 9 + .../airflow_breeze/utils/selective_checks.py | 9 +- dev/breeze/tests/test_selective_checks.py | 21 +- images/breeze/output-commands-hash.txt | 8 +- images/breeze/output-commands.svg | 108 +++++----- images/breeze/output_ci-image.svg | 24 +-- images/breeze/output_ci-image_build.svg | 174 ++++++++-------- images/breeze/output_prod-image.svg | 24 +-- images/breeze/output_prod-image_build.svg | 186 +++++++++--------- .../install_from_docker_context_files.sh | 45 ++++- 21 files changed, 428 insertions(+), 322 deletions(-) diff --git a/.github/actions/build-ci-images/action.yml b/.github/actions/build-ci-images/action.yml index ed613c29ce0c4..d43a42528449f 100644 --- a/.github/actions/build-ci-images/action.yml +++ b/.github/actions/build-ci-images/action.yml @@ -48,6 +48,12 @@ runs: cat "files/constraints-${PYTHON_VERSION}/*.md" >> $GITHUB_STEP_SUMMARY || true done if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false' + - name: "Upload constraint artifacts" + uses: actions/upload-artifact@v3 + with: + name: constraints + path: ./files/constraints-*/constraints-*.txt + retention-days: 7 - name: "Fix ownership" shell: bash run: breeze ci fix-ownership diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml index 0086345b69771..feac8c2ef270a 100644 --- a/.github/actions/build-prod-images/action.yml +++ b/.github/actions/build-prod-images/action.yml @@ -56,11 +56,18 @@ runs: - name: "Move dist packages to docker-context files" shell: bash run: mv -v ./dist/*.whl ./docker-context-files + - name: "Download constraints from the CI build" + uses: actions/download-artifact@v3 + with: + name: constraints + path: ./docker-context-files + if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false' - name: "Build & Push PROD images ${{ env.IMAGE_TAG }}:${{ env.PYTHON_VERSIONS }}" shell: bash run: > breeze prod-image build --tag-as-latest --run-in-parallel --push - --install-packages-from-context --upgrade-on-failure + --install-packages-from-context --airflow-constraints-mode constraints-source-providers + --use-constraints-for-context-packages env: COMMIT_SHA: ${{ github.sha }} - name: "Fix ownership" diff --git a/Dockerfile b/Dockerfile index 43f085ce6afc2..8a4d93211c46b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -593,17 +593,42 @@ function install_airflow_and_providers_from_docker_context_files(){ return fi - echo - echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with eager upgrade${COLOR_RESET}" - echo - # force reinstall all airflow + provider package local files with eager upgrade - set -x - pip install "${pip_flags[@]}" --root-user-action ignore --upgrade --upgrade-strategy eager \ - ${ADDITIONAL_PIP_INSTALL_FLAGS} \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \ - ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} - set +x + if [[ ${USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES=} == "true" ]]; then + local python_version + python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') + local local_constraints_file=/docker-context-files/constraints-"${python_version}"/${AIRFLOW_CONSTRAINTS_MODE}-"${python_version}".txt + if [[ -f "${local_constraints_file}" ]]; then + echo + echo "${COLOR_BLUE}Installing docker-context-files packages with constraints found in ${local_constraints_file}${COLOR_RESET}" + echo + # force reinstall all airflow + provider packages with constraints found in + set -x + pip install "${pip_flags[@]}" --root-user-action ignore --upgrade \ + ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint "${local_constraints_file}" \ + ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + set +x + else + echo + echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}" + echo + set -x + pip install "${pip_flags[@]}" --root-user-action ignore \ + ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" \ + ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + set +x + fi + else + echo + echo "${COLOR_BLUE}Installing docker-context-files packages without constraints${COLOR_RESET}" + echo + set -x + pip install "${pip_flags[@]}" --root-user-action ignore \ + ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + set +x + fi common::install_pip_version pip check } @@ -1280,6 +1305,12 @@ COPY --from=scripts common.sh install_pip_version.sh \ # is installed from docker-context files rather than from PyPI) ARG INSTALL_PACKAGES_FROM_CONTEXT="false" +# Normally constraints are not used when context packages are build - because we might have packages +# that are conflicting with Airflow constraints, however there are cases when we want to use constraints +# for example in CI builds when we already have source-package constraints - either from github branch or +# from eager-upgraded constraints by the CI builds +ARG USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES="false" + # In case of Production build image segment we want to pre-install main version of airflow # dependencies from GitHub so that we do not have to always reinstall it from the scratch. # The Airflow (and providers in case INSTALL_PROVIDERS_FROM_SOURCES is "false") @@ -1304,6 +1335,7 @@ ARG VERSION_SUFFIX_FOR_PYPI="" ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ INSTALL_PACKAGES_FROM_CONTEXT=${INSTALL_PACKAGES_FROM_CONTEXT} \ + USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES=${USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES} \ VERSION_SUFFIX_FOR_PYPI=${VERSION_SUFFIX_FOR_PYPI} WORKDIR ${AIRFLOW_HOME} diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 2eecbfff3ca81..9d343ed571943 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -43,6 +43,7 @@ option_airflow_constraints_mode_ci, option_airflow_constraints_reference_build, option_answer, + option_build_progress, option_build_timeout_minutes, option_builder, option_commit_sha, @@ -226,6 +227,7 @@ def kill_process_group(build_process_group_id: int): @option_additional_dev_apt_command @option_additional_dev_apt_env @option_builder +@option_build_progress @option_build_timeout_minutes @option_commit_sha @option_dev_apt_command diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py index 58509c3c5798c..6e88b70abe11d 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py @@ -36,6 +36,7 @@ "--tag-as-latest", "--docker-cache", "--force-build", + "--build-progress", ], }, { diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 7cbede2041f06..3c8b3b14cf503 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -39,6 +39,7 @@ option_airflow_constraints_location, option_airflow_constraints_mode_prod, option_airflow_constraints_reference_build, + option_build_progress, option_builder, option_commit_sha, option_debug_resources, @@ -67,8 +68,6 @@ option_runtime_apt_deps, option_skip_cleanup, option_tag_as_latest, - option_upgrade_on_failure, - option_upgrade_to_newer_dependencies, option_verbose, option_verify, option_version_suffix_for_pypi, @@ -156,8 +155,6 @@ def prod_image(): @option_debug_resources @option_include_success_outputs @option_python_versions -@option_upgrade_to_newer_dependencies -@option_upgrade_on_failure @option_platform_multiple @option_github_token @option_docker_cache @@ -178,6 +175,12 @@ def prod_image(): "Implies --disable-airflow-repo-cache.", is_flag=True, ) +@click.option( + "--use-constraints-for-context-packages", + help="Uses constraints for context packages installation - " + "either from constraints store in docker-context-files or from github.", + is_flag=True, +) @click.option( "--cleanup-context", help="Clean up docker context files before running build (cannot be used together" @@ -213,6 +216,7 @@ def prod_image(): @option_additional_runtime_apt_env @option_additional_runtime_apt_command @option_builder +@option_build_progress @option_dev_apt_command @option_dev_apt_deps @option_python_image @@ -434,7 +438,12 @@ def check_docker_context_files(install_packages_from_context: bool): :param install_packages_from_context: whether we want to install from docker-context-files """ context_file = DOCKER_CONTEXT_DIR.rglob("*") - any_context_files = any(context.is_file() and context.name != ".README.md" for context in context_file) + any_context_files = any( + context.is_file() + and context.name not in (".README.md", ".DS_Store") + and not context.parent.name.startswith("constraints") + for context in context_file + ) if not any_context_files and install_packages_from_context: get_console().print("[warning]\nERROR! You want to install packages from docker-context-files") get_console().print("[warning]\n but there are no packages to install in this folder.") @@ -501,24 +510,6 @@ def run_build_production_image( text=True, output=output, ) - if ( - build_command_result.returncode != 0 - and prod_image_params.upgrade_on_failure - and not prod_image_params.upgrade_to_newer_dependencies - ): - prod_image_params.upgrade_to_newer_dependencies = True - get_console().print("[warning]Attempting to build with upgrade_to_newer_dependencies on failure") - build_command_result = run_command( - prepare_docker_build_command( - image_params=prod_image_params, - ), - cwd=AIRFLOW_SOURCES_ROOT, - check=False, - text=True, - env=env, - output=output, - ) - if build_command_result.returncode == 0: - if prod_image_params.tag_as_latest: - build_command_result = tag_image_as_latest(image_params=prod_image_params, output=output) + if build_command_result.returncode == 0 and prod_image_params.tag_as_latest: + build_command_result = tag_image_as_latest(image_params=prod_image_params, output=output) return build_command_result.returncode, f"Image build: {prod_image_params.python}" diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py index 148788afc485c..8ffc636521631 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py @@ -31,11 +31,10 @@ "options": [ "--python", "--install-airflow-version", - "--upgrade-to-newer-dependencies", - "--upgrade-on-failure", "--image-tag", "--tag-as-latest", "--docker-cache", + "--build-progress", ], }, { @@ -79,6 +78,7 @@ "name": "Customization options (for specific customization needs)", "options": [ "--install-packages-from-context", + "--use-constraints-for-context-packages", "--cleanup-context", "--disable-mysql-client-installation", "--disable-mssql-client-installation", diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 8827a4962ffeb..9a09513cb6998 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -147,6 +147,7 @@ def all_helm_test_packages() -> list[str]: ALLOWED_INSTALLATION_PACKAGE_FORMATS = ["wheel", "sdist"] ALLOWED_INSTALLATION_METHODS = [".", "apache-airflow"] ALLOWED_BUILD_CACHE = ["registry", "local", "disabled"] +ALLOWED_BUILD_PROGRESS = ["auto", "plain", "tty"] MULTI_PLATFORM = "linux/amd64,linux/arm64" SINGLE_PLATFORMS = ["linux/amd64", "linux/arm64"] ALLOWED_PLATFORMS = [*SINGLE_PLATFORMS, MULTI_PLATFORM] diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py index 1f49f0597b53c..a713c280b54d9 100644 --- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -36,6 +36,8 @@ class BuildCiParams(CommonBuildParams): airflow_extras: str = "devel_ci" airflow_pre_cached_pip_packages: bool = True force_build: bool = False + upgrade_to_newer_dependencies: bool = False + upgrade_on_failure: bool = False eager_upgrade_additional_requirements: str = "" skip_provider_dependencies_check: bool = False @@ -66,7 +68,7 @@ def extra_docker_build_flags(self) -> list[str]: f"EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS={eager_upgrade_arg}", ] ) - return extra_ci_flags + return super().extra_docker_build_flags + extra_ci_flags @property def md5sum_cache_dir(self) -> Path: @@ -111,6 +113,7 @@ def optional_image_args(self) -> list[str]: "additional_python_deps", "version_suffix_for_pypi", "commit_sha", + "build_progress", ] def __post_init__(self): diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index e45eafa3265aa..585c7dc0c9690 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -52,6 +52,7 @@ class BuildProdParams(CommonBuildParams): install_airflow_reference: str = "" install_airflow_version: str = "" install_packages_from_context: bool = False + use_constraints_for_context_packages: bool = False installation_method: str = "." runtime_apt_command: str = "" runtime_apt_deps: str = "" @@ -159,7 +160,6 @@ def extra_docker_build_flags(self) -> list[str]: f"AIRFLOW_CONSTRAINTS_REFERENCE={self.airflow_constraints_reference}", ] ) - maintainers = json.dumps([{"name": "Apache Airflow PMC", "email": "dev@airflow.apache.org"}]) logo_url = "https://github.com/apache/airflow/raw/main/docs/apache-airflow/img/logos/wordmark_1.png" readme_url = "https://raw.githubusercontent.com/apache/airflow/main/docs/docker-stack/README.md" @@ -175,7 +175,7 @@ def extra_docker_build_flags(self) -> list[str]: f"io.artifacthub.package.logo-url={logo_url}", ] ) - return extra_build_flags + return super().extra_docker_build_flags + extra_build_flags @property def airflow_pre_cached_pip_packages(self) -> str: @@ -221,7 +221,6 @@ def required_image_args(self) -> list[str]: "install_postgres_client", "install_providers_from_sources", "python_base_image", - "upgrade_to_newer_dependencies", ] @property @@ -242,4 +241,6 @@ def optional_image_args(self) -> list[str]: "runtime_apt_deps", "version_suffix_for_pypi", "commit_sha", + "build_progress", + "use_constraints_for_context_packages", ] diff --git a/dev/breeze/src/airflow_breeze/params/common_build_params.py b/dev/breeze/src/airflow_breeze/params/common_build_params.py index 90874f67b33bf..fc1de2a4ed2d0 100644 --- a/dev/breeze/src/airflow_breeze/params/common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py @@ -22,7 +22,11 @@ from datetime import datetime from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH -from airflow_breeze.global_constants import APACHE_AIRFLOW_GITHUB_REPOSITORY, DOCKER_DEFAULT_PLATFORM +from airflow_breeze.global_constants import ( + ALLOWED_BUILD_PROGRESS, + APACHE_AIRFLOW_GITHUB_REPOSITORY, + DOCKER_DEFAULT_PLATFORM, +) from airflow_breeze.utils.console import get_console from airflow_breeze.utils.platforms import get_real_platform @@ -46,6 +50,7 @@ class CommonBuildParams: airflow_constraints_location: str = "" build_id: int = 0 builder: str = "autodetect" + build_progress: str = ALLOWED_BUILD_PROGRESS[0] constraints_github_repository: str = APACHE_AIRFLOW_GITHUB_REPOSITORY commit_sha: str = "" dev_apt_command: str = "" @@ -62,8 +67,6 @@ class CommonBuildParams: push: bool = False python: str = "3.8" tag_as_latest: bool = False - upgrade_to_newer_dependencies: bool = False - upgrade_on_failure: bool = False dry_run: bool = False version_suffix_for_pypi: str = "" verbose: bool = False @@ -96,7 +99,10 @@ def airflow_image_name(self): @property def extra_docker_build_flags(self) -> list[str]: - raise NotImplementedError() + extra_flass = [] + if self.build_progress: + extra_flass.append(f"--progress={self.build_progress}") + return extra_flass @property def docker_cache_directive(self) -> list[str]: diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index c5bfe688f23a3..01cff73e27b5a 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -25,6 +25,7 @@ ALL_HISTORICAL_PYTHON_VERSIONS, ALLOWED_BACKENDS, ALLOWED_BUILD_CACHE, + ALLOWED_BUILD_PROGRESS, ALLOWED_CELERY_BROKERS, ALLOWED_CONSTRAINTS_MODES_CI, ALLOWED_CONSTRAINTS_MODES_PROD, @@ -511,6 +512,14 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, show_default=True, default="autodetect", ) +option_build_progress = click.option( + "--build-progress", + help="Build progress.", + type=BetterChoice(ALLOWED_BUILD_PROGRESS), + envvar="BUILD_PROGRESS", + show_default=True, + default=ALLOWED_BUILD_PROGRESS[0], +) option_include_success_outputs = click.option( "--include-success-outputs", help="Whether to include outputs of successful parallel runs (skipped by default).", diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index c6b9aa00b4874..e6b350df19a8e 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -68,6 +68,7 @@ FULL_TESTS_NEEDED_LABEL = "full tests needed" DEBUG_CI_RESOURCES_LABEL = "debug ci resources" USE_PUBLIC_RUNNERS_LABEL = "use public runners" +UPGRADE_TO_NEWER_DEPENDENCIES_LABEL = "upgrade to newer dependencies" class FileGroupForCi(Enum): @@ -685,9 +686,11 @@ def basic_checks_only(self) -> bool: @cached_property def upgrade_to_newer_dependencies(self) -> bool: - return len( - self._matching_files(FileGroupForCi.SETUP_FILES, CI_FILE_GROUP_MATCHES) - ) > 0 or self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE] + return ( + len(self._matching_files(FileGroupForCi.SETUP_FILES, CI_FILE_GROUP_MATCHES)) > 0 + or self._github_event in [GithubEvents.PUSH, GithubEvents.SCHEDULE] + or UPGRADE_TO_NEWER_DEPENDENCIES_LABEL in self._pr_labels + ) @cached_property def docs_filter_list_as_string(self) -> str | None: diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 413fe8be71634..7711efc526975 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -900,13 +900,14 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): @pytest.mark.parametrize( - "files, expected_outputs,", + "files, expected_outputs, pr_labels", [ pytest.param( ("airflow/models/dag.py",), { "upgrade-to-newer-dependencies": "false", }, + (), id="Regular source changed", ), pytest.param( @@ -914,6 +915,7 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): { "upgrade-to-newer-dependencies": "true", }, + (), id="Setup.py changed", ), pytest.param( @@ -921,6 +923,7 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): { "upgrade-to-newer-dependencies": "true", }, + (), id="Setup.cfg changed", ), pytest.param( @@ -928,6 +931,7 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): { "upgrade-to-newer-dependencies": "false", }, + (), id="Provider.yaml changed", ), pytest.param( @@ -935,17 +939,28 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): { "upgrade-to-newer-dependencies": "true", }, + (), id="Generated provider_dependencies changed", ), + pytest.param( + ("airflow/models/dag.py",), + { + "upgrade-to-newer-dependencies": "true", + }, + ("upgrade to newer dependencies",), + id="Regular source changed", + ), ], ) -def test_upgrade_to_newer_dependencies(files: tuple[str, ...], expected_outputs: dict[str, str]): +def test_upgrade_to_newer_dependencies( + files: tuple[str, ...], expected_outputs: dict[str, str], pr_labels: tuple[str] +): stderr = SelectiveChecks( files=files, commit_ref="HEAD", github_event=GithubEvents.PULL_REQUEST, - pr_labels=(), default_branch="main", + pr_labels=pr_labels, ) assert_outputs_are_printed(expected_outputs, str(stderr)) diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index bed222b5cca4c..94498d7b19cee 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -10,10 +10,10 @@ ci:get-workflow-info:8246038093359b9c3c110043419473e2 ci:resource-check:bfcca92f18a403ca630955074eb5e9ad ci:selective-check:6657ed5d42affb7264b5efcc86f17a2a ci:5315c29bd9f68725ef92e4db8aff5cda -ci-image:build:5606d9d3c3db807228b6c6458e4c5018 +ci-image:build:0598b5423e61a2431b2449135475f9ff ci-image:pull:7f14482a588f018f76df84719e77723f ci-image:verify:c90dc7e20fce2351eb89d8d1ebbd35e7 -ci-image:627399747ac8a202cd8a8c03c9bdabbf +ci-image:6b058951af1e540d9c2f5c2c7af23183 cleanup:8d92d453a6700f6d8cb11fb6a8b50461 compile-www-assets:0963f1409f0aa1e3b137cddd4cc52e87 down:4580f5b3b178ea00182694f134a751f3 @@ -32,10 +32,10 @@ k8s:status:1529ccd444b41c4b0b5f943289957100 k8s:tests:2a1e2928faea2eddafaff94176a46690 k8s:upload-k8s-image:6b3a20cdeb692f3c3d727f6b9e68c901 k8s:c73e0ebdff75d89e35af6e324a3bc527 -prod-image:build:10cc859b7d898581bf9b2a24c19c1032 +prod-image:build:c1bd48b7f61eb2f0b82c0161235a4d66 prod-image:pull:76f1f27e6119928412abecf153fce4bb prod-image:verify:bd2b78738a7c388dbad6076c41a9f906 -prod-image:d9b47217a12f73f214bcd2938bbab84a +prod-image:887b1eb302942c62c40164f8e5e9d5f5 release-management:add-back-references:7e3a4e6d3f7932773cfb256059f7c2c8 release-management:create-minor-branch:a3834afc4aa5d1e98002c9e9e7a9931d release-management:generate-constraints:b8fcaf8f0acd35ed5dbd48659bdb6485 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 51829ca34b0dc..8b9d38fa91877 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -35,8 +35,8 @@ .breeze-help-r1 { fill: #c5c8c6;font-weight: bold } .breeze-help-r2 { fill: #c5c8c6 } .breeze-help-r3 { fill: #d0b344;font-weight: bold } -.breeze-help-r4 { fill: #68a0b3;font-weight: bold } -.breeze-help-r5 { fill: #868887 } +.breeze-help-r4 { fill: #868887 } +.breeze-help-r5 { fill: #68a0b3;font-weight: bold } .breeze-help-r6 { fill: #98a84b;font-weight: bold } .breeze-help-r7 { fill: #8d7b39 } @@ -217,59 +217,59 @@ -Usage: breeze [OPTIONS] COMMAND [ARGS]... +Usage: breeze [OPTIONS] COMMAND [ARGS]... -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ -│[default: 3.8]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8.0 | 8.1)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ -│statsd | statsd | trino)                                                                 │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│[default: autodetect]                                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        Build documents.                                                                                │ -│down              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing commands ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing        Tools that developers can use to run tests                                                         │ -│k8s            Tools that developers use to run Kubernetes tests                                                  │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Image commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ci-image         Tools that developers can use to manually manage CI images                                       │ -│prod-image       Tools that developers can use to manually manage PROD images                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Release management commands ────────────────────────────────────────────────────────────────────────────────────────╮ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│sbom                   Tools that release managers can use to prepare sbom information                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Other commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│setup     Tools that developers can use to configure Breeze                                                       │ -│ci        Tools that CI workflows use to cleanup/manage CI environment                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ +│[default: 3.8]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>11< | 12 | 13 | 14 | 15)[default: 11]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8.0 | 8.1)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(all | all-testable | cassandra | celery | kafka | kerberos | mongo | otel | pinot |     │ +│statsd | statsd | trino)                                                                 │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│[default: autodetect]                                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Developer commands ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        Build documents.                                                                                │ +│down              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing commands ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing        Tools that developers can use to run tests                                                         │ +│k8s            Tools that developers use to run Kubernetes tests                                                  │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Image commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ci-image         Tools that developers can use to manually manage CI images                                       │ +│prod-image       Tools that developers can use to manually manage PROD images                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Release management commands ────────────────────────────────────────────────────────────────────────────────────────╮ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│sbom                   Tools that release managers can use to prepare sbom information                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Other commands ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│setup     Tools that developers can use to configure Breeze                                                       │ +│ci        Tools that CI workflows use to cleanup/manage CI environment                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image.svg b/images/breeze/output_ci-image.svg index bf1df9347a162..3025e6e470c36 100644 --- a/images/breeze/output_ci-image.svg +++ b/images/breeze/output_ci-image.svg @@ -35,8 +35,8 @@ .breeze-ci-image-r1 { fill: #c5c8c6;font-weight: bold } .breeze-ci-image-r2 { fill: #c5c8c6 } .breeze-ci-image-r3 { fill: #d0b344;font-weight: bold } -.breeze-ci-image-r4 { fill: #68a0b3;font-weight: bold } -.breeze-ci-image-r5 { fill: #868887 } +.breeze-ci-image-r4 { fill: #868887 } +.breeze-ci-image-r5 { fill: #68a0b3;font-weight: bold } .breeze-ci-image-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze ci-image [OPTIONS] COMMAND [ARGS]... +Usage: breeze ci-image [OPTIONS] COMMAND [ARGS]... -Tools that developers can use to manually manage CI images +Tools that developers can use to manually manage CI images -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│build   Build CI image. Include building multiple images for all python versions.                                 │ -│pull    Pull and optionally verify CI images - possibly in parallel for all Python versions.                      │ -│verify  Verify CI image.                                                                                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│build   Build CI image. Include building multiple images for all python versions.                                 │ +│pull    Pull and optionally verify CI images - possibly in parallel for all Python versions.                      │ +│verify  Verify CI image.                                                                                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image_build.svg b/images/breeze/output_ci-image_build.svg index 1418e4c43c76b..2f6e1d26a790b 100644 --- a/images/breeze/output_ci-image_build.svg +++ b/images/breeze/output_ci-image_build.svg @@ -1,4 +1,4 @@ - + - + @@ -291,9 +291,12 @@ + + + - Command: ci-image build + Command: ci-image build @@ -304,88 +307,89 @@ -Usage: breeze ci-image build [OPTIONS] +Usage: breeze ci-image build [OPTIONS] -Build CI image. Include building multiple images for all python versions. +Build CI image. Include building multiple images for all python versions. -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.8< | 3.9 | 3.10 | 3.11)                                 │ -│[default: 3.8]                                              │ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ -│fails.                                                                         │ -│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -│--force-buildForce image build no matter if it is determined as needed.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.8 3.9 3.10 3.11]                                                  │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ -│file. It could be full remote URL to the location file, or local file placed in │ -│`docker-context-files` (in this case it has to start with                       │ -│/opt/airflow/docker-context-files).                                             │ -│(TEXT)                                                                          │ -│--airflow-constraints-modeMode of constraints for CI image building.                             │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye.                                   │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ -│itself).                                                                        │ -│(TEXT)                                                                          │ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ -│--commit-shaCommit SHA that is used to build the images.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Backtracking options ───────────────────────────────────────────────────────────────────────────────────────────────╮ -│--build-timeout-minutesOptional timeout for the build in minutes. Useful to detect `pip`        │ -│backtracking problems.                                                   │ -│(INTEGER)                                                                │ -│--eager-upgrade-additional-requirementsOptional additional requirements to upgrade eagerly to avoid backtracking│ -│(see `breeze ci find-backtracking-candidates`).                          │ -│(TEXT)                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│[default: autodetect]                                         │ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.8< | 3.9 | 3.10 | 3.11)                                 │ +│[default: 3.8]                                              │ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ +│fails.                                                                         │ +│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--build-progressBuild progress.(auto | plain | tty)[default: auto]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.8 3.9 3.10 3.11]                                                  │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ +│file. It could be full remote URL to the location file, or local file placed in │ +│`docker-context-files` (in this case it has to start with                       │ +│/opt/airflow/docker-context-files).                                             │ +│(TEXT)                                                                          │ +│--airflow-constraints-modeMode of constraints for CI image building.                             │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye.                                   │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ +│itself).                                                                        │ +│(TEXT)                                                                          │ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ +│--commit-shaCommit SHA that is used to build the images.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Backtracking options ───────────────────────────────────────────────────────────────────────────────────────────────╮ +│--build-timeout-minutesOptional timeout for the build in minutes. Useful to detect `pip`        │ +│backtracking problems.                                                   │ +│(INTEGER)                                                                │ +│--eager-upgrade-additional-requirementsOptional additional requirements to upgrade eagerly to avoid backtracking│ +│(see `breeze ci find-backtracking-candidates`).                          │ +│(TEXT)                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│[default: autodetect]                                         │ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image.svg b/images/breeze/output_prod-image.svg index 35bf4ae86bb6e..cd179a225efee 100644 --- a/images/breeze/output_prod-image.svg +++ b/images/breeze/output_prod-image.svg @@ -35,8 +35,8 @@ .breeze-prod-image-r1 { fill: #c5c8c6;font-weight: bold } .breeze-prod-image-r2 { fill: #c5c8c6 } .breeze-prod-image-r3 { fill: #d0b344;font-weight: bold } -.breeze-prod-image-r4 { fill: #68a0b3;font-weight: bold } -.breeze-prod-image-r5 { fill: #868887 } +.breeze-prod-image-r4 { fill: #868887 } +.breeze-prod-image-r5 { fill: #68a0b3;font-weight: bold } .breeze-prod-image-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze prod-image [OPTIONS] COMMAND [ARGS]... +Usage: breeze prod-image [OPTIONS] COMMAND [ARGS]... -Tools that developers can use to manually manage PROD images +Tools that developers can use to manually manage PROD images -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│build  Build Production image. Include building multiple images for all or selected Python versions sequentially. │ -│pull   Pull and optionally verify Production images - possibly in parallel for all Python versions.               │ -│verify Verify Production image.                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│build  Build Production image. Include building multiple images for all or selected Python versions sequentially. │ +│pull   Pull and optionally verify Production images - possibly in parallel for all Python versions.               │ +│verify Verify Production image.                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image_build.svg b/images/breeze/output_prod-image_build.svg index 3b3973b945729..1424dc9ee3cc6 100644 --- a/images/breeze/output_prod-image_build.svg +++ b/images/breeze/output_prod-image_build.svg @@ -35,8 +35,8 @@ .breeze-prod-image-build-r1 { fill: #c5c8c6;font-weight: bold } .breeze-prod-image-build-r2 { fill: #c5c8c6 } .breeze-prod-image-build-r3 { fill: #d0b344;font-weight: bold } -.breeze-prod-image-build-r4 { fill: #68a0b3;font-weight: bold } -.breeze-prod-image-build-r5 { fill: #868887 } +.breeze-prod-image-build-r4 { fill: #868887 } +.breeze-prod-image-build-r5 { fill: #68a0b3;font-weight: bold } .breeze-prod-image-build-r6 { fill: #98a84b;font-weight: bold } .breeze-prod-image-build-r7 { fill: #8d7b39 } @@ -340,100 +340,100 @@ -Usage: breeze prod-image build [OPTIONS] +Usage: breeze prod-image build [OPTIONS] Build Production image. Include building multiple images for all or selected Python versions sequentially. -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.8< | 3.9 | 3.10 | 3.11)                                 │ -│[default: 3.8]                                              │ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ -│fails.                                                                         │ -│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.8 3.9 3.10 3.11]                                                  │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-extrasExtras to install by default.                                                   │ -│(TEXT)                                                                          │ -│[default:                                                                       │ -│aiobotocore,amazon,async,celery,cncf.kubernetes,daskexecutor,docker,elasticsear…│ -│--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ -│file. It could be full remote URL to the location file, or local file placed in │ -│`docker-context-files` (in this case it has to start with                       │ -│/opt/airflow/docker-context-files).                                             │ -│(TEXT)                                                                          │ -│--airflow-constraints-modeMode of constraints for PROD image building.                           │ -│(constraints | constraints-no-providers | constraints-source-providers)│ -│[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye.                                   │ -│(TEXT)                                                                          │ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ -│itself).                                                                        │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ -│--commit-shaCommit SHA that is used to build the images.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ -│Implies --disable-airflow-repo-cache.                                     │ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│[default: autodetect]                                         │ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.8< | 3.9 | 3.10 | 3.11)                                 │ +│[default: 3.8]                                              │ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when │ +│you build or pull image with --image-tag.                                            │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--build-progressBuild progress.(auto | plain | tty)[default: auto]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.8 3.9 3.10 3.11]                                                  │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-extrasExtras to install by default.                                                   │ +│(TEXT)                                                                          │ +│[default:                                                                       │ +│aiobotocore,amazon,async,celery,cncf.kubernetes,daskexecutor,docker,elasticsear…│ +│--airflow-constraints-locationIf specified, it is used instead of calculating reference to the constraint     │ +│file. It could be full remote URL to the location file, or local file placed in │ +│`docker-context-files` (in this case it has to start with                       │ +│/opt/airflow/docker-context-files).                                             │ +│(TEXT)                                                                          │ +│--airflow-constraints-modeMode of constraints for PROD image building.                           │ +│(constraints | constraints-no-providers | constraints-source-providers)│ +│[default: constraints]                                                 │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye.                                   │ +│(TEXT)                                                                          │ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ +│itself).                                                                        │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +│--version-suffix-for-pypiVersion suffix used for PyPI packages (alpha, beta, rc1, etc.).(TEXT)│ +│--commit-shaCommit SHA that is used to build the images.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ +│Implies --disable-airflow-repo-cache.                                     │ +│--use-constraints-for-context-packagesUses constraints for context packages installation - either from          │ +│constraints store in docker-context-files or from github.                 │ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│[default: autodetect]                                         │ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh index fa08f47626f8a..cdb66875ac466 100644 --- a/scripts/docker/install_from_docker_context_files.sh +++ b/scripts/docker/install_from_docker_context_files.sh @@ -74,17 +74,42 @@ function install_airflow_and_providers_from_docker_context_files(){ return fi - echo - echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with eager upgrade${COLOR_RESET}" - echo - # force reinstall all airflow + provider package local files with eager upgrade - set -x - pip install "${pip_flags[@]}" --root-user-action ignore --upgrade --upgrade-strategy eager \ - ${ADDITIONAL_PIP_INSTALL_FLAGS} \ - ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} \ - ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=} - set +x + if [[ ${USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES=} == "true" ]]; then + local python_version + python_version=$(python -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') + local local_constraints_file=/docker-context-files/constraints-"${python_version}"/${AIRFLOW_CONSTRAINTS_MODE}-"${python_version}".txt + if [[ -f "${local_constraints_file}" ]]; then + echo + echo "${COLOR_BLUE}Installing docker-context-files packages with constraints found in ${local_constraints_file}${COLOR_RESET}" + echo + # force reinstall all airflow + provider packages with constraints found in + set -x + pip install "${pip_flags[@]}" --root-user-action ignore --upgrade \ + ${ADDITIONAL_PIP_INSTALL_FLAGS} --constraint "${local_constraints_file}" \ + ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + set +x + else + echo + echo "${COLOR_BLUE}Installing docker-context-files packages with constraints from GitHub${COLOR_RESET}" + echo + set -x + pip install "${pip_flags[@]}" --root-user-action ignore \ + ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" \ + ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + set +x + fi + else + echo + echo "${COLOR_BLUE}Installing docker-context-files packages without constraints${COLOR_RESET}" + echo + set -x + pip install "${pip_flags[@]}" --root-user-action ignore \ + ${ADDITIONAL_PIP_INSTALL_FLAGS} \ + ${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages} + set +x + fi common::install_pip_version pip check }